diff --git a/MAINTAINERS b/MAINTAINERS index ca1c59e1fc59..3ed1991cac46 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5992,6 +5992,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can.git T: git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can-next.git F: Documentation/devicetree/bindings/net/can/ F: Documentation/devicetree/bindings/phy/ti,tcan104x-can.yaml +F: Documentation/networking/device_drivers/can/ F: drivers/net/can/ F: drivers/phy/phy-can-transceiver.c F: include/linux/can/bittiming.h @@ -19600,6 +19601,10 @@ F: samples/pktgen/ F: tools/net/ F: tools/testing/selftests/net/ X: Documentation/networking/batman-adv.rst +X: Documentation/networking/can.rst +X: Documentation/networking/device_drivers/can/ +X: Documentation/networking/iso15765-2.rst +X: Documentation/networking/j1939.rst X: Documentation/networking/mac80211-injection.rst X: Documentation/networking/mac80211_hwsim/ X: Documentation/networking/regulatory.rst @@ -30680,6 +30685,7 @@ F: drivers/iio/adc/xilinx-ams.c XILINX AXI ETHERNET DRIVER M: Radhey Shyam Pandey +M: Suraj Gupta S: Maintained F: Documentation/devicetree/bindings/net/xlnx,axi-ethernet.yaml F: drivers/net/ethernet/xilinx/xilinx_axienet* diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 929018428c19..4cc5273fb175 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -1511,11 +1511,8 @@ static int btintel_read_debug_features(struct hci_dev *hdev, */ skb = __hci_cmd_sync(hdev, 0xfca6, sizeof(page_no), &page_no, HCI_INIT_TIMEOUT); - if (IS_ERR(skb)) { - bt_dev_err(hdev, "Reading supported features failed (%ld)", - PTR_ERR(skb)); + if (IS_ERR(skb)) return PTR_ERR(skb); - } if (skb->len != (sizeof(features->page1) + 3)) { bt_dev_err(hdev, "Supported features event size mismatch"); @@ -1632,8 +1629,10 @@ int btintel_set_quality_report(struct hci_dev *hdev, bool enable) * supported, need to load the additional DDC config to enable. */ err = btintel_read_debug_features(hdev, &features); - if (err) + if (err) { + bt_dev_err(hdev, "Reading supported features failed (%d)", err); return err; + } /* Set or reset the debug features. */ if (enable) @@ -1684,7 +1683,7 @@ static int btintel_register_devcoredump_support(struct hci_dev *hdev) err = btintel_read_debug_features(hdev, &features); if (err) { - bt_dev_info(hdev, "Error reading debug features"); + bt_dev_dbg(hdev, "Reading supported features failed (%d)", err); return err; } diff --git a/drivers/bluetooth/btintel_pcie.h b/drivers/bluetooth/btintel_pcie.h index 3030b4e8b750..5f315ebb4123 100644 --- a/drivers/bluetooth/btintel_pcie.h +++ b/drivers/bluetooth/btintel_pcie.h @@ -72,7 +72,7 @@ #define BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS (BTINTEL_PCIE_DBGC_BASE_ADDR + 0x1C) #define BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND (BTINTEL_PCIE_DBGC_BASE_ADDR + 0x2C) -#define BTINTEL_PCIE_DBGC_BASE_ADDR_SCP (0xf0d5d500) +#define BTINTEL_PCIE_DBGC_BASE_ADDR_SCP (0xf0d5d400) #define BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS_SCP (BTINTEL_PCIE_DBGC_BASE_ADDR_SCP + 0x1C) #define BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND_SCP (BTINTEL_PCIE_DBGC_BASE_ADDR_SCP + 0x2C) #define BTINTEL_PCIE_DBGC_ALLOCATION_OFFSET 0x100 diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c index 60735b70d89a..9f54e33e4250 100644 --- a/drivers/bluetooth/btqca.c +++ b/drivers/bluetooth/btqca.c @@ -983,6 +983,7 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate, switch (soc_type) { case QCA_WCN3991: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: /* get fw build info */ diff --git a/drivers/bluetooth/hci_mrvl.c b/drivers/bluetooth/hci_mrvl.c index 57e2d80827df..3e542956be1a 100644 --- a/drivers/bluetooth/hci_mrvl.c +++ b/drivers/bluetooth/hci_mrvl.c @@ -57,7 +57,6 @@ struct hci_mrvl_pkt { static int mrvl_open(struct hci_uart *hu) { struct mrvl_data *mrvl; - int ret; BT_DBG("hu %p", hu); @@ -75,17 +74,7 @@ static int mrvl_open(struct hci_uart *hu) hu->priv = mrvl; - if (hu->serdev) { - ret = serdev_device_open(hu->serdev); - if (ret) - goto err; - } - return 0; -err: - kfree(mrvl); - - return ret; } static int mrvl_close(struct hci_uart *hu) @@ -94,9 +83,6 @@ static int mrvl_close(struct hci_uart *hu) BT_DBG("hu %p", hu); - if (hu->serdev) - serdev_device_close(hu->serdev); - skb_queue_purge(&mrvl->txq); skb_queue_purge(&mrvl->rawq); kfree_skb(mrvl->rx_skb); diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index 4cebd0a9998d..158f294c1890 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -1132,6 +1132,7 @@ static void qca_controller_memdump(struct work_struct *work) if (!test_bit(QCA_MEMDUMP_COLLECTION, &qca->flags)) { bt_dev_err(hu->hdev, "QCA: Discarding other packets"); kfree(qca_memdump); + qca->qca_memdump = NULL; kfree_skb(skb); mutex_unlock(&qca->hci_memdump_lock); return; @@ -1386,6 +1387,7 @@ static int qca_set_baudrate(struct hci_dev *hdev, uint8_t baudrate) case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: usleep_range(1000, 10000); @@ -1473,6 +1475,7 @@ static int qca_check_speeds(struct hci_uart *hu) case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: if (!qca_get_speed(hu, QCA_INIT_SPEED) && @@ -1516,6 +1519,7 @@ static int qca_set_speed(struct hci_uart *hu, enum qca_speed_type speed_type) case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: hci_uart_set_flow_control(hu, true); @@ -1551,6 +1555,7 @@ error: case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: hci_uart_set_flow_control(hu, false); @@ -1750,6 +1755,27 @@ static bool qca_wakeup(struct hci_dev *hdev) return wakeup; } +static void qca_port_close(struct hci_uart *hu) +{ + /* HCI_UART_PROTO_READY tracks whether the port is open, so a port + * left closed by a failed qca_port_open() is not closed again by + * hci_uart_close() or hci_uart_unregister_device(). + */ + if (test_and_clear_bit(HCI_UART_PROTO_READY, &hu->flags)) + serdev_device_close(hu->serdev); +} + +static int qca_port_open(struct hci_uart *hu) +{ + int ret; + + ret = serdev_device_open(hu->serdev); + if (!ret) + set_bit(HCI_UART_PROTO_READY, &hu->flags); + + return ret; +} + static int qca_port_reopen(struct hci_uart *hu) { int ret; @@ -1759,8 +1785,8 @@ static int qca_port_reopen(struct hci_uart *hu) * Without this, we will have RTS and CTS synchronization * issues. */ - serdev_device_close(hu->serdev); - ret = serdev_device_open(hu->serdev); + qca_port_close(hu); + ret = qca_port_open(hu); if (ret) { bt_dev_err(hu->hdev, "failed to open port"); return ret; @@ -1784,12 +1810,12 @@ static int qca_regulator_init(struct hci_uart *hu) qcadev = serdev_device_get_drvdata(hu->serdev); if (!qcadev->bt_power->vregs_on) { - serdev_device_close(hu->serdev); + qca_port_close(hu); ret = qca_regulator_enable(qcadev); if (ret) return ret; - ret = serdev_device_open(hu->serdev); + ret = qca_port_open(hu); if (ret) { bt_dev_err(hu->hdev, "failed to open port"); return ret; @@ -1867,6 +1893,7 @@ static int qca_power_on(struct hci_dev *hdev) case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: ret = qca_regulator_init(hu); @@ -1989,6 +2016,7 @@ retry: case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: if (qcadev && qcadev->bdaddr_property_broken) @@ -2022,6 +2050,7 @@ retry: case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: break; @@ -2065,8 +2094,8 @@ out: if (retries < MAX_INIT_RETRIES) { bt_dev_warn(hdev, "Retry BT power ON:%d", retries); if (hu->serdev) { - serdev_device_close(hu->serdev); - ret = serdev_device_open(hu->serdev); + qca_port_close(hu); + ret = qca_port_open(hu); if (ret) { bt_dev_err(hdev, "failed to open port"); return ret; @@ -2284,6 +2313,7 @@ static void qca_power_off(struct hci_uart *hu) break; case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: gpiod_set_value_cansleep(qcadev->bt_en, 0); msleep(100); @@ -2438,6 +2468,7 @@ static int qca_serdev_probe(struct serdev_device *serdev) case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: qcadev->bt_power = devm_kzalloc_obj(&serdev->dev, @@ -2456,6 +2487,7 @@ static int qca_serdev_probe(struct serdev_device *serdev) case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: if (!device_property_present(&serdev->dev, "enable-gpios")) { @@ -2499,6 +2531,7 @@ static int qca_serdev_probe(struct serdev_device *serdev) if (!qcadev->bt_en && (data->soc_type == QCA_WCN6750 || + data->soc_type == QCA_QCA2066 || data->soc_type == QCA_WCN6855 || data->soc_type == QCA_WCN7850)) power_ctrl_enabled = false; @@ -2507,6 +2540,7 @@ static int qca_serdev_probe(struct serdev_device *serdev) GPIOD_IN); if (IS_ERR(qcadev->sw_ctrl) && (data->soc_type == QCA_WCN6750 || + data->soc_type == QCA_QCA2066 || data->soc_type == QCA_WCN6855 || data->soc_type == QCA_WCN7850)) { dev_err(&serdev->dev, "failed to acquire SW_CTRL gpio\n"); @@ -2591,6 +2625,7 @@ static void qca_serdev_remove(struct serdev_device *serdev) case QCA_WCN3991: case QCA_WCN3998: case QCA_WCN6750: + case QCA_QCA2066: case QCA_WCN6855: case QCA_WCN7850: if (power->vregs_on) diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c index 48ba017e219b..194f11251cc5 100644 --- a/drivers/bluetooth/hci_serdev.c +++ b/drivers/bluetooth/hci_serdev.c @@ -97,15 +97,26 @@ static int hci_uart_flush(struct hci_dev *hdev) BT_DBG("hdev %p serdev %p", hdev, hu->serdev); + /* Stop write_work and hold SENDING so wakeups only set TX_WAKEUP. */ + disable_work_sync(&hu->write_work); + set_bit(HCI_UART_SENDING, &hu->tx_state); + if (hu->tx_skb) { kfree_skb(hu->tx_skb); hu->tx_skb = NULL; } - /* Flush any pending characters in the driver and discipline. */ - serdev_device_write_flush(hu->serdev); + if (test_bit(HCI_UART_PROTO_READY, &hu->flags)) { + /* Flush any pending characters in the driver and discipline. */ + serdev_device_write_flush(hu->serdev); - if (test_bit(HCI_UART_PROTO_READY, &hu->flags)) hu->proto->flush(hu); + } + + /* Resume TX and replay a wakeup recorded while it was stopped. */ + enable_work(&hu->write_work); + clear_bit(HCI_UART_SENDING, &hu->tx_state); + if (test_bit(HCI_UART_TX_WAKEUP, &hu->tx_state)) + hci_uart_tx_wakeup(hu); return 0; } diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c index 8f7db0102b25..7fec02705e58 100644 --- a/drivers/bluetooth/hci_vhci.c +++ b/drivers/bluetooth/hci_vhci.c @@ -524,7 +524,9 @@ static inline ssize_t vhci_get_user(struct vhci_data *data, case HCI_ACLDATA_PKT: case HCI_SCODATA_PKT: case HCI_ISODATA_PKT: + mutex_lock(&data->open_mutex); if (!data->hdev) { + mutex_unlock(&data->open_mutex); kfree_skb(skb); return -ENODEV; } @@ -532,6 +534,7 @@ static inline ssize_t vhci_get_user(struct vhci_data *data, hci_skb_pkt_type(skb) = pkt_type; ret = hci_recv_frame(data->hdev, skb); + mutex_unlock(&data->open_mutex); break; case HCI_VENDOR_PKT: @@ -701,6 +704,8 @@ static int vhci_release(struct inode *inode, struct file *file) if (hdev) { if (!IS_ERR_OR_NULL(hdev->debugfs)) vhci_debugfs_remove(hdev); + /* Drain work queued by the last force_suspend writers. */ + flush_work(&data->suspend_work); hci_unregister_dev(hdev); hci_free_dev(hdev); } diff --git a/drivers/net/amt.c b/drivers/net/amt.c index 8b9fc3d11ece..b145ed3ca6ed 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -35,9 +35,19 @@ static struct workqueue_struct *amt_wq; static HLIST_HEAD(source_gc_list); -/* Lock for source_gc_list */ +/* Lock for source_gc_list, group_gc_list and the counts below */ static spinlock_t source_gc_lock; static struct delayed_work source_gc_wq; +static HLIST_HEAD(group_gc_list); +static struct delayed_work group_gc_wq; +/* Nodes parked on source_gc_list and group_gc_list, and the sources among + * them, so that a drain of either list can take its nodes off the count. + * Parked nodes are not charged to max_groups or max_sources, so + * Report/Leave churn could park them faster than the AMT_GC_INTERVAL tick + * frees them: once AMT_GC_KICK are parked, the drain runs at once. + */ +static unsigned int amt_gc_queued; +static unsigned int amt_gc_sources; static char *status_str[] = { "AMT_STATUS_INIT", "AMT_STATUS_SENT_DISCOVERY", @@ -88,11 +98,15 @@ static void __amt_source_gc_work(void) spin_lock_bh(&source_gc_lock); hlist_move_list(&source_gc_list, &gc_list); + amt_gc_queued -= amt_gc_sources; + amt_gc_sources = 0; spin_unlock_bh(&source_gc_lock); - hlist_for_each_entry_safe(snode, t, &gc_list, node) { - hlist_del_rcu(&snode->node); + hlist_for_each_entry_safe(snode, t, &gc_list, gc_node) { + cancel_delayed_work_sync(&snode->source_timer); + hlist_del_init(&snode->gc_node); kfree_rcu(snode, rcu); + cond_resched(); } } @@ -106,6 +120,62 @@ static void amt_source_gc_work(struct work_struct *work) spin_unlock_bh(&source_gc_lock); } +static void __amt_group_gc_work(void) +{ + struct amt_group_node *gnode; + struct hlist_head gc_list; + struct hlist_node *t; + + spin_lock_bh(&source_gc_lock); + hlist_move_list(&group_gc_list, &gc_list); + amt_gc_queued = amt_gc_sources; + spin_unlock_bh(&source_gc_lock); + + /* Group deletion queues all of its sources first. Reclaim them now + * rather than at the next periodic run; a source callback cannot + * reach a freed group because it resolves snode->gnode under RCU and + * the pointer is cleared before the source is queued. Call the GC + * here rather than flush source_gc_wq: a flush queues the work and + * waits for it to run, which needs a free amt_wq active slot, and + * when this runs on amt_wq every slot can be held by such waiters. + * The cancel_delayed_work_sync() calls in the GC need no slot: each + * takes a pending timer work off its timer or queue, or waits for a + * running one on a barrier that max_active does not count. + */ + __amt_source_gc_work(); + + hlist_for_each_entry_safe(gnode, t, &gc_list, gc_node) { + cancel_delayed_work_sync(&gnode->group_timer); + hlist_del_init(&gnode->gc_node); + kfree_rcu(gnode, rcu); + cond_resched(); + } +} + +static void amt_group_gc_work(struct work_struct *work) +{ + __amt_group_gc_work(); + + /* Run again at once if AMT_GC_KICK nodes were parked meanwhile: a + * kick that landed during the drain must not be pushed back to the + * interval. + */ + spin_lock_bh(&source_gc_lock); + mod_delayed_work(amt_wq, &group_gc_wq, + amt_gc_queued >= AMT_GC_KICK ? 0 : + msecs_to_jiffies(AMT_GC_INTERVAL)); + spin_unlock_bh(&source_gc_lock); +} + +/* Caller holds source_gc_lock and has just parked one node. This runs + * under tunnel->lock, often in softirq; mod_delayed_work() does not sleep. + */ +static void amt_gc_parked(void) +{ + if (++amt_gc_queued == AMT_GC_KICK) + mod_delayed_work(amt_wq, &group_gc_wq, 0); +} + static bool amt_addr_equal(union amt_addr *a, union amt_addr *b) { return !memcmp(a, b, sizeof(union amt_addr)); @@ -215,8 +285,13 @@ static struct amt_group_node *amt_lookup_group(struct amt_tunnel_list *tunnel, static void amt_destroy_source(struct amt_source_node *snode) { - struct amt_group_node *gnode = snode->gnode; struct amt_tunnel_list *tunnel; + struct amt_group_node *gnode; + + if (snode->dead) + return; + snode->dead = true; + gnode = snode->gnode; tunnel = gnode->tunnel_list; @@ -238,8 +313,11 @@ static void amt_destroy_source(struct amt_source_node *snode) hlist_del_init_rcu(&snode->node); tunnel->nr_sources--; gnode->nr_sources--; + WRITE_ONCE(snode->gnode, NULL); spin_lock_bh(&source_gc_lock); - hlist_add_head_rcu(&snode->node, &source_gc_list); + hlist_add_head(&snode->gc_node, &source_gc_list); + amt_gc_sources++; + amt_gc_parked(); spin_unlock_bh(&source_gc_lock); } @@ -249,6 +327,9 @@ static void amt_del_group(struct amt_dev *amt, struct amt_group_node *gnode) struct hlist_node *t; int i; + if (gnode->dead) + return; + gnode->dead = true; if (cancel_delayed_work(&gnode->group_timer)) dev_put(amt->dev); hlist_del_rcu(&gnode->node); @@ -266,10 +347,11 @@ static void amt_del_group(struct amt_dev *amt, struct amt_group_node *gnode) hlist_for_each_entry_safe(snode, t, &gnode->sources[i], node) amt_destroy_source(snode); - /* tunnel->lock was acquired outside of amt_del_group() - * But rcu_read_lock() was acquired too so It's safe. - */ - kfree_rcu(gnode, rcu); + WRITE_ONCE(gnode->tunnel_list, NULL); + spin_lock_bh(&source_gc_lock); + hlist_add_head(&gnode->gc_node, &group_gc_list); + amt_gc_parked(); + spin_unlock_bh(&source_gc_lock); } /* If a source timer expires with a router filter-mode for the group of @@ -282,13 +364,28 @@ static void amt_source_work(struct work_struct *work) struct amt_source_node *snode = container_of(to_delayed_work(work), struct amt_source_node, source_timer); - struct amt_group_node *gnode = snode->gnode; - struct amt_dev *amt = gnode->amt; struct amt_tunnel_list *tunnel; + struct amt_group_node *gnode; + struct amt_dev *amt; - tunnel = gnode->tunnel_list; - spin_lock_bh(&tunnel->lock); rcu_read_lock(); + gnode = READ_ONCE(snode->gnode); + if (!gnode) { + rcu_read_unlock(); + return; + } + amt = gnode->amt; + tunnel = READ_ONCE(gnode->tunnel_list); + if (!tunnel) { + rcu_read_unlock(); + return; + } + spin_lock_bh(&tunnel->lock); + if (snode->dead || gnode->dead) { + spin_unlock_bh(&tunnel->lock); + rcu_read_unlock(); + return; + } if (gnode->filter_mode == MCAST_INCLUDE) { amt_destroy_source(snode); if (!gnode->nr_sources) @@ -299,8 +396,8 @@ static void amt_source_work(struct work_struct *work) */ snode->status = AMT_SOURCE_STATUS_D_FWD; } - rcu_read_unlock(); spin_unlock_bh(&tunnel->lock); + rcu_read_unlock(); } static void amt_act_src(struct amt_tunnel_list *tunnel, @@ -368,6 +465,7 @@ static struct amt_source_node *amt_alloc_snode(struct amt_group_node *gnode, snode->status = AMT_SOURCE_STATUS_NONE; snode->flags = AMT_SOURCE_NEW; INIT_HLIST_NODE(&snode->node); + INIT_HLIST_NODE(&snode->gc_node); INIT_DELAYED_WORK(&snode->source_timer, amt_source_work); return snode; @@ -403,23 +501,31 @@ static void amt_group_work(struct work_struct *work) struct amt_group_node *gnode = container_of(to_delayed_work(work), struct amt_group_node, group_timer); - struct amt_tunnel_list *tunnel = gnode->tunnel_list; - struct amt_dev *amt = gnode->amt; + struct amt_tunnel_list *tunnel; struct amt_source_node *snode; bool delete_group = true; struct hlist_node *t; + struct amt_dev *amt; int i, buckets; + rcu_read_lock(); + tunnel = READ_ONCE(gnode->tunnel_list); + amt = gnode->amt; + if (!tunnel) + goto out; buckets = amt->hash_buckets; spin_lock_bh(&tunnel->lock); + if (gnode->dead) { + spin_unlock_bh(&tunnel->lock); + goto out; + } if (gnode->filter_mode == MCAST_INCLUDE) { /* Not Used */ spin_unlock_bh(&tunnel->lock); goto out; } - rcu_read_lock(); for (i = 0; i < buckets; i++) { hlist_for_each_entry_safe(snode, t, &gnode->sources[i], node) { @@ -436,9 +542,9 @@ static void amt_group_work(struct work_struct *work) amt_del_group(amt, gnode); else gnode->filter_mode = MCAST_INCLUDE; - rcu_read_unlock(); spin_unlock_bh(&tunnel->lock); out: + rcu_read_unlock(); dev_put(amt->dev); } @@ -479,6 +585,7 @@ static struct amt_group_node *amt_add_group(struct amt_dev *amt, gnode->tunnel_list = tunnel; gnode->filter_mode = MCAST_INCLUDE; INIT_HLIST_NODE(&gnode->node); + INIT_HLIST_NODE(&gnode->gc_node); INIT_DELAYED_WORK(&gnode->group_timer, amt_group_work); for (i = 0; i < amt->hash_buckets; i++) INIT_HLIST_HEAD(&gnode->sources[i]); @@ -593,14 +700,6 @@ static void __amt_update_relay_status(struct amt_tunnel_list *tunnel, tunnel->status = status; } -static void amt_update_relay_status(struct amt_tunnel_list *tunnel, - enum amt_status status, bool validate) -{ - spin_lock_bh(&tunnel->lock); - __amt_update_relay_status(tunnel, status, validate); - spin_unlock_bh(&tunnel->lock); -} - static void amt_send_discovery(struct amt_dev *amt) { struct amt_header_discovery *amtd; @@ -628,7 +727,7 @@ static void amt_send_discovery(struct amt_dev *amt) IPPROTO_UDP, 0, amt->stream_dev->ifindex); if (IS_ERR(rt)) { - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); goto out; } @@ -638,7 +737,7 @@ static void amt_send_discovery(struct amt_dev *amt) skb = netdev_alloc_skb_ip_align(amt->dev, len); if (!skb) { ip_rt_put(rt); - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); goto out; } @@ -683,7 +782,7 @@ static void amt_send_discovery(struct amt_dev *amt) ip_send_check(iph); err = ip_local_out(amt->net, sk, skb); if (unlikely(net_xmit_eval(err))) - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); amt_update_gw_status(amt, AMT_STATUS_SENT_DISCOVERY, true); out: @@ -719,7 +818,7 @@ static void amt_send_request(struct amt_dev *amt, bool v6) IPPROTO_UDP, 0, amt->stream_dev->ifindex); if (IS_ERR(rt)) { - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); goto out; } @@ -729,7 +828,7 @@ static void amt_send_request(struct amt_dev *amt, bool v6) skb = netdev_alloc_skb_ip_align(amt->dev, len); if (!skb) { ip_rt_put(rt); - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); goto out; } @@ -776,7 +875,7 @@ static void amt_send_request(struct amt_dev *amt, bool v6) ip_send_check(iph); err = ip_local_out(amt->net, sk, skb); if (unlikely(net_xmit_eval(err))) - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); out: rcu_read_unlock(); @@ -784,11 +883,16 @@ out: static bool amt_send_membership_query(struct amt_dev *amt, struct sk_buff *skb, - struct amt_tunnel_list *tunnel, + __be32 daddr, __be16 dport, + __be32 nonce, u64 mac, bool v6); -static void amt_send_igmp_gq(struct amt_dev *amt, - struct amt_tunnel_list *tunnel) +/* The destination is passed by value rather than as a tunnel: a Request is + * unauthenticated, so no tunnel state is created or changed for it - see + * amt_request_handler(). + */ +static void amt_send_igmp_gq(struct amt_dev *amt, __be32 daddr, __be16 dport, + __be32 nonce, u64 mac) { struct sk_buff *skb; @@ -797,8 +901,9 @@ static void amt_send_igmp_gq(struct amt_dev *amt, return; skb_pull(skb, sizeof(struct ethhdr)); - if (amt_send_membership_query(amt, skb, tunnel, false)) { - amt->dev->stats.tx_dropped++; + if (amt_send_membership_query(amt, skb, daddr, dport, nonce, mac, + false)) { + DEV_STATS_INC(amt->dev, tx_dropped); kfree_skb(skb); } } @@ -842,7 +947,7 @@ static struct sk_buff *amt_build_mld_gq(struct amt_dev *amt) if (ipv6_dev_get_saddr(amt->net, amt->dev, &ip6h->daddr, 0, &ip6h->saddr)) { - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); kfree_skb(skb); return NULL; } @@ -876,7 +981,8 @@ static struct sk_buff *amt_build_mld_gq(struct amt_dev *amt) return skb; } -static void amt_send_mld_gq(struct amt_dev *amt, struct amt_tunnel_list *tunnel) +static void amt_send_mld_gq(struct amt_dev *amt, __be32 daddr, __be16 dport, + __be32 nonce, u64 mac) { struct sk_buff *skb; @@ -885,13 +991,15 @@ static void amt_send_mld_gq(struct amt_dev *amt, struct amt_tunnel_list *tunnel) return; skb_pull(skb, sizeof(struct ethhdr)); - if (amt_send_membership_query(amt, skb, tunnel, true)) { - amt->dev->stats.tx_dropped++; + if (amt_send_membership_query(amt, skb, daddr, dport, nonce, mac, + true)) { + DEV_STATS_INC(amt->dev, tx_dropped); kfree_skb(skb); } } #else -static void amt_send_mld_gq(struct amt_dev *amt, struct amt_tunnel_list *tunnel) +static void amt_send_mld_gq(struct amt_dev *amt, __be32 daddr, __be16 dport, + __be32 nonce, u64 mac) { } #endif @@ -925,7 +1033,16 @@ static void amt_secret_work(struct work_struct *work) secret_wq); spin_lock_bh(&amt->lock); - get_random_bytes(&amt->key, sizeof(siphash_key_t)); + write_seqcount_begin(&amt->key_seq); + /* Keep three previous generations for a 180-240s MAC lifetime. + * Its 180s floor exceeds the advertised QQIC (amt->qi, 125s), + * allowing the gateway to renew its MAC with a periodic Request. + */ + amt->key[3] = amt->key[2]; + amt->key[2] = amt->key[1]; + amt->key[1] = amt->key[0]; + get_random_bytes(&amt->key[0], sizeof(siphash_key_t)); + write_seqcount_end(&amt->key_seq); spin_unlock_bh(&amt->lock); mod_delayed_work(amt_wq, &amt->secret_wq, msecs_to_jiffies(AMT_SECRET_TIMEOUT)); @@ -1125,9 +1242,16 @@ static void amt_send_multicast_data(struct amt_dev *amt, 0); } +/* Send a Membership Query in reply to an unauthenticated Request. + * + * No per-source state is read or created: the gateway echoes the nonce and + * the keyed MAC in its Membership Update, where the MAC is recomputed, so + * everything this needs is passed by value. + */ static bool amt_send_membership_query(struct amt_dev *amt, struct sk_buff *skb, - struct amt_tunnel_list *tunnel, + __be32 daddr, __be16 dport, + __be32 nonce, u64 mac, bool v6) { struct amt_header_membership_query *amtmq; @@ -1148,13 +1272,13 @@ static bool amt_send_membership_query(struct amt_dev *amt, skb_reset_inner_headers(skb); memset(&fl4, 0, sizeof(struct flowi4)); fl4.flowi4_oif = amt->stream_dev->ifindex; - fl4.daddr = tunnel->ip4; + fl4.daddr = daddr; fl4.saddr = amt->local_ip; fl4.flowi4_dscp = inet_dsfield_to_dscp(AMT_TOS); fl4.flowi4_proto = IPPROTO_UDP; rt = ip_route_output_key(amt->net, &fl4); if (IS_ERR(rt)) { - netdev_dbg(amt->dev, "no route to %pI4\n", &tunnel->ip4); + netdev_dbg(amt->dev, "no route to %pI4\n", &daddr); return true; } @@ -1164,8 +1288,8 @@ static bool amt_send_membership_query(struct amt_dev *amt, amtmq->reserved = 0; amtmq->l = 0; amtmq->g = 0; - amtmq->nonce = tunnel->nonce; - amtmq->response_mac = tunnel->mac; + amtmq->nonce = nonce; + amtmq->response_mac = mac; if (!v6) skb_set_inner_protocol(skb, htons(ETH_P_IP)); @@ -1178,11 +1302,10 @@ static bool amt_send_membership_query(struct amt_dev *amt, ip4_dst_hoplimit(&rt->dst), 0, amt->relay_port, - tunnel->source_port, + dport, false, false, 0); - amt_update_relay_status(tunnel, AMT_STATUS_SENT_QUERY, true); return false; } @@ -1246,7 +1369,7 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) group.ip6 = ipv6_hdr(skb)->daddr; #endif } else { - dev->stats.tx_errors++; + DEV_STATS_INC(dev, tx_errors); goto free; } @@ -1259,12 +1382,21 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) /* Gateway only passes IGMP/MLD packets */ if (!report) goto free; + /* A validated report can only be forwarded after the relay's + * family-specific Membership Query supplies the state echoed + * by the Membership Update. Log this readiness failure before + * the shared drop path accounts it. + */ if ((!v6 && !READ_ONCE(amt->ready4)) || - (v6 && !READ_ONCE(amt->ready6))) + (v6 && !READ_ONCE(amt->ready6))) { + netdev_dbg(dev, "drop %s report: waiting for Membership Query\n", + v6 ? "MLD" : "IGMP"); goto free; + } if (amt_send_membership_update(amt, skb, v6)) goto free; - goto unlock; + /* Sent: the tunnel consumed the skb. Not a drop. */ + return NETDEV_TX_OK; } else if (amt->mode == AMT_MODE_RELAY) { if (!data) goto free; @@ -1293,8 +1425,7 @@ found: return NETDEV_TX_OK; free: dev_kfree_skb(skb); -unlock: - dev->stats.tx_dropped++; + DEV_STATS_INC(dev, tx_dropped); return NETDEV_TX_OK; } @@ -1342,14 +1473,46 @@ static void amt_tunnel_expire(struct work_struct *work) struct amt_tunnel_list, gc_wq); struct amt_dev *amt = tunnel->amt; + unsigned long deadline, now; spin_lock_bh(&amt->lock); + spin_lock_bh(&tunnel->lock); + /* amt_dev_stop() has taken ownership of the teardown. */ + if (READ_ONCE(tunnel->dead)) { + spin_unlock_bh(&tunnel->lock); + spin_unlock_bh(&amt->lock); + return; + } + /* A Membership Update may have refreshed the tunnel after this work + * was queued. Re-arm from last_seen rather than letting + * amt_update_handler() re-arm gc_wq: this handler is then the only + * context that frees the tunnel on the timeout path; amt_dev_stop() + * still owns teardown during device shutdown. last_seen is read and + * dead is set under tunnel->lock, which the Update path holds while + * it refreshes last_seen, so an Update either keeps the tunnel alive + * or finds it dead. + */ + now = jiffies; + deadline = tunnel->last_seen + msecs_to_jiffies(amt_gmi(amt)) * 3; + if (time_before(now, deadline)) { + mod_delayed_work(amt_wq, &tunnel->gc_wq, deadline - now); + spin_unlock_bh(&tunnel->lock); + spin_unlock_bh(&amt->lock); + return; + } + WRITE_ONCE(tunnel->dead, true); + spin_unlock_bh(&tunnel->lock); rcu_read_lock(); list_del_rcu(&tunnel->list); amt->nr_tunnels--; amt_clear_groups(tunnel); rcu_read_unlock(); spin_unlock_bh(&amt->lock); + /* Free now, rather than at the next periodic run, the nodes that + * amt_clear_groups() queued. Call the GC rather than flush + * group_gc_wq, as explained in __amt_group_gc_work(). + */ + __amt_group_gc_work(); kfree_rcu(tunnel, rcu); } @@ -2382,6 +2545,10 @@ static bool amt_membership_query_handler(struct amt_dev *amt, if (amtmq->reserved || amtmq->version) return true; + /* A deferred query may outlive a relay address change. */ + if (ip_hdr(skb)->saddr != READ_ONCE(amt->remote_ip)) + return true; + if (amtmq->nonce != amt->nonce) return true; @@ -2474,16 +2641,165 @@ static bool amt_membership_query_handler(struct amt_dev *amt, return false; } +/* The live tunnel of the gateway endpoint @saddr:@sport, or NULL. Dead + * tunnels are skipped, so the retry in amt_update_tunnel() after losing a + * race with amt_tunnel_expire() creates the replacement. + */ +static struct amt_tunnel_list *amt_tunnel_lookup(struct amt_dev *amt, + __be32 saddr, __be16 sport) +{ + struct amt_tunnel_list *tunnel; + + list_for_each_entry_rcu(tunnel, &amt->tunnel_list, list) { + if (tunnel->ip4 == saddr && tunnel->source_port == sport && + !READ_ONCE(tunnel->dead)) + return tunnel; + } + return NULL; +} + +/* Most tunnels one source address may hold: an eighth of the table, at + * least one and at most AMT_MAX_TUNNELS_PER_SRC. + */ +static u32 amt_max_tunnels_per_src(const struct amt_dev *amt) +{ + return clamp_t(u32, amt->max_tunnels / 8, 1, + AMT_MAX_TUNNELS_PER_SRC); +} + +/* Look up the tunnel for a source whose Membership Update has already been + * authenticated, creating it on first contact. + * + * This is the only place tunnel state is allocated, so amt->max_tunnels + * bounds the number of *verified* gateways rather than the number of + * unverified Requests anyone can send. A per-source cap derived from the + * table, amt_max_tunnels_per_src(), additionally limits the tunnels one + * verified source address may hold. + */ +static struct amt_tunnel_list *amt_tunnel_get_or_create(struct amt_dev *amt, + __be32 saddr, + __be16 sport) +{ + struct amt_tunnel_list *tunnel; + u32 per_src = 0; + int i; + + tunnel = amt_tunnel_lookup(amt, saddr, sport); + if (tunnel) + return tunnel; + + spin_lock_bh(&amt->lock); + + /* Re-check under the lock in case a concurrent Update won the race, and + * count how many tunnels this source already holds on the same walk. + * Skip dead tunnels, as amt_tunnel_lookup() and the Request path do, so + * a tunnel being freed neither stands in for its endpoint nor counts + * against its address. + */ + list_for_each_entry(tunnel, &amt->tunnel_list, list) { + if (tunnel->ip4 != saddr || READ_ONCE(tunnel->dead)) + continue; + if (tunnel->source_port == sport) { + spin_unlock_bh(&amt->lock); + return tunnel; + } + per_src++; + } + + if (amt->nr_tunnels >= amt->max_tunnels || + per_src >= amt_max_tunnels_per_src(amt)) { + spin_unlock_bh(&amt->lock); + return NULL; + } + + if (!netif_running(amt->dev)) { + spin_unlock_bh(&amt->lock); + return NULL; + } + + tunnel = kzalloc(sizeof(*tunnel) + + (sizeof(struct hlist_head) * amt->hash_buckets), + GFP_ATOMIC); + if (!tunnel) { + spin_unlock_bh(&amt->lock); + return NULL; + } + + tunnel->source_port = sport; + tunnel->ip4 = saddr; + tunnel->amt = amt; + spin_lock_init(&tunnel->lock); + for (i = 0; i < amt->hash_buckets; i++) + INIT_HLIST_HEAD(&tunnel->groups[i]); + + tunnel->last_seen = jiffies; + INIT_DELAYED_WORK(&tunnel->gc_wq, amt_tunnel_expire); + __amt_update_relay_status(tunnel, AMT_STATUS_RECEIVED_UPDATE, false); + + /* Publish only once the entry is fully initialised. */ + list_add_tail_rcu(&tunnel->list, &amt->tunnel_list); + amt->nr_tunnels++; + mod_delayed_work(amt_wq, &tunnel->gc_wq, + msecs_to_jiffies(amt_gmi(amt)) * 3); + spin_unlock_bh(&amt->lock); + + return tunnel; +} + +/* Apply a valid, authenticated Membership Update to the tunnel of its + * endpoint, creating the tunnel on first contact. Returns false if the + * Update is dropped. + * + * amt_tunnel_expire() reads last_seen and sets dead under tunnel->lock, so + * the refresh below either keeps the tunnel alive or finds it dead. A + * tunnel that expired between the lookup and the lock is skipped by the + * second amt_tunnel_get_or_create() call, which creates its replacement. + */ +static bool amt_update_tunnel(struct amt_dev *amt, struct sk_buff *skb, + __be32 saddr, __be16 sport, bool v6) +{ + struct amt_tunnel_list *tunnel; + int i; + + for (i = 0; i < 2; i++) { + tunnel = amt_tunnel_get_or_create(amt, saddr, sport); + if (!tunnel) + return false; + + spin_lock_bh(&tunnel->lock); + if (!READ_ONCE(tunnel->dead)) { + /* gc_wq is re-armed by amt_tunnel_expire() itself + * from last_seen, never from here, so this cannot + * re-queue the work of a tunnel that is being freed. + */ + tunnel->last_seen = jiffies; +#if IS_ENABLED(CONFIG_IPV6) + if (v6) + amt_mld_report_handler(amt, skb, tunnel); + else +#endif + amt_igmp_report_handler(amt, skb, tunnel); + spin_unlock_bh(&tunnel->lock); + return true; + } + spin_unlock_bh(&tunnel->lock); + } + return false; +} + static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb) { struct amt_header_membership_update *amtmu; - struct amt_tunnel_list *tunnel; + bool verified = false; + siphash_key_t key[4]; + int len, hdr_size, i; struct ethhdr *eth; struct iphdr *iph; - int len, hdr_size; + unsigned int seq; u64 response_mac; __be32 saddr; __be32 nonce; + __be16 sport; saddr = ip_hdr(skb)->saddr; @@ -2497,30 +2813,45 @@ static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb) nonce = amtmu->nonce; response_mac = amtmu->response_mac; + /* Snapshot the tunnel endpoint port before the encap is stripped. */ + sport = udp_hdr(skb)->source; + /* Recompute the MAC handed out in the Membership Query. Four live + * secrets cover 180-240s, exceeding the advertised QQIC (amt->qi, + * 125s), so a periodic Request can renew the MAC before it expires. + * Authenticate before decapsulation or allocating tunnel state. + */ + do { + seq = read_seqcount_begin(&amt->key_seq); + for (i = 0; i < ARRAY_SIZE(key); i++) + key[i] = amt->key[i]; + } while (read_seqcount_retry(&amt->key_seq, seq)); + + for (i = 0; i < ARRAY_SIZE(key); i++) { + u64 mac = siphash_3u32((__force u32)saddr, + (__force u32)sport, + (__force u32)nonce, + &key[i]) >> 16; + + if (response_mac == mac) { + verified = true; + break; + } + } + + if (!verified) { + netdev_dbg(amt->dev, "Invalid MAC\n"); + return true; + } + + /* Create or refresh a tunnel only once the encapsulated message has + * passed ip_mc_check_igmp() or ipv6_mc_check_mld(). + */ if (iptunnel_pull_header(skb, hdr_size, skb->protocol, false)) return true; skb_reset_network_header(skb); - list_for_each_entry_rcu(tunnel, &amt->tunnel_list, list) { - if (tunnel->ip4 == saddr) { - if ((nonce == tunnel->nonce && - response_mac == tunnel->mac)) { - mod_delayed_work(amt_wq, &tunnel->gc_wq, - msecs_to_jiffies(amt_gmi(amt)) - * 3); - goto report; - } else { - netdev_dbg(amt->dev, "Invalid MAC\n"); - return true; - } - } - } - - return true; - -report: if (!pskb_may_pull(skb, sizeof(*iph))) return true; @@ -2534,9 +2865,8 @@ report: return true; } - spin_lock_bh(&tunnel->lock); - amt_igmp_report_handler(amt, skb, tunnel); - spin_unlock_bh(&tunnel->lock); + if (!amt_update_tunnel(amt, skb, saddr, sport, false)) + return true; skb_push(skb, sizeof(struct ethhdr)); skb_reset_mac_header(skb); @@ -2554,9 +2884,8 @@ report: return true; } - spin_lock_bh(&tunnel->lock); - amt_mld_report_handler(amt, skb, tunnel); - spin_unlock_bh(&tunnel->lock); + if (!amt_update_tunnel(amt, skb, saddr, sport, true)) + return true; skb_push(skb, sizeof(struct ethhdr)); skb_reset_mac_header(skb); @@ -2575,13 +2904,10 @@ report: skb->pkt_type = PACKET_MULTICAST; skb->ip_summed = CHECKSUM_NONE; len = skb->len; - if (__netif_rx(skb) == NET_RX_SUCCESS) { - amt_update_relay_status(tunnel, AMT_STATUS_RECEIVED_UPDATE, - true); + if (__netif_rx(skb) == NET_RX_SUCCESS) dev_sw_netstats_rx_add(amt->dev, len); - } else { + else amt->dev->stats.rx_dropped++; - } return false; } @@ -2614,7 +2940,7 @@ static void amt_send_advertisement(struct amt_dev *amt, __be32 nonce, IPPROTO_UDP, 0, amt->stream_dev->ifindex); if (IS_ERR(rt)) { - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); goto out; } @@ -2624,7 +2950,7 @@ static void amt_send_advertisement(struct amt_dev *amt, __be32 nonce, skb = netdev_alloc_skb_ip_align(amt->dev, len); if (!skb) { ip_rt_put(rt); - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); goto out; } @@ -2670,7 +2996,7 @@ static void amt_send_advertisement(struct amt_dev *amt, __be32 nonce, ip_send_check(iph); err = ip_local_out(amt->net, sk, skb); if (unlikely(net_xmit_eval(err))) - amt->dev->stats.tx_errors++; + DEV_STATS_INC(amt->dev, tx_errors); out: rcu_read_unlock(); @@ -2697,15 +3023,30 @@ static bool amt_discovery_handler(struct amt_dev *amt, struct sk_buff *skb) return false; } +/* Handle an AMT Request. + * + * The source of a Request is unauthenticated: it is a bare UDP datagram with a + * trivially spoofable source address. Allocating tunnel state here would let + * an attacker fill the table from spoofed addresses, or let one spoofed packet + * overwrite the nonce/MAC of an established tunnel and cut that gateway off. + * + * Nothing needs to be remembered at this point. Every input to the keyed MAC + * is carried in the packet or is device state, so the MAC is generated here, + * echoed back by the gateway in its Membership Update, and recomputed and + * verified there - see amt_update_handler(). Tunnel state is created only once + * that verification succeeds and the encapsulated message passes its IGMP or + * MLD check. + */ static bool amt_request_handler(struct amt_dev *amt, struct sk_buff *skb) { struct amt_header_request *amtrh; struct amt_tunnel_list *tunnel; - unsigned long long key; + bool full, have = false; struct udphdr *udph; struct iphdr *iph; + siphash_key_t key; + u32 per_src = 0; u64 mac; - int i; if (!pskb_may_pull(skb, sizeof(*udph) + sizeof(*amtrh))) return true; @@ -2717,59 +3058,45 @@ static bool amt_request_handler(struct amt_dev *amt, struct sk_buff *skb) if (amtrh->reserved1 || amtrh->reserved2 || amtrh->version) return true; - list_for_each_entry_rcu(tunnel, &amt->tunnel_list, list) - if (tunnel->ip4 == iph->saddr) - goto send; - - spin_lock_bh(&amt->lock); - if (amt->nr_tunnels >= amt->max_tunnels) { - spin_unlock_bh(&amt->lock); - icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); - return true; - } - - tunnel = kzalloc(sizeof(*tunnel) + - (sizeof(struct hlist_head) * amt->hash_buckets), - GFP_ATOMIC); - if (!tunnel) { - spin_unlock_bh(&amt->lock); - return true; - } - - tunnel->source_port = udph->source; - tunnel->ip4 = iph->saddr; - - memcpy(&key, &tunnel->key, sizeof(unsigned long long)); - tunnel->amt = amt; - spin_lock_init(&tunnel->lock); - for (i = 0; i < amt->hash_buckets; i++) - INIT_HLIST_HEAD(&tunnel->groups[i]); - - INIT_DELAYED_WORK(&tunnel->gc_wq, amt_tunnel_expire); - - list_add_tail_rcu(&tunnel->list, &amt->tunnel_list); - tunnel->key = amt->key; - __amt_update_relay_status(tunnel, AMT_STATUS_RECEIVED_REQUEST, true); - amt->nr_tunnels++; - mod_delayed_work(amt_wq, &tunnel->gc_wq, - msecs_to_jiffies(amt_gmi(amt))); - spin_unlock_bh(&amt->lock); - -send: - tunnel->nonce = amtrh->nonce; - mac = siphash_3u32((__force u32)tunnel->ip4, - (__force u32)tunnel->source_port, - (__force u32)tunnel->nonce, - &tunnel->key); - tunnel->mac = mac >> 16; - if (!netif_running(amt->dev) || !netif_running(amt->stream_dev)) return true; + /* Snapshot the secret and the table-full state under the lock and walk + * the endpoints outside it, so a Request holds amt->lock only for this + * O(1) snapshot. + */ + spin_lock_bh(&amt->lock); + key = amt->key[0]; + full = amt->nr_tunnels >= amt->max_tunnels; + spin_unlock_bh(&amt->lock); + + /* Skip dead tunnels, as amt_tunnel_lookup() does: each is already + * being torn down, so it must not count against a new endpoint. + */ + rcu_read_lock(); + list_for_each_entry_rcu(tunnel, &amt->tunnel_list, list) { + if (tunnel->ip4 != iph->saddr || READ_ONCE(tunnel->dead)) + continue; + per_src++; + if (tunnel->source_port == udph->source) + have = true; + } + rcu_read_unlock(); + + /* Withhold the Query when a new endpoint cannot get a tunnel. */ + if (!have && (full || per_src >= amt_max_tunnels_per_src(amt))) + return true; + + mac = siphash_3u32((__force u32)iph->saddr, + (__force u32)udph->source, + (__force u32)amtrh->nonce, + &key) >> 16; if (!amtrh->p) - amt_send_igmp_gq(amt, tunnel); + amt_send_igmp_gq(amt, iph->saddr, udph->source, amtrh->nonce, + mac); else - amt_send_mld_gq(amt, tunnel); + amt_send_mld_gq(amt, iph->saddr, udph->source, amtrh->nonce, + mac); return false; } @@ -2846,7 +3173,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb) } goto out; case AMT_MSG_MULTICAST_DATA: - if (saddr != remote_ip) { + if (!remote_ip || saddr != remote_ip) { netdev_dbg(amt->dev, "Invalid Relay IP\n"); err = true; goto drop; @@ -2857,7 +3184,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb) else goto out; case AMT_MSG_MEMBERSHIP_QUERY: - if (saddr != remote_ip) { + if (!remote_ip || saddr != remote_ip) { netdev_dbg(amt->dev, "Invalid Relay IP\n"); err = true; goto drop; @@ -3037,6 +3364,16 @@ static int amt_dev_open(struct net_device *dev) enable_delayed_work(&amt->discovery_wq); enable_delayed_work(&amt->req_wq); + /* Seed the keys before amt_socket_create() enables the encap receive + * path, so no Query MAC is generated, and no Update verified, with an + * all-zero (predictable) secret. + */ + spin_lock_bh(&amt->lock); + write_seqcount_begin(&amt->key_seq); + get_random_bytes(&amt->key, sizeof(amt->key)); + write_seqcount_end(&amt->key_seq); + spin_unlock_bh(&amt->lock); + err = amt_socket_create(amt); if (err) { disable_delayed_work(&amt->req_wq); @@ -3047,7 +3384,6 @@ static int amt_dev_open(struct net_device *dev) amt->req_cnt = 0; WRITE_ONCE(amt->remote_ip, 0); amt->nonce = 0; - get_random_bytes(&amt->key, sizeof(siphash_key_t)); amt->status = AMT_STATUS_INIT; if (amt->mode == AMT_MODE_GATEWAY) { @@ -3063,7 +3399,7 @@ static int amt_dev_open(struct net_device *dev) static int amt_dev_stop(struct net_device *dev) { struct amt_dev *amt = netdev_priv(dev); - struct amt_tunnel_list *tunnel, *tmp; + struct amt_tunnel_list *tunnel; struct sk_buff *skb; struct sock *sk; int i; @@ -3072,9 +3408,14 @@ static int amt_dev_stop(struct net_device *dev) disable_delayed_work_sync(&amt->discovery_wq); cancel_delayed_work_sync(&amt->secret_wq); - /* shutdown */ + /* shutdown. Detach the receive path as well, so that once the + * grace period ends no amt_rcv() can still see the device and queue + * work after the teardown below. + */ sk = rtnl_dereference(amt->sk); RCU_INIT_POINTER(amt->sk, NULL); + if (sk) + rcu_assign_sk_user_data(sk, NULL); synchronize_net(); if (sk) udp_tunnel_sock_release(sk); @@ -3092,11 +3433,37 @@ static int amt_dev_stop(struct net_device *dev) amt->req_cnt = 0; WRITE_ONCE(amt->remote_ip, 0); - list_for_each_entry_safe(tunnel, tmp, &amt->tunnel_list, list) { - list_del_rcu(&tunnel->list); - amt->nr_tunnels--; + /* Mark every tunnel dead first, so a concurrent amt_tunnel_expire() + * sees ->dead and bails instead of freeing a tunnel in parallel; this + * path is then the sole freer. + */ + spin_lock_bh(&amt->lock); + list_for_each_entry(tunnel, &amt->tunnel_list, list) { + spin_lock_bh(&tunnel->lock); + WRITE_ONCE(tunnel->dead, true); + spin_unlock_bh(&tunnel->lock); + } + spin_unlock_bh(&amt->lock); + + /* Unlink one tunnel at a time with list_del_rcu() rather than splicing + * the list onto a local head: a lockless reader walking tunnel_list + * must always terminate at &amt->tunnel_list, never at a stack head. + */ + for (;;) { + spin_lock_bh(&amt->lock); + tunnel = list_first_entry_or_null(&amt->tunnel_list, + struct amt_tunnel_list, list); + if (tunnel) { + list_del_rcu(&tunnel->list); + amt->nr_tunnels--; + } + spin_unlock_bh(&amt->lock); + if (!tunnel) + break; + cancel_delayed_work_sync(&tunnel->gc_wq); amt_clear_groups(tunnel); + __amt_group_gc_work(); kfree_rcu(tunnel, rcu); } @@ -3165,9 +3532,9 @@ static const struct nla_policy amt_policy[IFLA_AMT_MAX + 1] = { [IFLA_AMT_RELAY_PORT] = { .type = NLA_U16 }, [IFLA_AMT_GATEWAY_PORT] = { .type = NLA_U16 }, [IFLA_AMT_LINK] = { .type = NLA_U32 }, - [IFLA_AMT_LOCAL_IP] = { .len = sizeof_field(struct iphdr, daddr) }, + [IFLA_AMT_LOCAL_IP] = NLA_POLICY_EXACT_LEN(sizeof(__be32)), [IFLA_AMT_REMOTE_IP] = { .len = sizeof_field(struct iphdr, daddr) }, - [IFLA_AMT_DISCOVERY_IP] = { .len = sizeof_field(struct iphdr, daddr) }, + [IFLA_AMT_DISCOVERY_IP] = NLA_POLICY_EXACT_LEN(sizeof(__be32)), [IFLA_AMT_MAX_TUNNELS] = { .type = NLA_U32 }, }; @@ -3234,6 +3601,7 @@ static int amt_newlink(struct net_device *dev, amt->max_tunnels = AMT_MAX_TUNNELS; spin_lock_init(&amt->lock); + seqcount_spinlock_init(&amt->key_seq, &amt->lock); amt->max_groups = AMT_MAX_GROUP; amt->max_sources = AMT_MAX_SOURCE; amt->hash_buckets = AMT_HSIZE; @@ -3470,6 +3838,9 @@ static int __init amt_init(void) INIT_DELAYED_WORK(&source_gc_wq, amt_source_gc_work); mod_delayed_work(amt_wq, &source_gc_wq, msecs_to_jiffies(AMT_GC_INTERVAL)); + INIT_DELAYED_WORK(&group_gc_wq, amt_group_gc_work); + mod_delayed_work(amt_wq, &group_gc_wq, + msecs_to_jiffies(AMT_GC_INTERVAL)); spin_unlock_bh(&source_gc_lock); return 0; @@ -3490,6 +3861,8 @@ static void __exit amt_fini(void) unregister_netdevice_notifier(&amt_notifier_block); cancel_delayed_work_sync(&source_gc_wq); __amt_source_gc_work(); + cancel_delayed_work_sync(&group_gc_wq); + __amt_group_gc_work(); destroy_workqueue(amt_wq); } module_exit(amt_fini); diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index e9f5e753f677..042d7c75548f 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -1894,9 +1894,12 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, struct sockaddr_storage ss; int res = 0, i; - if (slave_dev->type == ARPHRD_CAN) { + if (slave_dev->type == ARPHRD_CAN || + slave_dev->type == ARPHRD_IEEE802154 || + slave_dev->type == ARPHRD_IEEE802154_MONITOR || + slave_dev->type == ARPHRD_6LOWPAN) { BOND_NL_ERR(bond_dev, extack, - "CAN devices cannot be enslaved"); + "CAN, IEEE 802.15.4 and 6LoWPAN devices cannot be enslaved"); return -EPERM; } diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c index 297fb36ab8c1..9b14cddf4432 100644 --- a/drivers/net/ethernet/amazon/ena/ena_com.c +++ b/drivers/net/ethernet/amazon/ena/ena_com.c @@ -1992,6 +1992,7 @@ int ena_com_admin_init(struct ena_com_dev *ena_dev, admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH; + admin_queue->ena_dev = ena_dev; admin_queue->q_dmadev = ena_dev->dmadev; admin_queue->polling = false; admin_queue->curr_cmd_id = 0; @@ -2045,7 +2046,6 @@ int ena_com_admin_init(struct ena_com_dev *ena_dev, if (ret) goto error; - admin_queue->ena_dev = ena_dev; admin_queue->running_state = true; return 0; diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index ecc6f1d827f9..560b20a5861c 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -2609,7 +2609,6 @@ error_report_dma_error: tx_info->skb = NULL; - tx_info->num_of_bufs += i; ena_unmap_tx_buff(tx_ring, tx_info); return -EINVAL; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 6846a27bfef4..e78cf7ad3cbc 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -11318,7 +11318,7 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) /* Filter for default vnic 0 */ rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); if (rc) { - if (BNXT_VF(bp) && rc == -ENODEV) + if (rc == -ECOMM) netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n"); else netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); @@ -12476,7 +12476,7 @@ int bnxt_update_link(struct bnxt *bp, bool chng_link_state) rc = hwrm_req_send(bp, req); if (rc) { hwrm_req_drop(bp, req); - if (BNXT_VF(bp) && rc == -ENODEV) { + if (rc == -ECOMM) { netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n"); rc = 0; } @@ -13953,7 +13953,7 @@ static int bnxt_cfg_rx_mode(struct bnxt *bp, struct netdev_hw_addr_list *uc, for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); if (rc) { - if (BNXT_VF(bp) && rc == -ENODEV) { + if (rc == -ECOMM) { netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); rc = -EAGAIN; } else if (rc == -EAGAIN) { @@ -17003,7 +17003,7 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) } rc = __bnxt_reserve_rings(bp); - if (rc && rc != -ENODEV) + if (rc && rc != -ECOMM) netdev_warn(bp->dev, "Unable to reserve tx rings\n"); bnxt_adj_tx_rings(bp); @@ -17013,7 +17013,7 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) /* Rings may have been trimmed, re-reserve the trimmed rings. */ if (bnxt_need_reserve_rings(bp)) { rc = __bnxt_reserve_rings(bp); - if (rc && rc != -ENODEV) + if (rc && rc != -ECOMM) netdev_warn(bp->dev, "2nd rings reservation failed.\n"); bnxt_adj_tx_rings(bp); } @@ -17039,7 +17039,7 @@ static int bnxt_init_dflt_ring_mode(struct bnxt *bp) bnxt_clear_int_mode(bp); rc = bnxt_set_dflt_rings(bp, true); if (rc) { - if (BNXT_VF(bp) && rc == -ENODEV) + if (rc == -ECOMM) netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); else netdev_err(bp->dev, "Not enough rings available.\n"); @@ -17417,7 +17417,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) bnxt_aux_devices_init(bp); rc = bnxt_set_dflt_rings(bp, true); if (rc) { - if (BNXT_VF(bp) && rc == -ENODEV) { + if (rc == -ECOMM) { netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); } else { netdev_err(bp->dev, "Not enough rings available.\n"); @@ -17573,21 +17573,50 @@ static int bnxt_suspend(struct device *device) return rc; } +#define BNXT_RESUME_MAX_RETRY 10 + static int bnxt_resume(struct device *device) { struct net_device *dev = dev_get_drvdata(device); struct bnxt *bp = netdev_priv(dev); - int rc = 0; + int i, rc; - rtnl_lock(); - netdev_lock(dev); rc = pci_enable_device(bp->pdev); if (rc) { netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", rc); - goto resume_exit; + return rc; } pci_set_master(bp->pdev); + if (BNXT_VF(bp)) { + for (i = 0; i < BNXT_RESUME_MAX_RETRY; i++) { + rc = bnxt_hwrm_poll(bp); + if (rc != -EAGAIN && rc != -EBUSY && rc != -ECOMM) + break; + + /* Wait briefly for -EAGAIN (FW busy), a bit longer for + * -ECOMM (PF down) to allow the PF to fully initialize. + * For -EBUSY, the message typically has timed out, so + * try again immediately. + */ + if (i < BNXT_RESUME_MAX_RETRY - 1) { + if (rc == -EAGAIN) + msleep(250); + else if (rc == -ECOMM) + msleep(750); + } + } + if (rc) { + netdev_err(dev, + "Firmware not responding during resume, rc = %d\n", + rc); + return -ENODEV; + } + } + + rtnl_lock(); + netdev_lock(dev); + if (bnxt_hwrm_ver_get(bp)) { rc = -ENODEV; goto resume_exit; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c index 5bfabdca7d0e..de02d1baa751 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c @@ -360,7 +360,7 @@ static int __hwrm_to_stderr(u32 hwrm_err) case HWRM_ERR_CODE_CMD_NOT_SUPPORTED: return -EOPNOTSUPP; case HWRM_ERR_CODE_PF_UNAVAILABLE: - return -ENODEV; + return -ECOMM; default: return -EIO; } @@ -716,6 +716,8 @@ exit: * ENOMEM: HWRM has no buffers. * EAGAIN: HWRM busy or reset in progress. * EOPNOTSUPP: Invalid request type. + * ENODEV: PCI device is disabled. + * ECOMM: The parent PF is unavailable when issued on VFs. * EIO: Any other error. * Error handling is orthogonal to request ownership. An unowned request will * still be consumed on error. If the caller owns the request, then the caller diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c index cf636fc5aafa..19010e08fc55 100644 --- a/drivers/net/ethernet/freescale/gianfar.c +++ b/drivers/net/ethernet/freescale/gianfar.c @@ -1092,6 +1092,7 @@ static void free_skb_rx_queue(struct gfar_priv_rx_q *rx_queue) struct rxbd8 *rxbdp = rx_queue->rx_bd_base; dev_kfree_skb(rx_queue->skb); + rx_queue->skb = NULL; for (i = 0; i < rx_queue->rx_ring_size; i++) { struct gfar_rx_buff *rxb = &rx_queue->rx_buff[i]; diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h index e509488a3d3e..143d8cd595aa 100644 --- a/drivers/net/ethernet/google/gve/gve.h +++ b/drivers/net/ethernet/google/gve/gve.h @@ -119,7 +119,7 @@ struct gve_rx_slot_page_info { void *page_address; u32 page_offset; /* offset to write to in page */ unsigned int buf_size; - int pagecnt_bias; /* expected pagecnt if only the driver has a ref */ + u16 pagecnt_bias; /* expected pagecnt if only the driver has a ref */ u16 pad; /* adjustment for rx padding */ u8 can_flip; /* tracks if the networking stack is using the page */ }; diff --git a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c index 6880d1531ba5..52223baf0303 100644 --- a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c @@ -149,8 +149,8 @@ int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx, buf_state->last_single_ref_offset = 0; /* The page already has 1 ref. */ - page_ref_add(buf_state->page_info.page, INT_MAX - 1); - buf_state->page_info.pagecnt_bias = INT_MAX; + page_ref_add(buf_state->page_info.page, USHRT_MAX - 1); + buf_state->page_info.pagecnt_bias = USHRT_MAX; return 0; } diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 0ee69bc11d83..35cf0f06a0f2 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -1798,6 +1798,12 @@ static int gve_verify_xdp_configuration(struct net_device *dev, struct gve_priv *priv = netdev_priv(dev); u16 max_xdp_mtu; + if (!(dev->xdp_features & NETDEV_XDP_ACT_BASIC)) { + NL_SET_ERR_MSG_MOD(extack, + "XDP is not supported on this queue format."); + return -EOPNOTSUPP; + } + if (dev->features & NETIF_F_GRO_HW) { NL_SET_ERR_MSG_MOD(extack, "XDP is not supported when HW-GRO is on."); diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c index 81ea800e66e9..3bed19866376 100644 --- a/drivers/net/ethernet/google/gve/gve_rx.c +++ b/drivers/net/ethernet/google/gve/gve_rx.c @@ -152,8 +152,8 @@ static void gve_setup_rx_buffer(struct gve_rx_ring *rx, page_info->buf_size = rx->packet_buffer_size; *slot_addr = cpu_to_be64(addr); /* The page already has 1 ref */ - page_ref_add(page, INT_MAX - 1); - page_info->pagecnt_bias = INT_MAX; + page_ref_add(page, USHRT_MAX - 1); + page_info->pagecnt_bias = USHRT_MAX; } static int gve_rx_alloc_buffer(struct gve_priv *priv, struct device *dev, @@ -230,8 +230,8 @@ static int gve_rx_prefill_pages(struct gve_rx_ring *rx, rx->qpl_copy_pool[j].buf_size = rx->packet_buffer_size; /* The page already has 1 ref. */ - page_ref_add(page, INT_MAX - 1); - rx->qpl_copy_pool[j].pagecnt_bias = INT_MAX; + page_ref_add(page, USHRT_MAX - 1); + rx->qpl_copy_pool[j].pagecnt_bias = USHRT_MAX; } } diff --git a/drivers/net/ethernet/google/gve/gve_utils.c b/drivers/net/ethernet/google/gve/gve_utils.c index caed38cd5f2b..930555ac8fe9 100644 --- a/drivers/net/ethernet/google/gve/gve_utils.c +++ b/drivers/net/ethernet/google/gve/gve_utils.c @@ -93,15 +93,11 @@ void gve_dec_pagecnt_bias(struct gve_rx_slot_page_info *page_info) { page_info->pagecnt_bias--; if (page_info->pagecnt_bias == 0) { - int pagecount = page_count(page_info->page); - - /* If we have run out of bias - set it back up to INT_MAX - * minus the existing refs. + /* Out of bias: take another batch. USHRT_MAX keeps this rare + * while leaving page_count() far below INT_MAX. */ - page_info->pagecnt_bias = INT_MAX - pagecount; - - /* Set pagecount back up to max. */ - page_ref_add(page_info->page, INT_MAX - pagecount); + page_info->pagecnt_bias = USHRT_MAX; + page_ref_add(page_info->page, USHRT_MAX); } } diff --git a/drivers/net/ethernet/google/gve/gve_utils.h b/drivers/net/ethernet/google/gve/gve_utils.h index bf2e9a0adb36..c686142d37ea 100644 --- a/drivers/net/ethernet/google/gve/gve_utils.h +++ b/drivers/net/ethernet/google/gve/gve_utils.h @@ -25,7 +25,7 @@ struct sk_buff *gve_rx_copy_data(struct net_device *dev, struct napi_struct *nap struct sk_buff *gve_rx_copy(struct net_device *dev, struct napi_struct *napi, struct gve_rx_slot_page_info *page_info, u16 len); -/* Decrement pagecnt_bias. Set it back to INT_MAX if it reached zero. */ +/* Decrement pagecnt_bias. Top it up by USHRT_MAX if it reached zero. */ void gve_dec_pagecnt_bias(struct gve_rx_slot_page_info *page_info); void gve_add_napi(struct gve_priv *priv, int ntfy_idx, diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index d7f5c6f16142..d7e279b739b8 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -1227,7 +1227,8 @@ err_mdio_ioremap: kfree(adapter->rx_ring); err_dma: err_sw_init: - iounmap(hw->ce4100_gbe_mdio_base_virt); + if (hw->ce4100_gbe_mdio_base_virt) + iounmap(hw->ce4100_gbe_mdio_base_virt); iounmap(hw->hw_addr); err_ioremap: disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags); diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 6a992df06fa1..58d842d67642 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -2143,6 +2143,8 @@ static int i40e_set_ringparam(struct net_device *netdev, i--; if (!i40e_active_tx_ring_index(vsi, i)) continue; + /* not live yet, skip its BQL reset on free */ + tx_rings[i].netdev = NULL; i40e_free_tx_resources(&tx_rings[i]); } kfree(tx_rings); @@ -2248,8 +2250,11 @@ free_tx: /* error cleanup if the Rx allocations failed after getting Tx */ if (tx_rings) { for (i = 0; i < tx_alloc_queue_pairs; i++) { - if (i40e_active_tx_ring_index(vsi, i)) - i40e_free_tx_resources(vsi->tx_rings[i]); + if (i40e_active_tx_ring_index(vsi, i)) { + /* not live yet, skip its BQL reset on free */ + tx_rings[i].netdev = NULL; + i40e_free_tx_resources(&tx_rings[i]); + } } kfree(tx_rings); tx_rings = NULL; diff --git a/drivers/net/ethernet/intel/iavf/iavf_adminq.c b/drivers/net/ethernet/intel/iavf/iavf_adminq.c index 6937b7dd44cb..40f76f9507f4 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_adminq.c +++ b/drivers/net/ethernet/intel/iavf/iavf_adminq.c @@ -60,6 +60,7 @@ static enum iavf_status iavf_alloc_adminq_arq_ring(struct iavf_hw *hw) **/ static void iavf_free_adminq_asq(struct iavf_hw *hw) { + iavf_free_virt_mem(hw, &hw->aq.asq.cmd_buf); iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf); } diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 2f8a80a11336..4de3ea8525b3 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -5638,6 +5638,7 @@ static void iavf_remove(struct pci_dev *pdev) iounmap(hw->hw_addr); pci_release_regions(pdev); kfree(adapter->vf_res); + kfree(adapter->qos_caps); spin_lock_bh(&adapter->mac_vlan_list_lock); /* If we got removed before an up/down sequence, we've got a filter * hanging out there that we need to get rid of. diff --git a/drivers/net/ethernet/intel/iavf/iavf_ptp.c b/drivers/net/ethernet/intel/iavf/iavf_ptp.c index 87b97e09df14..fcb273828bc1 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ptp.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ptp.c @@ -252,6 +252,16 @@ static int iavf_ptp_gettimex64(struct ptp_clock_info *info, return iavf_read_phc_indirect(adapter, ts, sts); } +static int iavf_ptp_adjfine(struct ptp_clock_info *info, long scaled_ppm) +{ + return -EOPNOTSUPP; +} + +static int iavf_ptp_adjtime(struct ptp_clock_info *info, s64 delta) +{ + return -EOPNOTSUPP; +} + static int iavf_ptp_settime64(struct ptp_clock_info *info, const struct timespec64 *ts) { @@ -326,6 +336,8 @@ static int iavf_ptp_register_clock(struct iavf_adapter *adapter) KBUILD_MODNAME, dev_name(dev)); ptp_info->owner = THIS_MODULE; ptp_info->gettimex64 = iavf_ptp_gettimex64; + ptp_info->adjfine = iavf_ptp_adjfine; + ptp_info->adjtime = iavf_ptp_adjtime; ptp_info->settime64 = iavf_ptp_settime64; ptp_info->do_aux_work = iavf_ptp_do_aux_work; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index d8563f863844..227695dfa51d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3421,6 +3421,23 @@ static int mlx5e_switch_priv_params(struct mlx5e_priv *priv, return 0; } +/* The old and the new channel of the same index are bound to the same + * XSK pool, which is lockless and expects a single user at a time. + * Close the old XSK queues before the new ones are activated, even + * when the closure of the rest of the old channels is deferred. + */ +static void mlx5e_close_channels_xsk(struct mlx5e_channels *chs) +{ + int i; + + for (i = 0; i < chs->num; i++) { + struct mlx5e_channel *c = chs->c[i]; + + if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) + mlx5e_close_xsk(c); + } +} + static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv, struct mlx5e_channels *old_chs, struct mlx5e_channels *new_chs, @@ -3452,6 +3469,8 @@ static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv, mlx5e_set_xdp_feature(priv); if (!MLX5_CAP_GEN(priv->mdev, tis_tir_td_order)) mlx5e_close_channels(old_chs); + else + mlx5e_close_channels_xsk(old_chs); priv->profile->update_rx(priv); mlx5e_selq_apply(&priv->selq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index c7b4a8a82773..ca5da48626f6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -506,10 +506,11 @@ static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep, u32 *sqns_array, int sqns_num) { + struct mlx5_devcom_comp_dev *devcom = esw->devcom; struct mlx5_flow_handle *flow_rule; struct mlx5e_rep_priv *rpriv; struct mlx5e_rep_sq *rep_sq; - bool devcom_locked = false; + bool peers_ready; int err; int i; @@ -518,9 +519,8 @@ static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, rpriv = mlx5e_rep_to_rep_priv(rep); - if (mlx5_devcom_comp_is_ready(esw->devcom) && - mlx5_devcom_for_each_peer_begin(esw->devcom)) - devcom_locked = true; + mlx5_devcom_comp_read_lock(devcom); + peers_ready = mlx5_devcom_comp_is_ready(devcom); for (i = 0; i < sqns_num; i++) { rep_sq = kzalloc_obj(*rep_sq); @@ -541,7 +541,7 @@ static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, rep_sq->sqn = sqns_array[i]; xa_init(&rep_sq->sq_peer); - if (devcom_locked) { + if (peers_ready) { err = mlx5e_sqs2vport_add_peers_rules(esw, rep, rep_sq, i); if (err) { mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule); @@ -554,16 +554,13 @@ static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, list_add(&rep_sq->list, &rpriv->vport_sqs_list); } - if (devcom_locked) - mlx5_devcom_for_each_peer_end(esw->devcom); + mlx5_devcom_comp_read_unlock(devcom); return 0; out_err: mlx5e_sqs2vport_stop(esw, rep); - - if (devcom_locked) - mlx5_devcom_for_each_peer_end(esw->devcom); + mlx5_devcom_comp_read_unlock(devcom); return err; } @@ -630,8 +627,12 @@ mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct mlx5e_rep_priv *rpriv = priv->ppriv; struct mlx5_eswitch_rep *rep = rpriv->rep; + struct mlx5_devcom_comp_dev *devcom; + devcom = esw->devcom; + mlx5_devcom_comp_read_lock(devcom); mlx5e_sqs2vport_stop(esw, rep); + mlx5_devcom_comp_read_unlock(devcom); } static int diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index 75855481522b..aed9e2a74162 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -456,6 +456,20 @@ void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom) up_write(&devcom->comp->sem); } +void mlx5_devcom_comp_read_lock(struct mlx5_devcom_comp_dev *devcom) +{ + if (!devcom) + return; + down_read(&devcom->comp->sem); +} + +void mlx5_devcom_comp_read_unlock(struct mlx5_devcom_comp_dev *devcom) +{ + if (!devcom) + return; + up_read(&devcom->comp->sem); +} + int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom) { if (!devcom) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index 7a704fafdbd3..94b0fe20cc26 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -79,6 +79,8 @@ void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom); void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom); +void mlx5_devcom_comp_read_lock(struct mlx5_devcom_comp_dev *devcom); +void mlx5_devcom_comp_read_unlock(struct mlx5_devcom_comp_dev *devcom); int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom); void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c index c2292cab4592..b953bbc4925c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c @@ -121,25 +121,29 @@ int mlx5_vxlan_add_port(struct mlx5_vxlan *vxlan, u16 port) return 0; } -int mlx5_vxlan_del_port(struct mlx5_vxlan *vxlan, u16 port) +static int __mlx5_vxlan_del_port(struct mlx5_vxlan *vxlan, u16 port) { struct mlx5_vxlan_port *vxlanp; - int ret = 0; - mutex_lock(&vxlan->sync_lock); + lockdep_assert_held(&vxlan->sync_lock); vxlanp = vxlan_lookup_port(vxlan, port); - if (WARN_ON(!vxlanp)) { - ret = -ENOENT; - goto out_unlock; - } + if (WARN_ON(!vxlanp)) + return -ENOENT; hash_del_rcu(&vxlanp->hlist); synchronize_rcu(); mlx5_vxlan_core_del_port_cmd(vxlan->mdev, port); kfree(vxlanp); + return 0; +} -out_unlock: +int mlx5_vxlan_del_port(struct mlx5_vxlan *vxlan, u16 port) +{ + int ret; + + mutex_lock(&vxlan->sync_lock); + ret = __mlx5_vxlan_del_port(vxlan, port); mutex_unlock(&vxlan->sync_lock); return ret; } @@ -185,12 +189,14 @@ void mlx5_vxlan_reset_to_default(struct mlx5_vxlan *vxlan) if (!mlx5_vxlan_allowed(vxlan)) return; + mutex_lock(&vxlan->sync_lock); hash_for_each_safe(vxlan->htable, bkt, tmp, vxlanp, hlist) { /* Don't delete default UDP port added by the HW. * Remove only user configured ports */ if (vxlanp->udp_port == IANA_VXLAN_UDP_PORT) continue; - mlx5_vxlan_del_port(vxlan, vxlanp->udp_port); + __mlx5_vxlan_del_port(vxlan, vxlanp->udp_port); } + mutex_unlock(&vxlan->sync_lock); } diff --git a/drivers/net/ethernet/microchip/encx24j600.c b/drivers/net/ethernet/microchip/encx24j600.c index b011bf5c2305..ebc067cb2c9d 100644 --- a/drivers/net/ethernet/microchip/encx24j600.c +++ b/drivers/net/ethernet/microchip/encx24j600.c @@ -893,9 +893,10 @@ static netdev_tx_t encx24j600_tx(struct sk_buff *skb, struct net_device *dev) static void encx24j600_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct encx24j600_priv *priv = netdev_priv(dev); + unsigned long j = jiffies; netif_err(priv, tx_err, dev, "TX timeout at %ld, latency %ld\n", - jiffies, jiffies - dev_trans_start(dev)); + j, j - dev_trans_start(dev)); dev->stats.tx_errors++; netif_wake_queue(dev); diff --git a/drivers/net/ethernet/microchip/sparx5/lan969x/lan969x_fdma.c b/drivers/net/ethernet/microchip/sparx5/lan969x/lan969x_fdma.c index 053d125eda65..fd532c89f911 100644 --- a/drivers/net/ethernet/microchip/sparx5/lan969x/lan969x_fdma.c +++ b/drivers/net/ethernet/microchip/sparx5/lan969x/lan969x_fdma.c @@ -113,8 +113,11 @@ static struct sk_buff *lan969x_fdma_rx_get_frame(struct sparx5 *sparx5, sparx5_ifh_parse(sparx5, page_address(page), &fi); port = fi.src_port < consts->n_ports ? sparx5->ports[fi.src_port] : NULL; - if (WARN_ON(!port)) + if (!port) { + dev_err_ratelimited(sparx5->dev, "Data on inactive port %d\n", + fi.src_port); goto free_page; + } skb = build_skb(page_address(page), fdma->db_size); if (unlikely(!skb)) @@ -258,10 +261,8 @@ int lan969x_fdma_napi_poll(struct napi_struct *napi, int weight) break; skb = lan969x_fdma_rx_get_frame(sparx5, rx); - if (!skb) - break; - - napi_gro_receive(&rx->napi, skb); + if (skb) + napi_gro_receive(&rx->napi, skb); fdma_db_advance(fdma); counter++; diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c b/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c index 15ec9b053824..8675702bf150 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c @@ -157,9 +157,15 @@ static bool sparx5_fdma_rx_get_frame(struct sparx5 *sparx5, struct sparx5_rx *rx sparx5->ports[fi.src_port] : NULL; if (!port || !port->ndev) { - dev_err(sparx5->dev, "Data on inactive port %d\n", fi.src_port); - sparx5_xtr_flush(sparx5, XTR_QUEUE); - return false; + dev_err_ratelimited(sparx5->dev, "Data on inactive port %d\n", + fi.src_port); + + /* Drop the frame, but report the DB as consumed so the + * caller advances past it and recycles the DCB. + */ + kfree_skb(skb); + + return true; } skb->dev = port->ndev; skb_pull(skb, IFH_LEN * sizeof(u32)); diff --git a/drivers/net/ethernet/qualcomm/qca_spi.c b/drivers/net/ethernet/qualcomm/qca_spi.c index 38a779f4b866..249b4b549a12 100644 --- a/drivers/net/ethernet/qualcomm/qca_spi.c +++ b/drivers/net/ethernet/qualcomm/qca_spi.c @@ -801,8 +801,8 @@ qcaspi_netdev_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct qcaspi *qca = netdev_priv(dev); - netdev_info(qca->net_dev, "Transmit timeout at %ld, latency %ld\n", - jiffies, jiffies - dev_trans_start(dev)); + netdev_info(qca->net_dev, "Transmit timeout, latency %ld\n", + jiffies - dev_trans_start(dev)); qca->net_dev->stats.tx_errors++; /* Trigger tx queue flush and QCA7000 reset */ set_bit(SPI_RESET, &qca->flags); diff --git a/drivers/net/ethernet/spacemit/k1_emac.c b/drivers/net/ethernet/spacemit/k1_emac.c index 71a80838bffd..d07d8a1aa5ee 100644 --- a/drivers/net/ethernet/spacemit/k1_emac.c +++ b/drivers/net/ethernet/spacemit/k1_emac.c @@ -1732,7 +1732,6 @@ static int emac_down(struct emac_priv *priv) napi_disable(&priv->napi); timer_delete_sync(&priv->txtimer); - cancel_work_sync(&priv->tx_timeout_task); timer_delete_sync(&priv->stats_timer); @@ -2010,8 +2009,9 @@ static void emac_remove(struct platform_device *pdev) { struct emac_priv *priv = platform_get_drvdata(pdev); + disable_work_sync(&priv->tx_timeout_task); + timer_shutdown_sync(&priv->txtimer); - cancel_work_sync(&priv->tx_timeout_task); timer_shutdown_sync(&priv->stats_timer); @@ -2024,16 +2024,23 @@ static int emac_resume(struct device *dev) struct net_device *ndev = priv->ndev; int ret; + enable_work(&priv->tx_timeout_task); + ret = clk_prepare_enable(priv->bus_clk); if (ret < 0) { dev_err(dev, "Failed to enable bus clock: %d\n", ret); return ret; } + rtnl_lock(); - if (!netif_running(ndev)) + if (!netif_running(ndev)) { + rtnl_unlock(); return 0; + } ret = emac_open(ndev); + rtnl_unlock(); + if (ret) { clk_disable_unprepare(priv->bus_clk); return ret; @@ -2051,12 +2058,18 @@ static int emac_suspend(struct device *dev) struct emac_priv *priv = dev_get_drvdata(dev); struct net_device *ndev = priv->ndev; + disable_work_sync(&priv->tx_timeout_task); + + rtnl_lock(); + if (!ndev || !netif_running(ndev)) { + rtnl_unlock(); clk_disable_unprepare(priv->bus_clk); return 0; } emac_stop(ndev); + rtnl_unlock(); clk_disable_unprepare(priv->bus_clk); netif_device_detach(ndev); diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index c74bc7c9750e..a972bee66504 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -382,6 +382,7 @@ enum dma_irq_dir { DMA_DIR_RX = 0x1, DMA_DIR_TX = 0x2, DMA_DIR_RXTX = 0x3, + DMA_DIR_ERR = 0x4, }; enum request_irq_err { diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h index 1de1f929d61c..55d9c39287f1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h @@ -162,7 +162,7 @@ enum rx_tx_priority_ratio { /* DMA CRS Control and Status Register Mapping */ /* DMA operation mode defines (start/stop tx/rx are placed in common header)*/ -/* Disable Drop TCP/IP csum error */ +#define DMA_CONTROL_DT 0x04000000 /* Disable Drop of TCP/IP csum errors */ #define DMA_CONTROL_RSF 0x02000000 /* Receive Store and Forward */ #define DMA_CONTROL_DFF 0x01000000 /* Disaable flushing */ /* Threshold for Activating the FC */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c index 3ac7a7949529..31d5f79ef560 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c @@ -15,6 +15,7 @@ #include #include "dwmac1000.h" #include "dwmac_dma.h" +#include "stmmac.h" static void dwmac1000_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi) { @@ -129,6 +130,13 @@ static void dwmac1000_dma_operation_mode_rx(struct stmmac_priv *priv, { u32 csr6 = readl(ioaddr + DMA_CHAN_CONTROL(channel)); + /* Deliver checksum-error frames only with enhanced + * descriptors, because the normal descriptor parser does not + * report a checksum-only error. + */ + if (priv->plat->enh_desc) + csr6 |= DMA_CONTROL_DT; + if (mode == SF_DMA_MODE) { pr_debug("GMAC: enable RX store and forward mode\n"); csr6 |= DMA_CONTROL_RSF | DMA_CONTROL_DFF; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h index 43b036d4e95b..a7caac3c98e6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h @@ -173,6 +173,11 @@ static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs, DMA_CHAN_STATUS_TI | \ DMA_CHAN_STATUS_MSK_COMMON) +#define DMA_CHAN_STATUS_MSK_ERR (DMA_CHAN_STATUS_AIS | \ + DMA_CHAN_STATUS_FBE | \ + DMA_CHAN_STATUS_RPS | \ + DMA_CHAN_STATUS_RBU) + int dwmac4_dma_reset(void __iomem *ioaddr); void dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan, bool rx, bool tx); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c index a0249715fafa..02aa1a3dd1e2 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c @@ -150,6 +150,8 @@ int dwmac4_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr, intr_status &= DMA_CHAN_STATUS_MSK_RX; else if (dir == DMA_DIR_TX) intr_status &= DMA_CHAN_STATUS_MSK_TX; + else if (dir == DMA_DIR_ERR) + intr_status &= DMA_CHAN_STATUS_MSK_ERR; /* ABNORMAL interrupts */ if (unlikely(intr_status & DMA_CHAN_STATUS_AIS)) { diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index 1a88cbaed70c..6b2bfc241109 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -859,10 +859,10 @@ dwxgmac3_safety_feat_config(void __iomem *ioaddr, unsigned int asp, value |= XGMAC_TCEIE; /* TSO Memory Correctable Error */ writel(value, ioaddr + XGMAC_DMA_ECC_INT_ENABLE); - /* 0x2: Without ECC or Parity Ports on External Application Interface - * 0x4: Only ECC Protection for External Memory feature is selected + /* 0x4: Only ECC Protection for External Memory feature is + * selected, without any on-chip parity protection */ - if (asp == 0x2 || asp == 0x4) + if (asp == 0x4) return 0; /* 4. Enable Parity and Timeout for FSM */ diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c index 7ece8d4157a8..d0622f2605aa 100644 --- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c @@ -216,9 +216,6 @@ static int enh_desc_get_rx_status(struct stmmac_extra_stats *x, if (unlikely(rdes0 & RDES0_OVERFLOW_ERROR)) x->rx_gmac_overflow++; - if (unlikely(rdes0 & RDES0_IPC_CSUM_ERROR)) - pr_err("\tIPC Csum Error/Giant frame\n"); - if (unlikely(rdes0 & RDES0_COLLISION)) x->rx_collision++; if (unlikely(rdes0 & RDES0_RECEIVE_WATCHDOG)) @@ -230,7 +227,10 @@ static int enh_desc_get_rx_status(struct stmmac_extra_stats *x, if (unlikely(rdes0 & RDES0_CRC_ERROR)) { x->rx_crc_errors++; } - ret = discard_frame; + if (rdes0 & (RDES0_DESCRIPTOR_ERROR | RDES0_OVERFLOW_ERROR | + RDES0_COLLISION | RDES0_RECEIVE_WATCHDOG | + RDES0_MII_ERROR | RDES0_CRC_ERROR)) + ret = discard_frame; } /* After a payload csum error, the ES bit is set. diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 25894771e9e9..a685ce4638f6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -3145,6 +3145,22 @@ static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) return status; } +static void stmmac_dma_err_interrupt(struct stmmac_priv *priv) +{ + u8 channels_to_check = max(priv->plat->rx_queues_to_use, + priv->plat->tx_queues_to_use); + int status; + u8 chan; + + for (chan = 0; chan < channels_to_check; chan++) { + status = stmmac_dma_interrupt_status(priv, priv->ioaddr, + &priv->xstats, chan, + DMA_DIR_ERR); + if (unlikely(status & tx_hard_error)) + stmmac_global_err(priv); + } +} + /** * stmmac_dma_interrupt - DMA ISR * @priv: driver private structure @@ -5547,7 +5563,7 @@ static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, struct dma_desc *p, struct dma_desc *np, - struct xdp_buff *xdp) + struct xdp_buff *xdp, int status) { struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; struct stmmac_channel *ch = &priv->channel[queue]; @@ -5575,7 +5591,8 @@ static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, skb->protocol = eth_type_trans(skb, priv->dev); - if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) + if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) || + (status & csum_none)) skb_checksum_none_assert(skb); else skb->ip_summed = CHECKSUM_UNNECESSARY; @@ -5789,7 +5806,8 @@ read_again: switch (res) { case STMMAC_XDP_PASS: - stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); + stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp, + status); xsk_buff_free(buf->xdp); break; case STMMAC_XDP_CONSUMED: @@ -6029,7 +6047,7 @@ read_again: buf->page); buf->page = NULL; rx_dropped++; - count++; + error = 1; goto drain_data; } @@ -6065,7 +6083,7 @@ drain_data: if (likely(status & rx_not_ls)) goto read_again; if (!skb) - continue; + goto next; /* Got entire packet into SKB. Finish it. */ @@ -6498,6 +6516,9 @@ static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) /* To handle Common interrupts */ stmmac_common_interrupt(priv); + if (priv->plat->core_type == DWMAC_CORE_GMAC4) + stmmac_dma_err_interrupt(priv); + return IRQ_HANDLED; } @@ -6741,9 +6762,9 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) }; static const char * const dwxgmac_safety_feature_desc[] = { "No", - "All Safety Features with ECC and Parity", - "All Safety Features without ECC or Parity", - "All Safety Features with Parity Only", + "All Safety Features with ECC and Parity Ports on External Application Interface", + "All Safety Features without ECC or Parity Ports on External Application Interface", + "All Safety Features with Parity Ports on External Application Interface", "ECC Only", "UNDEFINED", "UNDEFINED", diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 31a66e855d6b..6370bdc6c58b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -129,10 +129,19 @@ static int stmmac_set_time(struct ptp_clock_info *ptp, container_of(ptp, struct stmmac_priv, ptp_clock_ops); unsigned long flags; + mutex_lock(&priv->est_lock); + if (priv->est && priv->est->enable) + stmmac_est_configure(priv, priv, priv->est, + priv->plat->clk_ptp_rate, false); + write_lock_irqsave(&priv->ptp_lock, flags); stmmac_init_systime(priv, priv->ptpaddr, ts->tv_sec, ts->tv_nsec); write_unlock_irqrestore(&priv->ptp_lock, flags); + if (priv->est && priv->est->enable) + __stmmac_setup_est(priv); + mutex_unlock(&priv->est_lock); + return 0; } diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c index e56ebbdd428d..dbbbaf7f20ac 100644 --- a/drivers/net/ethernet/via/via-rhine.c +++ b/drivers/net/ethernet/via/via-rhine.c @@ -2461,6 +2461,17 @@ static void rhine_remove_one_platform(struct platform_device *pdev) free_netdev(dev); } +static void rhine_shutdown_platform(struct platform_device *pdev) +{ + struct net_device *dev = platform_get_drvdata(pdev); + + rtnl_lock(); + netif_device_detach(dev); + if (netif_running(dev)) + dev_close(dev); + rtnl_unlock(); +} + static void rhine_shutdown_pci(struct pci_dev *pdev) { struct net_device *dev = pci_get_drvdata(pdev); @@ -2580,6 +2591,7 @@ static struct pci_driver rhine_driver_pci = { static struct platform_driver rhine_driver_platform = { .probe = rhine_init_one_platform, .remove = rhine_remove_one_platform, + .shutdown = rhine_shutdown_platform, .driver = { .name = DRV_NAME, .of_match_table = rhine_of_tbl, diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c index ca2d515b77b3..a39ad18382f2 100644 --- a/drivers/net/phy/realtek/realtek_main.c +++ b/drivers/net/phy/realtek/realtek_main.c @@ -3483,7 +3483,7 @@ static struct phy_driver realtek_drvs[] = { .write_mmd = genphy_write_mmd_unsupported, }, { PHY_ID_MATCH_EXACT(RTL_8261C_CG), - .name = "Realtek RTL8261C/D 10Gbps PHY", + .name = "Realtek RTL8261C_RTL8261D 10Gbps PHY", .probe = rtl8261x_probe, .config_init = rtl8261x_config_init, .get_features = rtl8261x_get_features, diff --git a/drivers/net/team/team_core.c b/drivers/net/team/team_core.c index cfb833c307c1..65ac25e97ae3 100644 --- a/drivers/net/team/team_core.c +++ b/drivers/net/team/team_core.c @@ -2834,6 +2834,10 @@ int team_nl_options_set_doit(struct sk_buff *skb, struct genl_info *info) ctx.info = opt_inst_info; switch (opt_type) { case TEAM_OPTION_TYPE_U32: + if (nla_len(attr_data) != sizeof(u32)) { + err = -EINVAL; + goto team_put; + } ctx.data.u32_val = nla_get_u32(attr_data); break; case TEAM_OPTION_TYPE_STRING: @@ -2853,6 +2857,10 @@ int team_nl_options_set_doit(struct sk_buff *skb, struct genl_info *info) ctx.data.bool_val = attr_data ? true : false; break; case TEAM_OPTION_TYPE_S32: + if (nla_len(attr_data) != sizeof(s32)) { + err = -EINVAL; + goto team_put; + } ctx.data.s32_val = nla_get_s32(attr_data); break; default: diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 86fd44e10755..d4870bdc156f 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -3375,7 +3375,8 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd, ret = -EBUSY; } else if ((unsigned short)arg == ARPHRD_IEEE802154 || (unsigned short)arg == ARPHRD_IEEE802154_MONITOR || - (unsigned short)arg == ARPHRD_6LOWPAN) { + (unsigned short)arg == ARPHRD_6LOWPAN || + (unsigned short)arg == ARPHRD_MCTP) { ret = -EINVAL; } else { ret = call_netdevice_notifiers(NETDEV_PRE_TYPE_CHANGE, diff --git a/drivers/net/usb/smsc75xx.c b/drivers/net/usb/smsc75xx.c index 6f72a3f15907..d1eca6bf7770 100644 --- a/drivers/net/usb/smsc75xx.c +++ b/drivers/net/usb/smsc75xx.c @@ -2217,6 +2217,14 @@ static int smsc75xx_rx_fixup(struct usbnet *dev, struct sk_buff *skb) return 0; } + if (unlikely(size < ETH_HLEN + ETH_FCS_LEN)) { + netif_dbg(dev, rx_err, dev->net, + "size err rx_cmd_a=0x%08x\n", + rx_cmd_a); + dev->net->stats.rx_length_errors++; + return 0; + } + /* last frame in this batch */ if (skb->len == size) { smsc75xx_rx_csum_offload(dev, skb, rx_cmd_a, diff --git a/drivers/net/veth.c b/drivers/net/veth.c index a61842b2585b..faa837acea7b 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -984,7 +984,7 @@ static int veth_poll(struct napi_struct *napi, int budget) /* NAPI functions as RCU section */ peer_dev = rcu_dereference_check(priv->peer, rcu_read_lock_bh_held()); - peer_txq = (peer_dev && queue_idx < peer_dev->real_num_tx_queues) ? + peer_txq = (peer_dev && queue_idx < peer_dev->num_tx_queues) ? netdev_get_tx_queue(peer_dev, queue_idx) : NULL; xdp_set_return_frame_no_direct(); @@ -1087,6 +1087,7 @@ static int __veth_napi_enable(struct net_device *dev) static void veth_napi_del_range(struct net_device *dev, int start, int end) { struct veth_priv *priv = netdev_priv(dev); + struct net_device *peer; int i; for (i = start; i < end; i++) { @@ -1099,11 +1100,17 @@ static void veth_napi_del_range(struct net_device *dev, int start, int end) } synchronize_net(); + peer = rtnl_dereference(priv->peer); + for (i = start; i < end; i++) { struct veth_rq *rq = &priv->rq[i]; rq->rx_notify_masked = false; ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); + + /* veth_poll() can no longer wake a txq stopped on a full ring */ + if (peer && i < peer->num_tx_queues) + netif_wake_subqueue(peer, i); } for (i = start; i < end; i++) { diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index ed20e76fbc4c..1cd64f9e12e7 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -792,19 +792,32 @@ static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); } -static int check_mergeable_len(struct net_device *dev, void *mrg_ctx, - unsigned int len) +/* Length of the mergeable buffer that add_recvbuf_mergeable() posted. */ +static unsigned int mergeable_ctx_to_buflen(void *mrg_ctx) { - unsigned int headroom, tailroom, room, truesize; + unsigned int headroom, tailroom, room; - truesize = mergeable_ctx_to_truesize(mrg_ctx); headroom = mergeable_ctx_to_headroom(mrg_ctx); tailroom = headroom ? sizeof(struct skb_shared_info) : 0; room = SKB_DATA_ALIGN(headroom + tailroom); - if (len > truesize - room) { - pr_debug("%s: rx error: len %u exceeds truesize %lu\n", - dev->name, len, (unsigned long)(truesize - room)); + return mergeable_ctx_to_truesize(mrg_ctx) - room; +} + +/* Length of the small buffer that add_recvbuf_small() posts. */ +static unsigned int small_buflen(const struct virtnet_info *vi) +{ + return vi->hdr_len + GOOD_PACKET_LEN; +} + +static int check_mergeable_len(struct net_device *dev, void *mrg_ctx, + unsigned int len) +{ + unsigned int buflen = mergeable_ctx_to_buflen(mrg_ctx); + + if (len > buflen) { + pr_debug("%s: rx error: len %u exceeds buffer length %u\n", + dev->name, len, buflen); DEV_STATS_INC(dev, rx_length_errors); return -1; } @@ -1797,15 +1810,15 @@ static struct page *xdp_linearize_page(struct net_device *dev, p = virt_to_head_page(buf); off = buf - page_address(p); - if (rq->use_page_pool_dma) - page_pool_dma_sync_for_cpu(rq->page_pool, p, - off, buflen); - if (check_mergeable_len(dev, ctx, buflen)) { page_pool_put_page(rq->page_pool, p, -1, true); goto err_buf; } + if (rq->use_page_pool_dma) + page_pool_dma_sync_for_cpu(rq->page_pool, p, + off, buflen); + /* guard against a misconfigured or uncooperative backend that * is sending packet larger than the MTU. */ @@ -2165,15 +2178,15 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev, page = virt_to_head_page(buf); offset = buf - page_address(page); - if (rq->use_page_pool_dma) - page_pool_dma_sync_for_cpu(rq->page_pool, page, - offset, len); - if (check_mergeable_len(dev, ctx, len)) { page_pool_put_page(rq->page_pool, page, -1, true); goto err; } + if (rq->use_page_pool_dma) + page_pool_dma_sync_for_cpu(rq->page_pool, page, + offset, len); + truesize = mergeable_ctx_to_truesize(ctx); xdp_frags_truesz += truesize; @@ -2446,15 +2459,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, u64_stats_add(&stats->bytes, len); page = virt_to_head_page(buf); + if (check_mergeable_len(dev, ctx, len)) + goto err_skb; + if (rq->use_page_pool_dma) { offset = buf - page_address(page); page_pool_dma_sync_for_cpu(rq->page_pool, page, offset, len); } - if (check_mergeable_len(dev, ctx, len)) - goto err_skb; - truesize = mergeable_ctx_to_truesize(ctx); curr_skb = virtnet_skb_append_frag(rq, head_skb, curr_skb, page, buf, len, truesize); @@ -2574,12 +2587,22 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, /* Sync the memory before touching anything through buf, * unless virtio core did it already. + * + * len comes from the device and is checked against the buffer + * only later, so never sync past the buffer posted to the device. + * The sync still covers the header, which receive_buf() and the + * mergeable drop path read. Big buffers never come from the page + * pool, so a buffer here is either mergeable or small. */ if (rq->use_page_pool_dma) { struct page *page = virt_to_head_page(buf); int offset = buf - page_address(page); + unsigned int buflen = vi->mergeable_rx_bufs ? + mergeable_ctx_to_buflen(ctx) : + small_buflen(vi); - page_pool_dma_sync_for_cpu(rq->page_pool, page, offset, len); + page_pool_dma_sync_for_cpu(rq->page_pool, page, offset, + min(len, buflen)); } /* About the flags below: @@ -2655,7 +2678,7 @@ static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, buf += VIRTNET_RX_PAD + xdp_headroom; ctx = mergeable_len_to_ctx(alloc_len, xdp_headroom); - err = virtnet_rq_submit(rq, buf, vi->hdr_len + GOOD_PACKET_LEN, ctx, gfp); + err = virtnet_rq_submit(rq, buf, small_buflen(vi), ctx, gfp); if (err < 0) page_pool_put_page(rq->page_pool, virt_to_head_page(buf), -1, false); @@ -6017,7 +6040,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid) struct send_queue *sq; int err; - if (qid >= vi->curr_queue_pairs) + /* Removing XDP can leave the pool on a queue above curr_queue_pairs */ + if (qid >= vi->max_queue_pairs) return -EINVAL; sq = &vi->sq[qid]; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 697fc9387b73..37b6e47c9091 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -12453,8 +12453,8 @@ struct mlx5_ifc_lagc_bits { u8 reserved_at_18[0x5]; u8 lag_state[0x3]; - u8 reserved_at_20[0xc]; - u8 active_port[0x4]; + u8 reserved_at_20[0x8]; + u8 active_port[0x8]; u8 reserved_at_30[0x4]; u8 tx_remap_affinity_2[0x4]; u8 reserved_at_38[0x4]; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 6787fcf6ac3e..adc61f5e3e85 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -268,6 +268,7 @@ SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X)) #define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0)) #define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2)) +#define SKB_MAX_RX_HEADROOM (U16_MAX / 2) /* return minimum truesize of one skb containing X bytes of data */ #define SKB_TRUESIZE(X) ((X) + \ diff --git a/include/net/amt.h b/include/net/amt.h index 2846dde0cadc..9ba09712731d 100644 --- a/include/net/amt.h +++ b/include/net/amt.h @@ -240,12 +240,19 @@ struct amt_tunnel_list { u32 nr_sources; enum amt_status status; struct delayed_work gc_wq; + /* Last time a Membership Update refreshed this tunnel, under ->lock. + * gc_wq re-arms itself from this rather than being re-armed by the + * Update path, so the Update path cannot re-queue the work of a + * tunnel that amt_tunnel_expire() is freeing. + */ + unsigned long last_seen; + /* Set under amt->lock and tunnel->lock by whichever freer reaches the + * tunnel first; the other then leaves it alone, so the tunnel is freed + * exactly once. + */ + bool dead; __be16 source_port; __be32 ip4; - __be32 nonce; - siphash_key_t key; - u64 mac:48, - reserved:16; struct rcu_head rcu; struct hlist_head groups[]; }; @@ -275,6 +282,7 @@ enum amt_source_status { /* protected by gnode->lock */ struct amt_source_node { struct hlist_node node; + struct hlist_node gc_node; struct amt_group_node *gnode; struct delayed_work source_timer; union amt_addr source_addr; @@ -282,6 +290,8 @@ struct amt_source_node { #define AMT_SOURCE_OLD 0 #define AMT_SOURCE_NEW 1 u8 flags; + /* Protected by gnode->tunnel_list->lock. */ + bool dead; struct rcu_head rcu; }; @@ -295,7 +305,10 @@ struct amt_group_node { u32 nr_sources; struct amt_tunnel_list *tunnel_list; struct hlist_node node; + struct hlist_node gc_node; struct delayed_work group_timer; + /* Protected by tunnel_list->lock. */ + bool dead; struct rcu_head rcu; struct hlist_head sources[]; }; @@ -325,8 +338,12 @@ struct amt_dev { struct work_struct event_wq; /* AMT status */ enum amt_status status; - /* Generated key */ - siphash_key_t key; + /* Current key and three previous generations keep MACs valid for + * 180-240s, beyond the advertised QQIC (see amt_secret_work()). + */ + siphash_key_t key[4]; + /* For lockless readers of key; writers hold lock */ + seqcount_spinlock_t key_seq; struct sock __rcu *sk; u32 max_groups; u32 max_sources; @@ -370,6 +387,7 @@ struct amt_dev { #define AMT_IPHDR_OPTS 4 #define AMT_IP6HDR_OPTS 8 #define AMT_GC_INTERVAL (30 * 1000) +#define AMT_GC_KICK 2048 #define AMT_MAX_GROUP 32 #define AMT_MAX_SOURCE 128 #define AMT_HSIZE_SHIFT 8 @@ -383,6 +401,8 @@ struct amt_dev { #define AMT_SECRET_TIMEOUT 60000 #define IANA_AMT_UDP_PORT 2268 #define AMT_MAX_TUNNELS 128 +/* Upper bound of amt_max_tunnels_per_src(), the per-address tunnel cap */ +#define AMT_MAX_TUNNELS_PER_SRC 16 #define AMT_MAX_REQS 128 #define AMT_GW_HLEN (sizeof(struct iphdr) + \ sizeof(struct udphdr) + \ diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 6fb713d5e4be..fd4da751a24f 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -1972,6 +1972,7 @@ struct adv_info *hci_add_per_instance(struct hci_dev *hdev, u8 instance, u8 sid, int hci_set_adv_instance_data(struct hci_dev *hdev, u8 instance, u16 adv_data_len, u8 *adv_data, u16 scan_rsp_len, u8 *scan_rsp_data); +void hci_free_adv_instance(struct hci_dev *hdev, struct adv_info *adv_instance); int hci_remove_adv_instance(struct hci_dev *hdev, u8 instance); void hci_adv_instances_set_rpa_expired(struct hci_dev *hdev, bool rpa_expired); u32 hci_adv_instance_flags(struct hci_dev *hdev, u8 instance); diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 2a05bda8ab43..bb2839972cf3 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -636,6 +636,7 @@ struct l2cap_conn { struct sk_buff_head pending_rx; struct work_struct pending_rx_work; + bool pending_rx_blocked __guarded_by(&lock); struct delayed_work id_addr_timer; diff --git a/include/net/dropreason-core.h b/include/net/dropreason-core.h index fa2fe8990557..903d6a3a0ff1 100644 --- a/include/net/dropreason-core.h +++ b/include/net/dropreason-core.h @@ -128,6 +128,7 @@ FN(PSP_INPUT) \ FN(PSP_OUTPUT) \ FN(RECURSION_LIMIT) \ + FN(SKB_RX_HEADROOM) \ FNe(MAX) /** @@ -612,6 +613,11 @@ enum skb_drop_reason { * itself. */ SKB_DROP_REASON_RECURSION_LIMIT, + /** + * @SKB_DROP_REASON_SKB_RX_HEADROOM: accumulated receive headroom + * exceeds the header-offset budget. + */ + SKB_DROP_REASON_SKB_RX_HEADROOM, /** * @SKB_DROP_REASON_MAX: the maximum of core drop reasons, which * shouldn't be used as a real 'reason' - only for tracing code gen diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h index 6e2fe186d0dc..44b7e39a1245 100644 --- a/include/net/inet_hashtables.h +++ b/include/net/inet_hashtables.h @@ -87,6 +87,8 @@ struct inet_bind_bucket { __be32 fast_rcv_saddr; unsigned short fast_sk_family; bool fast_ipv6_only; + unsigned char fast_sk_reuse; + int fast_bound_dev_if; struct hlist_node node; struct hlist_head bhash2; struct rcu_head rcu; diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index 9a83bff32569..8dc80d5d42dc 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -1021,7 +1021,9 @@ struct ip_vs_dest { struct rcu_head rcu_head; struct list_head t_list; /* in dest_trash */ - unsigned int in_rs_table:1; /* we are in rs_table */ + unsigned long rcu_state; /* GP cookie on unlink (trash or rehash) */ + unsigned int in_rs_table:1, /* we are in rs_table */ + need_gp:1; /* Need GP on rehash */ }; /* The scheduler object */ @@ -1685,7 +1687,6 @@ static inline bool __ip_vs_conn_get(struct ip_vs_conn *cp) /* put back the conn without restarting its timer */ static inline void __ip_vs_conn_put(struct ip_vs_conn *cp) { - smp_mb__before_atomic(); refcount_dec(&cp->refcnt); } void ip_vs_conn_put(struct ip_vs_conn *cp); @@ -1923,7 +1924,7 @@ static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) kfree(dest); } -void ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload); +void ip_vs_dest_update_overload(struct ip_vs_dest *dest); /* Check if dest is in overloaded state */ static inline bool ip_vs_dest_is_overloaded(const struct ip_vs_dest *dest) diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 79a5a548a129..d6d691fec124 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -394,7 +394,7 @@ struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl, const void *key, struct net_device *dev); int pneigh_create(struct neigh_table *tbl, const void *key, struct net_device *dev, u32 flags, u8 protocol, - bool permanent); + bool permanent, bool update); int pneigh_delete(struct neigh_table *tbl, const void *key, struct net_device *dev); diff --git a/include/net/tcp.h b/include/net/tcp.h index 5c9203aae834..27de546dc802 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -164,6 +164,13 @@ static_assert((1 << ATO_BITS) > TCP_DELACK_MAX); #define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */ +/* Max RTT sample for tcp_rtt_estimator(). srtt_us holds 8 times the RTT, and + * mdev_us and rttvar_us up to 4 times its deviation, all in u32, like the sum + * in __tcp_set_rto(). A larger sample could not raise the RTO, which + * TCP_RTO_MAX bounds. + */ +#define TCP_RTT_MAX_US (TCP_RTO_MAX_SEC * USEC_PER_SEC) + #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now * used as a fallback RTO for the @@ -910,7 +917,9 @@ static inline void tcp_bound_rto(struct sock *sk) static inline u32 __tcp_set_rto(const struct tcp_sock *tp) { - return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); + u32 rto = usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); + + return max(rto, 1U); } static inline unsigned long tcp_reqsk_timeout(struct request_sock *req) @@ -930,7 +939,8 @@ static inline u32 tcp_rto_min(const struct sock *sk) u32 rto_min = READ_ONCE(inet_csk(sk)->icsk_rto_min); if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) - rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); + rto_min = clamp(dst_metric_rtt(dst, RTAX_RTO_MIN), + 1, TCP_RTO_MAX); return rto_min; } diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c index 8c51c20f6c4b..68ca2f49dadc 100644 --- a/net/bluetooth/6lowpan.c +++ b/net/bluetooth/6lowpan.c @@ -345,6 +345,7 @@ static int recv_pkt(struct sk_buff *skb, struct net_device *dev, return NET_RX_SUCCESS; drop: + kfree_skb(skb); dev->stats.rx_dropped++; return NET_RX_DROP; } @@ -356,21 +357,26 @@ static int chan_recv_cb(struct l2cap_chan *chan, struct sk_buff *skb) struct lowpan_peer *peer; int err; + /* The skb is always consumed, so never return an error that would + * make the caller free it again. + */ peer = lookup_peer(chan->conn); if (!peer) - return -ENOENT; + goto drop; dev = lookup_dev(chan->conn); if (!dev || !dev->netdev) - return -ENOENT; + goto drop; err = recv_pkt(skb, dev->netdev, peer); - if (err) { + if (err) BT_DBG("recv pkt %d", err); - err = -EAGAIN; - } - return err; + return 0; + +drop: + kfree_skb(skb); + return 0; } static int setup_header(struct sk_buff *skb, struct net_device *netdev, diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index eebd73988bf6..2fb37e078d73 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -1019,6 +1019,15 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, struct hci_conn *conn; struct smp_irk *irk = NULL; + lockdep_assert_held(&hdev->lock); + + /* hci_dev_close_sync() clears HCI_UP before hci_conn_hash_flush(). + * A connection added after that would never be flushed, so refuse + * it. The caller holds hdev->lock, which the flush also takes. + */ + if (!test_bit(HCI_UP, &hdev->flags)) + return ERR_PTR(-ENETDOWN); + switch (type) { case ACL_LINK: if (!hdev->acl_mtu) @@ -1166,6 +1175,9 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, hci_dev_hold(hdev); + /* Initialize the device before publishing it to RCU readers. */ + hci_conn_init_sysfs(conn); + hci_conn_hash_add(hdev, conn); /* The SCO and eSCO connections will only be notified when their @@ -1177,13 +1189,13 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, hdev->notify(hdev, HCI_NOTIFY_CONN_ADD); } - hci_conn_init_sysfs(conn); return conn; } struct hci_conn *hci_conn_add_unset(struct hci_dev *hdev, int type, bdaddr_t *dst, u8 dst_type, u8 role) { + struct hci_conn *conn; int handle; bt_dev_dbg(hdev, "dst %pMR", dst); @@ -1192,7 +1204,11 @@ struct hci_conn *hci_conn_add_unset(struct hci_dev *hdev, int type, if (unlikely(handle < 0)) return ERR_PTR(-ECONNREFUSED); - return __hci_conn_add(hdev, type, dst, dst_type, role, handle); + conn = __hci_conn_add(hdev, type, dst, dst_type, role, handle); + if (IS_ERR(conn)) + ida_free(&hdev->unset_handle_ida, handle); + + return conn; } struct hci_conn *hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t *dst, diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index c97aec3caba0..846e6eb2d1bc 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -1610,17 +1610,11 @@ struct adv_info *hci_get_next_instance(struct hci_dev *hdev, u8 instance) } /* This function requires the caller holds hdev->lock */ -int hci_remove_adv_instance(struct hci_dev *hdev, u8 instance) +void hci_free_adv_instance(struct hci_dev *hdev, struct adv_info *adv_instance) { - struct adv_info *adv_instance; + BT_DBG("%s removing %dMR", hdev->name, adv_instance->instance); - adv_instance = hci_find_adv_instance(hdev, instance); - if (!adv_instance) - return -ENOENT; - - BT_DBG("%s removing %dMR", hdev->name, instance); - - if (hdev->cur_adv_instance == instance) { + if (hdev->cur_adv_instance == adv_instance->instance) { if (hdev->adv_instance_timeout) { cancel_delayed_work(&hdev->adv_instance_expire); hdev->adv_instance_timeout = false; @@ -1634,6 +1628,18 @@ int hci_remove_adv_instance(struct hci_dev *hdev, u8 instance) kfree(adv_instance); hdev->adv_instance_cnt--; +} + +/* This function requires the caller holds hdev->lock */ +int hci_remove_adv_instance(struct hci_dev *hdev, u8 instance) +{ + struct adv_info *adv_instance; + + adv_instance = hci_find_adv_instance(hdev, instance); + if (!adv_instance) + return -ENOENT; + + hci_free_adv_instance(hdev, adv_instance); return 0; } diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 5e6b25eb5053..10c8b7b8e329 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -6158,6 +6158,26 @@ static void hci_le_pa_sync_lost_evt(struct hci_dev *hdev, void *data, hci_dev_unlock(hdev); } +/* The instance advertising on the set identified by handle. Handle 0x00 may + * be shared by Global Advertising and an instance, e.g. instance 1 when the + * controller supports a single advertising set, in which case the instance + * enabled on it, if any, owns the set (see hci_le_ext_adv_set_enabled()). + */ +static struct adv_info *hci_le_ext_adv_owner(struct hci_dev *hdev, u8 handle) +{ + struct adv_info *adv; + + if (handle) + return hci_find_adv_instance(hdev, handle); + + list_for_each_entry(adv, &hdev->adv_instances, list) { + if (!adv->handle && adv->enabled) + return adv; + } + + return NULL; +} + static void hci_le_ext_adv_term_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { @@ -6181,15 +6201,18 @@ static void hci_le_ext_adv_term_evt(struct hci_dev *hdev, void *data, hci_dev_lock(hdev); - adv = hci_find_adv_instance(hdev, ev->handle); + adv = hci_le_ext_adv_owner(hdev, ev->handle); if (ev->status) { + u8 instance; + if (!adv) goto unlock; /* Remove advertising as it has been terminated */ - hci_remove_adv_instance(hdev, ev->handle); - mgmt_advertising_removed(NULL, hdev, ev->handle); + instance = adv->instance; + hci_free_adv_instance(hdev, adv); + mgmt_advertising_removed(NULL, hdev, instance); list_for_each_entry_safe(adv, n, &hdev->adv_instances, list) { if (adv->enabled) @@ -6206,22 +6229,19 @@ static void hci_le_ext_adv_term_evt(struct hci_dev *hdev, void *data, conn = hci_conn_hash_lookup_handle(hdev, __le16_to_cpu(ev->conn_handle)); if (conn) { - /* Store handle in the connection so the correct advertising - * instance can be re-enabled when disconnected. + /* Store the instance in the connection so the correct + * advertising instance can be re-enabled when disconnected. */ - conn->adv_instance = ev->handle; + conn->adv_instance = adv ? adv->instance : ev->handle; if (hdev->adv_addr_type != ADDR_LE_DEV_RANDOM || bacmp(&conn->resp_addr, BDADDR_ANY)) goto unlock; - if (!ev->handle) { - bacpy(&conn->resp_addr, &hdev->random_addr); - goto unlock; - } - if (adv) bacpy(&conn->resp_addr, &adv->random_addr); + else if (!ev->handle) + bacpy(&conn->resp_addr, &hdev->random_addr); } unlock: diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 9213e3ef9f26..44ce776890d1 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -31,21 +31,23 @@ static void hci_cmd_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, hdev->req_result = result; WRITE_ONCE(hdev->req_status, HCI_REQ_DONE); - /* Free the request command so it is not used as response */ + /* Free the request command so it is not used as response, and drop + * the reference hci_cmd_sync_alloc() took on its sk, if set: the + * response skb doesn't carry it. + */ hci_dev_lock(hdev); + if (hdev->req_skb) { + struct sock *sk = hci_skb_sk(hdev->req_skb); + + if (sk) + sock_put(sk); + } kfree_skb(hdev->req_skb); hdev->req_skb = NULL; hci_dev_unlock(hdev); - if (skb) { - struct sock *sk = hci_skb_sk(skb); - - /* Drop sk reference if set */ - if (sk) - sock_put(sk); - + if (skb) hdev->req_rsp = skb_get(skb); - } wake_up_interruptible(&hdev->req_wait_q); } @@ -1258,6 +1260,8 @@ static int hci_set_adv_set_random_addr_sync(struct hci_dev *hdev, u8 instance, bdaddr_t *random_addr) { struct hci_cp_le_set_adv_set_rand_addr cp; + struct adv_info *adv; + u8 handle = instance; int err; if (!instance) { @@ -1269,15 +1273,46 @@ static int hci_set_adv_set_random_addr_sync(struct hci_dev *hdev, u8 instance, err = hci_set_random_addr_sync(hdev, random_addr); if (err) return err; + } else { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (adv) + handle = adv->handle; + hci_dev_unlock(hdev); } memset(&cp, 0, sizeof(cp)); - cp.handle = instance; + cp.handle = handle; bacpy(&cp.bdaddr, random_addr); - return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_SET_RAND_ADDR, - sizeof(cp), &cp, HCI_CMD_TIMEOUT); + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_SET_RAND_ADDR, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); + if (err || handle == instance) + return err; + + /* hci_cc_le_set_adv_set_random_addr() records the address of the + * instance numbered as the handle, and leaves handle 0x00 to + * hdev->random_addr, so record it here for an instance using another + * handle: instance 1 on handle 0x00, on a controller supporting a + * single advertising set. + */ + hci_dev_lock(hdev); + + adv = hci_find_adv_instance(hdev, instance); + if (adv) { + bacpy(&adv->random_addr, random_addr); + if (!bacmp(&hdev->rpa, random_addr)) { + adv->rpa_expired = false; + queue_delayed_work(hdev->workqueue, + &adv->rpa_expired_cb, + secs_to_jiffies(hdev->rpa_timeout)); + } + } + + hci_dev_unlock(hdev); + + return 0; } static int @@ -2077,6 +2112,9 @@ int hci_enable_advertising(struct hci_dev *hdev) int hci_remove_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance, struct sock *sk) { + struct adv_info *adv = NULL; + u8 handle = instance; + bool global = false; int err; if (!ext_adv_capable(hdev)) @@ -2086,13 +2124,53 @@ int hci_remove_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance, if (err) return err; - /* If request specifies an instance that doesn't exist, fail */ - if (instance > 0 && !hci_find_adv_instance(hdev, instance)) - return -EINVAL; + if (instance) { + hci_dev_lock(hdev); - return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_REMOVE_ADV_SET, - sizeof(instance), &instance, 0, - HCI_CMD_TIMEOUT, sk); + adv = hci_find_adv_instance(hdev, instance); + if (adv) { + handle = adv->handle; + + /* An instance sharing handle 0x00 with Global + * Advertising doesn't own the set while Global + * Advertising is on, and the set may still be enabled + * for it, which the controller would refuse to remove. + */ + global = !handle && + (hci_dev_test_flag(hdev, HCI_ADVERTISING) || + hci_dev_test_flag(hdev, HCI_LE_ADV_0)); + } + + hci_dev_unlock(hdev); + + /* If request specifies an instance that doesn't exist, fail */ + if (!adv) + return -EINVAL; + } + + if (!global) { + err = __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_REMOVE_ADV_SET, + sizeof(handle), &handle, 0, + HCI_CMD_TIMEOUT, sk); + if (err) + return err; + } + + /* hci_cc_le_remove_adv_set() removes the instance numbered as the + * handle it sent, so remove here an instance using another handle: + * instance 1 on handle 0x00, on a controller supporting a single + * advertising set. The completion can't tell it from instance 0. + */ + if (handle != instance) { + hci_dev_lock(hdev); + + if (!hci_remove_adv_instance(hdev, instance)) + mgmt_advertising_removed(sk, hdev, instance); + + hci_dev_unlock(hdev); + } + + return 0; } int hci_le_terminate_big_sync(struct hci_dev *hdev, u8 handle, u8 reason) @@ -5694,6 +5772,7 @@ static int hci_dev_shutdown(struct hci_dev *hdev) int hci_dev_close_sync(struct hci_dev *hdev) { + struct hci_conn *conn; bool auto_off; int err = 0; @@ -5761,6 +5840,22 @@ int hci_dev_close_sync(struct hci_dev *hdev) cancel_delayed_work_sync(&adv_instance->rpa_expired_cb); } + /* A delayed work whose timer was armed before close would queue + * itself while the workqueue is draining. Disable the connection + * works so that hci_conn_drop() and the timers can no longer queue + * them. The connections are deleted below, so nothing re-enables + * them. HCI_UP is clear, so no new connection can be added after + * this walk. + */ + hci_dev_lock(hdev); + list_for_each_entry(conn, &hdev->conn_hash.list, list) { + disable_delayed_work(&conn->disc_work); + disable_delayed_work(&conn->auto_accept_work); + disable_delayed_work(&conn->idle_work); + disable_delayed_work(&conn->le_conn_timeout); + } + hci_dev_unlock(hdev); + /* Avoid potential lockdep warnings from the *_flush() calls by * ensuring the workqueue is empty up front. */ diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 7dcb0586a10b..686e86c04d7e 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -1747,10 +1747,40 @@ static void l2cap_le_conn_ready(struct l2cap_conn *conn) } } -static void l2cap_conn_ready(struct l2cap_conn *conn) +static bool l2cap_is_blocked(struct hci_conn *hcon) + __must_hold(&hcon->hdev->lock) { + /* Since we can't actively block incoming LE connections we must + * at least ensure that we ignore incoming data from them. + */ + return hcon->type == LE_LINK && + hci_bdaddr_list_lookup(&hcon->hdev->reject_list, &hcon->dst, + bdaddr_dst_type(hcon)); +} + +static void l2cap_queue_pending_rx(struct hci_conn *hcon) + __must_hold(&hcon->hdev->lock) +{ + struct l2cap_conn *conn = hcon->l2cap_data; + + mutex_lock(&conn->lock); + + if (!skb_queue_empty(&conn->pending_rx)) { + /* Teardown holds hdev->lock while cancelling the worker. + * Pass the decision under conn->lock to avoid that lock. + */ + conn->pending_rx_blocked = l2cap_is_blocked(hcon); + queue_work(hcon->hdev->workqueue, &conn->pending_rx_work); + } + + mutex_unlock(&conn->lock); +} + +static void l2cap_conn_ready(struct hci_conn *hcon) + __must_hold(&hcon->hdev->lock) +{ + struct l2cap_conn *conn = hcon->l2cap_data; struct l2cap_chan *chan; - struct hci_conn *hcon = conn->hcon; BT_DBG("conn %p", conn); @@ -1782,7 +1812,7 @@ static void l2cap_conn_ready(struct l2cap_conn *conn) if (hcon->type == LE_LINK) l2cap_le_conn_ready(conn); - queue_work(hcon->hdev->workqueue, &conn->pending_rx_work); + l2cap_queue_pending_rx(hcon); } /* Notify sockets that we cannot guaranty reliability anymore */ @@ -5701,6 +5731,16 @@ static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn, break; } + /* + * Validate MPS and MTU per BT Core Spec + * Vol 3 Part A Sec 10.2 + */ + if (mtu < L2CAP_ECRED_MIN_MTU || + mps < L2CAP_ECRED_MIN_MPS) { + l2cap_chan_del(chan, ECONNREFUSED); + break; + } + chan->ident = 0; chan->dcid = dcid; chan->omtu = mtu; @@ -6186,6 +6226,7 @@ static void append_skb_frag(struct sk_buff *skb, struct sk_buff *new_frag, static int l2cap_reassemble_sdu(struct l2cap_chan *chan, struct sk_buff *skb, struct l2cap_ctrl *control) { + struct sk_buff *sdu; int err = -EINVAL; switch (control->sar) { @@ -6246,15 +6287,25 @@ static int l2cap_reassemble_sdu(struct l2cap_chan *chan, struct sk_buff *skb, if (chan->sdu->len != chan->sdu_len) break; - err = chan->ops->recv(chan, chan->sdu); + /* Reassembly complete. Hand the SDU over before the + * callback, for the reason given in l2cap_ecred_data_rcv(): + * the callback takes ownership and may free it, while + * l2cap_chan_rx_avail() -> l2cap_chan_le_send_credits() reads + * chan->sdu->len on the way out - it does so before the mode + * test that makes the rest of that function a no-op here. + * Return directly, since the state this leaves is what the + * trailing error block would otherwise set again. + */ + sdu = chan->sdu; + chan->sdu = NULL; + chan->sdu_last_frag = NULL; + chan->sdu_len = 0; - if (!err) { - /* Reassembly complete */ - chan->sdu = NULL; - chan->sdu_last_frag = NULL; - chan->sdu_len = 0; - } - break; + err = chan->ops->recv(chan, sdu); + if (err) + kfree_skb(sdu); + + return err; } if (err) { @@ -7099,6 +7150,9 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) pkt.cid = cpu_to_le16(chan->scid); pkt.credits = cpu_to_le16(return_credits); + /* Keeps l2cap_put_ident() from releasing ident before we do */ + lockdep_assert_held(&conn->lock); + ident = l2cap_get_ident(conn); l2cap_send_cmd(conn, ident, L2CAP_LE_CREDITS, sizeof(pkt), &pkt); @@ -7245,12 +7299,26 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) skb = NULL; if (chan->sdu->len == chan->sdu_len) { - err = l2cap_ecred_recv(chan, chan->sdu); - if (!err) { - chan->sdu = NULL; - chan->sdu_last_frag = NULL; - chan->sdu_len = 0; - } + struct sk_buff *sdu = chan->sdu; + + /* The receive callback takes ownership of a reassembled SDU and + * may free it, while l2cap_ecred_recv() goes on to update the + * credits - which reads chan->sdu->len. Drop the reassembly + * state before handing the SDU over, and release it here if the + * callback refused it, so that it is still freed exactly once. + * The reassembly state is already clear at that point, so + * return instead of falling into the failed: block, whose + * kfree_skb(chan->sdu) would have nothing left to free. + */ + chan->sdu = NULL; + chan->sdu_last_frag = NULL; + chan->sdu_len = 0; + + err = l2cap_ecred_recv(chan, sdu); + if (err) + kfree_skb(sdu); + + return 0; } failed: @@ -7381,7 +7449,8 @@ free_skb: kfree_skb(skb); } -static void l2cap_recv_frame(struct l2cap_conn *conn, struct sk_buff *skb) +static void l2cap_recv_frame(struct l2cap_conn *conn, struct sk_buff *skb, + bool blocked) __must_hold(&conn->lock) { struct l2cap_hdr *lh = (void *) skb->data; @@ -7408,12 +7477,7 @@ static void l2cap_recv_frame(struct l2cap_conn *conn, struct sk_buff *skb) return; } - /* Since we can't actively block incoming LE connections we must - * at least ensure that we ignore incoming data from them. - */ - if (hcon->type == LE_LINK && - hci_bdaddr_list_lookup(&hcon->hdev->reject_list, &hcon->dst, - bdaddr_dst_type(hcon))) { + if (blocked) { kfree_skb(skb); return; } @@ -7461,7 +7525,7 @@ static void process_pending_rx(struct work_struct *work) */ while (!test_bit(HCI_CONN_ENC_KEY_SIZE_PEND, conn->hcon->flags) && (skb = skb_dequeue(&conn->pending_rx))) - l2cap_recv_frame(conn, skb); + l2cap_recv_frame(conn, skb, conn->pending_rx_blocked); mutex_unlock(&conn->lock); } @@ -7913,7 +7977,7 @@ next: mutex_unlock(&conn->lock); - l2cap_conn_ready(conn); + l2cap_conn_ready(hcon); } int l2cap_disconn_ind(struct hci_conn *hcon) @@ -8052,8 +8116,7 @@ static void l2cap_security_cfm(struct hci_conn *hcon, u8 status, u8 encrypt) mutex_unlock(&conn->lock); /* Process data held back while the encryption key size was read */ - if (!skb_queue_empty(&conn->pending_rx)) - queue_work(hcon->hdev->workqueue, &conn->pending_rx_work); + l2cap_queue_pending_rx(hcon); } /* Append fragment into frame respecting the maximum len of rx_skb */ @@ -8142,6 +8205,7 @@ int l2cap_recv_acldata(struct hci_dev *hdev, u16 handle, { struct hci_conn *hcon; struct l2cap_conn *conn; + bool blocked; int len; /* Lock hdev for hci_conn, and race on l2cap_data vs. l2cap_conn_del */ @@ -8164,6 +8228,7 @@ int l2cap_recv_acldata(struct hci_dev *hdev, u16 handle, conn = l2cap_conn_add(hcon); conn = l2cap_conn_hold_unless_zero(conn); + blocked = l2cap_is_blocked(hcon); hcon = NULL; hci_dev_unlock(hdev); @@ -8200,7 +8265,7 @@ int l2cap_recv_acldata(struct hci_dev *hdev, u16 handle, if (len == skb->len) { /* Complete frame received */ - l2cap_recv_frame(conn, skb); + l2cap_recv_frame(conn, skb, blocked); goto unlock; } @@ -8224,7 +8289,7 @@ int l2cap_recv_acldata(struct hci_dev *hdev, u16 handle, * setting the socket unreliable. */ skb->len = len; - l2cap_recv_frame(conn, skb); + l2cap_recv_frame(conn, skb, blocked); l2cap_conn_unreliable(conn, ECOMM); goto unlock; } @@ -8274,7 +8339,7 @@ int l2cap_recv_acldata(struct hci_dev *hdev, u16 handle, */ struct sk_buff *rx_skb = conn->rx_skb; conn->rx_skb = NULL; - l2cap_recv_frame(conn, rx_skb); + l2cap_recv_frame(conn, rx_skb, blocked); } break; } diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index e5c15c083c1a..6b65710845dc 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -1229,11 +1229,11 @@ static void l2cap_publish_rx_avail(struct l2cap_chan *chan) /* Correct available memory by estimated sk_buff overhead. * This is significant due to small transfer sizes. However, accept - * at least one full packet if receive space is non-zero. + * at least one full SDU if receive space is non-zero. */ expected_skbs = DIV_ROUND_UP(avail, chan->mps); skb_overhead = expected_skbs * sizeof(struct sk_buff); - if (skb_overhead < avail) + if (avail - skb_overhead >= chan->imtu + L2CAP_SDULEN_SIZE) l2cap_chan_rx_avail(chan, avail - skb_overhead); else l2cap_chan_rx_avail(chan, -1); @@ -1292,6 +1292,7 @@ static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg, { struct sock *sk = sock->sk; struct l2cap_pinfo *pi = l2cap_pi(sk); + bool have_conn; int err; if (unlikely(flags & MSG_ERRQUEUE)) @@ -1312,9 +1313,12 @@ static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg, pi->chan->mode != L2CAP_MODE_EXT_FLOWCTL) return err; - lock_sock(sk); - + /* Returning credits must be serialized with the RX path */ + have_conn = l2cap_chan_lock_conn(pi->chan); l2cap_publish_rx_avail(pi->chan); + l2cap_chan_unlock_conn(pi->chan, have_conn); + + lock_sock(sk); /* Attempt to put pending rx data in the socket buffer */ while (!list_empty(&pi->rx_busy)) { diff --git a/net/bluetooth/mgmt_config.c b/net/bluetooth/mgmt_config.c index fdcc752c6f13..6a617656bfbe 100644 --- a/net/bluetooth/mgmt_config.c +++ b/net/bluetooth/mgmt_config.c @@ -362,6 +362,6 @@ int set_def_runtime_config(struct sock *sk, struct hci_dev *hdev, void *data, { bt_dev_dbg(hdev, "sock %p", sk); - return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_DEF_SYSTEM_CONFIG, + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_DEF_RUNTIME_CONFIG, MGMT_STATUS_INVALID_PARAMS); } diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index b7015e37e1f1..6ccb38b1d56d 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -1898,6 +1898,16 @@ static u8 sc_send_public_key(struct smp_chan *smp) } done: + /* Check if remote and local public keys are the same and debug key is + * not in use. + */ + if (!test_bit(SMP_FLAG_DEBUG_KEY, &smp->flags) && + !test_bit(SMP_FLAG_INITIATOR, &smp->flags) && + !crypto_memneq(smp->remote_pk, smp->local_pk, 32)) { + bt_dev_err(hdev, "Remote and local public keys are identical"); + return SMP_DHKEY_CHECK_FAILED; + } + SMP_DBG("Local Public Key X: %32phN", smp->local_pk); SMP_DBG("Local Public Key Y: %32phN", smp->local_pk + 32); @@ -2749,7 +2759,8 @@ static int smp_cmd_public_key(struct l2cap_conn *conn, struct sk_buff *skb) * not in use. */ if (!test_bit(SMP_FLAG_DEBUG_KEY, &smp->flags) && - !crypto_memneq(key, smp->local_pk, 64)) { + test_bit(SMP_FLAG_INITIATOR, &smp->flags) && + !crypto_memneq(key, smp->local_pk, 32)) { bt_dev_err(hdev, "Remote and local public keys are identical"); return SMP_DHKEY_CHECK_FAILED; } diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c index fc2e381c4fe0..b5bd9d6808fd 100644 --- a/net/bridge/br_input.c +++ b/net/bridge/br_input.c @@ -257,13 +257,18 @@ static void __br_handle_local_finish(struct sk_buff *skb) br_fdb_update(p->br, p, vlan, eth_hdr(skb)->h_source, 0); } -/* note: already called with rcu_read_lock */ +/* okfn of the link-local NF_BR_LOCAL_IN hook in br_handle_frame(). It is + * only called when a queued frame is reinjected, after br_handle_frame() + * returned RX_HANDLER_CONSUMED for it, so the frame can no longer be + * passed up on the port and has to be freed here. + * note: already called with rcu_read_lock + */ static int br_handle_local_finish(struct net *net, struct sock *sk, struct sk_buff *skb) { __br_handle_local_finish(skb); + kfree_skb(skb); - /* return 1 to signal the okfn() was called so it's ok to use the skb */ - return 1; + return 0; } static int nf_hook_bridge_pre(struct sk_buff *skb, struct sk_buff **pskb) @@ -424,11 +429,14 @@ static rx_handler_result_t br_handle_frame(struct sk_buff **pskb) /* The else clause should be hit when nf_hook(): * - returns < 0 (drop/error) * - returns = 0 (stolen/nf_queue) - * Thus return 1 from the okfn() to signal the skb is ok to pass + * On 1 the frame is passed on here. The okfn() only runs if a + * queued frame is reinjected later, and must consume it then. */ - if (NF_HOOK(NFPROTO_BRIDGE, NF_BR_LOCAL_IN, + if (nf_hook(NFPROTO_BRIDGE, NF_BR_LOCAL_IN, dev_net(skb->dev), NULL, skb, skb->dev, NULL, br_handle_local_finish) == 1) { + *pskb = skb; + __br_handle_local_finish(skb); return RX_HANDLER_PASS; } else { return RX_HANDLER_CONSUMED; diff --git a/net/core/dev.c b/net/core/dev.c index 3c58db732761..6140fd9d11cd 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -6223,6 +6223,10 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, trace_netif_receive_skb(skb); orig_dev = skb->dev; + if (unlikely(skb_headroom(skb) > SKB_MAX_RX_HEADROOM)) { + drop_reason = SKB_DROP_REASON_SKB_RX_HEADROOM; + goto drop; + } skb_reset_network_header(skb); #if !defined(CONFIG_DEBUG_NET) diff --git a/net/core/neighbour.c b/net/core/neighbour.c index d6e9c7c59a89..ae2db3912454 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -783,7 +783,7 @@ struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl, int pneigh_create(struct neigh_table *tbl, const void *pkey, struct net_device *dev, - u32 flags, u8 protocol, bool permanent) + u32 flags, u8 protocol, bool permanent, bool update) { struct pneigh_entry *n; unsigned int key_len; @@ -793,8 +793,11 @@ int pneigh_create(struct neigh_table *tbl, mutex_lock(&tbl->phash_lock); n = pneigh_lookup(tbl, pkey, dev); - if (n) + if (n) { + if (!update) + goto out; goto update; + } key_len = tbl->key_len; n = kzalloc(sizeof(*n) + key_len, GFP_KERNEL); @@ -2155,7 +2158,7 @@ static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh, } err = pneigh_create(tbl, dst, dev, ndm_flags, protocol, - !!(ndm->ndm_state & NUD_PERMANENT)); + !!(ndm->ndm_state & NUD_PERMANENT), true); goto out; } diff --git a/net/core/net_test.c b/net/core/net_test.c index 2b5080c22663..c36333b36211 100644 --- a/net/core/net_test.c +++ b/net/core/net_test.c @@ -834,13 +834,14 @@ static void netmem_ref_test_pp_page(struct kunit *test) #endif #if defined(CONFIG_PAGE_POOL) && defined(CONFIG_NET_DEVMEM) +static struct net_devmem_dmabuf_binding binding; + static void dummy_percpu_ref_release(struct percpu_ref *ref) { } static void netmem_ref_test_net_iov(struct kunit *test) { - struct net_devmem_dmabuf_binding binding = {}; struct page_pool_params pp_params = { .order = 0, .pool_size = 4, @@ -853,6 +854,7 @@ static void netmem_ref_test_net_iov(struct kunit *test) long base_ref; int err; + memset(&binding, 0, sizeof(binding)); err = percpu_ref_init(&binding.ref, dummy_percpu_ref_release, PERCPU_REF_INIT_ATOMIC, GFP_KERNEL); KUNIT_ASSERT_EQ(test, err, 0); @@ -995,7 +997,6 @@ static void netmem_ref_test_release_data(struct kunit *test) .nid = NUMA_NO_NODE, }; struct sk_buff *skb, *clone1, *clone2, *ext; - struct net_devmem_dmabuf_binding binding = {}; struct net_iov niov = {}; struct page_pool *pool; struct page *page; @@ -1004,6 +1005,7 @@ static void netmem_ref_test_release_data(struct kunit *test) int base_ref; int i; + memset(&binding, 0, sizeof(binding)); pool = page_pool_create(&pp_params); KUNIT_ASSERT_FALSE(test, IS_ERR(pool)); page = page_pool_dev_alloc_pages(pool); @@ -1185,7 +1187,6 @@ static void netmem_ref_test_skb_segment(struct kunit *test) .flags = SKBFL_ZEROCOPY_ENABLE, }; struct sk_buff *head, *list, *segs, *seg; - struct net_devmem_dmabuf_binding binding = {}; struct net_iov niov = {}; struct page *pp_page, *reg_page; netmem_ref pp_nm, niov_nm; @@ -1193,6 +1194,7 @@ static void netmem_ref_test_skb_segment(struct kunit *test) long base_pp_ref, base_iov_ref; int base_pp_page_ref, base_reg_ref; + memset(&binding, 0, sizeof(binding)); pool = page_pool_create(&pp_params); KUNIT_ASSERT_FALSE(test, IS_ERR(pool)); pp_page = page_pool_dev_alloc_pages(pool); @@ -1484,7 +1486,6 @@ static void netmem_ref_test_try_coalesce(struct kunit *test) .nid = NUMA_NO_NODE, }; struct sk_buff *to, *from, *clone; - struct net_devmem_dmabuf_binding binding = {}; struct net_iov niov = {}; struct page *pp_page, *reg_page; netmem_ref pp_nm, niov_nm; @@ -1494,6 +1495,7 @@ static void netmem_ref_test_try_coalesce(struct kunit *test) bool fragstolen; int delta; + memset(&binding, 0, sizeof(binding)); pool = page_pool_create(&pp_params); KUNIT_ASSERT_FALSE(test, IS_ERR(pool)); pp_page = page_pool_dev_alloc_pages(pool); @@ -1616,7 +1618,6 @@ static void netmem_ref_test_tcp_devmem_recv(struct kunit *test) .pool_size = 4, .nid = NUMA_NO_NODE, }; - struct net_devmem_dmabuf_binding binding = {}; struct msghdr msg = {}; struct dmabuf_token token; struct dmabuf_cmsg *cmsg; @@ -1630,6 +1631,7 @@ static void netmem_ref_test_tcp_devmem_recv(struct kunit *test) long base_iov_ref; int ret; + memset(&binding, 0, sizeof(binding)); KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock), diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 1ac1e327ac4c..74e9a96bc5df 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -4789,6 +4789,10 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb, } } + if (!err) + err = skb_cow_head(nskb, offset + tnl_hlen - + skb_network_offset(nskb)); + if (!tail) skb->next = nskb; else diff --git a/net/dcb/dcbnl.c b/net/dcb/dcbnl.c index ad0aebe0ad3d..95250658ecc1 100644 --- a/net/dcb/dcbnl.c +++ b/net/dcb/dcbnl.c @@ -2393,6 +2393,13 @@ static void dcbnl_flush_dev(struct net_device *dev) } } + list_for_each_entry_safe(itr, tmp, &dcb_rewr_list, list) { + if (itr->ifindex == dev->ifindex) { + list_del(&itr->list); + kfree(itr); + } + } + spin_unlock_bh(&dcb_lock); } diff --git a/net/ieee802154/socket.c b/net/ieee802154/socket.c index be8ca1f053a2..362457de8424 100644 --- a/net/ieee802154/socket.c +++ b/net/ieee802154/socket.c @@ -128,6 +128,10 @@ static int ieee802154_dev_ioctl(struct sock *sk, struct ifreq __user *arg, int ret = -ENOIOCTLCMD; struct net_device *dev; + if (cmd == SIOCSIFADDR && + !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) + return -EPERM; + if (get_user_ifreq(&ifr, NULL, arg)) return -EFAULT; diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index 1ca4ced280ae..3bc8db6ccd68 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -1099,7 +1099,7 @@ static int arp_req_set_public(struct net *net, struct arpreq *r, if (mask) { __be32 ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr; - return pneigh_create(tbl, &ip, dev, 0, 0, false); + return pneigh_create(tbl, &ip, dev, 0, 0, false, false); } return arp_req_set_proxy(net, dev, 1); diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c index 5c3c8f55d057..bff4dc488def 100644 --- a/net/ipv4/fou_core.c +++ b/net/ipv4/fou_core.c @@ -1071,6 +1071,9 @@ static int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, __be16 sport; int err; + if (skb_cow_head(skb, fou_encap_hlen(e))) + return -ENOMEM; + err = __fou_build_header(skb, e, protocol, &sport, type); if (err) return err; @@ -1083,16 +1086,24 @@ static int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, static int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e, u8 *protocol, struct flowi4 *fl4) { - int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM : - SKB_GSO_UDP_TUNNEL; + struct ip_tunnel_encap encap = { + .flags = READ_ONCE(e->flags), + .sport = READ_ONCE(e->sport), + .dport = READ_ONCE(e->dport), + }; + int type = encap.flags & TUNNEL_ENCAP_FLAG_CSUM ? + SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL; __be16 sport; int err; - err = __gue_build_header(skb, e, protocol, &sport, type); + if (skb_cow_head(skb, gue_encap_hlen(&encap))) + return -ENOMEM; + + err = __gue_build_header(skb, &encap, protocol, &sport, type); if (err) return err; - fou_build_udp(skb, e, fl4, protocol, sport); + fou_build_udp(skb, &encap, fl4, protocol, sport); return 0; } diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 6c079929e06f..9a3b05edb62b 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -420,13 +420,18 @@ static inline int sk_reuseport_match(struct inet_bind_bucket *tb, return 0; if (!uid_eq(tb->fastuid, sk_uid(sk))) return 0; - /* We only need to check the rcv_saddr if this tb was once marked + /* We only need to check the fast_* fields if this tb was once marked * without fastreuseport and then was reset, as we can only know that - * the fast_*rcv_saddr doesn't have any conflicts with the socks on the - * owners list. + * the socket they describe doesn't have any conflicts with the socks + * on the owners list. */ if (tb->fastreuseport == FASTREUSEPORT_ANY) return 1; + if (tb->fast_sk_family != sk->sk_family || + tb->fast_ipv6_only != ipv6_only_sock(sk) || + tb->fast_bound_dev_if != sk->sk_bound_dev_if || + tb->fast_sk_reuse != sk->sk_reuse) + return 0; #if IS_ENABLED(CONFIG_IPV6) if (tb->fast_sk_family == AF_INET6) return ipv6_rcv_saddr_equal(&tb->fast_v6_rcv_saddr, @@ -455,6 +460,8 @@ void inet_csk_update_fastreuse(const struct sock *sk, tb->fast_rcv_saddr = sk->sk_rcv_saddr; tb->fast_ipv6_only = ipv6_only_sock(sk); tb->fast_sk_family = sk->sk_family; + tb->fast_sk_reuse = sk->sk_reuse; + tb->fast_bound_dev_if = sk->sk_bound_dev_if; #if IS_ENABLED(CONFIG_IPV6) tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr; #endif @@ -482,6 +489,8 @@ void inet_csk_update_fastreuse(const struct sock *sk, tb->fast_rcv_saddr = sk->sk_rcv_saddr; tb->fast_ipv6_only = ipv6_only_sock(sk); tb->fast_sk_family = sk->sk_family; + tb->fast_sk_reuse = sk->sk_reuse; + tb->fast_bound_dev_if = sk->sk_bound_dev_if; #if IS_ENABLED(CONFIG_IPV6) tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr; #endif @@ -1331,6 +1340,22 @@ static int inet_ulp_can_listen(const struct sock *sk) return 0; } +/* Free all the reqs queued in rskq_rst_head. */ +static void reqsk_fastopen_purge_rst(struct request_sock_queue *queue) +{ + struct request_sock *next, *req; + + spin_lock_bh(&queue->fastopenq.lock); + req = queue->fastopenq.rskq_rst_head; + queue->fastopenq.rskq_rst_head = NULL; + spin_unlock_bh(&queue->fastopenq.lock); + while (req != NULL) { + next = req->dl_next; + reqsk_put(req); + req = next; + } +} + static void reqsk_queue_alloc(struct request_sock_queue *queue) { queue->fastopenq.rskq_rst_head = NULL; @@ -1373,6 +1398,12 @@ int inet_csk_listen_start(struct sock *sk) } inet_sk_set_state(sk, TCP_CLOSE); + + /* A Fast Open child accepted before a previous shutdown() of this + * listener might have queued its request in rskq_rst_head while + * we were transiently in TCP_LISTEN state. + */ + reqsk_fastopen_purge_rst(&icsk->icsk_accept_queue); return err; } @@ -1477,7 +1508,7 @@ void inet_csk_listen_stop(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); struct request_sock_queue *queue = &icsk->icsk_accept_queue; - struct request_sock *next, *req; + struct request_sock *req; /* Following specs, it would be better either to send FIN * (and enter FIN-WAIT-1, it is normal close) @@ -1533,18 +1564,7 @@ skip_child_forget: if (!has_current_bpf_ctx()) cond_resched(); } - if (queue->fastopenq.rskq_rst_head) { - /* Free all the reqs queued in rskq_rst_head. */ - spin_lock_bh(&queue->fastopenq.lock); - req = queue->fastopenq.rskq_rst_head; - queue->fastopenq.rskq_rst_head = NULL; - spin_unlock_bh(&queue->fastopenq.lock); - while (req != NULL) { - next = req->dl_next; - reqsk_put(req); - req = next; - } - } + reqsk_fastopen_purge_rst(queue); WARN_ON_ONCE(sk->sk_ack_backlog); } diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c index 9860178752b8..c74886b1f445 100644 --- a/net/ipv4/ip_input.c +++ b/net/ipv4/ip_input.c @@ -207,6 +207,12 @@ resubmit: ret = INDIRECT_CALL_2(ipprot->handler, tcp_v4_rcv, udp_rcv, skb); if (ret < 0) { + if (unlikely(skb_headroom(skb) > SKB_MAX_RX_HEADROOM)) { + __IP_INC_STATS(net, IPSTATS_MIB_INDISCARDS); + kfree_skb_reason(skb, + SKB_DROP_REASON_SKB_RX_HEADROOM); + return; + } protocol = -ret; goto resubmit; } diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index 0875474a578a..1052af6245df 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c @@ -610,6 +610,7 @@ void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, if (ip_tunnel_encap(skb, &tun_info->encap, &proto, &fl4) < 0) goto tx_error; + inner_iph = (const struct iphdr *)skb_inner_network_header(skb); use_cache = ip_tunnel_dst_cache_usable(skb, tun_info); if (use_cache) @@ -753,6 +754,8 @@ void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, } tos = tnl_params->tos; + df = tnl_params->frag_off; + ttl = tnl_params->ttl; if (tos & 0x1) { tos &= ~0x1; if (payload_protocol == htons(ETH_P_IP)) { @@ -771,6 +774,7 @@ void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, if (ip_tunnel_encap(skb, &tunnel->encap, &protocol, &fl4) < 0) goto tx_error; + inner_iph = (const struct iphdr *)skb_inner_network_header(skb); if (connected && md) { use_cache = ip_tunnel_dst_cache_usable(skb, tun_info); @@ -803,7 +807,6 @@ void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, goto tx_error; } - df = tnl_params->frag_off; if (payload_protocol == htons(ETH_P_IP) && !tunnel->ignore_df) df |= (inner_iph->frag_off & htons(IP_DF)); @@ -825,7 +828,6 @@ void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, } tos = ip_tunnel_ecn_encap(tos, inner_iph, skb); - ttl = tnl_params->ttl; if (ttl == 0) { if (payload_protocol == htons(ETH_P_IP)) ttl = inner_iph->ttl; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index a2074a1af422..d9a66a1342ef 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -857,7 +857,11 @@ void tcp_initialize_rcv_mss(struct sock *sk) static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep) { u32 new_sample, old_sample = tp->rcv_rtt_est.rtt_us; - long m = sample << 3; + long m; + + /* Keep sample << 3 and the smoothed estimate within u32. */ + sample = min(sample, TCP_RTT_MAX_US); + m = sample << 3; if (old_sample == 0 || m < old_sample) { new_sample = m; @@ -901,15 +905,16 @@ new_measure: tp->rcv_rtt_est.time = tp->tcp_mstamp; } +/* A sample from TSecr above TCP_RTT_MAX_US is bogus, not large. */ static s32 tcp_rtt_tsopt_us(const struct tcp_sock *tp, u32 min_delta) { u32 delta, delta_us; delta = tcp_time_stamp_ts(tp) - tp->rx_opt.rcv_tsecr; if (tp->tcp_usec_ts) - return delta; + return delta <= TCP_RTT_MAX_US ? delta : -1; - if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) { + if (likely(delta <= TCP_RTT_MAX_US / (USEC_PER_SEC / TCP_TS_HZ))) { if (!delta) delta = min_delta; delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ); @@ -1098,8 +1103,11 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) static void tcp_rtt_estimator(struct sock *sk, long mrtt_us) { struct tcp_sock *tp = tcp_sk(sk); - long m = mrtt_us; /* RTT */ u32 srtt = tp->srtt_us; + long m; /* RTT */ + + mrtt_us = min(mrtt_us, TCP_RTT_MAX_US); + m = mrtt_us; /* The following amusing code comes from Jacobson's * article in SIGCOMM '88. Note that rtt and mdev @@ -1221,8 +1229,9 @@ void tcp_set_rto(struct sock *sk) * with correct one. It is exactly, which we pretend to do. */ - /* NOTE: clamping at TCP_RTO_MIN is not required, current algo - * guarantees that rto is higher. + /* NOTE: clamping at tcp_rto_min() is not required, tcp_rtt_estimator() + * applies it to rttvar_us. But rto must never be 0, which exponential + * backoff could not leave: __tcp_set_rto() takes care of that. */ tcp_bound_rto(sk); } diff --git a/net/ipv6/ioam6_iptunnel.c b/net/ipv6/ioam6_iptunnel.c index cfb2c41634a0..e8be0d922f74 100644 --- a/net/ipv6/ioam6_iptunnel.c +++ b/net/ipv6/ioam6_iptunnel.c @@ -254,10 +254,11 @@ static int ioam6_do_fill(struct net *net, struct sk_buff *skb) } static int ioam6_do_inline(struct net *net, struct sk_buff *skb, - struct ioam6_lwt_encap *tuninfo, + const struct ioam6_lwt_encap *tuninfo, struct dst_entry *cache_dst) { struct ipv6hdr *oldhdr, *hdr; + struct ipv6_opt_hdr *opt_hdr; int hdrlen, err; hdrlen = (tuninfo->eh.hdrlen + 1) << 3; @@ -276,12 +277,13 @@ static int ioam6_do_inline(struct net *net, struct sk_buff *skb, hdr = ipv6_hdr(skb); memmove(hdr, oldhdr, sizeof(*oldhdr)); - tuninfo->eh.nexthdr = hdr->nexthdr; skb_set_transport_header(skb, sizeof(*hdr)); skb_postpush_rcsum(skb, hdr, sizeof(*hdr) + hdrlen); - memcpy(skb_transport_header(skb), (u8 *)tuninfo, hdrlen); + opt_hdr = (struct ipv6_opt_hdr *)skb_transport_header(skb); + memcpy(opt_hdr, tuninfo, hdrlen); + opt_hdr->nexthdr = hdr->nexthdr; hdr->nexthdr = NEXTHDR_HOP; hdr->payload_len = cpu_to_be16(skb->len - sizeof(*hdr)); @@ -290,7 +292,7 @@ static int ioam6_do_inline(struct net *net, struct sk_buff *skb, } static int ioam6_do_encap(struct net *net, struct sk_buff *skb, - struct ioam6_lwt_encap *tuninfo, + const struct ioam6_lwt_encap *tuninfo, bool has_tunsrc, struct in6_addr *tunsrc, struct in6_addr *tundst, @@ -298,6 +300,7 @@ static int ioam6_do_encap(struct net *net, struct sk_buff *skb, { struct dst_entry *dst = skb_dst(skb); struct ipv6hdr *hdr, *inner_hdr; + struct ipv6_opt_hdr *opt_hdr; int hdrlen, len, err; hdrlen = (tuninfo->eh.hdrlen + 1) << 3; @@ -314,8 +317,9 @@ static int ioam6_do_encap(struct net *net, struct sk_buff *skb, skb_mac_header_rebuild(skb); skb_set_transport_header(skb, sizeof(*hdr)); - tuninfo->eh.nexthdr = NEXTHDR_IPV6; - memcpy(skb_transport_header(skb), (u8 *)tuninfo, hdrlen); + opt_hdr = (struct ipv6_opt_hdr *)skb_transport_header(skb); + memcpy(opt_hdr, tuninfo, hdrlen); + opt_hdr->nexthdr = NEXTHDR_IPV6; hdr = ipv6_hdr(skb); memcpy(hdr, inner_hdr, sizeof(*hdr)); diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c index d332ec60f915..6dc1211eade6 100644 --- a/net/ipv6/ip6_input.c +++ b/net/ipv6/ip6_input.c @@ -479,6 +479,10 @@ resubmit_final: ret = INDIRECT_CALL_2(ipprot->handler, tcp_v6_rcv, udpv6_rcv, skb); if (ret > 0) { + if (unlikely(skb_headroom(skb) > SKB_MAX_RX_HEADROOM)) { + SKB_DR_SET(reason, SKB_RX_HEADROOM); + goto discard; + } if (ipprot->flags & INET6_PROTO_FINAL) { /* Not an extension header, most likely UDP * encapsulation. Use return value as nexthdr diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c index 700f1861d8eb..01642900c0fa 100644 --- a/net/ipv6/ip6_offload.c +++ b/net/ipv6/ip6_offload.c @@ -26,6 +26,7 @@ static int ipv6_gro_pull_exthdrs(struct sk_buff *skb, int off, int proto) { const struct net_offload *ops = NULL; struct ipv6_opt_hdr *opth; + int exthdr_cnt = 0; for (;;) { int len; @@ -38,6 +39,13 @@ static int ipv6_gro_pull_exthdrs(struct sk_buff *skb, int off, int proto) if (!(ops->flags & INET6_PROTO_GSO_EXTHDR)) break; + /* Same limit as ip6_protocol_deliver_rcu(). + * @proto is left pointing to an extension header, + * so that the caller does not aggregate this packet. + */ + if (unlikely(exthdr_cnt++ >= IP6_MAX_EXT_HDRS_CNT)) + break; + opth = skb_gro_header(skb, off + sizeof(*opth), off); if (unlikely(!opth)) break; diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index dd0638d3c3f9..b7ae220d6b21 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -1154,7 +1154,7 @@ static int ip6mr_cache_report(const struct mr_table *mrt, struct sk_buff *pkt, /* * Add our header */ - skb_put(skb, sizeof(*msg)); + skb_put_zero(skb, sizeof(*msg)); skb_reset_transport_header(skb); msg = (struct mrt6msg *)skb_transport_header(skb); diff --git a/net/ipv6/mcast_snoop.c b/net/ipv6/mcast_snoop.c index 04d5fcdfa6e0..7abf79815aa2 100644 --- a/net/ipv6/mcast_snoop.c +++ b/net/ipv6/mcast_snoop.c @@ -56,7 +56,8 @@ static int ipv6_mc_check_exthdrs(struct sk_buff *skb) if (nexthdr != IPPROTO_ICMPV6) return -ENOMSG; - skb_set_transport_header(skb, offset); + if (!skb_set_transport_header_careful(skb, offset)) + return -EINVAL; return 0; } diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 9dcefa162522..a6886a0432f9 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -651,12 +651,13 @@ static struct sock *__udp6_lib_err_encap(struct net *net, network_offset = skb_network_offset(skb); transport_offset = skb_transport_offset(skb); + /* Transport header needs to point to the UDP header */ + if (unlikely(!skb_set_transport_header_careful(skb, offset))) + return ERR_PTR(-EINVAL); + /* Network header needs to point to the outer IPv6 header inside ICMP */ skb_reset_network_header(skb); - /* Transport header needs to point to the UDP header */ - skb_set_transport_header(skb, offset); - if (sk) { up = udp_sk(sk); diff --git a/net/mac802154/llsec.c b/net/mac802154/llsec.c index 85452ef9a58c..32565f18897e 100644 --- a/net/mac802154/llsec.c +++ b/net/mac802154/llsec.c @@ -1019,6 +1019,8 @@ int mac802154_llsec_decrypt(struct mac802154_llsec *sec, struct sk_buff *skb) return 0; if (hdr.fc.version == 0) return -EINVAL; + if (hdr.sec.level == IEEE802154_SCF_SECLEVEL_NONE) + return -EINVAL; read_lock_bh(&sec->lock); if (!sec->params.enabled) { diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 8968db58083e..8a9efebf3497 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -3041,7 +3041,8 @@ void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout) return; close_timeout = (unsigned long)inet_csk(sk)->icsk_mtup.probe_timestamp - - tcp_jiffies32 + jiffies + mptcp_close_timeout(sk); + tcp_jiffies32; + close_timeout += jiffies + mptcp_close_timeout(sk); /* the close timeout takes precedence on the fail one, and here at least one of * them is active diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 94f25fb7d87e..83d8a9153a34 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -293,7 +293,6 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp) cp->flags |= IP_VS_CONN_F_HASHED; WRITE_ONCE(cp->hn0.hash_key, hash_key); WRITE_ONCE(cp->hn1.hash_key, hash_key2); - refcount_inc(&cp->refcnt); hlist_bl_add_head_rcu(&cp->hn0.node, head); if (use2) hlist_bl_add_head_rcu(&cp->hn1.node, head2); @@ -319,11 +318,27 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp) struct hlist_bl_head *head, *head2; u32 hash_key, hash_key2; struct ip_vs_rht *t; - bool ret = false; bool use2; + /* No more users and no running/pending timer except us? */ + if (!refcount_dec_if_one(&cp->refcnt)) + return false; + + /* The ONE_PACKET connection is special: + * - the flag is present only for normal connections, not for templates + * - its timer is never started + * - it is never hashed, so you can not find it and delete it + * - it is not synced + * - it has only one user + * - it is freed after use immediately + */ if (cp->flags & IP_VS_CONN_F_ONE_PACKET) - return refcount_dec_if_one(&cp->refcnt); + return true; + + /* Add the needed ACQUIRE barrier between the above + * refcount_dec_if_one() call and the read of cp fields + */ + smp_acquire__after_ctrl_dep(); rcu_read_lock(); local_bh_disable(); @@ -337,15 +352,11 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp) false /* new_hash2 */, &head, &head2); if (cp->flags & IP_VS_CONN_F_HASHED) { - /* Decrease refcnt and unlink conn only if we are last user */ - if (use2 == ip_vs_conn_use_hash2(cp) && - refcount_dec_if_one(&cp->refcnt)) { - hlist_bl_del_rcu(&cp->hn0.node); - if (use2) - hlist_bl_del_rcu(&cp->hn1.node); - cp->flags &= ~IP_VS_CONN_F_HASHED; - ret = true; - } + /* Unlink conn as we are the last user */ + hlist_bl_del_rcu(&cp->hn0.node); + if (use2) + hlist_bl_del_rcu(&cp->hn1.node); + cp->flags &= ~IP_VS_CONN_F_HASHED; } conn_tab_unlock(head, head2); @@ -353,7 +364,7 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp) local_bh_enable(); rcu_read_unlock(); - return ret; + return true; } @@ -579,6 +590,8 @@ struct ip_vs_conn *ip_vs_conn_out_get(const struct ip_vs_conn_param *p) ip_vs_addr_equal(p->af, p->vaddr, &cp->caddr) && ip_vs_addr_equal(p->af, p->caddr, saddr) && + (!p->vport ^ + (!(cp->flags & IP_VS_CONN_F_NO_CPORT))) && p->protocol == cp->protocol) { if (__ip_vs_conn_get(cp)) goto out; @@ -621,9 +634,12 @@ static void __ip_vs_conn_put_timer(struct ip_vs_conn *cp) { unsigned long t = (cp->flags & IP_VS_CONN_F_ONE_PACKET) ? 0 : cp->timeout; - mod_timer(&cp->timer, jiffies+t); - __ip_vs_conn_put(cp); + /* Handoff our reference to the timer if it was just armed, + * put the reference if timer was already pending. + */ + if (mod_timer(&cp->timer, jiffies + t)) + __ip_vs_conn_put(cp); } void ip_vs_conn_put(struct ip_vs_conn *cp) @@ -1154,7 +1170,7 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) atomic_inc(&dest->activeconns); tc = atomic_inc_return(&dest->totalconns); if (tc >= READ_ONCE(dest->u_threshold_val)) - ip_vs_dest_update_overload(dest, true); + ip_vs_dest_update_overload(dest); } else { /* It is a persistent connection/template, so increase the persistent connection counter */ @@ -1247,7 +1263,7 @@ static inline void ip_vs_unbind_dest(struct ip_vs_conn *cp) atomic_dec(&dest->activeconns); tc = atomic_fetch_dec(&dest->totalconns); if (tc <= READ_ONCE(dest->l_threshold_val)) - ip_vs_dest_update_overload(dest, false); + ip_vs_dest_update_overload(dest); } else { /* It is a persistent connection/template, so decrease the persistent connection counter */ @@ -1322,36 +1338,48 @@ static void ip_vs_conn_rcu_free(struct rcu_head *head) kmem_cache_free(ip_vs_conn_cachep, cp); } -/* Try to delete connection while not holding reference */ +/* Try to steal the reference from the pending timer. + * It can be called concurrently and always under RCU lock + * to protect from connection deletion from others. + */ +static bool ip_vs_conn_del_reference(struct ip_vs_conn *cp) +{ + /* Steal reference from the pending timer */ + if (!timer_delete(&cp->timer)) + return false; + + /* Drop cp->control chain too */ + if (cp->control) + cp->timeout = 0; + return true; +} + +/* Try to delete connection while not holding reference. + * Used when we need to remove the connection immediately + * and without overloading the timer processing. + * OTOH, we use ip_vs_conn_expire_now() when we prefer to + * leave the current context (packet processing) and to + * remove the connection from timer. + */ static void ip_vs_conn_del(struct ip_vs_conn *cp) { - if (timer_delete(&cp->timer)) { - /* Drop cp->control chain too */ - if (cp->control) - cp->timeout = 0; + if (ip_vs_conn_del_reference(cp)) ip_vs_conn_expire(&cp->timer); - } -} - -/* Try to delete connection while holding reference */ -static void ip_vs_conn_del_put(struct ip_vs_conn *cp) -{ - if (timer_delete(&cp->timer)) { - /* Drop cp->control chain too */ - if (cp->control) - cp->timeout = 0; - __ip_vs_conn_put(cp); - ip_vs_conn_expire(&cp->timer); - } else { - __ip_vs_conn_put(cp); - } } +/* Connection is removed in the following steps: + * - timer expires (with ref) or connection is deleted (with stolen ref) + * - there should be no more users except us (pending/running timer, + * n_control>0 or refcnt>1) + */ static void ip_vs_conn_expire(struct timer_list *t) { struct ip_vs_conn *cp = timer_container_of(cp, t, timer); struct netns_ipvs *ipvs = cp->ipvs; + rcu_read_lock(); + +repeat: /* * do I control anybody? */ @@ -1361,23 +1389,17 @@ static void ip_vs_conn_expire(struct timer_list *t) /* Unlink conn if not referenced anymore */ if (likely(ip_vs_conn_unlink(cp))) { struct ip_vs_conn *ct = cp->control; - - /* delete the timer if it is activated by other users */ - timer_delete(&cp->timer); + bool next = false; /* does anybody control me? */ if (ct) { - bool has_ref = !cp->timeout && __ip_vs_conn_get(ct); - ip_vs_control_del(cp); /* Drop CTL or non-assured TPL if not used anymore */ - if (has_ref && !atomic_read(&ct->n_control) && + if (!cp->timeout && !atomic_read(&ct->n_control) && (!(ct->flags & IP_VS_CONN_F_TEMPLATE) || !(ct->state & IP_VS_CTPL_S_ASSURED))) { IP_VS_DBG(4, "drop controlling connection\n"); - ip_vs_conn_del_put(ct); - } else if (has_ref) { - __ip_vs_conn_put(ct); + next = ip_vs_conn_del_reference(ct); } } @@ -1405,7 +1427,11 @@ static void ip_vs_conn_expire(struct timer_list *t) else call_rcu(&cp->rcu_head, ip_vs_conn_rcu_free); atomic_dec(&ipvs->conn_count); - return; + if (next) { + cp = ct; + goto repeat; + } + goto out; } expire_later: @@ -1413,13 +1439,16 @@ static void ip_vs_conn_expire(struct timer_list *t) refcount_read(&cp->refcnt), atomic_read(&cp->n_control)); - refcount_inc(&cp->refcnt); - cp->timeout = 60*HZ; + if (cp->timeout || atomic_read(&cp->n_control)) + cp->timeout = 60 * HZ; if (ipvs->sync_state & IP_VS_STATE_MASTER) ip_vs_sync_conn(ipvs, cp, sysctl_sync_threshold(ipvs)); __ip_vs_conn_put_timer(cp); + +out: + rcu_read_unlock(); } /* Modify timer, so that it expires as soon as possible. @@ -1432,7 +1461,7 @@ static void ip_vs_conn_expire(struct timer_list *t) void ip_vs_conn_expire_now(struct ip_vs_conn *cp) { /* Using mod_timer_pending will ensure the timer is not - * modified after the final timer_delete in ip_vs_conn_expire. + * started without giving it new reference. */ if (timer_pending(&cp->timer) && time_after(cp->timer.expires, jiffies)) @@ -1500,9 +1529,9 @@ ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, spin_lock_init(&cp->lock); /* - * Set the entry is referenced by the current thread before hashing - * it in the table, so that other thread run ip_vs_random_dropentry - * but cannot drop this entry. + * Set the entry as referenced by the current thread before starting + * the timer and giving it our reference to allow other threads to + * run ip_vs_random_dropentry() or any deletion method safely. */ refcount_set(&cp->refcnt, 1); diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index 8a3c880fdb26..f3eb72e95024 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -1305,6 +1305,7 @@ static inline bool is_new_conn_expected(const struct ip_vs_conn *cp, * In fwmark-VS actual vaddr and vport are unknown to IPVS * 2) Real Server and Virtual Service were NOT configured without port: * This is to allow match of different VS to the same RS ip-addr + * 3) Client port is not 0 */ struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, struct ip_vs_dest *dest, @@ -1328,7 +1329,7 @@ struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, /* check pre-requisites are satisfied */ if (svc->fwmark) return NULL; - if (!vport || !dport) + if (!vport || !dport || !cport) return NULL; /* for persistent service first create connection template */ diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index ce9fe20d6106..be818bf7d380 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -64,7 +64,8 @@ int ip_vs_get_debug_level(void) /* Protos */ -static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup); +static void __ip_vs_del_service(struct ip_vs_service *svc, + struct list_head *dest_trash, bool cleanup); #ifdef CONFIG_IP_VS_IPV6 @@ -918,6 +919,11 @@ static void ip_vs_rs_hash(struct netns_ipvs *ipvs, struct ip_vs_dest *dest) */ hash = ip_vs_rs_hashkey(dest->af, &dest->addr, port); + /* Need RCU grace period when rehashing */ + if (dest->need_gp) { + cond_synchronize_rcu(dest->rcu_state); + dest->need_gp = 0; + } hlist_add_head_rcu(&dest->d_list, &ipvs->rs_table[hash]); dest->in_rs_table = 1; } @@ -1111,6 +1117,8 @@ ip_vs_trash_get_dest(struct ip_vs_service *svc, int dest_af, struct ip_vs_dest *dest; struct netns_ipvs *ipvs = svc->ipvs; + lockdep_assert_held(&ipvs->service_mutex); + /* * Find the destination in trash */ @@ -1144,24 +1152,57 @@ out: return dest; } -/* Put destination in trash */ -static void ip_vs_trash_put_dest(struct netns_ipvs *ipvs, +/* Add destination in temp list for trash */ +static void ip_vs_trash_add_dest(struct netns_ipvs *ipvs, struct ip_vs_dest *dest, unsigned long istart, - bool cleanup) + struct list_head *dest_trash) { - spin_lock_bh(&ipvs->dest_trash_lock); IP_VS_DBG_BUF(3, "Moving dest %s:%u into trash, dest->refcnt=%d\n", IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port), refcount_read(&dest->refcnt)); + /* dest lives in trash with reference */ + list_add(&dest->t_list, dest_trash); + dest->idle_start = istart; +} + +/* Put destinations in trash */ +static void ip_vs_trash_put_dests(struct netns_ipvs *ipvs, + struct list_head *dest_trash, bool cleanup) +{ + spin_lock_bh(&ipvs->dest_trash_lock); if (list_empty(&ipvs->dest_trash) && !cleanup) mod_timer(&ipvs->dest_trash_timer, jiffies + (IP_VS_DEST_TRASH_PERIOD >> 1)); - /* dest lives in trash with reference */ - list_add(&dest->t_list, &ipvs->dest_trash); - dest->idle_start = istart; + list_splice(dest_trash, &ipvs->dest_trash); spin_unlock_bh(&ipvs->dest_trash_lock); } +/* Put destination back in trash */ +static void ip_vs_trash_put_back(struct netns_ipvs *ipvs, + struct ip_vs_dest *dest) +{ + LIST_HEAD(dest_trash); + + ip_vs_trash_add_dest(ipvs, dest, dest->idle_start, &dest_trash); + ip_vs_trash_put_dests(ipvs, &dest_trash, false); +} + +/* Put destinations in trash and save the RCU state */ +static void ip_vs_trash_save_dests(struct netns_ipvs *ipvs, + struct list_head *dest_trash, bool cleanup) +{ + struct ip_vs_dest *dest; + unsigned long rcu_state; + + if (!list_empty(dest_trash)) { + /* Remember when dests were removed and added to dest_trash */ + rcu_state = get_state_synchronize_rcu(); + list_for_each_entry(dest, dest_trash, t_list) + dest->rcu_state = rcu_state; + ip_vs_trash_put_dests(ipvs, dest_trash, cleanup); + } +} + static void ip_vs_dest_rcu_free(struct rcu_head *head) { struct ip_vs_dest *dest; @@ -1304,12 +1345,12 @@ void ip_vs_stats_free(struct ip_vs_stats *stats) } } -/* Update overload flag based on number of dest conns and lower/upper +/* Update overload state based on number of dest conns and lower/upper * connection thresholds: - * - conns reach u_threshold and exceed it: set the flag - * - conns go below l_threshold (or 75% of u_threshold): clear the flag + * - conns reach u_threshold and exceed it: set the overload state + * - conns go below l_threshold (or 75% of u_threshold): clear the state */ -static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload) +static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest) { int conns; u32 l, u; @@ -1320,10 +1361,18 @@ static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload) /* Low threshold defaults to 75% of upper threshold */ l = READ_ONCE(dest->l_threshold) ? : (u - (u >> 2)); conns = atomic_read(&dest->totalconns); - if (conns >= (overload ? l : u)) { + if (conns >= (ip_vs_dest_is_overloaded(dest) ? l : u)) { WRITE_ONCE(dest->u_threshold_val, INT_MAX); WRITE_ONCE(dest->l_threshold_val, l); - return; + /* Commit thresholds before repeating the check, + * it is paired with atomic_fetch_dec() in + * ip_vs_unbind_dest() to avoid entering the + * overload state forever. + */ + smp_mb(); + conns = atomic_read(&dest->totalconns); + if (conns >= l) + return; } } @@ -1331,10 +1380,10 @@ static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload) WRITE_ONCE(dest->l_threshold_val, 0); } -void ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload) +void ip_vs_dest_update_overload(struct ip_vs_dest *dest) { spin_lock_bh(&dest->dst_lock); - __ip_vs_dest_update_overload(dest, overload); + __ip_vs_dest_update_overload(dest); spin_unlock_bh(&dest->dst_lock); } @@ -1369,8 +1418,16 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, if ((udest->conn_flags & IP_VS_CONN_F_FWD_MASK) != IP_VS_DFWD_METHOD(dest) || udest->tun_type != dest->tun_type || - udest->tun_port != dest->tun_port) + udest->tun_port != dest->tun_port) { + bool unlinked = dest->in_rs_table; + ip_vs_rs_unhash(dest); + if (unlinked) { + dest->rcu_state = get_state_synchronize_rcu(); + /* Need grace period in this or next dest update */ + dest->need_gp = 1; + } + } /* set the tunnel info */ dest->tun_type = udest->tun_type; @@ -1406,13 +1463,10 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, if (READ_ONCE(dest->u_threshold) != udest->u_threshold || READ_ONCE(dest->l_threshold) != udest->l_threshold) { - bool overload; - spin_lock_bh(&dest->dst_lock); WRITE_ONCE(dest->u_threshold, udest->u_threshold); WRITE_ONCE(dest->l_threshold, udest->l_threshold); - overload = ip_vs_dest_is_overloaded(dest); - __ip_vs_dest_update_overload(dest, overload); + __ip_vs_dest_update_overload(dest); spin_unlock_bh(&dest->dst_lock); } @@ -1477,6 +1531,7 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) if (ret < 0) goto err_stats; + dest->u_threshold_val = INT_MAX; dest->af = udest->af; dest->protocol = svc->protocol; dest->vaddr = svc->addr; @@ -1556,6 +1611,11 @@ ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) dest = ip_vs_trash_get_dest(svc, udest->af, &daddr, dport); if (dest != NULL) { + /* Readers may still follow the n_list, d_list or estimator list + * links. Wait before reusing any of them. + */ + cond_synchronize_rcu(dest->rcu_state); + IP_VS_DBG_BUF(3, "Get destination %s:%u from trash, " "dest->refcnt=%d, service %u/%s:%u\n", IP_VS_DBG_ADDR(udest->af, &daddr), ntohs(dport), @@ -1566,11 +1626,14 @@ ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) ret = ip_vs_start_estimator(svc->ipvs, &dest->stats); /* On error put back dest into the trash */ - if (ret < 0) - ip_vs_trash_put_dest(svc->ipvs, dest, dest->idle_start, - false); - else + if (ret < 0) { + ip_vs_trash_put_back(svc->ipvs, dest); + } else { + /* need_gp matters only on dest edit, reinit it */ + dest->need_gp = 0; + __ip_vs_update_dest(svc, dest, udest, 1); + } } else { /* * Allocate and initialize the dest structure @@ -1634,7 +1697,7 @@ ip_vs_edit_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) * Delete a destination (must be already unlinked from the service) */ static void __ip_vs_del_dest(struct netns_ipvs *ipvs, struct ip_vs_dest *dest, - bool cleanup) + struct list_head *dest_trash, bool cleanup) { ip_vs_stop_estimator(ipvs, &dest->stats); @@ -1643,7 +1706,7 @@ static void __ip_vs_del_dest(struct netns_ipvs *ipvs, struct ip_vs_dest *dest, */ ip_vs_rs_unhash(dest); - ip_vs_trash_put_dest(ipvs, dest, 0, cleanup); + ip_vs_trash_add_dest(ipvs, dest, 0, dest_trash); /* Queue up delayed work to expire all no destination connections. * No-op when CONFIG_SYSCTL is disabled. @@ -1693,6 +1756,7 @@ ip_vs_del_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) { struct ip_vs_dest *dest; __be16 dport = udest->port; + LIST_HEAD(dest_trash); /* We use function that requires RCU lock */ rcu_read_lock(); @@ -1712,7 +1776,9 @@ ip_vs_del_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) /* * Delete the destination */ - __ip_vs_del_dest(svc->ipvs, dest, false); + __ip_vs_del_dest(svc->ipvs, dest, &dest_trash, false); + + ip_vs_trash_save_dests(svc->ipvs, &dest_trash, false); return 0; } @@ -2057,7 +2123,8 @@ out: * - The service must be unlinked, unlocked and not referenced! * - We are called under _bh lock */ -static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) +static void __ip_vs_del_service(struct ip_vs_service *svc, + struct list_head *dest_trash, bool cleanup) { struct ip_vs_dest *dest, *nxt; struct ip_vs_scheduler *old_sched; @@ -2091,7 +2158,7 @@ static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) */ list_for_each_entry_safe(dest, nxt, &svc->destinations, n_list) { __ip_vs_unlink_dest(svc, dest, 0); - __ip_vs_del_dest(svc->ipvs, dest, cleanup); + __ip_vs_del_dest(svc->ipvs, dest, dest_trash, cleanup); } /* @@ -2114,7 +2181,8 @@ static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) /* * Unlink a service from list and try to delete it if its refcnt reached 0 */ -static void ip_vs_unlink_service(struct ip_vs_service *svc, bool cleanup) +static void ip_vs_unlink_service(struct ip_vs_service *svc, + struct list_head *dest_trash, bool cleanup) { ip_vs_unregister_conntrack(svc); /* Hold svc to avoid double release from dest_trash */ @@ -2124,7 +2192,7 @@ static void ip_vs_unlink_service(struct ip_vs_service *svc, bool cleanup) */ ip_vs_svc_unhash(svc); - __ip_vs_del_service(svc, cleanup); + __ip_vs_del_service(svc, dest_trash, cleanup); } /* @@ -2134,12 +2202,14 @@ static int ip_vs_del_service(struct ip_vs_service *svc) { struct netns_ipvs *ipvs; struct ip_vs_rht *t, *p; + LIST_HEAD(dest_trash); int ns; if (svc == NULL) return -EEXIST; ipvs = svc->ipvs; - ip_vs_unlink_service(svc, false); + ip_vs_unlink_service(svc, &dest_trash, false); + ip_vs_trash_save_dests(ipvs, &dest_trash, false); /* Drop the table if no more services */ ns = ip_vs_get_num_services(ipvs); @@ -2190,6 +2260,7 @@ static int ip_vs_flush(struct netns_ipvs *ipvs, bool cleanup) struct hlist_bl_node *ne; struct hlist_bl_node *e; struct ip_vs_rht *t, *p; + LIST_HEAD(dest_trash); /* Stop the resizer and drop the tables */ if (!test_and_set_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags)) @@ -2199,8 +2270,9 @@ static int ip_vs_flush(struct netns_ipvs *ipvs, bool cleanup) if (ip_vs_get_num_services(ipvs)) { ip_vs_rht_walk_buckets(ipvs->svc_table, head) { hlist_bl_for_each_entry_safe(svc, e, ne, head, s_list) - ip_vs_unlink_service(svc, cleanup); + ip_vs_unlink_service(svc, &dest_trash, cleanup); } + ip_vs_trash_save_dests(ipvs, &dest_trash, cleanup); } /* Unregister the hash table and release it after RCU grace period */ diff --git a/net/netfilter/ipvs/ip_vs_ftp.c b/net/netfilter/ipvs/ip_vs_ftp.c index 9e3e005a8263..44de99f4d187 100644 --- a/net/netfilter/ipvs/ip_vs_ftp.c +++ b/net/netfilter/ipvs/ip_vs_ftp.c @@ -62,6 +62,17 @@ static unsigned short ports[IP_VS_APP_MAX_PORTS] = {21, 0}; module_param_array(ports, ushort, &ports_count, 0444); MODULE_PARM_DESC(ports, "Ports to monitor for FTP control commands"); +static bool is_control_port(u16 port) +{ + unsigned int i; + + for (i = 0; i < ports_count; i++) { + if (ports[i] == port) + return true; + } + return false; +} + static char *ip_vs_ftp_data_ptr(struct sk_buff *skb, struct ip_vs_iphdr *ipvsh) { @@ -319,6 +330,10 @@ static int ip_vs_ftp_out(struct ip_vs_app *app, struct ip_vs_conn *cp, return 1; } + /* Do not redirect data to control ports */ + if (!port || is_control_port(ntohs(port))) + return 0; + /* Now update or create a connection entry for it */ { struct ip_vs_conn_param p; @@ -368,7 +383,7 @@ static int ip_vs_ftp_out(struct ip_vs_app *app, struct ip_vs_conn *cp, buf_len = strlen(buf); ct = nf_ct_get(skb, &ctinfo); - if (ct) { + if (ct && !nf_ct_is_template(ct)) { bool mangled; /* If mangling fails this function will return 0 @@ -529,6 +544,9 @@ static int ip_vs_ftp_in(struct ip_vs_app *app, struct ip_vs_conn *cp, return 1; } + if (!port || is_control_port(ntohs(cp->vport) - 1)) + return 0; + /* Passive mode off */ cp->app_data = (void *) IP_VS_FTP_ACTIVE; diff --git a/net/netfilter/ipvs/ip_vs_lblc.c b/net/netfilter/ipvs/ip_vs_lblc.c index e1111c3b2721..f5ce1f2afb86 100644 --- a/net/netfilter/ipvs/ip_vs_lblc.c +++ b/net/netfilter/ipvs/ip_vs_lblc.c @@ -467,10 +467,11 @@ is_overloaded(struct ip_vs_dest *dest, struct ip_vs_service *svc) struct ip_vs_dest *d; list_for_each_entry_rcu(d, &svc->destinations, n_list) { - if (atomic_read(&d->activeconns)*2 - < atomic_read(&d->weight)) { + if (ip_vs_dest_is_overloaded(d)) + continue; + if (atomic_read(&d->activeconns) * 2 < + atomic_read(&d->weight)) return 1; - } } } return 0; @@ -507,6 +508,7 @@ ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, dest = en->dest; if ((dest->cflags & IP_VS_DEST_CF_AVAILABLE) && + !ip_vs_dest_is_overloaded(dest) && atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc)) goto out; } diff --git a/net/netfilter/ipvs/ip_vs_lblcr.c b/net/netfilter/ipvs/ip_vs_lblcr.c index dfb5bf83750d..4046608922e7 100644 --- a/net/netfilter/ipvs/ip_vs_lblcr.c +++ b/net/netfilter/ipvs/ip_vs_lblcr.c @@ -630,10 +630,11 @@ is_overloaded(struct ip_vs_dest *dest, struct ip_vs_service *svc) struct ip_vs_dest *d; list_for_each_entry_rcu(d, &svc->destinations, n_list) { - if (atomic_read(&d->activeconns)*2 - < atomic_read(&d->weight)) { + if (ip_vs_dest_is_overloaded(d)) + continue; + if (atomic_read(&d->activeconns) * 2 < + atomic_read(&d->weight)) return 1; - } } } return 0; @@ -658,9 +659,6 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, if (en) { en->lastuse = jiffies; - /* Get the least loaded destination */ - dest = ip_vs_dest_set_min(&en->set); - /* More than one destination + enough time passed by, cleanup */ if (atomic_read(&en->set.size) > 1 && time_after(jiffies, en->set.lastmod + @@ -676,6 +674,9 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, spin_unlock_bh(&svc->sched_lock); } + /* Get the least loaded destination */ + dest = ip_vs_dest_set_min(&en->set); + /* If the destination is not overloaded, use it */ if (dest && !is_overloaded(dest, svc)) goto out; diff --git a/net/netfilter/ipvs/ip_vs_proto_sctp.c b/net/netfilter/ipvs/ip_vs_proto_sctp.c index fb8af6b15a39..cb34bcceb4f6 100644 --- a/net/netfilter/ipvs/ip_vs_proto_sctp.c +++ b/net/netfilter/ipvs/ip_vs_proto_sctp.c @@ -30,6 +30,7 @@ sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, sch = skb_header_pointer(skb, iph->len + sizeof(_sctph), sizeof(_schunkh), &_schunkh); if (sch) { + /* Accept packet without scheduling? */ if (sch->type == SCTP_CID_ABORT || !(sysctl_sloppy_sctp(ipvs) || sch->type == SCTP_CID_INIT)) @@ -56,6 +57,12 @@ sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, if (svc) { int ignored; + /* Do not create connection for port 0 */ + if (!ports[0] || !ports[1]) { + *verdict = NF_DROP; + return 0; + } + if (ip_vs_todrop(ipvs)) { /* * It seems that we are very loaded. diff --git a/net/netfilter/ipvs/ip_vs_proto_tcp.c b/net/netfilter/ipvs/ip_vs_proto_tcp.c index fec0e8b47b71..25d4576d0e12 100644 --- a/net/netfilter/ipvs/ip_vs_proto_tcp.c +++ b/net/netfilter/ipvs/ip_vs_proto_tcp.c @@ -48,6 +48,7 @@ tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, if (likely(!ip_vs_iph_icmp(iph))) { th = skb_header_pointer(skb, iph->len, sizeof(_tcph), &_tcph); if (th) { + /* Accept packet without scheduling? */ if (th->rst || !(sysctl_sloppy_tcp(ipvs) || th->syn)) return 1; ports = &th->source; @@ -74,6 +75,12 @@ tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, if (svc) { int ignored; + /* Do not create connection for port 0 */ + if (!ports[0] || !ports[1]) { + *verdict = NF_DROP; + return 0; + } + if (ip_vs_todrop(ipvs)) { /* * It seems that we are very loaded. diff --git a/net/netfilter/ipvs/ip_vs_proto_udp.c b/net/netfilter/ipvs/ip_vs_proto_udp.c index f32785682402..ab3ab9524168 100644 --- a/net/netfilter/ipvs/ip_vs_proto_udp.c +++ b/net/netfilter/ipvs/ip_vs_proto_udp.c @@ -62,6 +62,12 @@ udp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, if (svc) { int ignored; + /* Do not create connection for port 0 */ + if (!ports[0] || !ports[1]) { + *verdict = NF_DROP; + return 0; + } + if (ip_vs_todrop(ipvs)) { /* * It seems that we are very loaded. diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c index 157f5c8361ab..7e562e745ab5 100644 --- a/net/netfilter/ipvs/ip_vs_sync.c +++ b/net/netfilter/ipvs/ip_vs_sync.c @@ -539,6 +539,7 @@ set: static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts) { + struct ip_vs_sync_conn_options opts; struct ip_vs_sync_mesg_v0 *m; struct ip_vs_sync_conn_v0 *s; struct ip_vs_sync_buff *buff; @@ -557,6 +558,14 @@ static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, if (!ip_vs_sync_conn_needed(ipvs, cp, pkts)) return; + if (flags & IP_VS_CONN_F_SEQ_MASK) { + /* Take atomic snapshot of flags and seqs */ + spin_lock_bh(&cp->lock); + flags = cp->flags; + memcpy(&opts, &cp->sync_conn_opt, sizeof(opts)); + spin_unlock_bh(&cp->lock); + } + spin_lock_bh(&ipvs->sync_buff_lock); if (!(ipvs->sync_state & IP_VS_STATE_MASTER)) { spin_unlock_bh(&ipvs->sync_buff_lock); @@ -604,7 +613,7 @@ static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, if (seq_mask) { struct ip_vs_sync_conn_options *opt = (struct ip_vs_sync_conn_options *)&s[1]; - memcpy(opt, &cp->sync_conn_opt, sizeof(*opt)); + memcpy(opt, &opts, sizeof(*opt)); } m->nr_conns++; @@ -630,6 +639,7 @@ static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, */ void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts) { + struct ip_vs_sync_conn_options opts; struct ip_vs_sync_mesg *m; union ip_vs_sync_conn *s; struct ip_vs_sync_buff *buff; @@ -651,6 +661,13 @@ sloop: if (!ip_vs_sync_conn_needed(ipvs, cp, pkts)) goto control; flags = READ_ONCE(cp->flags); + if (flags & IP_VS_CONN_F_SEQ_MASK) { + /* Take atomic snapshot of flags and seqs */ + spin_lock_bh(&cp->lock); + flags = cp->flags; + memcpy(&opts, &cp->sync_conn_opt, sizeof(opts)); + spin_unlock_bh(&cp->lock); + } /* Sanity checks */ pe_name_len = 0; @@ -732,7 +749,7 @@ sloop: s->v4.vport = cp->vport; s->v4.dport = cp->dport; s->v4.fwmark = htonl(cp->fwmark); - s->v4.timeout = htonl(cp->timeout / HZ); + s->v4.timeout = htonl(max(1UL, cp->timeout / HZ)); m->nr_conns++; #ifdef CONFIG_IP_VS_IPV6 @@ -752,9 +769,9 @@ sloop: if (seq_mask) { *(p++) = IPVS_OPT_SEQ_DATA; *(p++) = sizeof(struct ip_vs_sync_conn_options); - hton_seq(&cp->in_seq, (struct ip_vs_seq *)p); + hton_seq(&opts.in_seq, (struct ip_vs_seq *)p); p += sizeof(struct ip_vs_seq); - hton_seq(&cp->out_seq, (struct ip_vs_seq *)p); + hton_seq(&opts.out_seq, (struct ip_vs_seq *)p); p += sizeof(struct ip_vs_seq); } /* Handle pe data */ @@ -850,9 +867,29 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa unsigned long timeout, __u32 fwmark, struct ip_vs_sync_conn_options *opt) { + struct ip_vs_conn *cp_new = NULL; struct ip_vs_dest *dest; struct ip_vs_conn *cp; + /* Templates require cport 0. Ordinary connections require + * NO_CPORT exactly when cport is 0. + */ + if (flags & IP_VS_CONN_F_TEMPLATE) { + if (param->cport) { + IP_VS_DBG(2, "BACKUP, template with cport dropped\n"); + kfree(param->pe_data); + return; + } + } else if (!param->cport && !(flags & IP_VS_CONN_F_NO_CPORT)) { + IP_VS_DBG(2, "BACKUP, conn without cport dropped\n"); + kfree(param->pe_data); + return; + } else if (param->cport && (flags & IP_VS_CONN_F_NO_CPORT)) { + IP_VS_DBG(2, "BACKUP, conn with cport and NO_CPORT dropped\n"); + kfree(param->pe_data); + return; + } + if (!(flags & IP_VS_CONN_F_TEMPLATE)) { cp = ip_vs_conn_in_get(param); if (cp && ((cp->dport != dport) || @@ -891,9 +928,6 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa flags &= IP_VS_CONN_F_BACKUP_UPD_MASK; flags |= cp->flags & ~IP_VS_CONN_F_BACKUP_UPD_MASK; cp->flags = flags; - spin_unlock_bh(&cp->lock); - if (!dest) - ip_vs_try_bind_dest(cp); } else { /* * Find the appropriate destination for the connection. @@ -920,6 +954,8 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa } if (!(flags & IP_VS_CONN_F_TEMPLATE)) kfree(param->pe_data); + cp_new = cp; + spin_lock_bh(&cp->lock); } if (opt) { @@ -951,6 +987,9 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa else cp->timeout = (3*60*HZ); } + spin_unlock_bh(&cp->lock); + if (!dest && !cp_new) + ip_vs_try_bind_dest(cp); ip_vs_conn_put(cp); } diff --git a/net/openvswitch/flow.c b/net/openvswitch/flow.c index 3734416b518d..5fbeb0cca9dc 100644 --- a/net/openvswitch/flow.c +++ b/net/openvswitch/flow.c @@ -438,7 +438,8 @@ static int parse_ipv6hdr(struct sk_buff *skb, struct sw_flow_key *key) return -EPROTO; nh_len = payload_ofs - nh_ofs; - skb_set_transport_header(skb, nh_ofs + nh_len); + if (unlikely(!skb_set_transport_header_careful(skb, payload_ofs))) + return -EPROTO; key->ip.proto = nexthdr; return nh_len; } diff --git a/net/sctp/ipv6.c b/net/sctp/ipv6.c index 97390569c4bd..ad5c951efdf0 100644 --- a/net/sctp/ipv6.c +++ b/net/sctp/ipv6.c @@ -170,8 +170,11 @@ static int sctp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, /* Fix up skb to look at the embedded net header. */ saveip = skb->network_header; savesctp = skb->transport_header; + if (unlikely(!skb_set_transport_header_careful(skb, offset))) { + __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), ICMP6_MIB_INERRORS); + return -EINVAL; + } skb_reset_network_header(skb); - skb_set_transport_header(skb, offset); sk = sctp_err_lookup(net, AF_INET6, skb, sctp_hdr(skb), &asoc, &transport); /* Put back, the original pointers. */ skb->network_header = saveip; diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index 15c2d87575a8..ba0474c2a0bc 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -2870,7 +2870,6 @@ struct sctp_chunk *sctp_make_asconf_update_ip(struct sctp_association *asoc, int addr_param_len = 0; union sctp_addr *addr; int totallen = 0, i; - int del_pickup = 0; struct sctp_af *af; void *addr_buf; @@ -2885,16 +2884,21 @@ struct sctp_chunk *sctp_make_asconf_update_ip(struct sctp_association *asoc, totallen += addr_param_len; addr_buf += af->sockaddr_len; - if (asoc->asconf_addr_del_pending && !del_pickup) { - /* reuse the parameter length from the same scope one */ - totallen += paramlen; - totallen += addr_param_len; - del_pickup = 1; + } - pr_debug("%s: picked same-scope del_pending addr, " - "totallen for all addresses is %d\n", - __func__, totallen); - } + if (asoc->asconf_addr_del_pending) { + /* The DEL_IP parameter is serialized from the pending address + * itself below; account for its own address family, which may + * differ from the added ones. + */ + addr = asoc->asconf_addr_del_pending; + af = sctp_get_af_specific(addr->v4.sin_family); + totallen += paramlen; + totallen += af->to_addr_param(addr, &addr_param); + + pr_debug("%s: picked del_pending addr, " + "totallen for all addresses is %d\n", + __func__, totallen); } /* Create an asconf chunk with the required length. */ @@ -2917,7 +2921,7 @@ struct sctp_chunk *sctp_make_asconf_update_ip(struct sctp_association *asoc, addr_buf += af->sockaddr_len; } - if (flags == SCTP_PARAM_ADD_IP && del_pickup) { + if (flags == SCTP_PARAM_ADD_IP && asoc->asconf_addr_del_pending) { addr = asoc->asconf_addr_del_pending; af = sctp_get_af_specific(addr->v4.sin_family); addr_param_len = af->to_addr_param(addr, &addr_param); diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 64519fd8f943..6a4205de0aa6 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -6372,7 +6372,8 @@ static struct sctp_sackhdr *sctp_sm_pull_sack(struct sctp_chunk *chunk) num_dup_tsns = ntohs(sack->num_dup_tsns); len = sizeof(struct sctp_sackhdr); len += (num_blocks + num_dup_tsns) * sizeof(__u32); - if (len > chunk->skb->len) + if (len > ntohs(chunk->chunk_hdr->length) - + sizeof(struct sctp_chunkhdr)) return NULL; skb_pull(chunk->skb, len); diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 8e647737b256..bc68af88e271 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -2063,6 +2063,11 @@ static int sctp_sendmsg(struct sock *sk, struct msghdr *msg, size_t msg_len) asoc = transport->asoc; new = true; + /* sctp_sendmsg_to_asoc() may release the socket lock, + * and the new asoc can be peeled off meanwhile. Hold + * it, and free it on error only if sk still owns it. + */ + sctp_association_hold(asoc); } if (!sctp_style(sk, TCP) && !(sflags & SCTP_ADDR_OVER)) @@ -2084,8 +2089,11 @@ static int sctp_sendmsg(struct sock *sk, struct msghdr *msg, size_t msg_len) /* Send msg to the asoc */ err = sctp_sendmsg_to_asoc(asoc, msg, msg_len, transport, sinfo); - if (err < 0 && err != -ESRCH && new) - sctp_association_free(asoc); + if (new) { + if (err < 0 && err != -ESRCH && asoc->base.sk == sk) + sctp_association_free(asoc); + sctp_association_put(asoc); + } out_unlock: release_sock(sk); diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 4cdfbb57ecff..fa7170cd4fd1 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -1350,6 +1350,8 @@ static int tls_device_down(struct net_device *netdev) spin_unlock_irqrestore(&tls_device_lock, flags); list_for_each_entry_safe(ctx, tmp, &list, list) { + bool free_ctx; + /* Stop offloaded TX and switch to the fallback. * tls_is_skb_tx_device_offloaded will return false. */ @@ -1387,17 +1389,19 @@ static int tls_device_down(struct net_device *netdev) */ spin_lock_irqsave(&tls_device_lock, flags); list_move_tail(&ctx->list, &tls_device_down_list); + /* Serialize the final removal with sk_destruct. */ + free_ctx = refcount_dec_and_test(&ctx->refcount); + if (free_ctx) + list_del(&ctx->list); spin_unlock_irqrestore(&tls_device_lock, flags); /* Device contexts for RX and TX will be freed in on sk_destruct * by tls_device_free_ctx. rx_conf and tx_conf stay in TLS_HW. - * Now release the ref taken above. */ - if (refcount_dec_and_test(&ctx->refcount)) { + if (free_ctx) { /* sk_destruct ran after tls_device_down took a ref, and * it returned early. Complete the destruction here. */ - list_del(&ctx->list); tls_device_free_ctx(ctx); } } diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index f847cf1fbeaf..91c1d6cad497 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -609,6 +609,28 @@ static void unix_write_space(struct sock *sk) rcu_read_unlock(); } +static void scm_stat_add(struct sock *sk, struct sk_buff *skb) +{ + struct scm_fp_list *fp = UNIXCB(skb).fp; + struct unix_sock *u = unix_sk(sk); + + if (unlikely(fp && fp->count)) { + atomic_add(fp->count, &u->scm_stat.nr_fds); + unix_add_edges(fp, u); + } +} + +static void scm_stat_del(struct sock *sk, struct sk_buff *skb) +{ + struct scm_fp_list *fp = UNIXCB(skb).fp; + struct unix_sock *u = unix_sk(sk); + + if (unlikely(fp && fp->count)) { + atomic_sub(fp->count, &u->scm_stat.nr_fds); + unix_del_edges(fp); + } +} + /* When dgram socket disconnects (or changes its peer), we clear its receive * queue of packets arrived from previous peer. First, it allows to do * flow control based only on wmem_alloc; second, sk connected to peer @@ -616,8 +638,19 @@ static void unix_write_space(struct sock *sk) static void unix_dgram_disconnected(struct sock *sk, struct sock *other) { if (!skb_queue_empty(&sk->sk_receive_queue)) { - skb_queue_purge_reason(&sk->sk_receive_queue, - SKB_DROP_REASON_UNIX_DISCONNECT); + struct sk_buff_head tmp; + struct sk_buff *skb; + + __skb_queue_head_init(&tmp); + + spin_lock(&sk->sk_receive_queue.lock); + skb_queue_splice_init(&sk->sk_receive_queue, &tmp); + spin_unlock(&sk->sk_receive_queue.lock); + + while ((skb = __skb_dequeue(&tmp)) != NULL) { + scm_stat_del(sk, skb); + kfree_skb_reason(skb, SKB_DROP_REASON_UNIX_DISCONNECT); + } wake_up_interruptible_all(&unix_sk(sk)->peer_wait); @@ -2059,28 +2092,6 @@ static bool unix_skb_scm_eq(const struct sock *sk, struct sk_buff *skb, unix_secdata_eq(scm, skb); } -static void scm_stat_add(struct sock *sk, struct sk_buff *skb) -{ - struct scm_fp_list *fp = UNIXCB(skb).fp; - struct unix_sock *u = unix_sk(sk); - - if (unlikely(fp && fp->count)) { - atomic_add(fp->count, &u->scm_stat.nr_fds); - unix_add_edges(fp, u); - } -} - -static void scm_stat_del(struct sock *sk, struct sk_buff *skb) -{ - struct scm_fp_list *fp = UNIXCB(skb).fp; - struct unix_sock *u = unix_sk(sk); - - if (unlikely(fp && fp->count)) { - atomic_sub(fp->count, &u->scm_stat.nr_fds); - unix_del_edges(fp); - } -} - static void unix_orphan_scm(struct sock *sk, struct sk_buff *skb) { scm_stat_del(sk, skb); diff --git a/tools/testing/selftests/net/af_unix/scm_rights.c b/tools/testing/selftests/net/af_unix/scm_rights.c index c165f250220a..19aa26c60f3b 100644 --- a/tools/testing/selftests/net/af_unix/scm_rights.c +++ b/tools/testing/selftests/net/af_unix/scm_rights.c @@ -395,4 +395,79 @@ TEST_F(scm_rights, mixed_lowpoint) close_sockets(6); } +static int count_scm_fds(struct __test_metadata *_metadata, int fd) +{ + char path[64], *line = NULL; + int scm_fds = -1, ret; + size_t unused = 0; + FILE *f; + + snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", fd); + + f = fopen(path, "r"); + ASSERT_NE(NULL, f); + + while (getline(&line, &unused, f) != -1) { + if (strncmp(line, "scm_fds:", 8)) + continue; + + ret = sscanf(line + 8, "%d", &scm_fds); + ASSERT_EQ(1, ret); + + break; + } + + free(line); + + ret = fclose(f); + ASSERT_EQ(0, ret); + + return scm_fds; +} + +TEST(dgram_disconnect_fdinfo) +{ + const FIXTURE_VARIANT(scm_rights) variant = { + .name = "UNIX ", + .type = SOCK_DGRAM, + }; + FIXTURE_DATA(scm_rights) self = {}; + struct sockaddr addr = { + .sa_family = AF_UNSPEC, + }; + int ret; + + ret = unshare(CLONE_NEWNET); + ASSERT_EQ(0, ret); + + create_socketpairs(_metadata, &self, &variant, 2); + + /* 1. Disconnect via connect(AF_UNSPEC) */ + __send_fd(_metadata, &self, &variant, 0, 0); + ASSERT_EQ(2, count_scm_fds(_metadata, self.fd[0])); + + ret = connect(self.fd[0], &addr, sizeof(addr.sa_family)); + ASSERT_EQ(0, ret); + ASSERT_EQ(0, count_scm_fds(_metadata, self.fd[0])); + + /* 2. Dead-peer disconnect via send() */ + __send_fd(_metadata, &self, &variant, 1, 1); + ASSERT_EQ(2, count_scm_fds(_metadata, self.fd[2])); + + ret = close(self.fd[3]); + ASSERT_EQ(0, ret); + + ret = send(self.fd[2], "x", 1, 0); + ASSERT_EQ(-1, ret); + ASSERT_EQ(ECONNREFUSED, errno); + ASSERT_EQ(0, count_scm_fds(_metadata, self.fd[2])); + + ret = close(self.fd[0]); + ASSERT_EQ(0, ret); + ret = close(self.fd[1]); + ASSERT_EQ(0, ret); + ret = close(self.fd[2]); + ASSERT_EQ(0, ret); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/net/bind_wildcard.c b/tools/testing/selftests/net/bind_wildcard.c index be3298c791b2..334a0cf87edf 100644 --- a/tools/testing/selftests/net/bind_wildcard.c +++ b/tools/testing/selftests/net/bind_wildcard.c @@ -811,4 +811,58 @@ TEST_F(bind_wildcard, reuseport) bind_socket(_metadata, self, variant, i, SO_REUSEPORT); } +/* The SO_REUSEPORT bind cache (tb->fast_*) filled by a v6only [::] socket + * must not let a dual-stack [::] socket skip the conflict check against a + * 0.0.0.0 listener. + */ +TEST(reuseport_v6only_cache) +{ + struct sockaddr_in6 addr6 = { + .sin6_family = AF_INET6, + .sin6_addr = in6addr_any, + }; + struct sockaddr_in addr4 = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_ANY), + }; + socklen_t addrlen; + int fd[3], ret; + int on = 1; + + ASSERT_EQ(unshare(CLONE_NEWNET), 0); + + /* bind(0.0.0.0:P), listen() */ + fd[0] = socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GT(fd[0], 0); + ASSERT_EQ(bind(fd[0], (struct sockaddr *)&addr4, sizeof(addr4)), 0); + ASSERT_EQ(listen(fd[0], 1), 0); + + addrlen = sizeof(addr4); + ASSERT_EQ(getsockname(fd[0], (struct sockaddr *)&addr4, &addrlen), 0); + addr6.sin6_port = addr4.sin_port; + + /* SO_REUSEPORT + IPV6_V6ONLY, bind([::]:P) fills the cache. */ + fd[1] = socket(AF_INET6, SOCK_STREAM, 0); + ASSERT_GT(fd[1], 0); + ret = setsockopt(fd[1], SOL_IPV6, IPV6_V6ONLY, &on, sizeof(on)); + ASSERT_EQ(ret, 0); + ret = setsockopt(fd[1], SOL_SOCKET, SO_REUSEPORT, &on, sizeof(on)); + ASSERT_EQ(ret, 0); + ASSERT_EQ(bind(fd[1], (struct sockaddr *)&addr6, sizeof(addr6)), 0); + + /* SO_REUSEPORT, bind([::]:P) conflicts with 0.0.0.0:P. */ + fd[2] = socket(AF_INET6, SOCK_STREAM, 0); + ASSERT_GT(fd[2], 0); + ret = setsockopt(fd[2], SOL_SOCKET, SO_REUSEPORT, &on, sizeof(on)); + ASSERT_EQ(ret, 0); + + ret = bind(fd[2], (struct sockaddr *)&addr6, sizeof(addr6)); + ASSERT_EQ(ret, -1); + ASSERT_EQ(errno, EADDRINUSE); + + close(fd[2]); + close(fd[1]); + close(fd[0]); +} + TEST_HARNESS_MAIN