diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index b513baf3cbb296..072133b4a4416d 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -146,6 +146,7 @@ config IXGBE tristate "Intel(R) 10GbE PCI Express adapters support" depends on PCI depends on PTP_1588_CLOCK_OPTIONAL + select LIBETH_XDP select LIBIE_FWLOG if DEBUG_FS select MDIO select NET_DEVLINK diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index dce4936708eb44..64a3cbabba728f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -33,6 +33,8 @@ #include "ixgbe_ipsec.h" #include +#include +#include /* common prefix used by pr_<> macros */ #undef pr_fmt @@ -69,65 +71,12 @@ /* Supported Rx Buffer Sizes */ #define IXGBE_RXBUFFER_256 256 /* Used for skb receive header */ -#define IXGBE_RXBUFFER_1536 1536 -#define IXGBE_RXBUFFER_2K 2048 #define IXGBE_RXBUFFER_3K 3072 #define IXGBE_RXBUFFER_4K 4096 #define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */ -#define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)) - -/* Attempt to maximize the headroom available for incoming frames. We - * use a 2K buffer for receives and need 1536/1534 to store the data for - * the frame. This leaves us with 512 bytes of room. From that we need - * to deduct the space needed for the shared info and the padding needed - * to IP align the frame. - * - * Note: For cache line sizes 256 or larger this value is going to end - * up negative. In these cases we should fall back to the 3K - * buffers. - */ -#if (PAGE_SIZE < 8192) -#define IXGBE_MAX_2K_FRAME_BUILD_SKB (IXGBE_RXBUFFER_1536 - NET_IP_ALIGN) -#define IXGBE_2K_TOO_SMALL_WITH_PADDING \ -((NET_SKB_PAD + IXGBE_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K)) - -static inline int ixgbe_compute_pad(int rx_buf_len) -{ - int page_size, pad_size; - - page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2); - pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len; - - return pad_size; -} - -static inline int ixgbe_skb_pad(void) -{ - int rx_buf_len; - - /* If a 2K buffer cannot handle a standard Ethernet frame then - * optimize padding for a 3K buffer instead of a 1.5K buffer. - * - * For a 3K buffer we need to add enough padding to allow for - * tailroom due to NET_IP_ALIGN possibly shifting us out of - * cache-line alignment. - */ - if (IXGBE_2K_TOO_SMALL_WITH_PADDING) - rx_buf_len = IXGBE_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN); - else - rx_buf_len = IXGBE_RXBUFFER_1536; - - /* if needed make room for NET_IP_ALIGN */ - rx_buf_len -= NET_IP_ALIGN; - - return ixgbe_compute_pad(rx_buf_len); -} - -#define IXGBE_SKB_PAD ixgbe_skb_pad() -#else -#define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) -#endif +#define IXGBE_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) /* * NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we @@ -271,20 +220,9 @@ struct ixgbe_tx_buffer { u32 tx_flags; }; -struct ixgbe_rx_buffer { - union { - struct { - struct sk_buff *skb; - dma_addr_t dma; - struct page *page; - __u32 page_offset; - __u16 pagecnt_bias; - }; - struct { - bool discard; - struct xdp_buff *xdp; - }; - }; +struct ixgbe_xsk_rx_buffer { + bool discard; + struct xdp_buff *xdp; }; struct ixgbe_queue_stats { @@ -311,8 +249,6 @@ struct ixgbe_rx_queue_stats { #define IXGBE_TS_HDR_LEN 8 enum ixgbe_ring_state_t { - __IXGBE_RX_3K_BUFFER, - __IXGBE_RX_BUILD_SKB_ENABLED, __IXGBE_RX_RSC_ENABLED, __IXGBE_RX_CSUM_UDP_ZERO_ERR, __IXGBE_RX_FCOE, @@ -357,17 +293,22 @@ struct ixgbe_ring { struct ixgbe_ring *next; /* pointer to next ring in q_vector */ struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */ struct net_device *netdev; /* netdev ring belongs to */ - struct bpf_prog *xdp_prog; - struct device *dev; /* device for DMA mapping */ + struct bpf_prog __rcu *xdp_prog; + union { + struct page_pool *pp; /* Rx ring */ + struct device *dev; /* Tx ring */ + }; void *desc; /* descriptor ring memory */ union { + struct libeth_fqe *rx_fqes; + struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; struct ixgbe_tx_buffer *tx_buffer_info; - struct ixgbe_rx_buffer *rx_buffer_info; }; unsigned long state; u8 __iomem *tail; dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ + u32 truesize; u16 count; /* amount of descriptors */ @@ -382,12 +323,9 @@ struct ixgbe_ring { unsigned long last_rx_timestamp; - union { - u16 next_to_alloc; - struct { - u8 atr_sample_rate; - u8 atr_count; - }; + struct { + u8 atr_sample_rate; + u8 atr_count; }; u8 dcb_tc; @@ -397,12 +335,12 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; - u16 rx_offset; struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ - u16 rx_buf_len; + u32 rx_buf_len; + struct libeth_xdp_buff_stash xdp_stash; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { @@ -446,31 +384,6 @@ struct ixgbe_ring_feature { #define IXGBE_82599_VMDQ_4Q_MASK 0x7C #define IXGBE_82599_VMDQ_2Q_MASK 0x7E -/* - * FCoE requires that all Rx buffers be over 2200 bytes in length. Since - * this is twice the size of a half page we need to double the page order - * for FCoE enabled Rx queues. - */ -static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) -{ - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - return IXGBE_RXBUFFER_3K; -#if (PAGE_SIZE < 8192) - if (ring_uses_build_skb(ring)) - return IXGBE_MAX_2K_FRAME_BUILD_SKB; -#endif - return IXGBE_RXBUFFER_2K; -} - -static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - return 1; -#endif - return 0; -} -#define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring)) #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 @@ -919,7 +832,6 @@ struct ixgbe_cb { }; dma_addr_t dma; u16 append_cnt; - bool page_released; }; #define IXGBE_CB(skb) ((struct ixgbe_cb *)(skb)->cb) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 2ad81f687a844a..d8406e08e54be9 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1951,7 +1951,6 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter) /* Setup Rx Descriptor ring and Rx buffers */ rx_ring->count = IXGBE_DEFAULT_RXD; rx_ring->queue_index = 0; - rx_ring->dev = &adapter->pdev->dev; rx_ring->netdev = adapter->netdev; rx_ring->reg_idx = adapter->rx_ring[0]->reg_idx; @@ -1963,6 +1962,10 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter) hw->mac.ops.disable_rx(hw); + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + ixgbe_configure_rx_ring(adapter, rx_ring); rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXCTRL); @@ -2058,14 +2061,16 @@ static void ixgbe_create_lbtest_frame(struct sk_buff *skb, skb->data[frame_size + 12] = 0xAF; } -static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer, +static bool ixgbe_check_lbtest_frame(const struct libeth_fqe *rx_buffer, unsigned int frame_size) { + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; unsigned char *data; frame_size >>= 1; - data = page_address(rx_buffer->page) + rx_buffer->page_offset; + data = page_address(__netmem_to_page(rx_buffer->netmem)) + + rx_buffer->offset + hr; return data[3] == 0xFF && data[frame_size + 10] == 0xBE && data[frame_size + 12] == 0xAF; @@ -2113,16 +2118,13 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, } while (rx_desc->wb.upper.length) { - struct ixgbe_rx_buffer *rx_buffer; + struct libeth_fqe *rx_buffer; /* check Rx buffer */ - rx_buffer = &rx_ring->rx_buffer_info[rx_ntc]; + rx_buffer = &rx_ring->rx_fqes[rx_ntc]; /* sync Rx buffer for CPU read */ - dma_sync_single_for_cpu(rx_ring->dev, - rx_buffer->dma, - ixgbe_rx_bufsz(rx_ring), - DMA_FROM_DEVICE); + libeth_rx_sync_for_cpu(rx_buffer, rx_ring->rx_buf_len); /* verify contents of skb */ if (ixgbe_check_lbtest_frame(rx_buffer, size)) @@ -2130,11 +2132,8 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, else break; - /* sync Rx buffer for device write */ - dma_sync_single_for_device(rx_ring->dev, - rx_buffer->dma, - ixgbe_rx_bufsz(rx_ring), - DMA_FROM_DEVICE); + /* recycle the page back to the pool */ + libeth_rx_recycle_slow(rx_buffer->netmem); /* increment Rx next to clean counter */ rx_ntc++; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c index a1d04914fbbca8..2d139d2217fac4 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c @@ -965,7 +965,6 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, while (rxr_count) { /* assign generic ring traits */ - ring->dev = &adapter->pdev->dev; ring->netdev = adapter->netdev; /* configure backlink on ring */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c58051e4350be2..5439f6e06aa60d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -54,13 +55,13 @@ char ixgbe_driver_name[] = "ixgbe"; static const char ixgbe_driver_string[] = - "Intel(R) 10 Gigabit PCI Express Network Driver"; + "Intel(R) 10 Gigabit PCI Express Network Driver"; #ifdef IXGBE_FCOE char ixgbe_default_device_descr[] = - "Intel(R) 10 Gigabit Network Connection"; + "Intel(R) 10 Gigabit Network Connection"; #else static char ixgbe_default_device_descr[] = - "Intel(R) 10 Gigabit Network Connection"; + "Intel(R) 10 Gigabit Network Connection"; #endif static const char ixgbe_copyright[] = "Copyright (c) 1999-2016 Intel Corporation."; @@ -147,7 +148,7 @@ MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, - void *p); + void *p); static struct notifier_block dca_notifier = { .notifier_call = ixgbe_notify_dca, .next = NULL, @@ -174,6 +175,8 @@ MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); MODULE_IMPORT_NS("LIBIE_FWLOG"); MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); +MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL v2"); DEFINE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key); @@ -212,7 +215,7 @@ static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter, pcie_capability_read_word(parent_dev, reg, value); if (*value == IXGBE_FAILED_READ_CFG_WORD && - ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) + ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) return -1; return 0; } @@ -264,7 +267,7 @@ static bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw) } static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, - int expected_gts) + int expected_gts) { struct ixgbe_hw *hw = &adapter->hw; struct pci_dev *pdev; @@ -288,8 +291,8 @@ static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && - !test_bit(__IXGBE_REMOVING, &adapter->state) && - !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) + !test_bit(__IXGBE_REMOVING, &adapter->state) && + !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) queue_work(ixgbe_wq, &adapter->service_task); } @@ -354,7 +357,7 @@ u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg) if (ixgbe_removed(reg_addr)) return IXGBE_FAILED_READ_REG; if (unlikely(hw->phy.nw_mng_if_sel & - IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) { + IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) { struct ixgbe_adapter *adapter; int i; @@ -401,7 +404,7 @@ u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg) return IXGBE_FAILED_READ_CFG_WORD; pci_read_config_word(adapter->pdev, reg, &value); if (value == IXGBE_FAILED_READ_CFG_WORD && - ixgbe_check_cfg_remove(hw, adapter->pdev)) + ixgbe_check_cfg_remove(hw, adapter->pdev)) return IXGBE_FAILED_READ_CFG_WORD; return value; } @@ -416,7 +419,7 @@ static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg) return IXGBE_FAILED_READ_CFG_DWORD; pci_read_config_dword(adapter->pdev, reg, &value); if (value == IXGBE_FAILED_READ_CFG_DWORD && - ixgbe_check_cfg_remove(hw, adapter->pdev)) + ixgbe_check_cfg_remove(hw, adapter->pdev)) return IXGBE_FAILED_READ_CFG_DWORD; return value; } @@ -592,7 +595,6 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) struct my_u0 { u64 a; u64 b; } *u0; struct ixgbe_ring *rx_ring; union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *rx_buffer_info; int i = 0; if (!netif_msg_hw(adapter)) @@ -613,7 +615,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) dev_info(&adapter->pdev->dev, "Register Dump\n"); pr_info(" Register Name Value\n"); for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl; - reginfo->name; reginfo++) { + reginfo->name; reginfo++) { ixgbe_regdump(hw, reginfo); } @@ -694,7 +696,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) const char *ring_desc; if (i == ring->next_to_use && - i == ring->next_to_clean) + i == ring->next_to_clean) ring_desc = " NTC/U"; else if (i == ring->next_to_use) ring_desc = " NTU"; @@ -714,7 +716,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) ring_desc); if (netif_msg_pktdata(adapter) && - tx_buffer->skb) + tx_buffer->skb) print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 1, tx_buffer->skb->data, @@ -790,14 +792,12 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) pr_info("------------------------------------\n"); pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); pr_info("------------------------------------\n"); - pr_info("%s%s%s\n", + pr_info("%s%s\n", "R [desc] [ PktBuf A0] ", - "[ HeadBuf DD] [bi->dma ] [bi->skb ] ", - "<-- Adv Rx Read format"); - pr_info("%s%s%s\n", + "[ HeadBuf DD] <-- Adv Rx Read format"); + pr_info("%s%s\n", "RWB[desc] [PcsmIpSHl PtRs] ", - "[vl er S cks ln] ---------------- [bi->skb ] ", - "<-- Adv Rx Write-Back format"); + "[vl er S cks ln] <-- Adv Rx Write-Back format"); for (i = 0; i < rx_ring->count; i++) { const char *ring_desc; @@ -809,34 +809,20 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) else ring_desc = ""; - rx_buffer_info = &rx_ring->rx_buffer_info[i]; rx_desc = IXGBE_RX_DESC(rx_ring, i); u0 = (struct my_u0 *)rx_desc; if (rx_desc->wb.upper.length) { - /* Descriptor Done */ - pr_info("RWB[0x%03X] %016llX %016llX ---------------- %p%s\n", + pr_info("RWB[0x%03X] %016llX %016llX%s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - rx_buffer_info->skb, ring_desc); } else { - pr_info("R [0x%03X] %016llX %016llX %016llX %p%s\n", + pr_info("R [0x%03X] %016llX %016llX %s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - (u64)rx_buffer_info->dma, - rx_buffer_info->skb, ring_desc); - - if (netif_msg_pktdata(adapter) && - rx_buffer_info->dma) { - print_hex_dump(KERN_INFO, "", - DUMP_PREFIX_ADDRESS, 16, 1, - page_address(rx_buffer_info->page) + - rx_buffer_info->page_offset, - ixgbe_rx_bufsz(rx_ring), true); - } } } } @@ -917,7 +903,7 @@ static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, } void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, - u64 qmask) + u64 qmask) { u32 mask; @@ -950,7 +936,7 @@ static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) u32 data; if ((hw->fc.current_mode != ixgbe_fc_full) && - (hw->fc.current_mode != ixgbe_fc_rx_pause)) + (hw->fc.current_mode != ixgbe_fc_rx_pause)) return; switch (hw->mac.type) { @@ -1134,7 +1120,7 @@ static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) * @maxrate: desired maximum transmit bitrate **/ static int ixgbe_tx_maxrate(struct net_device *netdev, - int queue_index, u32 maxrate) + int queue_index, u32 maxrate) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -1206,7 +1192,7 @@ void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring, * Prints a message containing details about the tx hang. */ static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring, - unsigned int next) + unsigned int next) { struct ixgbe_adapter *adapter = netdev_priv(tx_ring->netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -1219,14 +1205,14 @@ static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring, "tx_buffer_info[next_to_clean]\n" " time_stamp <%lx>\n" " jiffies <%lx>\n", - tx_ring->queue_index, - IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), - IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), - tx_ring->next_to_use, next, - tx_ring->tx_buffer_info[next].time_stamp, jiffies); + tx_ring->queue_index, + IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), + IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), + tx_ring->next_to_use, next, + tx_ring->tx_buffer_info[next].time_stamp, jiffies); netif_stop_subqueue(tx_ring->netdev, - tx_ring->queue_index); + tx_ring->queue_index); } /** @@ -1245,8 +1231,8 @@ static void ixgbe_vf_handle_tx_hang(struct ixgbe_adapter *adapter, u16 vf) return; e_warn(drv, - "Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM", - hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses); + "Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM", + hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses); adapter->tx_hang_count[vf]++; if (adapter->tx_hang_count[vf] == IXGBE_MAX_TX_VF_HANGS) { @@ -1273,7 +1259,7 @@ static u32 ixgbe_poll_tx_icache(struct ixgbe_hw *hw, u16 queue, u16 idx) * Returns: true if queue contain illegal packet. */ static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter, - u16 queue) + u16 queue) { u32 hdr_len_reg, mss_len_reg, type_reg; struct ixgbe_hw *hw = &adapter->hw; @@ -1296,13 +1282,13 @@ static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter, /* check if Advanced Context Descriptor */ if (FIELD_GET(IXGBE_ADVTXD_DTYP_MASK, type_reg) != - IXGBE_ADVTXD_DTYP_CTXT) + IXGBE_ADVTXD_DTYP_CTXT) continue; /* check for illegal MSS and Header length */ mss_len = FIELD_GET(IXGBE_ADVTXD_MSS_MASK, mss_len_reg); header_len = FIELD_GET(IXGBE_ADVTXD_HEADER_LEN_MASK, - hdr_len_reg); + hdr_len_reg); if ((mss_len + header_len) > SZ_16K) { e_warn(probe, "mss len + header len too long\n"); return true; @@ -1333,7 +1319,7 @@ static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter, /* malicious vf not detected */ for (q = 0; q < IXGBE_MAX_TX_QUEUES; q++) { if (ixgbe_check_illegal_queue(adapter, q) && - !ixgbe_get_vf_idx(adapter, q, &vf)) + !ixgbe_get_vf_idx(adapter, q, &vf)) /* illegal queue detected */ ixgbe_vf_handle_tx_hang(adapter, vf); } @@ -1347,7 +1333,7 @@ static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter, * @napi_budget: Used to determine if we are in netpoll **/ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring *tx_ring, int napi_budget) + struct ixgbe_ring *tx_ring, int napi_budget) { struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_tx_buffer *tx_buffer; @@ -1416,9 +1402,9 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) { dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buffer, len, 0); } } @@ -1456,7 +1442,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, ixgbe_pf_handle_tx_hang(tx_ring, i); e_info(probe, - "tx hang %d detected on queue %d, resetting adapter\n", + "tx hang %d detected on queue %d, resetting adapter\n", adapter->tx_timeout_count + 1, tx_ring->queue_index); /* schedule immediate reset if we believe we hung */ @@ -1525,7 +1511,7 @@ static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, u8 reg_idx = rx_ring->reg_idx; if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) - rxctrl = dca3_get_tag(rx_ring->dev, cpu); + rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu); switch (hw->mac.type) { case ixgbe_mac_82599EB: @@ -1643,8 +1629,8 @@ static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, return; skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), - (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); + (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); } #ifdef IXGBE_FCOE @@ -1656,14 +1642,14 @@ static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, * Returns : true if it is FCoE pkt */ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, - union ixgbe_adv_rx_desc *rx_desc) + union ixgbe_adv_rx_desc *rx_desc) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; return test_bit(__IXGBE_RX_FCOE, &ring->state) && - ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == + ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << - IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); + IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); } #endif /* IXGBE_FCOE */ @@ -1674,8 +1660,8 @@ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, * @skb: skb currently being received and modified **/ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; bool encap_pkt = false; @@ -1694,7 +1680,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, /* if IP and error */ if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && - ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { + ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { ring->rx_stats.csum_err++; return; } @@ -1708,7 +1694,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, * checksum errors. */ if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && - test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) + test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) return; ring->rx_stats.csum_err++; @@ -1730,55 +1716,6 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, } } -static unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) -{ - return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0; -} - -static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *bi) -{ - struct page *page = bi->page; - dma_addr_t dma; - - /* since we are recycling buffers we should seldom need to alloc */ - if (likely(page)) - return true; - - /* alloc new page for storage */ - page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring)); - if (unlikely(!page)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - - /* - * if mapping failed free memory back to system since - * there isn't much point in holding memory we can't use - */ - if (dma_mapping_error(rx_ring->dev, dma)) { - __free_pages(page, ixgbe_rx_pg_order(rx_ring)); - - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - bi->dma = dma; - bi->page = page; - bi->page_offset = rx_ring->rx_offset; - page_ref_add(page, USHRT_MAX - 1); - bi->pagecnt_bias = USHRT_MAX; - rx_ring->rx_stats.alloc_rx_page++; - - return true; -} - /** * ixgbe_alloc_rx_buffers - Replace used receive buffers * @rx_ring: ring to place buffers on @@ -1787,42 +1724,34 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; - u16 i = rx_ring->next_to_use; - u16 bufsz; + const struct libeth_fq_fp fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + .truesize = rx_ring->truesize, + .count = rx_ring->count, + }; + u16 ntu = rx_ring->next_to_use; /* nothing to do */ - if (!cleaned_count) + if (unlikely(!cleaned_count)) return; - - rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; - i -= rx_ring->count; - - bufsz = ixgbe_rx_bufsz(rx_ring); + rx_desc = IXGBE_RX_DESC(rx_ring, ntu); do { - if (!ixgbe_alloc_mapped_page(rx_ring, bi)) + dma_addr_t addr; + + addr = libeth_rx_alloc(&fq, ntu); + if (addr == DMA_MAPPING_ERROR) break; - /* sync the buffer for use by the device */ - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, bufsz, - DMA_FROM_DEVICE); - - /* - * Refresh the desc even if buffer_addrs didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); + rx_desc->read.pkt_addr = cpu_to_le64(addr); rx_desc++; - bi++; - i++; - if (unlikely(!i)) { + ntu++; + + if (unlikely(ntu == fq.count)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; - i -= rx_ring->count; + ntu = 0; } /* clear the length for the next_to_use descriptor */ @@ -1831,21 +1760,16 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) cleaned_count--; } while (cleaned_count); - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { - rx_ring->next_to_use = i; - - /* update next to alloc since we have filled the ring */ - rx_ring->next_to_alloc = i; - + if (likely(rx_ring->next_to_use != ntu)) { + /* record the next descriptor to use */ + rx_ring->next_to_use = ntu; /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, * such as IA-64). */ wmb(); - writel(i, rx_ring->tail); + writel(ntu, rx_ring->tail); } } @@ -1887,8 +1811,8 @@ static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring, * other fields within the skb. **/ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { struct net_device *dev = rx_ring->netdev; u32 flags = rx_ring->q_vector->adapter->flags; @@ -1903,7 +1827,7 @@ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb); if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && - ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { + ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); } @@ -1917,8 +1841,6 @@ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, else macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true, false); - - skb->protocol = eth_type_trans(skb, dev); } void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, @@ -1931,7 +1853,7 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * ixgbe_is_non_eop - process handling of non-EOP buffers * @rx_ring: Rx ring being processed * @rx_desc: Rx descriptor for current buffer - * @skb: Current socket buffer containing buffer in progress + * @rsc_append_cnt: pointer to RSC append count accumulator * * This function updates next to clean. If the buffer is an EOP buffer * this function exits returning false, otherwise it will place the @@ -1940,7 +1862,7 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, **/ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + u32 *rsc_append_cnt) { u32 ntc = rx_ring->next_to_clean + 1; @@ -1953,13 +1875,13 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, /* update RSC append count if present */ if (ring_is_rsc_enabled(rx_ring)) { __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data & - cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); + cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); if (unlikely(rsc_enabled)) { u32 rsc_cnt = le32_to_cpu(rsc_enabled); rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; - IXGBE_CB(skb)->append_cnt += rsc_cnt - 1; + *rsc_append_cnt += rsc_cnt - 1; /* update ntc based on RSC value */ ntc = le32_to_cpu(rx_desc->wb.upper.status_error); @@ -1972,8 +1894,6 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) return false; - /* place skb in next buffer to be received */ - rx_ring->rx_buffer_info[ntc].skb = skb; rx_ring->rx_stats.non_eop_descs++; return true; @@ -1992,7 +1912,7 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, * truesize for the skb. */ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, - struct sk_buff *skb) + struct sk_buff *skb) { skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; unsigned char *va; @@ -2021,47 +1941,6 @@ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, skb->tail += pull_len; } -/** - * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB - * @rx_ring: rx descriptor ring packet is being transacted on - * @skb: pointer to current skb being updated - * - * This function provides a basic DMA sync up for the first fragment of an - * skb. The reason for doing this is that the first fragment cannot be - * unmapped until we have reached the end of packet descriptor for a buffer - * chain. - */ -static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, - struct sk_buff *skb) -{ - if (ring_uses_build_skb(rx_ring)) { - unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; - unsigned long offset = (unsigned long)(skb->data) & mask; - - dma_sync_single_range_for_cpu(rx_ring->dev, - IXGBE_CB(skb)->dma, - offset, - skb_headlen(skb), - DMA_FROM_DEVICE); - } else { - skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; - - dma_sync_single_range_for_cpu(rx_ring->dev, - IXGBE_CB(skb)->dma, - skb_frag_off(frag), - skb_frag_size(frag), - DMA_FROM_DEVICE); - } - - /* If the page was released, just unmap it. */ - if (unlikely(IXGBE_CB(skb)->page_released)) { - dma_unmap_page_attrs(rx_ring->dev, IXGBE_CB(skb)->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - } -} - /** * ixgbe_cleanup_headers - Correct corrupted or empty headers * @rx_ring: rx descriptor ring packet is being transacted on @@ -2094,9 +1973,9 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, * errors that would be unacceptable to the netdev. */ if (!netdev || - (unlikely(ixgbe_test_staterr(rx_desc, + (unlikely(ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_FRAME_ERR_MASK) && - !(netdev->features & NETIF_F_RXALL)))) { + !(netdev->features & NETIF_F_RXALL)))) { dev_kfree_skb_any(skb); return true; } @@ -2111,295 +1990,13 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; #endif - /* if eth_skb_pad returns an error the skb was freed */ - if (eth_skb_pad(skb)) - return true; return false; } -/** - * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring - * @rx_ring: rx descriptor ring to store buffers on - * @old_buff: donor buffer to have page reused - * - * Synchronizes page for reuse by the adapter - **/ -static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *old_buff) -{ - struct ixgbe_rx_buffer *new_buff; - u16 nta = rx_ring->next_to_alloc; - - new_buff = &rx_ring->rx_buffer_info[nta]; - - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* Transfer page from old buffer to new buffer. - * Move each member individually to avoid possible store - * forwarding stalls and unnecessary copy of skb. - */ - new_buff->dma = old_buff->dma; - new_buff->page = old_buff->page; - new_buff->page_offset = old_buff->page_offset; - new_buff->pagecnt_bias = old_buff->pagecnt_bias; -} - -static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, - int rx_buffer_pgcnt) -{ - unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; - struct page *page = rx_buffer->page; - - /* avoid re-using remote and pfmemalloc pages */ - if (!dev_page_is_reusable(page)) - return false; - -#if (PAGE_SIZE < 8192) - /* if we are only owner of page we can reuse it */ - if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) - return false; -#else - /* The last offset is a bit aggressive in that we assume the - * worst case of FCoE being enabled and using a 3K buffer. - * However this should have minimal impact as the 1K extra is - * still less than one buffer in size. - */ -#define IXGBE_LAST_OFFSET \ - (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K) - if (rx_buffer->page_offset > IXGBE_LAST_OFFSET) - return false; -#endif - - /* If we have drained the page fragment pool we need to update - * the pagecnt_bias and page count so that we fully restock the - * number of references the driver holds. - */ - if (unlikely(pagecnt_bias == 1)) { - page_ref_add(page, USHRT_MAX - 1); - rx_buffer->pagecnt_bias = USHRT_MAX; - } - - return true; -} - -/** - * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff - * @rx_ring: rx descriptor ring to transact packets on - * @rx_buffer: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: size of data in rx_buffer - * - * This function will add the data contained in rx_buffer->page to the skb. - * This is done either through a direct copy if the data in the buffer is - * less than the skb header size, otherwise it will just attach the page as - * a frag to the skb. - * - * The function will then update the page offset if necessary and return - * true if the buffer can be reused by the adapter. - **/ -static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct sk_buff *skb, - unsigned int size) -{ -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = rx_ring->rx_offset ? - SKB_DATA_ALIGN(rx_ring->rx_offset + size) : - SKB_DATA_ALIGN(size); -#endif - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, - rx_buffer->page_offset, size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif -} - -static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff **skb, - const unsigned int size, - int *rx_buffer_pgcnt) -{ - struct ixgbe_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - *rx_buffer_pgcnt = -#if (PAGE_SIZE < 8192) - page_count(rx_buffer->page); -#else - 0; -#endif - prefetchw(rx_buffer->page); - *skb = rx_buffer->skb; - - /* Delay unmapping of the first packet. It carries the header - * information, HW may still access the header after the writeback. - * Only unmap it when EOP is reached - */ - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) { - if (!*skb) - goto skip_sync; - } else { - if (*skb) - ixgbe_dma_sync_frag(rx_ring, *skb); - } - - /* we are reusing so sync this buffer for CPU use */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - size, - DMA_FROM_DEVICE); -skip_sync: - rx_buffer->pagecnt_bias--; - - return rx_buffer; -} - -static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct sk_buff *skb, - int rx_buffer_pgcnt) -{ - if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) { - /* hand second half of page back to the ring */ - ixgbe_reuse_rx_page(rx_ring, rx_buffer); - } else { - if (skb && IXGBE_CB(skb)->dma == rx_buffer->dma) { - /* the page has been released from the ring */ - IXGBE_CB(skb)->page_released = true; - } else { - /* we are not reusing the buffer so unmap it */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - } - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - } - - /* clear contents of rx_buffer */ - rx_buffer->page = NULL; - rx_buffer->skb = NULL; -} - -static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int size = xdp->data_end - xdp->data; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); -#endif - struct sk_buff *skb; - - /* prefetch first cache line of first page */ - net_prefetch(xdp->data); - - /* Note, we get here by enabling legacy-rx via: - * - * ethtool --set-priv-flags legacy-rx on - * - * In this mode, we currently get 0 extra XDP headroom as - * opposed to having legacy-rx off, where we process XDP - * packets going to stack via ixgbe_build_skb(). The latter - * provides us currently with 192 bytes of headroom. - * - * For ixgbe_construct_skb() mode it means that the - * xdp->data_meta will always point to xdp->data, since - * the helper cannot expand the head. Should this ever - * change in future for legacy-rx mode on, then lets also - * add xdp->data_meta handling here. - */ - - /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE); - if (unlikely(!skb)) - return NULL; - - if (size > IXGBE_RX_HDR_SIZE) { - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) - IXGBE_CB(skb)->dma = rx_buffer->dma; - - skb_add_rx_frag(skb, 0, rx_buffer->page, - xdp->data - page_address(rx_buffer->page), - size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - } else { - memcpy(__skb_put(skb, size), - xdp->data, ALIGN(size, sizeof(long))); - rx_buffer->pagecnt_bias++; - } - - return skb; -} - -static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int metasize = xdp->data - xdp->data_meta; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); -#endif - struct sk_buff *skb; - - /* Prefetch first cache line of first page. If xdp->data_meta - * is unused, this points exactly as xdp->data, otherwise we - * likely have a consumer accessing first few bytes of meta - * data, and then actual data. - */ - net_prefetch(xdp->data_meta); - - /* build an skb to around the page buffer */ - skb = napi_build_skb(xdp->data_hard_start, truesize); - if (unlikely(!skb)) - return NULL; - - /* update pointers within the skb to store the data */ - skb_reserve(skb, xdp->data - xdp->data_hard_start); - __skb_put(skb, xdp->data_end - xdp->data); - if (metasize) - skb_metadata_set(skb, metasize); - - /* record DMA address if this is the start of a chain of buffers */ - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) - IXGBE_CB(skb)->dma = rx_buffer->dma; - - /* update buffer offset */ -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - - return skb; -} - static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, - struct xdp_buff *xdp) + struct libeth_xdp_buff *xdp) { int err, result = IXGBE_XDP_PASS; struct bpf_prog *xdp_prog; @@ -2412,14 +2009,14 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, if (!xdp_prog) goto xdp_out; - prefetchw(xdp->data_hard_start); /* xdp_frame write */ + prefetchw(xdp->base.data_hard_start); /* xdp_frame write */ - act = bpf_prog_run_xdp(xdp_prog, xdp); + act = bpf_prog_run_xdp(xdp_prog, &xdp->base); switch (act) { case XDP_PASS: break; case XDP_TX: - xdpf = xdp_convert_buff_to_frame(xdp); + xdpf = xdp_convert_buff_to_frame(&xdp->base); if (unlikely(!xdpf)) goto out_failure; ring = ixgbe_determine_xdp_ring(adapter); @@ -2432,7 +2029,7 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, goto out_failure; break; case XDP_REDIRECT: - err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); + err = xdp_do_redirect(adapter->netdev, &xdp->base, xdp_prog); if (err) goto out_failure; result = IXGBE_XDP_REDIR; @@ -2446,40 +2043,13 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBE_XDP_CONSUMED; + libeth_xdp_return_buff(xdp); break; } xdp_out: return result; } -static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring, - unsigned int size) -{ - unsigned int truesize; - -#if (PAGE_SIZE < 8192) - truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ -#else - truesize = rx_ring->rx_offset ? - SKB_DATA_ALIGN(rx_ring->rx_offset + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : - SKB_DATA_ALIGN(size); -#endif - return truesize; -} - -static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - unsigned int size) -{ - unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif -} - /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @q_vector: structure containing interrupt and ring information @@ -2494,32 +2064,27 @@ static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring, * Returns amount of work completed **/ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring *rx_ring, - const int budget) + struct ixgbe_ring *rx_ring, + const int budget) { - unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; #ifdef IXGBE_FCOE int ddp_bytes; unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); - unsigned int offset = rx_ring->rx_offset; + LIBETH_XDP_ONSTACK_BUFF(xdp); unsigned int xdp_xmit = 0; - struct xdp_buff xdp; + u32 rsc_append_cnt = 0; int xdp_res = 0; - /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ -#if (PAGE_SIZE < 8192) - frame_sz = ixgbe_rx_frame_truesize(rx_ring, 0); -#endif - xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); + libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *rx_buffer; + struct libeth_fqe *rx_buffer; struct sk_buff *skb; - int rx_buffer_pgcnt; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2530,7 +2095,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -2539,62 +2104,47 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size, &rx_buffer_pgcnt); - - /* retrieve a buffer from the ring */ - if (!skb) { - unsigned char *hard_start; + rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + libeth_xdp_process_buff(xdp, rx_buffer, size); - hard_start = page_address(rx_buffer->page) + - rx_buffer->page_offset - offset; - xdp_prepare_buff(&xdp, hard_start, offset, size, true); - xdp_buff_clear_frags_flag(&xdp); -#if (PAGE_SIZE > 4096) - /* At larger PAGE_SIZE, frame_sz depend on len size */ - xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size); -#endif - xdp_res = ixgbe_run_xdp(adapter, rx_ring, &xdp); - } + cleaned_count++; + /* fetch next buffer in frame if non-eop */ + if (ixgbe_is_non_eop(rx_ring, rx_desc, &rsc_append_cnt) || + unlikely(!xdp->data)) + continue; + xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); if (xdp_res) { - if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) { + if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; - ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size); - } else { - rx_buffer->pagecnt_bias++; - } + total_rx_packets++; - total_rx_bytes += size; - } else if (skb) { - ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size); - } else if (ring_uses_build_skb(rx_ring)) { - skb = ixgbe_build_skb(rx_ring, rx_buffer, - &xdp, rx_desc); - } else { - skb = ixgbe_construct_skb(rx_ring, rx_buffer, - &xdp, rx_desc); + total_rx_bytes += xdp_get_buff_len(&xdp->base); + rsc_append_cnt = 0; + xdp->data = NULL; + continue; } + skb = xdp_build_skb_from_buff(&xdp->base); + /* exit if we failed to retrieve a buffer */ - if (!xdp_res && !skb) { + if (unlikely(!skb)) { + rsc_append_cnt = 0; + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; - rx_buffer->pagecnt_bias++; break; } + xdp->data = NULL; - ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt); - cleaned_count++; - - /* place incomplete frames back on ring for completion */ - if (ixgbe_is_non_eop(rx_ring, rx_desc, skb)) - continue; + /* apply RSC append count to skb if accumulated */ + IXGBE_CB(skb)->append_cnt = rsc_append_cnt; + rsc_append_cnt = 0; /* verify the packet layout is correct */ - if (xdp_res || ixgbe_cleanup_headers(rx_ring, rx_desc, skb)) + if (unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + skb = NULL; continue; - - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; + } /* populate checksum, timestamp, VLAN, and protocol */ ixgbe_process_skb_fields(rx_ring, rx_desc, skb); @@ -2619,17 +2169,22 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } if (!ddp_bytes) { dev_kfree_skb_any(skb); + skb = NULL; continue; } } #endif /* IXGBE_FCOE */ - ixgbe_rx_skb(q_vector, skb); - - /* update budget accounting */ + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; total_rx_packets++; + + ixgbe_rx_skb(q_vector, skb); } + /* place incomplete frames back on ring for completion */ + libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); + if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -2684,7 +2239,7 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, - v_idx); + v_idx); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: @@ -2725,7 +2280,7 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) * while increasing bulk throughput. **/ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring_container *ring_container) + struct ixgbe_ring_container *ring_container) { unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS | IXGBE_ITR_ADAPTIVE_LATENCY; @@ -2874,8 +2429,8 @@ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, case IXGBE_LINK_SPEED_100_FULL: default: itr += DIV_ROUND_UP(avg_wire_size, - IXGBE_ITR_ADAPTIVE_MIN_INC * 256) * - IXGBE_ITR_ADAPTIVE_MIN_INC; + IXGBE_ITR_ADAPTIVE_MIN_INC * 256) * + IXGBE_ITR_ADAPTIVE_MIN_INC; break; case IXGBE_LINK_SPEED_2_5GB_FULL: case IXGBE_LINK_SPEED_1GB_FULL: @@ -2883,8 +2438,8 @@ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, if (avg_wire_size > 8064) avg_wire_size = 8064; itr += DIV_ROUND_UP(avg_wire_size, - IXGBE_ITR_ADAPTIVE_MIN_INC * 64) * - IXGBE_ITR_ADAPTIVE_MIN_INC; + IXGBE_ITR_ADAPTIVE_MIN_INC * 64) * + IXGBE_ITR_ADAPTIVE_MIN_INC; break; } @@ -2986,7 +2541,7 @@ static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) * check if we got a LSC */ if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) && - !(eicr & IXGBE_EICR_LSC)) + !(eicr & IXGBE_EICR_LSC)) return; if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { @@ -3026,7 +2581,7 @@ static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) struct ixgbe_hw *hw = &adapter->hw; if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && - (eicr & IXGBE_EICR_GPI_SDP1(hw))) { + (eicr & IXGBE_EICR_GPI_SDP1(hw))) { e_crit(probe, "Fan has stopped, replace the adapter\n"); /* write to clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); @@ -3047,8 +2602,8 @@ static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) * on service task */ if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) || - (eicr & IXGBE_EICR_LSC)) && - (!test_bit(__IXGBE_DOWN, &adapter->state))) { + (eicr & IXGBE_EICR_LSC)) && + (!test_bit(__IXGBE_DOWN, &adapter->state))) { adapter->interrupt_event = eicr; adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; ixgbe_service_event_schedule(adapter); @@ -3123,7 +2678,7 @@ static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) } if (adapter->hw.mac.type == ixgbe_mac_82599EB && - (eicr & IXGBE_EICR_GPI_SDP1(hw))) { + (eicr & IXGBE_EICR_GPI_SDP1(hw))) { /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); if (!test_bit(__IXGBE_DOWN, &adapter->state)) { @@ -3155,7 +2710,7 @@ static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) * Check if external PHY FW load failed and print an error message if it did. */ static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { if (!(link_cfg_err & IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE)) { adapter->flags2 &= ~IXGBE_FLAG2_PHY_FW_LOAD_FAILED; @@ -3180,11 +2735,11 @@ static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter, * and print error messages if module power level is not supported. */ static void ixgbe_check_module_power(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { /* If module power level is supported, clear the flag. */ if (!(link_cfg_err & (IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT | - IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) { + IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) { adapter->flags2 &= ~IXGBE_FLAG2_MOD_POWER_UNSUPPORTED; return; } @@ -3213,7 +2768,7 @@ static void ixgbe_check_module_power(struct ixgbe_adapter *adapter, * link_cfg_err parameter in the link info structure. */ static void ixgbe_check_link_cfg_err(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { ixgbe_check_module_power(adapter, link_cfg_err); ixgbe_check_phy_fw_load(adapter, link_cfg_err); @@ -3252,12 +2807,12 @@ ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up, /* Turn off PHY if media was removed. */ if (!(adapter->flags2 & IXGBE_FLAG2_NO_MEDIA) && - !(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) + !(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) adapter->flags2 |= IXGBE_FLAG2_NO_MEDIA; if (link_up == adapter->link_up && - link_up == netif_carrier_ok(adapter->netdev) && - link_speed == adapter->link_speed) + link_up == netif_carrier_ok(adapter->netdev) && + link_speed == adapter->link_speed) return 0; adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; @@ -3279,7 +2834,7 @@ ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up, */ static void ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter, - struct ixgbe_aci_event *e) + struct ixgbe_aci_event *e) { struct ixgbe_aci_cmd_get_link_status_data *link_data; u16 link_speed; @@ -3304,8 +2859,8 @@ ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter, static void ixgbe_schedule_fw_event(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && - !test_bit(__IXGBE_REMOVING, &adapter->state) && - !test_bit(__IXGBE_RESETTING, &adapter->state)) { + !test_bit(__IXGBE_REMOVING, &adapter->state) && + !test_bit(__IXGBE_RESETTING, &adapter->state)) { adapter->flags2 |= IXGBE_FLAG2_FW_ASYNC_EVENT; ixgbe_service_event_schedule(adapter); } @@ -3358,7 +2913,7 @@ static void ixgbe_handle_fw_event(struct ixgbe_adapter *adapter) break; case libie_aqc_opc_fw_logs_event: libie_get_fwlog_data(&hw->fwlog, event.msg_buf, - le16_to_cpu(event.desc.datalen)); + le16_to_cpu(event.desc.datalen)); break; default: e_warn(hw, "unknown FW async event captured\n"); @@ -3404,7 +2959,7 @@ static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, * @flush: flush register write **/ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, - bool flush) + bool flush) { struct ixgbe_hw *hw = &adapter->hw; u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); @@ -3442,8 +2997,8 @@ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, fallthrough; case ixgbe_mac_x550em_a: if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP || - adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || - adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) + adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || + adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw); if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t) mask |= IXGBE_EICR_GPI_SDP0_X540; @@ -3455,7 +3010,7 @@ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, } if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && - !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) + !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) mask |= IXGBE_EIMS_FLOW_DIR; IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); @@ -3507,7 +3062,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) case ixgbe_mac_x550em_a: case ixgbe_mac_e610: if (hw->phy.type == ixgbe_phy_x550em_ext_t && - (eicr & IXGBE_EICR_GPI_SDP0_X540)) { + (eicr & IXGBE_EICR_GPI_SDP0_X540)) { adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT; ixgbe_service_event_schedule(adapter); IXGBE_WRITE_REG(hw, IXGBE_EICR, @@ -3526,7 +3081,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *ring = adapter->tx_ring[i]; if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, - &ring->state)) + &ring->state)) reinit_count++; } if (reinit_count) { @@ -3610,9 +3165,9 @@ int ixgbe_poll(struct napi_struct *napi, int budget) ixgbe_for_each_ring(ring, q_vector->rx) { int cleaned = ring->xsk_pool ? - ixgbe_clean_rx_irq_zc(q_vector, ring, - per_ring_budget) : - ixgbe_clean_rx_irq(q_vector, ring, + ixgbe_clean_rx_irq_zc(q_vector, ring, + per_ring_budget) : + ixgbe_clean_rx_irq(q_vector, ring, per_ring_budget); work_done += cleaned; @@ -3671,7 +3226,7 @@ static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) q_vector->name, q_vector); if (err) { e_err(probe, "request_irq failed for MSIX interrupt " - "Error: %d\n", err); + "Error: %d\n", err); goto free_queue_irqs; } /* If Flow Director is enabled, set interrupt affinity */ @@ -3901,7 +3456,7 @@ static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) * Configure the Tx descriptor ring after a reset. **/ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 tdba = ring->dma; @@ -3963,22 +3518,22 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, if (q_vector) netif_set_xps_queue(ring->netdev, - &q_vector->affinity_mask, - ring->queue_index); + &q_vector->affinity_mask, + ring->queue_index); } clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); /* reinitialize tx_buffer_info */ memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbe_tx_buffer) * ring->count); + sizeof(struct ixgbe_tx_buffer) * ring->count); /* enable queue */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* poll to verify queue is enabled */ @@ -4123,7 +3678,7 @@ static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) * and performance reasons. */ if (adapter->num_vfs || (adapter->num_rx_queues > 1 && - !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { + !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { for (i = 0; i < adapter->num_rx_queues; i++) ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]); } else { @@ -4138,11 +3693,11 @@ static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; - u8 reg_idx = rx_ring->reg_idx; + u8 reg_idx = ring->reg_idx; if (hw->mac.type == ixgbe_mac_82598EB) { u16 mask = adapter->ring_feature[RING_F_RSS].mask; @@ -4157,27 +3712,12 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, /* configure header buffer length, needed for RSC */ srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - /* configure the packet buffer length */ - if (rx_ring->xsk_pool) { - u32 xsk_buf_len = xsk_pool_get_rx_frame_size(rx_ring->xsk_pool); - - /* If the MAC support setting RXDCTL.RLPML, the - * SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and - * RXDCTL.RLPML is set to the actual UMEM buffer - * size. If not, then we are stuck with a 1k buffer - * size resolution. In this case frames larger than - * the UMEM buffer size viewed in a 1k resolution will - * be dropped. - */ - if (hw->mac.type != ixgbe_mac_82599EB) - srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - else - srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) { - srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else { - srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } + if (ring->xsk_pool) + srrctl |= DIV_ROUND_UP(xsk_pool_get_rx_frame_size(ring->xsk_pool), + IXGBE_SRRCTL_BSIZEPKT_STEP); + else + srrctl |= DIV_ROUND_UP(ring->rx_buf_len, + IXGBE_SRRCTL_BSIZEPKT_STEP); /* configure descriptor type */ srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; @@ -4408,9 +3948,9 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) /* Perform hash on these packet types */ rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 | - IXGBE_MRQC_RSS_FIELD_IPV4_TCP | - IXGBE_MRQC_RSS_FIELD_IPV6 | - IXGBE_MRQC_RSS_FIELD_IPV6_TCP; + IXGBE_MRQC_RSS_FIELD_IPV4_TCP | + IXGBE_MRQC_RSS_FIELD_IPV6 | + IXGBE_MRQC_RSS_FIELD_IPV6_TCP; if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; @@ -4418,7 +3958,7 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; if ((hw->mac.type >= ixgbe_mac_X550) && - (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { + (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { u16 pool = adapter->num_rx_pools; /* Enable VF RSS mode */ @@ -4469,7 +4009,7 @@ static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, #define IXGBE_MAX_RX_DESC_POLL 10 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBE_MAX_RX_DESC_POLL; @@ -4480,7 +4020,7 @@ static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, return; /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; do { @@ -4490,12 +4030,12 @@ static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, if (!wait_loop) { e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within " - "the polling period\n", reg_idx); + "the polling period\n", reg_idx); } } void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; @@ -4511,8 +4051,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, NULL)); xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); } else { - WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_PAGE_SHARED, NULL)); + xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, ring->pp); } /* disable queue to avoid use of these values while updating state */ @@ -4534,7 +4073,6 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx); - ixgbe_configure_srrctl(adapter, ring); ixgbe_configure_rscctl(adapter, ring); if (hw->mac.type == ixgbe_mac_82598EB) { @@ -4551,35 +4089,19 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, /* RXDCTL.RLPML does not work on 82599 */ } else if (hw->mac.type != ixgbe_mac_82599EB) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); - - /* Limit the maximum frame size so we don't overrun the skb. - * This can happen in SRIOV mode when the MTU of the VF is - * higher than the MTU of the PF. - */ - if (ring_uses_build_skb(ring) && - !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB | - IXGBE_RXDCTL_RLPML_EN; -#endif + IXGBE_RXDCTL_RLPML_EN); } - - ring->rx_offset = ixgbe_rx_offset(ring); - +#endif if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { - u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); + u32 pkt_len = + READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; - rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); - rxdctl |= xsk_buf_len | IXGBE_RXDCTL_RLPML_EN; - - ring->rx_buf_len = xsk_buf_len; + rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); + if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) + rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; } - /* initialize rx_buffer_info */ - memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbe_rx_buffer) * ring->count); - + ixgbe_configure_srrctl(adapter, ring); /* initialize Rx descriptor 0 */ rx_desc = IXGBE_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -4603,10 +4125,10 @@ static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) /* PSRTYPE must be initialized in non 82598 adapters */ u32 psrtype = IXGBE_PSRTYPE_TCPHDR | - IXGBE_PSRTYPE_UDPHDR | - IXGBE_PSRTYPE_IPV4HDR | - IXGBE_PSRTYPE_L2HDR | - IXGBE_PSRTYPE_IPV6HDR; + IXGBE_PSRTYPE_UDPHDR | + IXGBE_PSRTYPE_IPV4HDR | + IXGBE_PSRTYPE_L2HDR | + IXGBE_PSRTYPE_IPV6HDR; if (hw->mac.type == ixgbe_mac_82598EB) return; @@ -4691,73 +4213,6 @@ static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) } } -static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) -{ - struct ixgbe_hw *hw = &adapter->hw; - struct net_device *netdev = adapter->netdev; - int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - struct ixgbe_ring *rx_ring; - int i; - u32 mhadd, hlreg0; - -#ifdef IXGBE_FCOE - /* adjust max frame to be able to do baby jumbo for FCoE */ - if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && - (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) - max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; - -#endif /* IXGBE_FCOE */ - - /* adjust max frame to be at least the size of a standard frame */ - if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) - max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); - - mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); - if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { - mhadd &= ~IXGBE_MHADD_MFS_MASK; - mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; - - IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); - } - - hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); - /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ - hlreg0 |= IXGBE_HLREG0_JUMBOEN; - IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); - - /* - * Setup the HW Rx Head and Tail Descriptor Pointers and - * the Base and Length of the Rx Descriptor Ring - */ - for (i = 0; i < adapter->num_rx_queues; i++) { - rx_ring = adapter->rx_ring[i]; - - clear_ring_rsc_enabled(rx_ring); - clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); - - if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_ring_rsc_enabled(rx_ring); - - if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state)) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - - if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) - continue; - - set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); - -#if (PAGE_SIZE < 8192) - if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - - if (IXGBE_2K_TOO_SMALL_WITH_PADDING || - (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN))) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); -#endif - } -} - static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; @@ -4811,8 +4266,12 @@ static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; + struct net_device *netdev = adapter->netdev; + int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; + struct ixgbe_ring *rx_ring; int i; u32 rxctrl, rfctl; + u32 mhadd, hlreg0; /* disable receives while setting up the descriptors */ hw->mac.ops.disable_rx(hw); @@ -4833,15 +4292,45 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) /* Program registers for the distribution of queues */ ixgbe_setup_mrqc(adapter); - /* set_rx_buffer_len must be called before ring initialization */ - ixgbe_set_rx_buffer_len(adapter); +#ifdef IXGBE_FCOE + /* adjust max frame to be able to do baby jumbo for FCoE */ + if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && + (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) + max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; +#endif /* IXGBE_FCOE */ + + /* adjust max frame to be at least the size of a standard frame */ + if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) + max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); + + mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); + if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { + mhadd &= ~IXGBE_MHADD_MFS_MASK; + mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; + + IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); + } + + /* set jumbo enable since MHADD.MFS is keeping size locked at + * max_frame + */ + hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); + hlreg0 |= IXGBE_HLREG0_JUMBOEN; + IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); /* * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ - for (i = 0; i < adapter->num_rx_queues; i++) - ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); + for (i = 0; i < adapter->num_rx_queues; i++) { + rx_ring = adapter->rx_ring[i]; + + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + + ixgbe_configure_rx_ring(adapter, rx_ring); + } rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); /* disable drop enable for 82598 parts */ @@ -5105,7 +4594,7 @@ static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) || - hw->mac.type == ixgbe_mac_82598EB) + hw->mac.type == ixgbe_mac_82598EB) return; /* We are not in VLAN promisc, nothing to do */ @@ -5170,9 +4659,9 @@ void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter) if (mac_table->state & IXGBE_MAC_STATE_IN_USE) hw->mac.ops.set_rar(hw, i, - mac_table->addr, - mac_table->pool, - IXGBE_RAH_AV); + mac_table->addr, + mac_table->pool, + IXGBE_RAH_AV); else hw->mac.ops.clear_rar(hw, i); } @@ -5193,9 +4682,9 @@ static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter) if (mac_table->state & IXGBE_MAC_STATE_IN_USE) hw->mac.ops.set_rar(hw, i, - mac_table->addr, - mac_table->pool, - IXGBE_RAH_AV); + mac_table->addr, + mac_table->pool, + IXGBE_RAH_AV); else hw->mac.ops.clear_rar(hw, i); } @@ -5250,7 +4739,7 @@ static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter) mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE; hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool, - IXGBE_RAH_AV); + IXGBE_RAH_AV); } int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, @@ -5271,7 +4760,7 @@ int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, mac_table->pool = pool; mac_table->state |= IXGBE_MAC_STATE_MODIFIED | - IXGBE_MAC_STATE_IN_USE; + IXGBE_MAC_STATE_IN_USE; ixgbe_sync_mac_table(adapter); @@ -5562,7 +5051,7 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && - (pb == ixgbe_fcoe_get_tc(adapter))) + (pb == ixgbe_fcoe_get_tc(adapter))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5596,8 +5085,8 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) */ if (marker < 0) { e_warn(drv, "Packet Buffer(%i) can not provide enough" - "headroom to support flow control." - "Decrease MTU or number of traffic classes\n", pb); + "headroom to support flow control." + "Decrease MTU or number of traffic classes\n", pb); marker = tc + 1; } @@ -5623,7 +5112,7 @@ static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && - (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) + (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5677,7 +5166,7 @@ static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) u8 tc = adapter->hw_tcs; if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || - adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) + adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) hdrm = 32 << adapter->fdir_pballoc; else hdrm = 0; @@ -5708,13 +5197,13 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) if (!vf && (ring >= adapter->num_rx_queues)) { e_err(drv, "FDIR restore failed without VF, ring: %u\n", - ring); + ring); continue; } else if (vf && ((vf > adapter->num_vfs) || - ring >= adapter->num_rx_queues_per_pool)) { + ring >= adapter->num_rx_queues_per_pool)) { e_err(drv, "FDIR restore failed with VF, vf: %hhu, ring: %u\n", - vf, ring); + vf, ring); continue; } @@ -5739,60 +5228,29 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) **/ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) { - u16 i = rx_ring->next_to_clean; - struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i]; - if (rx_ring->xsk_pool) { ixgbe_xsk_clean_rx_ring(rx_ring); goto skip_free; } - /* Free all the Rx ring sk_buffs */ - while (i != rx_ring->next_to_alloc) { - if (rx_buffer->skb) { - struct sk_buff *skb = rx_buffer->skb; - if (IXGBE_CB(skb)->page_released) - dma_unmap_page_attrs(rx_ring->dev, - IXGBE_CB(skb)->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - dev_kfree_skb(skb); - } + /* Free Rx ring xdp stash */ + libeth_xdp_return_stash(&rx_ring->xdp_stash); - /* Invalidate cache lines that may have been written to by - * device so that we avoid corrupting memory. - */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - ixgbe_rx_bufsz(rx_ring), - DMA_FROM_DEVICE); - - /* free resources associated with mapping */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); + for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { + const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; - i++; - rx_buffer++; - if (i == rx_ring->count) { + libeth_rx_recycle_slow(rx_fqe->netmem); + if (unlikely(++i == rx_ring->count)) i = 0; - rx_buffer = rx_ring->rx_buffer_info; - } } skip_free: - rx_ring->next_to_alloc = 0; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; } static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, - struct ixgbe_fwd_adapter *accel) + struct ixgbe_fwd_adapter *accel) { u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; int num_tc = netdev_get_num_tc(adapter->netdev); @@ -5810,7 +5268,7 @@ static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, /* record configuration for macvlan interface in vdev */ for (i = 0; i < num_tc; i++) netdev_bind_sb_channel_queue(adapter->netdev, vdev, - i, rss_i, baseq + (rss_i * i)); + i, rss_i, baseq + (rss_i * i)); for (i = 0; i < adapter->num_rx_queues_per_pool; i++) adapter->rx_ring[baseq + i]->netdev = vdev; @@ -5847,7 +5305,7 @@ static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, } static int ixgbe_macvlan_up(struct net_device *vdev, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data; struct ixgbe_fwd_adapter *accel; @@ -5871,7 +5329,7 @@ static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter) }; netdev_walk_all_upper_dev_rcu(adapter->netdev, - ixgbe_macvlan_up, &priv); + ixgbe_macvlan_up, &priv); } static void ixgbe_configure(struct ixgbe_adapter *adapter) @@ -5906,7 +5364,7 @@ static void ixgbe_configure(struct ixgbe_adapter *adapter) adapter->fdir_pballoc); } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { ixgbe_init_fdir_perfect_82599(&adapter->hw, - adapter->fdir_pballoc); + adapter->fdir_pballoc); ixgbe_fdir_filter_restore(adapter); } @@ -6007,7 +5465,7 @@ static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) { struct ixgbe_adapter *adapter = container_of(hw, struct ixgbe_adapter, - hw); + hw); u16 mask = ~((u16)(IXGBE_ACI_LINK_EVENT_UPDOWN | IXGBE_ACI_LINK_EVENT_MEDIA_NA | IXGBE_ACI_LINK_EVENT_MODULE_QUAL_FAIL | @@ -6131,7 +5589,7 @@ static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | - IXGBE_GPIE_OCD; + IXGBE_GPIE_OCD; gpie |= IXGBE_GPIE_EIAME; /* * use EIAM to auto-mask when MSI-X interrupt is asserted @@ -6365,7 +5823,7 @@ void ixgbe_disable_rx(struct ixgbe_adapter *adapter) /* RXDCTL.EN may not change on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* Determine our minimum delay interval. We will increase this value @@ -6406,7 +5864,7 @@ void ixgbe_disable_rx(struct ixgbe_adapter *adapter) } e_err(drv, - "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); + "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); } void ixgbe_disable_tx(struct ixgbe_adapter *adapter) @@ -6487,7 +5945,7 @@ void ixgbe_disable_tx(struct ixgbe_adapter *adapter) } e_err(drv, - "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); + "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); dma_engine_disable: /* Disable the Tx DMA engine on 82599 and later MAC */ @@ -6521,7 +5979,7 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) /* clear all SFP and link config related flags while holding SFP_INIT */ adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | - IXGBE_FLAG2_SFP_NEEDS_RESET); + IXGBE_FLAG2_SFP_NEEDS_RESET); adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; err = hw->mac.ops.init_hw(hw); @@ -6617,9 +6075,9 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); } /* move us one more past the eop_desc for start of next pkt */ @@ -6833,7 +6291,7 @@ static void ixgbe_init_dcb(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE) adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg, - sizeof(adapter->temp_dcb_cfg)); + sizeof(adapter->temp_dcb_cfg)); } #endif @@ -6905,8 +6363,8 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter, adapter->jump_tables[i] = NULL; adapter->mac_table = kcalloc(hw->mac.num_rar_entries, - sizeof(struct ixgbe_mac_addr), - GFP_KERNEL); + sizeof(struct ixgbe_mac_addr), + GFP_KERNEL); if (!adapter->mac_table) return -ENOMEM; @@ -7142,13 +6600,6 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) return err; } -static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) -{ - struct ixgbe_q_vector *q_vector = rx_ring->q_vector; - - return q_vector ? q_vector->napi.napi_id : 0; -} - /** * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) * @adapter: pointer to ixgbe_adapter @@ -7157,56 +6608,82 @@ static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) * Returns 0 on success, negative on failure **/ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *rx_ring) { - struct device *dev = rx_ring->dev; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBE_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); + struct napi_struct *napi_dev; int ring_node = NUMA_NO_NODE; - int size; - - size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; + int ret; - if (rx_ring->q_vector) + if (rx_ring->q_vector) { + fq.nid = rx_ring->q_vector->numa_node; ring_node = rx_ring->q_vector->numa_node; + napi_dev = &rx_ring->q_vector->napi; + } else { + napi_dev = NULL; + } - rx_ring->rx_buffer_info = vmalloc_node(size, ring_node); - if (!rx_ring->rx_buffer_info) - rx_ring->rx_buffer_info = vmalloc(size); - if (!rx_ring->rx_buffer_info) - goto err; + ret = libeth_rx_fq_create(&fq, napi_dev); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); set_dev_node(dev, ring_node); - rx_ring->desc = dma_alloc_coherent(dev, - rx_ring->size, - &rx_ring->dma, - GFP_KERNEL); + rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, + &rx_ring->dma, GFP_KERNEL); set_dev_node(dev, orig_node); if (!rx_ring->desc) rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, - &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) - goto err; + &rx_ring->dma, + GFP_KERNEL); + if (!rx_ring->desc) { + ret = -ENOMEM; + dev_err(&adapter->pdev->dev, + "Unable to allocate memory for the Rx descriptor ring\n"); + goto err_destroy_fq; + } rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; /* XDP RX-queue info */ - if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring)) < 0) - goto err; - - WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog); + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret < 0) + goto err_free_desc; + + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; -err: - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; - dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); - return -ENOMEM; + +err_free_desc: + dma_free_coherent(dev, rx_ring->size, + rx_ring->desc, rx_ring->dma); + rx_ring->desc = NULL; +err_destroy_fq: + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + return ret; } /** @@ -7293,21 +6770,32 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) **/ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { + struct libeth_fq fq = { + .fqes = rx_ring->rx_fqes, + .pp = rx_ring->pp, + }; + ixgbe_clean_rx_ring(rx_ring); - rx_ring->xdp_prog = NULL; + rcu_assign_pointer(rx_ring->xdp_prog, NULL); + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; /* if not set, then don't free */ if (!rx_ring->desc) return; - dma_free_coherent(rx_ring->dev, rx_ring->size, - rx_ring->desc, rx_ring->dma); + dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, + rx_ring->dma); rx_ring->desc = NULL; + + kvfree(rx_ring->rx_xsk_buffer_info); + rx_ring->rx_xsk_buffer_info = NULL; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; } /** @@ -7335,10 +6823,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) */ static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter) { - if (PAGE_SIZE >= 8192 || adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) - return IXGBE_RXBUFFER_2K; - else - return IXGBE_RXBUFFER_3K; + return IXGBE_RXBUFFER_3K; } /** @@ -7353,7 +6838,7 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); if (ixgbe_enabled_xdp_adapter(adapter)) { - int new_frame_size = new_mtu + IXGBE_PKT_HDR_PAD; + int new_frame_size = new_mtu + LIBETH_RX_LL_LEN; if (new_frame_size > ixgbe_max_xdp_frame_size(adapter)) { e_warn(probe, "Requested MTU size is not supported with XDP\n"); @@ -7367,8 +6852,8 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) * warning that legacy VFs will be disabled. */ if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && - (adapter->hw.mac.type == ixgbe_mac_82599EB) && - (new_mtu > ETH_DATA_LEN)) + (adapter->hw.mac.type == ixgbe_mac_82599EB) && + (new_mtu > ETH_DATA_LEN)) e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n"); netdev_dbg(netdev, "changing MTU from %d to %d\n", @@ -7666,7 +7151,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_RESETTING, &adapter->state)) return; if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { @@ -7767,11 +7252,11 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); if (hw->mac.type == ixgbe_mac_82599EB || - hw->mac.type == ixgbe_mac_X540 || - hw->mac.type == ixgbe_mac_X550 || - hw->mac.type == ixgbe_mac_X550EM_x || - hw->mac.type == ixgbe_mac_x550em_a || - hw->mac.type == ixgbe_mac_e610) { + hw->mac.type == ixgbe_mac_X540 || + hw->mac.type == ixgbe_mac_X550 || + hw->mac.type == ixgbe_mac_X550EM_x || + hw->mac.type == ixgbe_mac_x550em_a || + hw->mac.type == ixgbe_mac_e610) { hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */ hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); @@ -7807,7 +7292,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) case ixgbe_mac_82599EB: for (i = 0; i < 16; i++) adapter->hw_rx_no_dma_resources += - IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); + IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); @@ -7954,7 +7439,7 @@ static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); } else { e_err(probe, "failed to finish FDIR re-initialization, " - "ignored adding FDIR ATR filters\n"); + "ignored adding FDIR ATR filters\n"); } } @@ -7975,8 +7460,8 @@ static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) /* If we're down, removing or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) return; /* Force detection of hung controller */ @@ -8036,7 +7521,7 @@ static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) } if (link_up || - time_after(jiffies, (adapter->link_check_timeout + + time_after(jiffies, (adapter->link_check_timeout + IXGBE_TRY_LINK_TIMEOUT))) { adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); @@ -8052,9 +7537,9 @@ static void ixgbe_update_default_up(struct ixgbe_adapter *adapter) #ifdef CONFIG_IXGBE_DCB struct net_device *netdev = adapter->netdev; struct dcb_app app = { - .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, - .protocol = 0, - }; + .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, + .protocol = 0, + }; u8 up = 0; if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE) @@ -8138,9 +7623,9 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) break; } e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str, - ((flow_rx && flow_tx) ? "RX/TX" : - (flow_rx ? "RX" : - (flow_tx ? "TX" : "None")))); + ((flow_rx && flow_tx) ? "RX/TX" : + (flow_rx ? "RX" : + (flow_tx ? "TX" : "None")))); netif_carrier_on(netdev); ixgbe_check_vf_rate_limit(adapter); @@ -8243,7 +7728,7 @@ static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) { if (!netif_carrier_ok(adapter->netdev)) { if (ixgbe_ring_tx_pending(adapter) || - ixgbe_vf_tx_pending(adapter)) { + ixgbe_vf_tx_pending(adapter)) { /* We've lost link, so the controller stops DMA, * but we've got queued Tx work that's never going * to get done, so reset controller to flush Tx. @@ -8261,18 +7746,18 @@ static void ixgbe_bad_vf_abort(struct ixgbe_adapter *adapter, u32 vf) struct ixgbe_hw *hw = &adapter->hw; if (adapter->hw.mac.type == ixgbe_mac_82599EB && - adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) { + adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) { adapter->vfinfo[vf].primary_abort_count++; if (adapter->vfinfo[vf].primary_abort_count == - IXGBE_PRIMARY_ABORT_LIMIT) { + IXGBE_PRIMARY_ABORT_LIMIT) { ixgbe_set_vf_link_state(adapter, vf, IFLA_VF_LINK_STATE_DISABLE); adapter->vfinfo[vf].primary_abort_count = 0; e_info(drv, - "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on", - hw->bus.func, vf, - adapter->vfinfo[vf].vf_mac_addresses); + "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on", + hw->bus.func, vf, + adapter->vfinfo[vf].vf_mac_addresses); } } } @@ -8308,7 +7793,7 @@ static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) continue; pci_read_config_word(vfdev, PCI_STATUS, &status_reg); if (status_reg != IXGBE_FAILED_READ_CFG_WORD && - status_reg & PCI_STATUS_REC_MASTER_ABORT) { + status_reg & PCI_STATUS_REC_MASTER_ABORT) { ixgbe_bad_vf_abort(adapter, vf); pcie_flr(vfdev); } @@ -8321,7 +7806,7 @@ static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) /* Do not perform spoof check for 82598 or if not in IOV mode */ if (adapter->hw.mac.type == ixgbe_mac_82598EB || - adapter->num_vfs == 0) + adapter->num_vfs == 0) return; ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); @@ -8355,8 +7840,8 @@ static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) { /* if interface is down, removing or resetting, do nothing */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) return; ixgbe_watchdog_update_link(adapter); @@ -8384,11 +7869,11 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) /* not searching for SFP so there is nothing to do here */ if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && - !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) + !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) return; if (adapter->sfp_poll_time && - time_after(adapter->sfp_poll_time, jiffies)) + time_after(adapter->sfp_poll_time, jiffies)) return; /* If not yet time to poll for SFP */ /* someone else is in init, wait until next service event */ @@ -8437,7 +7922,7 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); if (err == -EOPNOTSUPP && - adapter->netdev->reg_state == NETREG_REGISTERED) { + adapter->netdev->reg_state == NETREG_REGISTERED) { e_dev_err("failed to initialize because an unsupported " "SFP+ module type was detected.\n"); e_dev_err("Reload the driver after installing a " @@ -8473,7 +7958,7 @@ static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) speed = IXGBE_LINK_SPEED_10GB_FULL; else speed = cap_speed & (IXGBE_LINK_SPEED_10GB_FULL | - IXGBE_LINK_SPEED_1GB_FULL); + IXGBE_LINK_SPEED_1GB_FULL); if (hw->mac.ops.setup_link) hw->mac.ops.setup_link(hw, speed, true); @@ -8531,8 +8016,8 @@ static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) rtnl_lock(); /* If we're already down, removing or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) { + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) { rtnl_unlock(); return; } @@ -8590,7 +8075,7 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) /* skip if E610's FW is reloading, warning in that case may be misleading */ if (fwsm & IXGBE_FWSM_EXT_ERR_IND_MASK || - (!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610))) + (!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610))) e_dev_warn("Warning firmware error detected FWSM: 0x%08X\n", fwsm); @@ -8616,7 +8101,7 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) goto no_version; if (hw->mac.ops.get_nvm_ver && - hw->mac.ops.get_nvm_ver(hw, nvm_info)) + hw->mac.ops.get_nvm_ver(hw, nvm_info)) goto no_version; snprintf(ver_buff, sizeof(ver_buff), @@ -8636,8 +8121,8 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) static void ixgbe_recovery_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, - struct ixgbe_adapter, - service_task); + struct ixgbe_adapter, + service_task); ixgbe_handle_fw_event(adapter); ixgbe_service_event_complete(adapter); @@ -8652,8 +8137,8 @@ static void ixgbe_recovery_service_task(struct work_struct *work) static void ixgbe_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, - struct ixgbe_adapter, - service_task); + struct ixgbe_adapter, + service_task); if (ixgbe_removed(adapter->hw.hw_addr)) { if (!test_bit(__IXGBE_DOWN, &adapter->state)) { rtnl_lock(); @@ -8699,9 +8184,9 @@ static void ixgbe_service_task(struct work_struct *work) } static int ixgbe_tso(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, - u8 *hdr_len, - struct ixgbe_ipsec_tx_data *itd) + struct ixgbe_tx_buffer *first, + u8 *hdr_len, + struct ixgbe_ipsec_tx_data *itd) { u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; struct sk_buff *skb = first->skb; @@ -8737,7 +8222,7 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ? - IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP; + IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP; /* initialize outer IP header fields */ if (ip.v4->version == 4) { @@ -8774,12 +8259,12 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, /* compute length of segmentation header */ *hdr_len = (l4.tcp->doff * 4) + l4_offset; csum_replace_by_diff(&l4.tcp->check, - (__force __wsum)htonl(paylen)); + (__force __wsum)htonl(paylen)); } else { /* compute length of segmentation header */ *hdr_len = sizeof(*l4.udp) + l4_offset; csum_replace_by_diff(&l4.udp->check, - (__force __wsum)htonl(paylen)); + (__force __wsum)htonl(paylen)); } /* update gso size and bytecount with header size */ @@ -8863,8 +8348,8 @@ static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) { /* set type for advanced descriptor with frame checksum insertion */ u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS; /* set HW vlan bit if vlan is present */ cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN, @@ -8918,7 +8403,7 @@ static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) { if (!netif_subqueue_try_stop(tx_ring->netdev, tx_ring->queue_index, - ixgbe_desc_unused(tx_ring), size)) + ixgbe_desc_unused(tx_ring), size)) return -EBUSY; ++tx_ring->tx_stats.restart_queue; @@ -9018,7 +8503,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, data_len -= size; dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, - DMA_TO_DEVICE); + DMA_TO_DEVICE); tx_buffer = &tx_ring->tx_buffer_info[i]; } @@ -9068,9 +8553,9 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, tx_buffer = &tx_ring->tx_buffer_info[i]; if (dma_unmap_len(tx_buffer, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buffer, len, 0); if (tx_buffer == first) break; @@ -9088,7 +8573,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, } static void ixgbe_atr(struct ixgbe_ring *ring, - struct ixgbe_tx_buffer *first) + struct ixgbe_tx_buffer *first) { struct ixgbe_q_vector *q_vector = ring->q_vector; union ixgbe_atr_hash_dword input = { .dword = 0 }; @@ -9116,7 +8601,7 @@ static void ixgbe_atr(struct ixgbe_ring *ring, /* currently only IPv4/IPv6 with TCP is supported */ if ((first->protocol != htons(ETH_P_IP)) && - (first->protocol != htons(ETH_P_IPV6))) + (first->protocol != htons(ETH_P_IPV6))) return; /* snag network header to get L4 type and address */ @@ -9125,21 +8610,21 @@ static void ixgbe_atr(struct ixgbe_ring *ring, if (unlikely(hdr.network <= skb->data)) return; if (skb->encapsulation && - first->protocol == htons(ETH_P_IP) && - hdr.ipv4->protocol == IPPROTO_UDP) { + first->protocol == htons(ETH_P_IP) && + hdr.ipv4->protocol == IPPROTO_UDP) { struct ixgbe_adapter *adapter = q_vector->adapter; if (unlikely(skb_tail_pointer(skb) < hdr.network + - vxlan_headroom(0))) + vxlan_headroom(0))) return; /* verify the port is recognized as VXLAN */ if (adapter->vxlan_port && - udp_hdr(skb)->dest == adapter->vxlan_port) + udp_hdr(skb)->dest == adapter->vxlan_port) hdr.network = skb_inner_network_header(skb); if (adapter->geneve_port && - udp_hdr(skb)->dest == adapter->geneve_port) + udp_hdr(skb)->dest == adapter->geneve_port) hdr.network = skb_inner_network_header(skb); } @@ -9169,7 +8654,7 @@ static void ixgbe_atr(struct ixgbe_ring *ring, return; if (unlikely(skb_tail_pointer(skb) < hdr.network + - hlen + sizeof(struct tcphdr))) + hlen + sizeof(struct tcphdr))) return; th = (struct tcphdr *)(hdr.network + hlen); @@ -9214,13 +8699,13 @@ static void ixgbe_atr(struct ixgbe_ring *ring, case 6: input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ - hdr.ipv6->saddr.s6_addr32[1] ^ - hdr.ipv6->saddr.s6_addr32[2] ^ - hdr.ipv6->saddr.s6_addr32[3] ^ - hdr.ipv6->daddr.s6_addr32[0] ^ - hdr.ipv6->daddr.s6_addr32[1] ^ - hdr.ipv6->daddr.s6_addr32[2] ^ - hdr.ipv6->daddr.s6_addr32[3]; + hdr.ipv6->saddr.s6_addr32[1] ^ + hdr.ipv6->saddr.s6_addr32[2] ^ + hdr.ipv6->saddr.s6_addr32[3] ^ + hdr.ipv6->daddr.s6_addr32[0] ^ + hdr.ipv6->daddr.s6_addr32[1] ^ + hdr.ipv6->daddr.s6_addr32[2] ^ + hdr.ipv6->daddr.s6_addr32[3]; break; default: break; @@ -9231,12 +8716,12 @@ static void ixgbe_atr(struct ixgbe_ring *ring, /* This assumes the Rx queue and Tx queue are bound to the same CPU */ ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, - input, common, ring->queue_index); + input, common, ring->queue_index); } #ifdef IXGBE_FCOE static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev) + struct net_device *sb_dev) { struct ixgbe_adapter *adapter; struct ixgbe_ring_feature *f; @@ -9349,8 +8834,8 @@ int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, tx_buff = &ring->tx_buffer_info[index]; if (dma_unmap_len(tx_buff, len)) dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma), - dma_unmap_len(tx_buff, len), - DMA_TO_DEVICE); + dma_unmap_len(tx_buff, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buff, len, 0); if (tx_buff == tx_head) break; @@ -9416,9 +8901,9 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, protocol = vlan_get_protocol(skb); if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - adapter->ptp_clock) { + adapter->ptp_clock) { if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && - !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, + !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, &adapter->state)) { skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; tx_flags |= IXGBE_TX_FLAGS_TSTAMP; @@ -9443,8 +8928,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #endif /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */ if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && - ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || - (skb->priority != TC_PRIO_CONTROL))) { + ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || + (skb->priority != TC_PRIO_CONTROL))) { tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; tx_flags |= (skb->priority & 0x7) << IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; @@ -9468,7 +8953,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #ifdef IXGBE_FCOE /* setup tx offload for FCoE */ if ((protocol == htons(ETH_P_FCOE)) && - (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { + (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { tso = ixgbe_fso(tx_ring, first, &hdr_len); if (tso < 0) goto out_drop; @@ -9480,7 +8965,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #ifdef CONFIG_IXGBE_IPSEC if (xfrm_offload(skb) && - !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx)) + !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx)) goto out_drop; #endif tso = ixgbe_tso(tx_ring, first, &hdr_len, &ipsec_tx); @@ -9516,8 +9001,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, } static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, - struct net_device *netdev, - struct ixgbe_ring *ring) + struct net_device *netdev, + struct ixgbe_ring *ring) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_ring *tx_ring; @@ -9537,7 +9022,7 @@ static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, } static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, - struct net_device *netdev) + struct net_device *netdev) { return __ixgbe_xmit_frame(skb, netdev, NULL); } @@ -9593,7 +9078,7 @@ ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr) } static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad, - u16 addr, u16 value) + u16 addr, u16 value) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -9690,7 +9175,7 @@ static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats, } static void ixgbe_get_stats64(struct net_device *netdev, - struct rtnl_link_stats64 *stats) + struct rtnl_link_stats64 *stats) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); int i; @@ -9814,7 +9299,7 @@ static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter) #endif /* CONFIG_IXGBE_DCB */ static int ixgbe_reassign_macvlan_pool(struct net_device *vdev, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data; struct ixgbe_fwd_adapter *accel; @@ -9862,7 +9347,7 @@ static void ixgbe_defrag_macvlan_pools(struct net_device *dev) /* walk through upper devices reassigning pools */ netdev_walk_all_upper_dev_rcu(dev, ixgbe_reassign_macvlan_pool, - &priv); + &priv); } /** @@ -9942,7 +9427,7 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc) } static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 hdl = cls->knode.handle; u32 uhtid = TC_U32_USERHTID(cls->knode.handle); @@ -9978,8 +9463,8 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, continue; spin_lock(&adapter->fdir_perfect_lock); err = ixgbe_update_ethtool_fdir_entry(adapter, - NULL, - j + 1); + NULL, + j + 1); spin_unlock(&adapter->fdir_perfect_lock); clear_bit(j, jump->child_loc_map); } @@ -9999,7 +9484,7 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, } static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); @@ -10017,7 +9502,7 @@ static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, } static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); @@ -10037,7 +9522,7 @@ struct upper_walk_data { }; static int get_macvlan_queue(struct net_device *upper, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { if (netif_is_macvlan(upper)) { struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(upper); @@ -10096,7 +9581,7 @@ static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex, } static int parse_tc_actions(struct ixgbe_adapter *adapter, - struct tcf_exts *exts, u64 *action, u8 *queue) + struct tcf_exts *exts, u64 *action, u8 *queue) { const struct tc_action *a; int i; @@ -10119,7 +9604,7 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter, if (!dev) return -EINVAL; return handle_redirect_action(adapter, dev->ifindex, - queue, action); + queue, action); } return -EINVAL; @@ -10129,17 +9614,17 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter, } #else static int parse_tc_actions(struct ixgbe_adapter *adapter, - struct tcf_exts *exts, u64 *action, u8 *queue) + struct tcf_exts *exts, u64 *action, u8 *queue) { return -EINVAL; } #endif /* CONFIG_NET_CLS_ACT */ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, - union ixgbe_atr_input *mask, - struct tc_cls_u32_offload *cls, - struct ixgbe_mat_field *field_ptr, - struct ixgbe_nexthdr *nexthdr) + union ixgbe_atr_input *mask, + struct tc_cls_u32_offload *cls, + struct ixgbe_mat_field *field_ptr, + struct ixgbe_nexthdr *nexthdr) { int i, j, off; __be32 val, m; @@ -10162,10 +9647,10 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, } if (nexthdr) { if (nexthdr->off == cls->knode.sel->keys[i].off && - nexthdr->val == - (__force u32)cls->knode.sel->keys[i].val && - nexthdr->mask == - (__force u32)cls->knode.sel->keys[i].mask) + nexthdr->val == + (__force u32)cls->knode.sel->keys[i].val && + nexthdr->mask == + (__force u32)cls->knode.sel->keys[i].mask) found_jump_field = true; else continue; @@ -10179,7 +9664,7 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, return 0; mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK | - IXGBE_ATR_L4TYPE_MASK; + IXGBE_ATR_L4TYPE_MASK; if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4) mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK; @@ -10260,17 +9745,17 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, * and link to it. */ if (adapter->jump_tables[link_uhtid] && - (adapter->jump_tables[link_uhtid])->link_hdl) { + (adapter->jump_tables[link_uhtid])->link_hdl) { e_err(drv, "Link filter exists for link: %x\n", - link_uhtid); + link_uhtid); return err; } for (i = 0; nexthdr[i].jump; i++) { if (nexthdr[i].o != cls->knode.sel->offoff || - nexthdr[i].s != cls->knode.sel->offshift || - nexthdr[i].m != - (__force u32)cls->knode.sel->offmask) + nexthdr[i].s != cls->knode.sel->offshift || + nexthdr[i].m != + (__force u32)cls->knode.sel->offmask) return err; jump = kzalloc(sizeof(*jump), GFP_KERNEL); @@ -10291,7 +9776,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, jump->link_hdl = cls->knode.handle; err = ixgbe_clsu32_build_input(input, mask, cls, - field_ptr, &nexthdr[i]); + field_ptr, &nexthdr[i]); if (!err) { jump->mat = nexthdr[i].jump; adapter->jump_tables[link_uhtid] = jump; @@ -10317,10 +9802,10 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) { if ((adapter->jump_tables[uhtid])->input) memcpy(input, (adapter->jump_tables[uhtid])->input, - sizeof(*input)); + sizeof(*input)); if ((adapter->jump_tables[uhtid])->mask) memcpy(mask, (adapter->jump_tables[uhtid])->mask, - sizeof(*mask)); + sizeof(*mask)); /* Lookup in all child hash tables if this location is already * filled with a filter @@ -10330,7 +9815,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, if (link && (test_bit(loc - 1, link->child_loc_map))) { e_err(drv, "Filter exists in location: %x\n", - loc); + loc); err = -EINVAL; goto err_out; } @@ -10341,7 +9826,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, goto err_out; err = parse_tc_actions(adapter, cls->knode.exts, &input->action, - &queue); + &queue); if (err < 0) goto err_out; @@ -10361,7 +9846,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, ixgbe_atr_compute_perfect_hash_82599(&input->filter, mask); err = ixgbe_fdir_write_perfect_filter_82599(hw, &input->filter, - input->sw_idx, queue); + input->sw_idx, queue); if (err) goto err_out_w_lock; @@ -10429,7 +9914,7 @@ static int ixgbe_setup_tc_mqprio(struct net_device *dev, static LIST_HEAD(ixgbe_block_cb_list); static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type, - void *type_data) + void *type_data) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); @@ -10468,7 +9953,7 @@ void ixgbe_do_reset(struct net_device *netdev) } static netdev_features_t ixgbe_fix_features(struct net_device *netdev, - netdev_features_t features) + netdev_features_t features) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); @@ -10496,7 +9981,7 @@ static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter) /* go back to full RSS if we're not running SR-IOV */ if (!adapter->ring_feature[RING_F_VMDQ].offset) adapter->flags &= ~(IXGBE_FLAG_VMDQ_ENABLED | - IXGBE_FLAG_SRIOV_ENABLED); + IXGBE_FLAG_SRIOV_ENABLED); adapter->ring_feature[RING_F_RSS].limit = rss; adapter->ring_feature[RING_F_VMDQ].limit = 1; @@ -10505,7 +9990,7 @@ static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter) } static int ixgbe_set_features(struct net_device *netdev, - netdev_features_t features) + netdev_features_t features) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); netdev_features_t changed = netdev->features ^ features; @@ -10519,12 +10004,12 @@ static int ixgbe_set_features(struct net_device *netdev, } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { if (adapter->rx_itr_setting == 1 || - adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { + adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; need_reset = true; } else if ((changed ^ features) & NETIF_F_LRO) { e_info(probe, "rx-usecs set too low, " - "disabling RSC\n"); + "disabling RSC\n"); } } @@ -10548,12 +10033,12 @@ static int ixgbe_set_features(struct net_device *netdev, /* We cannot enable ATR if SR-IOV is enabled */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED || - /* We cannot enable ATR if we have 2 or more tcs */ - (adapter->hw_tcs > 1) || - /* We cannot enable ATR if RSS is disabled */ - (adapter->ring_feature[RING_F_RSS].limit <= 1) || - /* A sample rate of 0 indicates ATR disabled */ - (!adapter->atr_sample_rate)) + /* We cannot enable ATR if we have 2 or more tcs */ + (adapter->hw_tcs > 1) || + /* We cannot enable ATR if RSS is disabled */ + (adapter->ring_feature[RING_F_RSS].limit <= 1) || + /* A sample rate of 0 indicates ATR disabled */ + (!adapter->atr_sample_rate)) ; /* do nothing not supported */ else /* otherwise supported and set the flag */ adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; @@ -10569,17 +10054,17 @@ static int ixgbe_set_features(struct net_device *netdev, else if (need_reset) ixgbe_do_reset(netdev); else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_FILTER)) + NETIF_F_HW_VLAN_CTAG_FILTER)) ixgbe_set_rx_mode(netdev); return 1; } static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], - struct net_device *dev, - const unsigned char *addr, u16 vid, - u16 flags, bool *notified, - struct netlink_ext_ack *extack) + struct net_device *dev, + const unsigned char *addr, u16 vid, + u16 flags, bool *notified, + struct netlink_ext_ack *extack) { /* guarantee we can provide a unique filter for the unicast address */ if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { @@ -10601,7 +10086,7 @@ static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], * Configure some settings require for various bridge modes. **/ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, - __u16 mode) + __u16 mode) { struct ixgbe_hw *hw = &adapter->hw; unsigned int p, num_pools; @@ -10627,8 +10112,8 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, for (p = 0; p < num_pools; p++) { if (hw->mac.ops.set_source_address_pruning) hw->mac.ops.set_source_address_pruning(hw, - true, - p); + true, + p); } break; case BRIDGE_MODE_VEB: @@ -10651,8 +10136,8 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, for (p = 0; p < num_pools; p++) { if (hw->mac.ops.set_source_address_pruning) hw->mac.ops.set_source_address_pruning(hw, - false, - p); + false, + p); } break; default: @@ -10662,14 +10147,14 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, adapter->bridge_mode = mode; e_info(drv, "enabling bridge mode: %s\n", - mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); + mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); return 0; } static int ixgbe_ndo_bridge_setlink(struct net_device *dev, - struct nlmsghdr *nlh, u16 flags, - struct netlink_ext_ack *extack) + struct nlmsghdr *nlh, u16 flags, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct nlattr *attr, *br_spec; @@ -10696,8 +10181,8 @@ static int ixgbe_ndo_bridge_setlink(struct net_device *dev, } static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, - struct net_device *dev, - u32 filter_mask, int nlflags) + struct net_device *dev, + u32 filter_mask, int nlflags) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); @@ -10705,8 +10190,8 @@ static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, return 0; return ndo_dflt_bridge_getlink(skb, pid, seq, dev, - adapter->bridge_mode, 0, 0, nlflags, - filter_mask, NULL); + adapter->bridge_mode, 0, 0, nlflags, + filter_mask, NULL); } static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) @@ -10741,8 +10226,8 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) u16 reserved_pools; if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && - adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) || - adapter->num_rx_pools > IXGBE_MAX_MACVLANS) + adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) || + adapter->num_rx_pools > IXGBE_MAX_MACVLANS) return ERR_PTR(-EBUSY); /* Hardware has a limited number of available pools. Each VF, @@ -10762,12 +10247,12 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) */ if (used_pools < 32 && adapter->num_rx_pools < 16) reserved_pools = min_t(u16, - 32 - used_pools, - 16 - adapter->num_rx_pools); + 32 - used_pools, + 16 - adapter->num_rx_pools); else if (adapter->num_rx_pools < 32) reserved_pools = min_t(u16, - 64 - used_pools, - 32 - adapter->num_rx_pools); + 64 - used_pools, + 32 - adapter->num_rx_pools); else reserved_pools = 64 - used_pools; @@ -10814,7 +10299,7 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv) /* delete unicast filter associated with offloaded interface */ ixgbe_del_mac_filter(adapter, accel->netdev->dev_addr, - VMDQ_P(accel->pool)); + VMDQ_P(accel->pool)); /* Allow remaining Rx packets to get flushed out of the * Rx FIFO before we drop the netdev for the ring. @@ -10846,7 +10331,7 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv) static netdev_features_t ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, - netdev_features_t features) + netdev_features_t features) { unsigned int network_hdr_len, mac_hdr_len; @@ -10854,19 +10339,19 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, mac_hdr_len = skb_network_offset(skb); if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN)) return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_GSO_UDP_L4 | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_TSO | - NETIF_F_TSO6); + NETIF_F_SCTP_CRC | + NETIF_F_GSO_UDP_L4 | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_TSO | + NETIF_F_TSO6); network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN)) return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_GSO_UDP_L4 | - NETIF_F_TSO | - NETIF_F_TSO6); + NETIF_F_SCTP_CRC | + NETIF_F_GSO_UDP_L4 | + NETIF_F_TSO | + NETIF_F_TSO6); /* We can only support IPV4 TSO in tunnels if we can mangle the * inner IP ID field, so strip TSO if MANGLEID is not supported. @@ -10883,13 +10368,15 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) +static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) { - int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + u32 frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct bpf_prog *old_prog; bool need_reset; int num_queues; + bool requires_mbuf; if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) return -EINVAL; @@ -10898,14 +10385,18 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) return -EINVAL; /* verify ixgbe ring attributes are sufficient for XDP */ - for (i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_rx_queues; i++) { struct ixgbe_ring *ring = adapter->rx_ring[i]; if (ring_is_rsc_enabled(ring)) return -EINVAL; + } - if (frame_size > ixgbe_rx_bufsz(ring)) - return -EINVAL; + requires_mbuf = frame_size > IXGBE_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { + NL_SET_ERR_MSG_MOD(extack, + "Configured MTU requires non-linear frames and XDP prog does not support frags"); + return -EOPNOTSUPP; } /* if the number of cpus is much larger than the maximum of queues, @@ -10931,7 +10422,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (!prog) xdp_features_clear_redirect_target(dev); } else { - for (i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_rx_queues; i++) { WRITE_ONCE(adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); } @@ -10946,10 +10437,10 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (need_reset && prog) { num_queues = min_t(int, adapter->num_rx_queues, adapter->num_xdp_queues); - for (i = 0; i < num_queues; i++) + for (int i = 0; i < num_queues; i++) if (adapter->xdp_ring[i]->xsk_pool) (void)ixgbe_xsk_wakeup(adapter->netdev, i, - XDP_WAKEUP_RX); + XDP_WAKEUP_RX); xdp_features_set_redirect_target(dev, true); } @@ -10962,10 +10453,10 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: - return ixgbe_xdp_setup(dev, xdp->prog); + return ixgbe_xdp_setup(dev, xdp->prog, xdp->extack); case XDP_SETUP_XSK_POOL: return ixgbe_xsk_pool_setup(adapter, xdp->xsk.pool, - xdp->xsk.queue_id); + xdp->xsk.queue_id); default: return -EINVAL; @@ -11002,7 +10493,7 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, return -ENETDOWN; if (!netif_carrier_ok(adapter->netdev) || - !netif_running(adapter->netdev)) + !netif_running(adapter->netdev)) return -ENETDOWN; if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) @@ -11119,7 +10610,7 @@ static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter, } static void ixgbe_disable_txr(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring) + struct ixgbe_ring *tx_ring) { set_bit(__IXGBE_TX_DISABLED, &tx_ring->state); ixgbe_disable_txr_hw(adapter, tx_ring); @@ -11143,7 +10634,7 @@ static void ixgbe_disable_rxr_hw(struct ixgbe_adapter *adapter, /* RXDCTL.EN may not change on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* delay mechanism from ixgbe_disable_rx */ @@ -11273,6 +10764,11 @@ void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring) ixgbe_configure_tx_ring(adapter, tx_ring); if (xdp_ring) ixgbe_configure_tx_ring(adapter, xdp_ring); + + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + ixgbe_configure_rx_ring(adapter, rx_ring); clear_bit(__IXGBE_TX_DISABLED, &tx_ring->state); @@ -11318,7 +10814,7 @@ static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter) * this case, simply return -1 to indicate this. */ if ((entry->vendor != pdev->vendor) || - (entry->device != pdev->device)) + (entry->device != pdev->device)) return -1; physfns++; @@ -11350,8 +10846,8 @@ bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, /* check eeprom to see if WOL is enabled for X540 and newer */ if (hw->mac.type >= ixgbe_mac_X540) { if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || - ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && - (hw->bus.func == 0))) + ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && + (hw->bus.func == 0))) return true; } @@ -11538,7 +11034,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) */ if (pdev->is_virtfn) { WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", - pci_name(pdev), pdev->vendor, pdev->device); + pci_name(pdev), pdev->vendor, pdev->device); return -EINVAL; } @@ -11598,7 +11094,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); hw->hw_addr = ioremap(pci_resource_start(pdev, 0), - pci_resource_len(pdev, 0)); + pci_resource_len(pdev, 0)); adapter->io_addr = hw->hw_addr; if (!hw->hw_addr) { err = -EIO; @@ -11741,15 +11237,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_CSUM; #define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ - NETIF_F_GSO_GRE_CSUM | \ - NETIF_F_GSO_IPXIP4 | \ - NETIF_F_GSO_IPXIP6 | \ - NETIF_F_GSO_UDP_TUNNEL | \ - NETIF_F_GSO_UDP_TUNNEL_CSUM) + NETIF_F_GSO_GRE_CSUM | \ + NETIF_F_GSO_IPXIP4 | \ + NETIF_F_GSO_IPXIP6 | \ + NETIF_F_GSO_UDP_TUNNEL | \ + NETIF_F_GSO_UDP_TUNNEL_CSUM) netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES; netdev->features |= NETIF_F_GSO_PARTIAL | - IXGBE_GSO_PARTIAL_FEATURES; + IXGBE_GSO_PARTIAL_FEATURES; if (hw->mac.type >= ixgbe_mac_82599EB) netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4; @@ -11764,15 +11260,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) #endif /* copy netdev features into list of user selectable features */ netdev->hw_features |= netdev->features | - NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_RXALL | - NETIF_F_HW_L2FW_DOFFLOAD; + NETIF_F_HW_VLAN_CTAG_FILTER | + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_RXALL | + NETIF_F_HW_L2FW_DOFFLOAD; if (hw->mac.type >= ixgbe_mac_82599EB) netdev->hw_features |= NETIF_F_NTUPLE | - NETIF_F_HW_TC; + NETIF_F_HW_TC; netdev->features |= NETIF_F_HIGHDMA; @@ -11786,14 +11282,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* set this bit last since it cannot be part of vlan_features */ netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_TX; + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; netdev->priv_flags |= IFF_SUPP_NOFCS; netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY; + NETDEV_XDP_ACT_XSK_ZEROCOPY | + NETDEV_XDP_ACT_RX_SG; /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU; @@ -11819,7 +11316,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->ring_feature[RING_F_FCOE].limit = fcoe_l; netdev->features |= NETIF_F_FSO | - NETIF_F_FCOE_CRC; + NETIF_F_FCOE_CRC; netdev->vlan_features |= NETIF_F_FSO | NETIF_F_FCOE_CRC; @@ -11838,7 +11335,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } eth_platform_get_mac_address(&adapter->pdev->dev, - adapter->hw.mac.perm_addr); + adapter->hw.mac.perm_addr); eth_hw_addr_set(netdev, hw->mac.perm_addr); @@ -12156,7 +11653,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, u16 req_id, pf_func; if (adapter->hw.mac.type == ixgbe_mac_82598EB || - adapter->num_vfs == 0) + adapter->num_vfs == 0) goto skip_bad_vf_detection; bdev = pdev->bus->self; @@ -12221,7 +11718,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, if (vfdev->devfn == (req_id & 0xFF)) break; vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, - device_id, vfdev); + device_id, vfdev); } /* * There's a slim chance the VF could have been hot plugged, @@ -12408,7 +11905,7 @@ static void __exit ixgbe_exit_module(void) #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, - void *p) + void *p) { int ret_val; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index b1bfeb21537acc..825cf17fa1878d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -2452,7 +2452,7 @@ enum { #define IXGBE_PSRTYPE_L2HDR 0x00001000 /* SRRCTL bit definitions */ -#define IXGBE_SRRCTL_BSIZEPKT_SHIFT 10 /* so many KBs */ +#define IXGBE_SRRCTL_BSIZEPKT_STEP 1024 #define IXGBE_SRRCTL_RDMTS_SHIFT 22 #define IXGBE_SRRCTL_RDMTS_MASK 0x01C00000 #define IXGBE_SRRCTL_DROP_EN 0x10000000 diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 7b941505a9d024..15425d3c367872 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -151,7 +151,7 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; u16 i = rx_ring->next_to_use; dma_addr_t dma; bool ok = true; @@ -161,7 +161,7 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) return true; rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; + bi = &rx_ring->rx_xsk_buffer_info[i]; i -= rx_ring->count; do { @@ -183,7 +183,7 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) i++; if (unlikely(!i)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; + bi = rx_ring->rx_xsk_buffer_info; i -= rx_ring->count; } @@ -257,7 +257,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -279,17 +279,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + bi = &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) { - struct ixgbe_rx_buffer *next_bi; + struct ixgbe_xsk_rx_buffer *next_bi; xsk_buff_free(bi->xdp); bi->xdp = NULL; ixgbe_inc_ntc(rx_ring); next_bi = - &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; next_bi->discard = true; continue; } @@ -345,6 +345,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, total_rx_bytes += skb->len; total_rx_packets++; + skb->protocol = eth_type_trans(skb, rx_ring->netdev); ixgbe_process_skb_fields(rx_ring, rx_desc, skb); ixgbe_rx_skb(q_vector, skb); } @@ -374,11 +375,11 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) { - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; u16 i; for (i = 0; i < rx_ring->count; i++) { - bi = &rx_ring->rx_buffer_info[i]; + bi = &rx_ring->rx_xsk_buffer_info[i]; if (!bi->xdp) continue;