1100 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2/* Copyright(c) 1999 - 2018 Intel Corporation. */3 4#ifndef _IXGBE_H_5#define _IXGBE_H_6 7#include <linux/bitops.h>8#include <linux/types.h>9#include <linux/pci.h>10#include <linux/netdevice.h>11#include <linux/cpumask.h>12#include <linux/if_vlan.h>13#include <linux/jiffies.h>14#include <linux/phy.h>15 16#include <linux/timecounter.h>17#include <linux/net_tstamp.h>18#include <linux/ptp_clock_kernel.h>19 20#include "ixgbe_type.h"21#include "ixgbe_common.h"22#include "ixgbe_dcb.h"23#if IS_ENABLED(CONFIG_FCOE)24#define IXGBE_FCOE25#include "ixgbe_fcoe.h"26#endif /* IS_ENABLED(CONFIG_FCOE) */27#ifdef CONFIG_IXGBE_DCA28#include <linux/dca.h>29#endif30#include "ixgbe_ipsec.h"31 32#include <net/xdp.h>33 34/* common prefix used by pr_<> macros */35#undef pr_fmt36#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt37 38/* TX/RX descriptor defines */39#define IXGBE_DEFAULT_TXD 51240#define IXGBE_DEFAULT_TX_WORK 25641#define IXGBE_MAX_TXD_82598 409642#define IXGBE_MAX_TXD_82599 819243#define IXGBE_MAX_TXD_X540 819244#define IXGBE_MAX_TXD_X550 3276845#define IXGBE_MIN_TXD 6446 47#if (PAGE_SIZE < 8192)48#define IXGBE_DEFAULT_RXD 51249#else50#define IXGBE_DEFAULT_RXD 12851#endif52#define IXGBE_MAX_RXD_82598 409653#define IXGBE_MAX_RXD_82599 819254#define IXGBE_MAX_RXD_X540 819255#define IXGBE_MAX_RXD_X550 3276856#define IXGBE_MIN_RXD 6457 58/* flow control */59#define IXGBE_MIN_FCRTL 0x4060#define IXGBE_MAX_FCRTL 0x7FF8061#define IXGBE_MIN_FCRTH 0x60062#define IXGBE_MAX_FCRTH 0x7FFF063#define IXGBE_DEFAULT_FCPAUSE 0xFFFF64#define IXGBE_MIN_FCPAUSE 065#define IXGBE_MAX_FCPAUSE 0xFFFF66 67/* Supported Rx Buffer Sizes */68#define IXGBE_RXBUFFER_256 256 /* Used for skb receive header */69#define IXGBE_RXBUFFER_1536 153670#define IXGBE_RXBUFFER_2K 204871#define IXGBE_RXBUFFER_3K 307272#define IXGBE_RXBUFFER_4K 409673#define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */74 75#define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2))76 77/* Attempt to maximize the headroom available for incoming frames. We78 * use a 2K buffer for receives and need 1536/1534 to store the data for79 * the frame. This leaves us with 512 bytes of room. From that we need80 * to deduct the space needed for the shared info and the padding needed81 * to IP align the frame.82 *83 * Note: For cache line sizes 256 or larger this value is going to end84 * up negative. In these cases we should fall back to the 3K85 * buffers.86 */87#if (PAGE_SIZE < 8192)88#define IXGBE_MAX_2K_FRAME_BUILD_SKB (IXGBE_RXBUFFER_1536 - NET_IP_ALIGN)89#define IXGBE_2K_TOO_SMALL_WITH_PADDING \90((NET_SKB_PAD + IXGBE_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K))91 92static inline int ixgbe_compute_pad(int rx_buf_len)93{94 int page_size, pad_size;95 96 page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2);97 pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len;98 99 return pad_size;100}101 102static inline int ixgbe_skb_pad(void)103{104 int rx_buf_len;105 106 /* If a 2K buffer cannot handle a standard Ethernet frame then107 * optimize padding for a 3K buffer instead of a 1.5K buffer.108 *109 * For a 3K buffer we need to add enough padding to allow for110 * tailroom due to NET_IP_ALIGN possibly shifting us out of111 * cache-line alignment.112 */113 if (IXGBE_2K_TOO_SMALL_WITH_PADDING)114 rx_buf_len = IXGBE_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN);115 else116 rx_buf_len = IXGBE_RXBUFFER_1536;117 118 /* if needed make room for NET_IP_ALIGN */119 rx_buf_len -= NET_IP_ALIGN;120 121 return ixgbe_compute_pad(rx_buf_len);122}123 124#define IXGBE_SKB_PAD ixgbe_skb_pad()125#else126#define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN)127#endif128 129/*130 * NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we131 * reserve 64 more, and skb_shared_info adds an additional 320 bytes more,132 * this adds up to 448 bytes of extra data.133 *134 * Since netdev_alloc_skb now allocates a page fragment we can use a value135 * of 256 and the resultant skb will have a truesize of 960 or less.136 */137#define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256138 139/* How many Rx Buffers do we bundle into one write to the hardware ? */140#define IXGBE_RX_BUFFER_WRITE 16 /* Must be power of 2 */141 142#define IXGBE_RX_DMA_ATTR \143 (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)144 145enum ixgbe_tx_flags {146 /* cmd_type flags */147 IXGBE_TX_FLAGS_HW_VLAN = 0x01,148 IXGBE_TX_FLAGS_TSO = 0x02,149 IXGBE_TX_FLAGS_TSTAMP = 0x04,150 151 /* olinfo flags */152 IXGBE_TX_FLAGS_CC = 0x08,153 IXGBE_TX_FLAGS_IPV4 = 0x10,154 IXGBE_TX_FLAGS_CSUM = 0x20,155 IXGBE_TX_FLAGS_IPSEC = 0x40,156 157 /* software defined flags */158 IXGBE_TX_FLAGS_SW_VLAN = 0x80,159 IXGBE_TX_FLAGS_FCOE = 0x100,160};161 162/* VLAN info */163#define IXGBE_TX_FLAGS_VLAN_MASK 0xffff0000164#define IXGBE_TX_FLAGS_VLAN_PRIO_MASK 0xe0000000165#define IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT 29166#define IXGBE_TX_FLAGS_VLAN_SHIFT 16167 168#define IXGBE_MAX_VF_MC_ENTRIES 30169#define IXGBE_MAX_VF_FUNCTIONS 64170#define IXGBE_MAX_VFTA_ENTRIES 128171#define MAX_EMULATION_MAC_ADDRS 16172#define IXGBE_MAX_PF_MACVLANS 15173#define VMDQ_P(p) ((p) + adapter->ring_feature[RING_F_VMDQ].offset)174#define IXGBE_82599_VF_DEVICE_ID 0x10ED175#define IXGBE_X540_VF_DEVICE_ID 0x1515176 177#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \178 { \179 u32 current_counter = IXGBE_READ_REG(hw, reg); \180 if (current_counter < last_counter) \181 counter += 0x100000000LL; \182 last_counter = current_counter; \183 counter &= 0xFFFFFFFF00000000LL; \184 counter |= current_counter; \185 }186 187#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \188 { \189 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \190 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \191 u64 current_counter = (current_counter_msb << 32) | \192 current_counter_lsb; \193 if (current_counter < last_counter) \194 counter += 0x1000000000LL; \195 last_counter = current_counter; \196 counter &= 0xFFFFFFF000000000LL; \197 counter |= current_counter; \198 }199 200struct vf_stats {201 u64 gprc;202 u64 gorc;203 u64 gptc;204 u64 gotc;205 u64 mprc;206};207 208struct vf_data_storage {209 struct pci_dev *vfdev;210 unsigned char vf_mac_addresses[ETH_ALEN];211 u16 vf_mc_hashes[IXGBE_MAX_VF_MC_ENTRIES];212 u16 num_vf_mc_hashes;213 bool clear_to_send;214 struct vf_stats vfstats;215 struct vf_stats last_vfstats;216 struct vf_stats saved_rst_vfstats;217 bool pf_set_mac;218 u16 pf_vlan; /* When set, guest VLAN config not allowed. */219 u16 pf_qos;220 u16 tx_rate;221 int link_enable;222 int link_state;223 u8 spoofchk_enabled;224 bool rss_query_enabled;225 u8 trusted;226 int xcast_mode;227 unsigned int vf_api;228 u8 primary_abort_count;229};230 231enum ixgbevf_xcast_modes {232 IXGBEVF_XCAST_MODE_NONE = 0,233 IXGBEVF_XCAST_MODE_MULTI,234 IXGBEVF_XCAST_MODE_ALLMULTI,235 IXGBEVF_XCAST_MODE_PROMISC,236};237 238struct vf_macvlans {239 struct list_head l;240 int vf;241 bool free;242 bool is_macvlan;243 u8 vf_macvlan[ETH_ALEN];244};245 246#define IXGBE_MAX_TXD_PWR 14247#define IXGBE_MAX_DATA_PER_TXD (1u << IXGBE_MAX_TXD_PWR)248 249/* Tx Descriptors needed, worst case */250#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)251#define DESC_NEEDED (MAX_SKB_FRAGS + 4)252 253/* wrapper around a pointer to a socket buffer,254 * so a DMA handle can be stored along with the buffer */255struct ixgbe_tx_buffer {256 union ixgbe_adv_tx_desc *next_to_watch;257 unsigned long time_stamp;258 union {259 struct sk_buff *skb;260 struct xdp_frame *xdpf;261 };262 unsigned int bytecount;263 unsigned short gso_segs;264 __be16 protocol;265 DEFINE_DMA_UNMAP_ADDR(dma);266 DEFINE_DMA_UNMAP_LEN(len);267 u32 tx_flags;268};269 270struct ixgbe_rx_buffer {271 union {272 struct {273 struct sk_buff *skb;274 dma_addr_t dma;275 struct page *page;276 __u32 page_offset;277 __u16 pagecnt_bias;278 };279 struct {280 bool discard;281 struct xdp_buff *xdp;282 };283 };284};285 286struct ixgbe_queue_stats {287 u64 packets;288 u64 bytes;289};290 291struct ixgbe_tx_queue_stats {292 u64 restart_queue;293 u64 tx_busy;294 u64 tx_done_old;295};296 297struct ixgbe_rx_queue_stats {298 u64 rsc_count;299 u64 rsc_flush;300 u64 non_eop_descs;301 u64 alloc_rx_page;302 u64 alloc_rx_page_failed;303 u64 alloc_rx_buff_failed;304 u64 csum_err;305};306 307#define IXGBE_TS_HDR_LEN 8308 309enum ixgbe_ring_state_t {310 __IXGBE_RX_3K_BUFFER,311 __IXGBE_RX_BUILD_SKB_ENABLED,312 __IXGBE_RX_RSC_ENABLED,313 __IXGBE_RX_CSUM_UDP_ZERO_ERR,314 __IXGBE_RX_FCOE,315 __IXGBE_TX_FDIR_INIT_DONE,316 __IXGBE_TX_XPS_INIT_DONE,317 __IXGBE_TX_DETECT_HANG,318 __IXGBE_HANG_CHECK_ARMED,319 __IXGBE_TX_XDP_RING,320 __IXGBE_TX_DISABLED,321};322 323#define ring_uses_build_skb(ring) \324 test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state)325 326struct ixgbe_fwd_adapter {327 unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];328 struct net_device *netdev;329 unsigned int tx_base_queue;330 unsigned int rx_base_queue;331 int pool;332};333 334#define check_for_tx_hang(ring) \335 test_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state)336#define set_check_for_tx_hang(ring) \337 set_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state)338#define clear_check_for_tx_hang(ring) \339 clear_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state)340#define ring_is_rsc_enabled(ring) \341 test_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state)342#define set_ring_rsc_enabled(ring) \343 set_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state)344#define clear_ring_rsc_enabled(ring) \345 clear_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state)346#define ring_is_xdp(ring) \347 test_bit(__IXGBE_TX_XDP_RING, &(ring)->state)348#define set_ring_xdp(ring) \349 set_bit(__IXGBE_TX_XDP_RING, &(ring)->state)350#define clear_ring_xdp(ring) \351 clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state)352struct ixgbe_ring {353 struct ixgbe_ring *next; /* pointer to next ring in q_vector */354 struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */355 struct net_device *netdev; /* netdev ring belongs to */356 struct bpf_prog *xdp_prog;357 struct device *dev; /* device for DMA mapping */358 void *desc; /* descriptor ring memory */359 union {360 struct ixgbe_tx_buffer *tx_buffer_info;361 struct ixgbe_rx_buffer *rx_buffer_info;362 };363 unsigned long state;364 u8 __iomem *tail;365 dma_addr_t dma; /* phys. address of descriptor ring */366 unsigned int size; /* length in bytes */367 368 u16 count; /* amount of descriptors */369 370 u8 queue_index; /* needed for multiqueue queue management */371 u8 reg_idx; /* holds the special value that gets372 * the hardware register offset373 * associated with this ring, which is374 * different for DCB and RSS modes375 */376 u16 next_to_use;377 u16 next_to_clean;378 379 unsigned long last_rx_timestamp;380 381 union {382 u16 next_to_alloc;383 struct {384 u8 atr_sample_rate;385 u8 atr_count;386 };387 };388 389 u8 dcb_tc;390 struct ixgbe_queue_stats stats;391 struct u64_stats_sync syncp;392 union {393 struct ixgbe_tx_queue_stats tx_stats;394 struct ixgbe_rx_queue_stats rx_stats;395 };396 u16 rx_offset;397 struct xdp_rxq_info xdp_rxq;398 spinlock_t tx_lock; /* used in XDP mode */399 struct xsk_buff_pool *xsk_pool;400 u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */401 u16 rx_buf_len;402} ____cacheline_internodealigned_in_smp;403 404enum ixgbe_ring_f_enum {405 RING_F_NONE = 0,406 RING_F_VMDQ, /* SR-IOV uses the same ring feature */407 RING_F_RSS,408 RING_F_FDIR,409#ifdef IXGBE_FCOE410 RING_F_FCOE,411#endif /* IXGBE_FCOE */412 413 RING_F_ARRAY_SIZE /* must be last in enum set */414};415 416#define IXGBE_MAX_RSS_INDICES 16417#define IXGBE_MAX_RSS_INDICES_X550 63418#define IXGBE_MAX_VMDQ_INDICES 64419#define IXGBE_MAX_FDIR_INDICES 63 /* based on q_vector limit */420#define IXGBE_MAX_FCOE_INDICES 8421#define MAX_RX_QUEUES (IXGBE_MAX_FDIR_INDICES + 1)422#define MAX_TX_QUEUES (IXGBE_MAX_FDIR_INDICES + 1)423#define IXGBE_MAX_XDP_QS (IXGBE_MAX_FDIR_INDICES + 1)424#define IXGBE_MAX_L2A_QUEUES 4425#define IXGBE_BAD_L2A_QUEUE 3426#define IXGBE_MAX_MACVLANS 63427 428DECLARE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key);429 430struct ixgbe_ring_feature {431 u16 limit; /* upper limit on feature indices */432 u16 indices; /* current value of indices */433 u16 mask; /* Mask used for feature to ring mapping */434 u16 offset; /* offset to start of feature */435} ____cacheline_internodealigned_in_smp;436 437#define IXGBE_82599_VMDQ_8Q_MASK 0x78438#define IXGBE_82599_VMDQ_4Q_MASK 0x7C439#define IXGBE_82599_VMDQ_2Q_MASK 0x7E440 441/*442 * FCoE requires that all Rx buffers be over 2200 bytes in length. Since443 * this is twice the size of a half page we need to double the page order444 * for FCoE enabled Rx queues.445 */446static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring)447{448 if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))449 return IXGBE_RXBUFFER_3K;450#if (PAGE_SIZE < 8192)451 if (ring_uses_build_skb(ring))452 return IXGBE_MAX_2K_FRAME_BUILD_SKB;453#endif454 return IXGBE_RXBUFFER_2K;455}456 457static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring)458{459#if (PAGE_SIZE < 8192)460 if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))461 return 1;462#endif463 return 0;464}465#define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring))466 467#define IXGBE_ITR_ADAPTIVE_MIN_INC 2468#define IXGBE_ITR_ADAPTIVE_MIN_USECS 10469#define IXGBE_ITR_ADAPTIVE_MAX_USECS 126470#define IXGBE_ITR_ADAPTIVE_LATENCY 0x80471#define IXGBE_ITR_ADAPTIVE_BULK 0x00472 473struct ixgbe_ring_container {474 struct ixgbe_ring *ring; /* pointer to linked list of rings */475 unsigned long next_update; /* jiffies value of last update */476 unsigned int total_bytes; /* total bytes processed this int */477 unsigned int total_packets; /* total packets processed this int */478 u16 work_limit; /* total work allowed per interrupt */479 u8 count; /* total number of rings in vector */480 u8 itr; /* current ITR setting for ring */481};482 483/* iterator for handling rings in ring container */484#define ixgbe_for_each_ring(pos, head) \485 for (pos = (head).ring; pos != NULL; pos = pos->next)486 487#define MAX_RX_PACKET_BUFFERS ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) \488 ? 8 : 1)489#define MAX_TX_PACKET_BUFFERS MAX_RX_PACKET_BUFFERS490 491/* MAX_Q_VECTORS of these are allocated,492 * but we only use one per queue-specific vector.493 */494struct ixgbe_q_vector {495 struct ixgbe_adapter *adapter;496#ifdef CONFIG_IXGBE_DCA497 int cpu; /* CPU for DCA */498#endif499 u16 v_idx; /* index of q_vector within array, also used for500 * finding the bit in EICR and friends that501 * represents the vector for this ring */502 u16 itr; /* Interrupt throttle rate written to EITR */503 struct ixgbe_ring_container rx, tx;504 505 struct napi_struct napi;506 cpumask_t affinity_mask;507 int numa_node;508 struct rcu_head rcu; /* to avoid race with update stats on free */509 char name[IFNAMSIZ + 9];510 511 /* for dynamic allocation of rings associated with this q_vector */512 struct ixgbe_ring ring[] ____cacheline_internodealigned_in_smp;513};514 515#ifdef CONFIG_IXGBE_HWMON516 517#define IXGBE_HWMON_TYPE_LOC 0518#define IXGBE_HWMON_TYPE_TEMP 1519#define IXGBE_HWMON_TYPE_CAUTION 2520#define IXGBE_HWMON_TYPE_MAX 3521 522struct hwmon_attr {523 struct device_attribute dev_attr;524 struct ixgbe_hw *hw;525 struct ixgbe_thermal_diode_data *sensor;526 char name[12];527};528 529struct hwmon_buff {530 struct attribute_group group;531 const struct attribute_group *groups[2];532 struct attribute *attrs[IXGBE_MAX_SENSORS * 4 + 1];533 struct hwmon_attr hwmon_list[IXGBE_MAX_SENSORS * 4];534 unsigned int n_hwmon;535};536#endif /* CONFIG_IXGBE_HWMON */537 538/*539 * microsecond values for various ITR rates shifted by 2 to fit itr register540 * with the first 3 bits reserved 0541 */542#define IXGBE_MIN_RSC_ITR 24543#define IXGBE_100K_ITR 40544#define IXGBE_20K_ITR 200545#define IXGBE_12K_ITR 336546 547/* ixgbe_test_staterr - tests bits in Rx descriptor status and error fields */548static inline __le32 ixgbe_test_staterr(union ixgbe_adv_rx_desc *rx_desc,549 const u32 stat_err_bits)550{551 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);552}553 554static inline u16 ixgbe_desc_unused(struct ixgbe_ring *ring)555{556 u16 ntc = ring->next_to_clean;557 u16 ntu = ring->next_to_use;558 559 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;560}561 562#define IXGBE_RX_DESC(R, i) \563 (&(((union ixgbe_adv_rx_desc *)((R)->desc))[i]))564#define IXGBE_TX_DESC(R, i) \565 (&(((union ixgbe_adv_tx_desc *)((R)->desc))[i]))566#define IXGBE_TX_CTXTDESC(R, i) \567 (&(((struct ixgbe_adv_tx_context_desc *)((R)->desc))[i]))568 569#define IXGBE_MAX_JUMBO_FRAME_SIZE 9728 /* Maximum Supported Size 9.5KB */570#ifdef IXGBE_FCOE571/* Use 3K as the baby jumbo frame size for FCoE */572#define IXGBE_FCOE_JUMBO_FRAME_SIZE 3072573#endif /* IXGBE_FCOE */574 575#define OTHER_VECTOR 1576#define NON_Q_VECTORS (OTHER_VECTOR)577 578#define MAX_MSIX_VECTORS_82599 64579#define MAX_Q_VECTORS_82599 64580#define MAX_MSIX_VECTORS_82598 18581#define MAX_Q_VECTORS_82598 16582 583struct ixgbe_mac_addr {584 u8 addr[ETH_ALEN];585 u16 pool;586 u16 state; /* bitmask */587};588 589#define IXGBE_MAC_STATE_DEFAULT 0x1590#define IXGBE_MAC_STATE_MODIFIED 0x2591#define IXGBE_MAC_STATE_IN_USE 0x4592 593#define MAX_Q_VECTORS MAX_Q_VECTORS_82599594#define MAX_MSIX_COUNT MAX_MSIX_VECTORS_82599595 596#define MIN_MSIX_Q_VECTORS 1597#define MIN_MSIX_COUNT (MIN_MSIX_Q_VECTORS + NON_Q_VECTORS)598 599/* default to trying for four seconds */600#define IXGBE_TRY_LINK_TIMEOUT (4 * HZ)601#define IXGBE_SFP_POLL_JIFFIES (2 * HZ) /* SFP poll every 2 seconds */602 603#define IXGBE_PRIMARY_ABORT_LIMIT 5604 605/* board specific private data structure */606struct ixgbe_adapter {607 unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];608 /* OS defined structs */609 struct net_device *netdev;610 struct bpf_prog *xdp_prog;611 struct pci_dev *pdev;612 struct mii_bus *mii_bus;613 614 unsigned long state;615 616 /* Some features need tri-state capability,617 * thus the additional *_CAPABLE flags.618 */619 u32 flags;620#define IXGBE_FLAG_MSI_ENABLED BIT(1)621#define IXGBE_FLAG_MSIX_ENABLED BIT(3)622#define IXGBE_FLAG_RX_1BUF_CAPABLE BIT(4)623#define IXGBE_FLAG_RX_PS_CAPABLE BIT(5)624#define IXGBE_FLAG_RX_PS_ENABLED BIT(6)625#define IXGBE_FLAG_DCA_ENABLED BIT(8)626#define IXGBE_FLAG_DCA_CAPABLE BIT(9)627#define IXGBE_FLAG_IMIR_ENABLED BIT(10)628#define IXGBE_FLAG_MQ_CAPABLE BIT(11)629#define IXGBE_FLAG_DCB_ENABLED BIT(12)630#define IXGBE_FLAG_VMDQ_CAPABLE BIT(13)631#define IXGBE_FLAG_VMDQ_ENABLED BIT(14)632#define IXGBE_FLAG_FAN_FAIL_CAPABLE BIT(15)633#define IXGBE_FLAG_NEED_LINK_UPDATE BIT(16)634#define IXGBE_FLAG_NEED_LINK_CONFIG BIT(17)635#define IXGBE_FLAG_FDIR_HASH_CAPABLE BIT(18)636#define IXGBE_FLAG_FDIR_PERFECT_CAPABLE BIT(19)637#define IXGBE_FLAG_FCOE_CAPABLE BIT(20)638#define IXGBE_FLAG_FCOE_ENABLED BIT(21)639#define IXGBE_FLAG_SRIOV_CAPABLE BIT(22)640#define IXGBE_FLAG_SRIOV_ENABLED BIT(23)641#define IXGBE_FLAG_RX_HWTSTAMP_ENABLED BIT(25)642#define IXGBE_FLAG_RX_HWTSTAMP_IN_REGISTER BIT(26)643#define IXGBE_FLAG_DCB_CAPABLE BIT(27)644 645 u32 flags2;646#define IXGBE_FLAG2_RSC_CAPABLE BIT(0)647#define IXGBE_FLAG2_RSC_ENABLED BIT(1)648#define IXGBE_FLAG2_TEMP_SENSOR_CAPABLE BIT(2)649#define IXGBE_FLAG2_TEMP_SENSOR_EVENT BIT(3)650#define IXGBE_FLAG2_SEARCH_FOR_SFP BIT(4)651#define IXGBE_FLAG2_SFP_NEEDS_RESET BIT(5)652#define IXGBE_FLAG2_FDIR_REQUIRES_REINIT BIT(7)653#define IXGBE_FLAG2_RSS_FIELD_IPV4_UDP BIT(8)654#define IXGBE_FLAG2_RSS_FIELD_IPV6_UDP BIT(9)655#define IXGBE_FLAG2_PTP_PPS_ENABLED BIT(10)656#define IXGBE_FLAG2_PHY_INTERRUPT BIT(11)657#define IXGBE_FLAG2_VLAN_PROMISC BIT(13)658#define IXGBE_FLAG2_EEE_CAPABLE BIT(14)659#define IXGBE_FLAG2_EEE_ENABLED BIT(15)660#define IXGBE_FLAG2_RX_LEGACY BIT(16)661#define IXGBE_FLAG2_IPSEC_ENABLED BIT(17)662#define IXGBE_FLAG2_VF_IPSEC_ENABLED BIT(18)663#define IXGBE_FLAG2_AUTO_DISABLE_VF BIT(19)664 665 /* Tx fast path data */666 int num_tx_queues;667 u16 tx_itr_setting;668 u16 tx_work_limit;669 u64 tx_ipsec;670 671 /* Rx fast path data */672 int num_rx_queues;673 u16 rx_itr_setting;674 u64 rx_ipsec;675 676 /* Port number used to identify VXLAN traffic */677 __be16 vxlan_port;678 __be16 geneve_port;679 680 /* XDP */681 int num_xdp_queues;682 struct ixgbe_ring *xdp_ring[IXGBE_MAX_XDP_QS];683 unsigned long *af_xdp_zc_qps; /* tracks AF_XDP ZC enabled rings */684 685 /* TX */686 struct ixgbe_ring *tx_ring[MAX_TX_QUEUES] ____cacheline_aligned_in_smp;687 688 u64 restart_queue;689 u64 lsc_int;690 u32 tx_timeout_count;691 692 /* RX */693 struct ixgbe_ring *rx_ring[MAX_RX_QUEUES];694 int num_rx_pools; /* == num_rx_queues in 82598 */695 int num_rx_queues_per_pool; /* 1 if 82598, can be many if 82599 */696 u64 hw_csum_rx_error;697 u64 hw_rx_no_dma_resources;698 u64 rsc_total_count;699 u64 rsc_total_flush;700 u64 non_eop_descs;701 u32 alloc_rx_page;702 u32 alloc_rx_page_failed;703 u32 alloc_rx_buff_failed;704 705 struct ixgbe_q_vector *q_vector[MAX_Q_VECTORS];706 707 /* DCB parameters */708 struct ieee_pfc *ixgbe_ieee_pfc;709 struct ieee_ets *ixgbe_ieee_ets;710 struct ixgbe_dcb_config dcb_cfg;711 struct ixgbe_dcb_config temp_dcb_cfg;712 u8 hw_tcs;713 u8 dcb_set_bitmap;714 u8 dcbx_cap;715 enum ixgbe_fc_mode last_lfc_mode;716 717 int num_q_vectors; /* current number of q_vectors for device */718 int max_q_vectors; /* true count of q_vectors for device */719 struct ixgbe_ring_feature ring_feature[RING_F_ARRAY_SIZE];720 struct msix_entry *msix_entries;721 722 u32 test_icr;723 struct ixgbe_ring test_tx_ring;724 struct ixgbe_ring test_rx_ring;725 726 /* structs defined in ixgbe_hw.h */727 struct ixgbe_hw hw;728 u16 msg_enable;729 struct ixgbe_hw_stats stats;730 731 u64 tx_busy;732 unsigned int tx_ring_count;733 unsigned int xdp_ring_count;734 unsigned int rx_ring_count;735 736 u32 link_speed;737 bool link_up;738 unsigned long sfp_poll_time;739 unsigned long link_check_timeout;740 741 struct timer_list service_timer;742 struct work_struct service_task;743 744 struct hlist_head fdir_filter_list;745 unsigned long fdir_overflow; /* number of times ATR was backed off */746 union ixgbe_atr_input fdir_mask;747 int fdir_filter_count;748 u32 fdir_pballoc;749 u32 atr_sample_rate;750 spinlock_t fdir_perfect_lock;751 752#ifdef IXGBE_FCOE753 struct ixgbe_fcoe fcoe;754#endif /* IXGBE_FCOE */755 u8 __iomem *io_addr; /* Mainly for iounmap use */756 u32 wol;757 758 u16 bridge_mode;759 760 char eeprom_id[NVM_VER_SIZE];761 u16 eeprom_cap;762 763 u32 interrupt_event;764 u32 led_reg;765 766 struct ptp_clock *ptp_clock;767 struct ptp_clock_info ptp_caps;768 struct work_struct ptp_tx_work;769 struct sk_buff *ptp_tx_skb;770 struct hwtstamp_config tstamp_config;771 unsigned long ptp_tx_start;772 unsigned long last_overflow_check;773 unsigned long last_rx_ptp_check;774 unsigned long last_rx_timestamp;775 spinlock_t tmreg_lock;776 struct cyclecounter hw_cc;777 struct timecounter hw_tc;778 u32 base_incval;779 u32 tx_hwtstamp_timeouts;780 u32 tx_hwtstamp_skipped;781 u32 rx_hwtstamp_cleared;782 void (*ptp_setup_sdp)(struct ixgbe_adapter *);783 784 /* SR-IOV */785 DECLARE_BITMAP(active_vfs, IXGBE_MAX_VF_FUNCTIONS);786 unsigned int num_vfs;787 struct vf_data_storage *vfinfo;788 int vf_rate_link_speed;789 struct vf_macvlans vf_mvs;790 struct vf_macvlans *mv_list;791 792 u32 timer_event_accumulator;793 u32 vferr_refcount;794 struct ixgbe_mac_addr *mac_table;795 struct kobject *info_kobj;796#ifdef CONFIG_IXGBE_HWMON797 struct hwmon_buff *ixgbe_hwmon_buff;798#endif /* CONFIG_IXGBE_HWMON */799#ifdef CONFIG_DEBUG_FS800 struct dentry *ixgbe_dbg_adapter;801#endif /*CONFIG_DEBUG_FS*/802 803 u8 default_up;804 /* Bitmask indicating in use pools */805 DECLARE_BITMAP(fwd_bitmask, IXGBE_MAX_MACVLANS + 1);806 807#define IXGBE_MAX_LINK_HANDLE 10808 struct ixgbe_jump_table *jump_tables[IXGBE_MAX_LINK_HANDLE];809 unsigned long tables;810 811/* maximum number of RETA entries among all devices supported by ixgbe812 * driver: currently it's x550 device in non-SRIOV mode813 */814#define IXGBE_MAX_RETA_ENTRIES 512815 u8 rss_indir_tbl[IXGBE_MAX_RETA_ENTRIES];816 817#define IXGBE_RSS_KEY_SIZE 40 /* size of RSS Hash Key in bytes */818 u32 *rss_key;819 820#ifdef CONFIG_IXGBE_IPSEC821 struct ixgbe_ipsec *ipsec;822#endif /* CONFIG_IXGBE_IPSEC */823 spinlock_t vfs_lock;824};825 826static inline int ixgbe_determine_xdp_q_idx(int cpu)827{828 if (static_key_enabled(&ixgbe_xdp_locking_key))829 return cpu % IXGBE_MAX_XDP_QS;830 else831 return cpu;832}833 834static inline835struct ixgbe_ring *ixgbe_determine_xdp_ring(struct ixgbe_adapter *adapter)836{837 int index = ixgbe_determine_xdp_q_idx(smp_processor_id());838 839 return adapter->xdp_ring[index];840}841 842static inline u8 ixgbe_max_rss_indices(struct ixgbe_adapter *adapter)843{844 switch (adapter->hw.mac.type) {845 case ixgbe_mac_82598EB:846 case ixgbe_mac_82599EB:847 case ixgbe_mac_X540:848 return IXGBE_MAX_RSS_INDICES;849 case ixgbe_mac_X550:850 case ixgbe_mac_X550EM_x:851 case ixgbe_mac_x550em_a:852 return IXGBE_MAX_RSS_INDICES_X550;853 default:854 return 0;855 }856}857 858struct ixgbe_fdir_filter {859 struct hlist_node fdir_node;860 union ixgbe_atr_input filter;861 u16 sw_idx;862 u64 action;863};864 865enum ixgbe_state_t {866 __IXGBE_TESTING,867 __IXGBE_RESETTING,868 __IXGBE_DOWN,869 __IXGBE_DISABLED,870 __IXGBE_REMOVING,871 __IXGBE_SERVICE_SCHED,872 __IXGBE_SERVICE_INITED,873 __IXGBE_IN_SFP_INIT,874 __IXGBE_PTP_RUNNING,875 __IXGBE_PTP_TX_IN_PROGRESS,876 __IXGBE_RESET_REQUESTED,877};878 879struct ixgbe_cb {880 union { /* Union defining head/tail partner */881 struct sk_buff *head;882 struct sk_buff *tail;883 };884 dma_addr_t dma;885 u16 append_cnt;886 bool page_released;887};888#define IXGBE_CB(skb) ((struct ixgbe_cb *)(skb)->cb)889 890enum ixgbe_boards {891 board_82598,892 board_82599,893 board_X540,894 board_X550,895 board_X550EM_x,896 board_x550em_x_fw,897 board_x550em_a,898 board_x550em_a_fw,899};900 901extern const struct ixgbe_info ixgbe_82598_info;902extern const struct ixgbe_info ixgbe_82599_info;903extern const struct ixgbe_info ixgbe_X540_info;904extern const struct ixgbe_info ixgbe_X550_info;905extern const struct ixgbe_info ixgbe_X550EM_x_info;906extern const struct ixgbe_info ixgbe_x550em_x_fw_info;907extern const struct ixgbe_info ixgbe_x550em_a_info;908extern const struct ixgbe_info ixgbe_x550em_a_fw_info;909#ifdef CONFIG_IXGBE_DCB910extern const struct dcbnl_rtnl_ops ixgbe_dcbnl_ops;911#endif912 913extern char ixgbe_driver_name[];914#ifdef IXGBE_FCOE915extern char ixgbe_default_device_descr[];916#endif /* IXGBE_FCOE */917 918int ixgbe_open(struct net_device *netdev);919int ixgbe_close(struct net_device *netdev);920void ixgbe_up(struct ixgbe_adapter *adapter);921void ixgbe_down(struct ixgbe_adapter *adapter);922void ixgbe_reinit_locked(struct ixgbe_adapter *adapter);923void ixgbe_reset(struct ixgbe_adapter *adapter);924void ixgbe_set_ethtool_ops(struct net_device *netdev);925int ixgbe_setup_rx_resources(struct ixgbe_adapter *, struct ixgbe_ring *);926int ixgbe_setup_tx_resources(struct ixgbe_ring *);927void ixgbe_free_rx_resources(struct ixgbe_ring *);928void ixgbe_free_tx_resources(struct ixgbe_ring *);929void ixgbe_configure_rx_ring(struct ixgbe_adapter *, struct ixgbe_ring *);930void ixgbe_configure_tx_ring(struct ixgbe_adapter *, struct ixgbe_ring *);931void ixgbe_disable_rx(struct ixgbe_adapter *adapter);932void ixgbe_disable_tx(struct ixgbe_adapter *adapter);933void ixgbe_update_stats(struct ixgbe_adapter *adapter);934int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter);935bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id,936 u16 subdevice_id);937#ifdef CONFIG_PCI_IOV938void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter);939#endif940int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter,941 const u8 *addr, u16 queue);942int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter,943 const u8 *addr, u16 queue);944void ixgbe_update_pf_promisc_vlvf(struct ixgbe_adapter *adapter, u32 vid);945void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter);946netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *, struct ixgbe_adapter *,947 struct ixgbe_ring *);948void ixgbe_alloc_rx_buffers(struct ixgbe_ring *, u16);949void ixgbe_write_eitr(struct ixgbe_q_vector *);950int ixgbe_poll(struct napi_struct *napi, int budget);951int ethtool_ioctl(struct ifreq *ifr);952int ixgbe_reinit_fdir_tables_82599(struct ixgbe_hw *hw);953int ixgbe_init_fdir_signature_82599(struct ixgbe_hw *hw, u32 fdirctrl);954int ixgbe_init_fdir_perfect_82599(struct ixgbe_hw *hw, u32 fdirctrl);955int ixgbe_fdir_add_signature_filter_82599(struct ixgbe_hw *hw,956 union ixgbe_atr_hash_dword input,957 union ixgbe_atr_hash_dword common,958 u8 queue);959int ixgbe_fdir_set_input_mask_82599(struct ixgbe_hw *hw,960 union ixgbe_atr_input *input_mask);961int ixgbe_fdir_write_perfect_filter_82599(struct ixgbe_hw *hw,962 union ixgbe_atr_input *input,963 u16 soft_id, u8 queue);964int ixgbe_fdir_erase_perfect_filter_82599(struct ixgbe_hw *hw,965 union ixgbe_atr_input *input,966 u16 soft_id);967void ixgbe_atr_compute_perfect_hash_82599(union ixgbe_atr_input *input,968 union ixgbe_atr_input *mask);969int ixgbe_update_ethtool_fdir_entry(struct ixgbe_adapter *adapter,970 struct ixgbe_fdir_filter *input,971 u16 sw_idx);972void ixgbe_set_rx_mode(struct net_device *netdev);973#ifdef CONFIG_IXGBE_DCB974void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter);975#endif976int ixgbe_setup_tc(struct net_device *dev, u8 tc);977void ixgbe_tx_ctxtdesc(struct ixgbe_ring *, u32, u32, u32, u32);978void ixgbe_do_reset(struct net_device *netdev);979#ifdef CONFIG_IXGBE_HWMON980void ixgbe_sysfs_exit(struct ixgbe_adapter *adapter);981int ixgbe_sysfs_init(struct ixgbe_adapter *adapter);982#endif /* CONFIG_IXGBE_HWMON */983#ifdef IXGBE_FCOE984void ixgbe_configure_fcoe(struct ixgbe_adapter *adapter);985int ixgbe_fso(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first,986 u8 *hdr_len);987int ixgbe_fcoe_ddp(struct ixgbe_adapter *adapter,988 union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb);989int ixgbe_fcoe_ddp_get(struct net_device *netdev, u16 xid,990 struct scatterlist *sgl, unsigned int sgc);991int ixgbe_fcoe_ddp_target(struct net_device *netdev, u16 xid,992 struct scatterlist *sgl, unsigned int sgc);993int ixgbe_fcoe_ddp_put(struct net_device *netdev, u16 xid);994int ixgbe_setup_fcoe_ddp_resources(struct ixgbe_adapter *adapter);995void ixgbe_free_fcoe_ddp_resources(struct ixgbe_adapter *adapter);996int ixgbe_fcoe_enable(struct net_device *netdev);997int ixgbe_fcoe_disable(struct net_device *netdev);998int ixgbe_fcoe_get_wwn(struct net_device *netdev, u64 *wwn, int type);999int ixgbe_fcoe_get_hbainfo(struct net_device *netdev,1000 struct netdev_fcoe_hbainfo *info);1001u8 ixgbe_fcoe_get_tc(struct ixgbe_adapter *adapter);1002#endif /* IXGBE_FCOE */1003#ifdef CONFIG_DEBUG_FS1004void ixgbe_dbg_adapter_init(struct ixgbe_adapter *adapter);1005void ixgbe_dbg_adapter_exit(struct ixgbe_adapter *adapter);1006void ixgbe_dbg_init(void);1007void ixgbe_dbg_exit(void);1008#else1009static inline void ixgbe_dbg_adapter_init(struct ixgbe_adapter *adapter) {}1010static inline void ixgbe_dbg_adapter_exit(struct ixgbe_adapter *adapter) {}1011static inline void ixgbe_dbg_init(void) {}1012static inline void ixgbe_dbg_exit(void) {}1013#endif /* CONFIG_DEBUG_FS */1014static inline struct netdev_queue *txring_txq(const struct ixgbe_ring *ring)1015{1016 return netdev_get_tx_queue(ring->netdev, ring->queue_index);1017}1018 1019void ixgbe_ptp_init(struct ixgbe_adapter *adapter);1020void ixgbe_ptp_suspend(struct ixgbe_adapter *adapter);1021void ixgbe_ptp_stop(struct ixgbe_adapter *adapter);1022void ixgbe_ptp_overflow_check(struct ixgbe_adapter *adapter);1023void ixgbe_ptp_rx_hang(struct ixgbe_adapter *adapter);1024void ixgbe_ptp_tx_hang(struct ixgbe_adapter *adapter);1025void ixgbe_ptp_rx_pktstamp(struct ixgbe_q_vector *, struct sk_buff *);1026void ixgbe_ptp_rx_rgtstamp(struct ixgbe_q_vector *, struct sk_buff *skb);1027static inline void ixgbe_ptp_rx_hwtstamp(struct ixgbe_ring *rx_ring,1028 union ixgbe_adv_rx_desc *rx_desc,1029 struct sk_buff *skb)1030{1031 if (unlikely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_TSIP))) {1032 ixgbe_ptp_rx_pktstamp(rx_ring->q_vector, skb);1033 return;1034 }1035 1036 if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_TS)))1037 return;1038 1039 ixgbe_ptp_rx_rgtstamp(rx_ring->q_vector, skb);1040 1041 /* Update the last_rx_timestamp timer in order to enable watchdog check1042 * for error case of latched timestamp on a dropped packet.1043 */1044 rx_ring->last_rx_timestamp = jiffies;1045}1046 1047int ixgbe_ptp_set_ts_config(struct ixgbe_adapter *adapter, struct ifreq *ifr);1048int ixgbe_ptp_get_ts_config(struct ixgbe_adapter *adapter, struct ifreq *ifr);1049void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter);1050void ixgbe_ptp_reset(struct ixgbe_adapter *adapter);1051void ixgbe_ptp_check_pps_event(struct ixgbe_adapter *adapter);1052#ifdef CONFIG_PCI_IOV1053void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter);1054#endif1055 1056netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,1057 struct ixgbe_adapter *adapter,1058 struct ixgbe_ring *tx_ring);1059u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter);1060void ixgbe_store_key(struct ixgbe_adapter *adapter);1061void ixgbe_store_reta(struct ixgbe_adapter *adapter);1062int ixgbe_negotiate_fc(struct ixgbe_hw *hw, u32 adv_reg, u32 lp_reg,1063 u32 adv_sym, u32 adv_asm, u32 lp_sym, u32 lp_asm);1064#ifdef CONFIG_IXGBE_IPSEC1065void ixgbe_init_ipsec_offload(struct ixgbe_adapter *adapter);1066void ixgbe_stop_ipsec_offload(struct ixgbe_adapter *adapter);1067void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter);1068void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring,1069 union ixgbe_adv_rx_desc *rx_desc,1070 struct sk_buff *skb);1071int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first,1072 struct ixgbe_ipsec_tx_data *itd);1073void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter, u32 vf);1074int ixgbe_ipsec_vf_add_sa(struct ixgbe_adapter *adapter, u32 *mbuf, u32 vf);1075int ixgbe_ipsec_vf_del_sa(struct ixgbe_adapter *adapter, u32 *mbuf, u32 vf);1076#else1077static inline void ixgbe_init_ipsec_offload(struct ixgbe_adapter *adapter) { }1078static inline void ixgbe_stop_ipsec_offload(struct ixgbe_adapter *adapter) { }1079static inline void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter) { }1080static inline void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring,1081 union ixgbe_adv_rx_desc *rx_desc,1082 struct sk_buff *skb) { }1083static inline int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring,1084 struct ixgbe_tx_buffer *first,1085 struct ixgbe_ipsec_tx_data *itd) { return 0; }1086static inline void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter,1087 u32 vf) { }1088static inline int ixgbe_ipsec_vf_add_sa(struct ixgbe_adapter *adapter,1089 u32 *mbuf, u32 vf) { return -EACCES; }1090static inline int ixgbe_ipsec_vf_del_sa(struct ixgbe_adapter *adapter,1091 u32 *mbuf, u32 vf) { return -EACCES; }1092#endif /* CONFIG_IXGBE_IPSEC */1093 1094static inline bool ixgbe_enabled_xdp_adapter(struct ixgbe_adapter *adapter)1095{1096 return !!adapter->xdp_prog;1097}1098 1099#endif /* _IXGBE_H_ */1100