456 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */2/*3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.4 */5 6#ifndef ENA_H7#define ENA_H8 9#include <linux/bitops.h>10#include <linux/dim.h>11#include <linux/etherdevice.h>12#include <linux/if_vlan.h>13#include <linux/inetdevice.h>14#include <linux/interrupt.h>15#include <linux/netdevice.h>16#include <linux/skbuff.h>17#include <net/xdp.h>18#include <uapi/linux/bpf.h>19 20#include "ena_com.h"21#include "ena_eth_com.h"22 23#define DRV_MODULE_GEN_MAJOR 224#define DRV_MODULE_GEN_MINOR 125#define DRV_MODULE_GEN_SUBMINOR 026 27#define DRV_MODULE_NAME "ena"28 29#define DEVICE_NAME "Elastic Network Adapter (ENA)"30 31/* 1 for AENQ + ADMIN */32#define ENA_ADMIN_MSIX_VEC 133#define ENA_MAX_MSIX_VEC(io_queues) (ENA_ADMIN_MSIX_VEC + (io_queues))34 35/* The ENA buffer length fields is 16 bit long. So when PAGE_SIZE == 64kB the36 * driver passes 0.37 * Since the max packet size the ENA handles is ~9kB limit the buffer length to38 * 16kB.39 */40#if PAGE_SIZE > SZ_16K41#define ENA_PAGE_SIZE (_AC(SZ_16K, UL))42#else43#define ENA_PAGE_SIZE PAGE_SIZE44#endif45 46#define ENA_MIN_MSIX_VEC 247 48#define ENA_REG_BAR 049#define ENA_MEM_BAR 250#define ENA_BAR_MASK (BIT(ENA_REG_BAR) | BIT(ENA_MEM_BAR))51 52#define ENA_DEFAULT_RING_SIZE (1024)53#define ENA_MIN_RING_SIZE (256)54 55#define ENA_MIN_RX_BUF_SIZE (2048)56 57#define ENA_MIN_NUM_IO_QUEUES (1)58 59#define ENA_TX_WAKEUP_THRESH (MAX_SKB_FRAGS + 2)60#define ENA_DEFAULT_RX_COPYBREAK (256 - NET_IP_ALIGN)61 62#define ENA_MIN_MTU 12863 64#define ENA_NAME_MAX_LEN 2065#define ENA_IRQNAME_SIZE 4066 67#define ENA_PKT_MAX_BUFS 1968 69#define ENA_RX_RSS_TABLE_LOG_SIZE 770#define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE)71 72/* The number of tx packet completions that will be handled each NAPI poll73 * cycle is ring_size / ENA_TX_POLL_BUDGET_DIVIDER.74 */75#define ENA_TX_POLL_BUDGET_DIVIDER 476 77/* Refill Rx queue when number of required descriptors is above78 * QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER or ENA_RX_REFILL_THRESH_PACKET79 */80#define ENA_RX_REFILL_THRESH_DIVIDER 881#define ENA_RX_REFILL_THRESH_PACKET 25682 83/* Number of queues to check for missing queues per timer service */84#define ENA_MONITORED_TX_QUEUES 485/* Max timeout packets before device reset */86#define MAX_NUM_OF_TIMEOUTED_PACKETS 12887 88#define ENA_TX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1))89 90#define ENA_RX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1))91#define ENA_RX_RING_IDX_ADD(idx, n, ring_size) \92 (((idx) + (n)) & ((ring_size) - 1))93 94#define ENA_IO_TXQ_IDX(q) (2 * (q))95#define ENA_IO_RXQ_IDX(q) (2 * (q) + 1)96#define ENA_IO_TXQ_IDX_TO_COMBINED_IDX(q) ((q) / 2)97#define ENA_IO_RXQ_IDX_TO_COMBINED_IDX(q) (((q) - 1) / 2)98 99#define ENA_MGMNT_IRQ_IDX 0100#define ENA_IO_IRQ_FIRST_IDX 1101#define ENA_IO_IRQ_IDX(q) (ENA_IO_IRQ_FIRST_IDX + (q))102 103#define ENA_ADMIN_POLL_DELAY_US 100104 105/* ENA device should send keep alive msg every 1 sec.106 * We wait for 6 sec just to be on the safe side.107 */108#define ENA_DEVICE_KALIVE_TIMEOUT (6 * HZ)109#define ENA_MAX_NO_INTERRUPT_ITERATIONS 3110 111#define ENA_MMIO_DISABLE_REG_READ BIT(0)112 113struct ena_irq {114 irq_handler_t handler;115 void *data;116 int cpu;117 u32 vector;118 cpumask_t affinity_hint_mask;119 char name[ENA_IRQNAME_SIZE];120};121 122struct ena_napi {123 u8 first_interrupt ____cacheline_aligned;124 u8 interrupts_masked;125 struct napi_struct napi;126 struct ena_ring *tx_ring;127 struct ena_ring *rx_ring;128 u32 qid;129 struct dim dim;130};131 132struct ena_tx_buffer {133 union {134 struct sk_buff *skb;135 /* XDP buffer structure which is used for sending packets in136 * the xdp queues137 */138 struct xdp_frame *xdpf;139 };140 /* num of ena desc for this specific skb141 * (includes data desc and metadata desc)142 */143 u32 tx_descs;144 /* num of buffers used by this skb */145 u32 num_of_bufs;146 147 /* Total size of all buffers in bytes */148 u32 total_tx_size;149 150 /* Indicate if bufs[0] map the linear data of the skb. */151 u8 map_linear_data;152 153 /* Used for detect missing tx packets to limit the number of prints */154 u8 print_once;155 /* Save the last jiffies to detect missing tx packets156 *157 * sets to non zero value on ena_start_xmit and set to zero on158 * napi and timer_Service_routine.159 *160 * while this value is not protected by lock,161 * a given packet is not expected to be handled by ena_start_xmit162 * and by napi/timer_service at the same time.163 */164 unsigned long last_jiffies;165 struct ena_com_buf bufs[ENA_PKT_MAX_BUFS];166} ____cacheline_aligned;167 168struct ena_rx_buffer {169 struct sk_buff *skb;170 struct page *page;171 dma_addr_t dma_addr;172 u32 page_offset;173 u32 buf_offset;174 struct ena_com_buf ena_buf;175} ____cacheline_aligned;176 177struct ena_stats_tx {178 u64 cnt;179 u64 bytes;180 u64 queue_stop;181 u64 prepare_ctx_err;182 u64 queue_wakeup;183 u64 dma_mapping_err;184 u64 linearize;185 u64 linearize_failed;186 u64 napi_comp;187 u64 tx_poll;188 u64 doorbells;189 u64 bad_req_id;190 u64 llq_buffer_copy;191 u64 missed_tx;192 u64 unmask_interrupt;193 u64 last_napi_jiffies;194};195 196struct ena_stats_rx {197 u64 cnt;198 u64 bytes;199 u64 rx_copybreak_pkt;200 u64 csum_good;201 u64 refil_partial;202 u64 csum_bad;203 u64 page_alloc_fail;204 u64 skb_alloc_fail;205 u64 dma_mapping_err;206 u64 bad_desc_num;207 u64 bad_req_id;208 u64 empty_rx_ring;209 u64 csum_unchecked;210 u64 xdp_aborted;211 u64 xdp_drop;212 u64 xdp_pass;213 u64 xdp_tx;214 u64 xdp_invalid;215 u64 xdp_redirect;216};217 218struct ena_ring {219 /* Holds the empty requests for TX/RX220 * out of order completions221 */222 u16 *free_ids;223 224 union {225 struct ena_tx_buffer *tx_buffer_info;226 struct ena_rx_buffer *rx_buffer_info;227 };228 229 /* cache ptr to avoid using the adapter */230 struct device *dev;231 struct pci_dev *pdev;232 struct napi_struct *napi;233 struct net_device *netdev;234 struct ena_com_dev *ena_dev;235 struct ena_adapter *adapter;236 struct ena_com_io_cq *ena_com_io_cq;237 struct ena_com_io_sq *ena_com_io_sq;238 struct bpf_prog *xdp_bpf_prog;239 struct xdp_rxq_info xdp_rxq;240 spinlock_t xdp_tx_lock; /* synchronize XDP TX/Redirect traffic */241 /* Used for rx queues only to point to the xdp tx ring, to242 * which traffic should be redirected from this rx ring.243 */244 struct ena_ring *xdp_ring;245 246 u16 next_to_use;247 u16 next_to_clean;248 u16 rx_copybreak;249 u16 rx_headroom;250 u16 qid;251 u16 mtu;252 u16 sgl_size;253 254 /* The maximum header length the device can handle */255 u8 tx_max_header_size;256 257 bool disable_meta_caching;258 u16 no_interrupt_event_cnt;259 260 /* cpu and NUMA for TPH */261 int cpu;262 int numa_node;263 264 /* number of tx/rx_buffer_info's entries */265 int ring_size;266 267 enum ena_admin_placement_policy_type tx_mem_queue_type;268 269 struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS];270 u32 smoothed_interval;271 u32 per_napi_packets;272 u16 non_empty_napi_events;273 struct u64_stats_sync syncp;274 union {275 struct ena_stats_tx tx_stats;276 struct ena_stats_rx rx_stats;277 };278 279 u8 *push_buf_intermediate_buf;280 int empty_rx_queue;281} ____cacheline_aligned;282 283struct ena_stats_dev {284 u64 tx_timeout;285 u64 suspend;286 u64 resume;287 u64 wd_expired;288 u64 interface_up;289 u64 interface_down;290 u64 admin_q_pause;291 u64 rx_drops;292 u64 tx_drops;293 u64 reset_fail;294};295 296enum ena_flags_t {297 ENA_FLAG_DEVICE_RUNNING,298 ENA_FLAG_DEV_UP,299 ENA_FLAG_LINK_UP,300 ENA_FLAG_MSIX_ENABLED,301 ENA_FLAG_TRIGGER_RESET,302 ENA_FLAG_ONGOING_RESET303};304 305/* adapter specific private data structure */306struct ena_adapter {307 struct ena_com_dev *ena_dev;308 /* OS defined structs */309 struct net_device *netdev;310 struct pci_dev *pdev;311 312 /* rx packets that shorter that this len will be copied to the skb313 * header314 */315 u32 rx_copybreak;316 u32 max_mtu;317 318 u32 num_io_queues;319 u32 max_num_io_queues;320 321 int msix_vecs;322 323 u32 missing_tx_completion_threshold;324 325 u32 requested_tx_ring_size;326 u32 requested_rx_ring_size;327 328 u32 max_tx_ring_size;329 u32 max_rx_ring_size;330 331 u32 msg_enable;332 333 /* large_llq_header_enabled is used for two purposes:334 * 1. Indicates that large LLQ has been requested.335 * 2. Indicates whether large LLQ is set or not after device336 * initialization / configuration.337 */338 bool large_llq_header_enabled;339 bool large_llq_header_supported;340 341 u16 max_tx_sgl_size;342 u16 max_rx_sgl_size;343 344 u8 mac_addr[ETH_ALEN];345 346 unsigned long keep_alive_timeout;347 unsigned long missing_tx_completion_to;348 349 char name[ENA_NAME_MAX_LEN];350 351 unsigned long flags;352 /* TX */353 struct ena_ring tx_ring[ENA_MAX_NUM_IO_QUEUES]354 ____cacheline_aligned_in_smp;355 356 /* RX */357 struct ena_ring rx_ring[ENA_MAX_NUM_IO_QUEUES]358 ____cacheline_aligned_in_smp;359 360 struct ena_napi ena_napi[ENA_MAX_NUM_IO_QUEUES];361 362 struct ena_irq irq_tbl[ENA_MAX_MSIX_VEC(ENA_MAX_NUM_IO_QUEUES)];363 364 /* timer service */365 struct work_struct reset_task;366 struct timer_list timer_service;367 368 bool wd_state;369 bool dev_up_before_reset;370 bool disable_meta_caching;371 unsigned long last_keep_alive_jiffies;372 373 struct u64_stats_sync syncp;374 struct ena_stats_dev dev_stats;375 struct ena_admin_eni_stats eni_stats;376 struct ena_admin_ena_srd_info ena_srd_info;377 378 /* last queue index that was checked for uncompleted tx packets */379 u32 last_monitored_tx_qid;380 381 enum ena_regs_reset_reason_types reset_reason;382 383 struct bpf_prog *xdp_bpf_prog;384 u32 xdp_first_ring;385 u32 xdp_num_queues;386};387 388void ena_set_ethtool_ops(struct net_device *netdev);389 390void ena_dump_stats_to_dmesg(struct ena_adapter *adapter);391 392void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf);393 394 395int ena_update_queue_params(struct ena_adapter *adapter,396 u32 new_tx_size,397 u32 new_rx_size,398 u32 new_llq_header_len);399 400int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count);401 402int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak);403 404int ena_get_sset_count(struct net_device *netdev, int sset);405 406static inline void ena_reset_device(struct ena_adapter *adapter,407 enum ena_regs_reset_reason_types reset_reason)408{409 adapter->reset_reason = reset_reason;410 /* Make sure reset reason is set before triggering the reset */411 smp_mb__before_atomic();412 set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);413}414 415int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,416 struct ena_tx_buffer *tx_info, bool is_xdp);417 418/* Increase a stat by cnt while holding syncp seqlock on 32bit machines */419static inline void ena_increase_stat(u64 *statp, u64 cnt,420 struct u64_stats_sync *syncp)421{422 u64_stats_update_begin(syncp);423 (*statp) += cnt;424 u64_stats_update_end(syncp);425}426 427static inline void ena_ring_tx_doorbell(struct ena_ring *tx_ring)428{429 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);430 ena_increase_stat(&tx_ring->tx_stats.doorbells, 1, &tx_ring->syncp);431}432 433int ena_xmit_common(struct ena_adapter *adapter,434 struct ena_ring *ring,435 struct ena_tx_buffer *tx_info,436 struct ena_com_tx_ctx *ena_tx_ctx,437 u16 next_to_use,438 u32 bytes);439void ena_unmap_tx_buff(struct ena_ring *tx_ring,440 struct ena_tx_buffer *tx_info);441void ena_init_io_rings(struct ena_adapter *adapter,442 int first_index, int count);443int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,444 int first_index, int count);445int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,446 int first_index, int count);447void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter,448 int first_index, int count);449void ena_free_all_io_tx_resources(struct ena_adapter *adapter);450void ena_down(struct ena_adapter *adapter);451int ena_up(struct ena_adapter *adapter);452void ena_unmask_interrupt(struct ena_ring *tx_ring, struct ena_ring *rx_ring);453void ena_update_ring_numa_node(struct ena_ring *tx_ring,454 struct ena_ring *rx_ring);455#endif /* !(ENA_H) */456