1063 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/* Copyright (C) 2023 Intel Corporation */3 4#ifndef _IDPF_TXRX_H_5#define _IDPF_TXRX_H_6 7#include <linux/dim.h>8 9#include <net/libeth/cache.h>10#include <net/tcp.h>11#include <net/netdev_queues.h>12 13#include "idpf_lan_txrx.h"14#include "virtchnl2_lan_desc.h"15 16#define IDPF_LARGE_MAX_Q 25617#define IDPF_MAX_Q 1618#define IDPF_MIN_Q 219/* Mailbox Queue */20#define IDPF_MAX_MBXQ 121 22#define IDPF_MIN_TXQ_DESC 6423#define IDPF_MIN_RXQ_DESC 6424#define IDPF_MIN_TXQ_COMPLQ_DESC 25625#define IDPF_MAX_QIDS 25626 27/* Number of descriptors in a queue should be a multiple of 32. RX queue28 * descriptors alone should be a multiple of IDPF_REQ_RXQ_DESC_MULTIPLE29 * to achieve BufQ descriptors aligned to 3230 */31#define IDPF_REQ_DESC_MULTIPLE 3232#define IDPF_REQ_RXQ_DESC_MULTIPLE (IDPF_MAX_BUFQS_PER_RXQ_GRP * 32)33#define IDPF_MIN_TX_DESC_NEEDED (MAX_SKB_FRAGS + 6)34#define IDPF_TX_WAKE_THRESH ((u16)IDPF_MIN_TX_DESC_NEEDED * 2)35 36#define IDPF_MAX_DESCS 816037#define IDPF_MAX_TXQ_DESC ALIGN_DOWN(IDPF_MAX_DESCS, IDPF_REQ_DESC_MULTIPLE)38#define IDPF_MAX_RXQ_DESC ALIGN_DOWN(IDPF_MAX_DESCS, IDPF_REQ_RXQ_DESC_MULTIPLE)39#define MIN_SUPPORT_TXDID (\40 VIRTCHNL2_TXDID_FLEX_FLOW_SCHED |\41 VIRTCHNL2_TXDID_FLEX_TSO_CTX)42 43#define IDPF_DFLT_SINGLEQ_TX_Q_GROUPS 144#define IDPF_DFLT_SINGLEQ_RX_Q_GROUPS 145#define IDPF_DFLT_SINGLEQ_TXQ_PER_GROUP 446#define IDPF_DFLT_SINGLEQ_RXQ_PER_GROUP 447 48#define IDPF_COMPLQ_PER_GROUP 149#define IDPF_SINGLE_BUFQ_PER_RXQ_GRP 150#define IDPF_MAX_BUFQS_PER_RXQ_GRP 251#define IDPF_BUFQ2_ENA 152#define IDPF_NUMQ_PER_CHUNK 153 54#define IDPF_DFLT_SPLITQ_TXQ_PER_GROUP 155#define IDPF_DFLT_SPLITQ_RXQ_PER_GROUP 156 57/* Default vector sharing */58#define IDPF_MBX_Q_VEC 159#define IDPF_MIN_Q_VEC 160 61#define IDPF_DFLT_TX_Q_DESC_COUNT 51262#define IDPF_DFLT_TX_COMPLQ_DESC_COUNT 51263#define IDPF_DFLT_RX_Q_DESC_COUNT 51264 65/* IMPORTANT: We absolutely _cannot_ have more buffers in the system than a66 * given RX completion queue has descriptors. This includes _ALL_ buffer67 * queues. E.g.: If you have two buffer queues of 512 descriptors and buffers,68 * you have a total of 1024 buffers so your RX queue _must_ have at least that69 * many descriptors. This macro divides a given number of RX descriptors by70 * number of buffer queues to calculate how many descriptors each buffer queue71 * can have without overrunning the RX queue.72 *73 * If you give hardware more buffers than completion descriptors what will74 * happen is that if hardware gets a chance to post more than ring wrap of75 * descriptors before SW gets an interrupt and overwrites SW head, the gen bit76 * in the descriptor will be wrong. Any overwritten descriptors' buffers will77 * be gone forever and SW has no reasonable way to tell that this has happened.78 * From SW perspective, when we finally get an interrupt, it looks like we're79 * still waiting for descriptor to be done, stalling forever.80 */81#define IDPF_RX_BUFQ_DESC_COUNT(RXD, NUM_BUFQ) ((RXD) / (NUM_BUFQ))82 83#define IDPF_RX_BUFQ_WORKING_SET(rxq) ((rxq)->desc_count - 1)84 85#define IDPF_RX_BUMP_NTC(rxq, ntc) \86do { \87 if (unlikely(++(ntc) == (rxq)->desc_count)) { \88 ntc = 0; \89 idpf_queue_change(GEN_CHK, rxq); \90 } \91} while (0)92 93#define IDPF_SINGLEQ_BUMP_RING_IDX(q, idx) \94do { \95 if (unlikely(++(idx) == (q)->desc_count)) \96 idx = 0; \97} while (0)98 99#define IDPF_RX_BUF_STRIDE 32100#define IDPF_RX_BUF_POST_STRIDE 16101#define IDPF_LOW_WATERMARK 64102 103#define IDPF_TX_TSO_MIN_MSS 88104 105/* Minimum number of descriptors between 2 descriptors with the RE bit set;106 * only relevant in flow scheduling mode107 */108#define IDPF_TX_SPLITQ_RE_MIN_GAP 64109 110#define IDPF_RX_BI_GEN_M BIT(16)111#define IDPF_RX_BI_BUFID_M GENMASK(15, 0)112 113#define IDPF_RXD_EOF_SPLITQ VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_EOF_M114#define IDPF_RXD_EOF_SINGLEQ VIRTCHNL2_RX_BASE_DESC_STATUS_EOF_M115 116#define IDPF_DESC_UNUSED(txq) \117 ((((txq)->next_to_clean > (txq)->next_to_use) ? 0 : (txq)->desc_count) + \118 (txq)->next_to_clean - (txq)->next_to_use - 1)119 120#define IDPF_TX_BUF_RSV_UNUSED(txq) ((txq)->stash->buf_stack.top)121#define IDPF_TX_BUF_RSV_LOW(txq) (IDPF_TX_BUF_RSV_UNUSED(txq) < \122 (txq)->desc_count >> 2)123 124#define IDPF_TX_COMPLQ_OVERFLOW_THRESH(txcq) ((txcq)->desc_count >> 1)125/* Determine the absolute number of completions pending, i.e. the number of126 * completions that are expected to arrive on the TX completion queue.127 */128#define IDPF_TX_COMPLQ_PENDING(txq) \129 (((txq)->num_completions_pending >= (txq)->complq->num_completions ? \130 0 : U32_MAX) + \131 (txq)->num_completions_pending - (txq)->complq->num_completions)132 133#define IDPF_TX_SPLITQ_COMPL_TAG_WIDTH 16134/* Adjust the generation for the completion tag and wrap if necessary */135#define IDPF_TX_ADJ_COMPL_TAG_GEN(txq) \136 ((++(txq)->compl_tag_cur_gen) >= (txq)->compl_tag_gen_max ? \137 0 : (txq)->compl_tag_cur_gen)138 139#define IDPF_TXD_LAST_DESC_CMD (IDPF_TX_DESC_CMD_EOP | IDPF_TX_DESC_CMD_RS)140 141#define IDPF_TX_FLAGS_TSO BIT(0)142#define IDPF_TX_FLAGS_IPV4 BIT(1)143#define IDPF_TX_FLAGS_IPV6 BIT(2)144#define IDPF_TX_FLAGS_TUNNEL BIT(3)145 146union idpf_tx_flex_desc {147 struct idpf_flex_tx_desc q; /* queue based scheduling */148 struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */149};150 151#define idpf_tx_buf libeth_sqe152 153/**154 * struct idpf_buf_lifo - LIFO for managing OOO completions155 * @top: Used to know how many buffers are left156 * @size: Total size of LIFO157 * @bufs: Backing array158 */159struct idpf_buf_lifo {160 u16 top;161 u16 size;162 struct idpf_tx_stash **bufs;163};164 165/**166 * struct idpf_tx_offload_params - Offload parameters for a given packet167 * @tx_flags: Feature flags enabled for this packet168 * @hdr_offsets: Offset parameter for single queue model169 * @cd_tunneling: Type of tunneling enabled for single queue model170 * @tso_len: Total length of payload to segment171 * @mss: Segment size172 * @tso_segs: Number of segments to be sent173 * @tso_hdr_len: Length of headers to be duplicated174 * @td_cmd: Command field to be inserted into descriptor175 */176struct idpf_tx_offload_params {177 u32 tx_flags;178 179 u32 hdr_offsets;180 u32 cd_tunneling;181 182 u32 tso_len;183 u16 mss;184 u16 tso_segs;185 u16 tso_hdr_len;186 187 u16 td_cmd;188};189 190/**191 * struct idpf_tx_splitq_params192 * @dtype: General descriptor info193 * @eop_cmd: Type of EOP194 * @compl_tag: Associated tag for completion195 * @td_tag: Descriptor tunneling tag196 * @offload: Offload parameters197 */198struct idpf_tx_splitq_params {199 enum idpf_tx_desc_dtype_value dtype;200 u16 eop_cmd;201 union {202 u16 compl_tag;203 u16 td_tag;204 };205 206 struct idpf_tx_offload_params offload;207};208 209enum idpf_tx_ctx_desc_eipt_offload {210 IDPF_TX_CTX_EXT_IP_NONE = 0x0,211 IDPF_TX_CTX_EXT_IP_IPV6 = 0x1,212 IDPF_TX_CTX_EXT_IP_IPV4_NO_CSUM = 0x2,213 IDPF_TX_CTX_EXT_IP_IPV4 = 0x3214};215 216/* Checksum offload bits decoded from the receive descriptor. */217struct idpf_rx_csum_decoded {218 u32 l3l4p : 1;219 u32 ipe : 1;220 u32 eipe : 1;221 u32 eudpe : 1;222 u32 ipv6exadd : 1;223 u32 l4e : 1;224 u32 pprs : 1;225 u32 nat : 1;226 u32 raw_csum_inv : 1;227 u32 raw_csum : 16;228};229 230struct idpf_rx_extracted {231 unsigned int size;232 u16 rx_ptype;233};234 235#define IDPF_TX_COMPLQ_CLEAN_BUDGET 256236#define IDPF_TX_MIN_PKT_LEN 17237#define IDPF_TX_DESCS_FOR_SKB_DATA_PTR 1238#define IDPF_TX_DESCS_PER_CACHE_LINE (L1_CACHE_BYTES / \239 sizeof(struct idpf_flex_tx_desc))240#define IDPF_TX_DESCS_FOR_CTX 1241/* TX descriptors needed, worst case */242#define IDPF_TX_DESC_NEEDED (MAX_SKB_FRAGS + IDPF_TX_DESCS_FOR_CTX + \243 IDPF_TX_DESCS_PER_CACHE_LINE + \244 IDPF_TX_DESCS_FOR_SKB_DATA_PTR)245 246/* The size limit for a transmit buffer in a descriptor is (16K - 1).247 * In order to align with the read requests we will align the value to248 * the nearest 4K which represents our maximum read request size.249 */250#define IDPF_TX_MAX_READ_REQ_SIZE SZ_4K251#define IDPF_TX_MAX_DESC_DATA (SZ_16K - 1)252#define IDPF_TX_MAX_DESC_DATA_ALIGNED \253 ALIGN_DOWN(IDPF_TX_MAX_DESC_DATA, IDPF_TX_MAX_READ_REQ_SIZE)254 255#define idpf_rx_buf libeth_fqe256 257#define IDPF_RX_MAX_PTYPE_PROTO_IDS 32258#define IDPF_RX_MAX_PTYPE_SZ (sizeof(struct virtchnl2_ptype) + \259 (sizeof(u16) * IDPF_RX_MAX_PTYPE_PROTO_IDS))260#define IDPF_RX_PTYPE_HDR_SZ sizeof(struct virtchnl2_get_ptype_info)261#define IDPF_RX_MAX_PTYPES_PER_BUF \262 DIV_ROUND_DOWN_ULL((IDPF_CTLQ_MAX_BUF_LEN - IDPF_RX_PTYPE_HDR_SZ), \263 IDPF_RX_MAX_PTYPE_SZ)264 265#define IDPF_GET_PTYPE_SIZE(p) struct_size((p), proto_id, (p)->proto_id_count)266 267#define IDPF_TUN_IP_GRE (\268 IDPF_PTYPE_TUNNEL_IP |\269 IDPF_PTYPE_TUNNEL_IP_GRENAT)270 271#define IDPF_TUN_IP_GRE_MAC (\272 IDPF_TUN_IP_GRE |\273 IDPF_PTYPE_TUNNEL_IP_GRENAT_MAC)274 275#define IDPF_RX_MAX_PTYPE 1024276#define IDPF_RX_MAX_BASE_PTYPE 256277#define IDPF_INVALID_PTYPE_ID 0xFFFF278 279enum idpf_tunnel_state {280 IDPF_PTYPE_TUNNEL_IP = BIT(0),281 IDPF_PTYPE_TUNNEL_IP_GRENAT = BIT(1),282 IDPF_PTYPE_TUNNEL_IP_GRENAT_MAC = BIT(2),283};284 285struct idpf_ptype_state {286 bool outer_ip:1;287 bool outer_frag:1;288 u8 tunnel_state:6;289};290 291/**292 * enum idpf_queue_flags_t293 * @__IDPF_Q_GEN_CHK: Queues operating in splitq mode use a generation bit to294 * identify new descriptor writebacks on the ring. HW sets295 * the gen bit to 1 on the first writeback of any given296 * descriptor. After the ring wraps, HW sets the gen bit of297 * those descriptors to 0, and continues flipping298 * 0->1 or 1->0 on each ring wrap. SW maintains its own299 * gen bit to know what value will indicate writebacks on300 * the next pass around the ring. E.g. it is initialized301 * to 1 and knows that reading a gen bit of 1 in any302 * descriptor on the initial pass of the ring indicates a303 * writeback. It also flips on every ring wrap.304 * @__IDPF_Q_RFL_GEN_CHK: Refill queues are SW only, so Q_GEN acts as the HW305 * bit and Q_RFL_GEN is the SW bit.306 * @__IDPF_Q_FLOW_SCH_EN: Enable flow scheduling307 * @__IDPF_Q_SW_MARKER: Used to indicate TX queue marker completions308 * @__IDPF_Q_POLL_MODE: Enable poll mode309 * @__IDPF_Q_CRC_EN: enable CRC offload in singleq mode310 * @__IDPF_Q_HSPLIT_EN: enable header split on Rx (splitq)311 * @__IDPF_Q_FLAGS_NBITS: Must be last312 */313enum idpf_queue_flags_t {314 __IDPF_Q_GEN_CHK,315 __IDPF_Q_RFL_GEN_CHK,316 __IDPF_Q_FLOW_SCH_EN,317 __IDPF_Q_SW_MARKER,318 __IDPF_Q_POLL_MODE,319 __IDPF_Q_CRC_EN,320 __IDPF_Q_HSPLIT_EN,321 322 __IDPF_Q_FLAGS_NBITS,323};324 325#define idpf_queue_set(f, q) __set_bit(__IDPF_Q_##f, (q)->flags)326#define idpf_queue_clear(f, q) __clear_bit(__IDPF_Q_##f, (q)->flags)327#define idpf_queue_change(f, q) __change_bit(__IDPF_Q_##f, (q)->flags)328#define idpf_queue_has(f, q) test_bit(__IDPF_Q_##f, (q)->flags)329 330#define idpf_queue_has_clear(f, q) \331 __test_and_clear_bit(__IDPF_Q_##f, (q)->flags)332#define idpf_queue_assign(f, q, v) \333 __assign_bit(__IDPF_Q_##f, (q)->flags, v)334 335/**336 * struct idpf_vec_regs337 * @dyn_ctl_reg: Dynamic control interrupt register offset338 * @itrn_reg: Interrupt Throttling Rate register offset339 * @itrn_index_spacing: Register spacing between ITR registers of the same340 * vector341 */342struct idpf_vec_regs {343 u32 dyn_ctl_reg;344 u32 itrn_reg;345 u32 itrn_index_spacing;346};347 348/**349 * struct idpf_intr_reg350 * @dyn_ctl: Dynamic control interrupt register351 * @dyn_ctl_intena_m: Mask for dyn_ctl interrupt enable352 * @dyn_ctl_intena_msk_m: Mask for dyn_ctl interrupt enable mask353 * @dyn_ctl_itridx_s: Register bit offset for ITR index354 * @dyn_ctl_itridx_m: Mask for ITR index355 * @dyn_ctl_intrvl_s: Register bit offset for ITR interval356 * @dyn_ctl_wb_on_itr_m: Mask for WB on ITR feature357 * @rx_itr: RX ITR register358 * @tx_itr: TX ITR register359 * @icr_ena: Interrupt cause register offset360 * @icr_ena_ctlq_m: Mask for ICR361 */362struct idpf_intr_reg {363 void __iomem *dyn_ctl;364 u32 dyn_ctl_intena_m;365 u32 dyn_ctl_intena_msk_m;366 u32 dyn_ctl_itridx_s;367 u32 dyn_ctl_itridx_m;368 u32 dyn_ctl_intrvl_s;369 u32 dyn_ctl_wb_on_itr_m;370 void __iomem *rx_itr;371 void __iomem *tx_itr;372 void __iomem *icr_ena;373 u32 icr_ena_ctlq_m;374};375 376/**377 * struct idpf_q_vector378 * @vport: Vport back pointer379 * @num_rxq: Number of RX queues380 * @num_txq: Number of TX queues381 * @num_bufq: Number of buffer queues382 * @num_complq: number of completion queues383 * @rx: Array of RX queues to service384 * @tx: Array of TX queues to service385 * @bufq: Array of buffer queues to service386 * @complq: array of completion queues387 * @intr_reg: See struct idpf_intr_reg388 * @napi: napi handler389 * @total_events: Number of interrupts processed390 * @wb_on_itr: whether WB on ITR is enabled391 * @tx_dim: Data for TX net_dim algorithm392 * @tx_itr_value: TX interrupt throttling rate393 * @tx_intr_mode: Dynamic ITR or not394 * @tx_itr_idx: TX ITR index395 * @rx_dim: Data for RX net_dim algorithm396 * @rx_itr_value: RX interrupt throttling rate397 * @rx_intr_mode: Dynamic ITR or not398 * @rx_itr_idx: RX ITR index399 * @v_idx: Vector index400 * @affinity_mask: CPU affinity mask401 */402struct idpf_q_vector {403 __cacheline_group_begin_aligned(read_mostly);404 struct idpf_vport *vport;405 406 u16 num_rxq;407 u16 num_txq;408 u16 num_bufq;409 u16 num_complq;410 struct idpf_rx_queue **rx;411 struct idpf_tx_queue **tx;412 struct idpf_buf_queue **bufq;413 struct idpf_compl_queue **complq;414 415 struct idpf_intr_reg intr_reg;416 __cacheline_group_end_aligned(read_mostly);417 418 __cacheline_group_begin_aligned(read_write);419 struct napi_struct napi;420 u16 total_events;421 bool wb_on_itr;422 423 struct dim tx_dim;424 u16 tx_itr_value;425 bool tx_intr_mode;426 u32 tx_itr_idx;427 428 struct dim rx_dim;429 u16 rx_itr_value;430 bool rx_intr_mode;431 u32 rx_itr_idx;432 __cacheline_group_end_aligned(read_write);433 434 __cacheline_group_begin_aligned(cold);435 u16 v_idx;436 437 cpumask_var_t affinity_mask;438 __cacheline_group_end_aligned(cold);439};440libeth_cacheline_set_assert(struct idpf_q_vector, 112,441 424 + 2 * sizeof(struct dim),442 8 + sizeof(cpumask_var_t));443 444struct idpf_rx_queue_stats {445 u64_stats_t packets;446 u64_stats_t bytes;447 u64_stats_t rsc_pkts;448 u64_stats_t hw_csum_err;449 u64_stats_t hsplit_pkts;450 u64_stats_t hsplit_buf_ovf;451 u64_stats_t bad_descs;452};453 454struct idpf_tx_queue_stats {455 u64_stats_t packets;456 u64_stats_t bytes;457 u64_stats_t lso_pkts;458 u64_stats_t linearize;459 u64_stats_t q_busy;460 u64_stats_t skb_drops;461 u64_stats_t dma_map_errs;462};463 464#define IDPF_ITR_DYNAMIC 1465#define IDPF_ITR_MAX 0x1FE0466#define IDPF_ITR_20K 0x0032467#define IDPF_ITR_GRAN_S 1 /* Assume ITR granularity is 2us */468#define IDPF_ITR_MASK 0x1FFE /* ITR register value alignment mask */469#define ITR_REG_ALIGN(setting) ((setting) & IDPF_ITR_MASK)470#define IDPF_ITR_IS_DYNAMIC(itr_mode) (itr_mode)471#define IDPF_ITR_TX_DEF IDPF_ITR_20K472#define IDPF_ITR_RX_DEF IDPF_ITR_20K473/* Index used for 'No ITR' update in DYN_CTL register */474#define IDPF_NO_ITR_UPDATE_IDX 3475#define IDPF_ITR_IDX_SPACING(spacing, dflt) (spacing ? spacing : dflt)476#define IDPF_DIM_DEFAULT_PROFILE_IX 1477 478/**479 * struct idpf_txq_stash - Tx buffer stash for Flow-based scheduling mode480 * @buf_stack: Stack of empty buffers to store buffer info for out of order481 * buffer completions. See struct idpf_buf_lifo482 * @sched_buf_hash: Hash table to store buffers483 */484struct idpf_txq_stash {485 struct idpf_buf_lifo buf_stack;486 DECLARE_HASHTABLE(sched_buf_hash, 12);487} ____cacheline_aligned;488 489/**490 * struct idpf_rx_queue - software structure representing a receive queue491 * @rx: universal receive descriptor array492 * @single_buf: buffer descriptor array in singleq493 * @desc_ring: virtual descriptor ring address494 * @bufq_sets: Pointer to the array of buffer queues in splitq mode495 * @napi: NAPI instance corresponding to this queue (splitq)496 * @rx_buf: See struct &libeth_fqe497 * @pp: Page pool pointer in singleq mode498 * @netdev: &net_device corresponding to this queue499 * @tail: Tail offset. Used for both queue models single and split.500 * @flags: See enum idpf_queue_flags_t501 * @idx: For RX queue, it is used to index to total RX queue across groups and502 * used for skb reporting.503 * @desc_count: Number of descriptors504 * @rxdids: Supported RX descriptor ids505 * @rx_ptype_lkup: LUT of Rx ptypes506 * @next_to_use: Next descriptor to use507 * @next_to_clean: Next descriptor to clean508 * @next_to_alloc: RX buffer to allocate at509 * @skb: Pointer to the skb510 * @truesize: data buffer truesize in singleq511 * @stats_sync: See struct u64_stats_sync512 * @q_stats: See union idpf_rx_queue_stats513 * @q_id: Queue id514 * @size: Length of descriptor ring in bytes515 * @dma: Physical address of ring516 * @q_vector: Backreference to associated vector517 * @rx_buffer_low_watermark: RX buffer low watermark518 * @rx_hbuf_size: Header buffer size519 * @rx_buf_size: Buffer size520 * @rx_max_pkt_size: RX max packet size521 */522struct idpf_rx_queue {523 __cacheline_group_begin_aligned(read_mostly);524 union {525 union virtchnl2_rx_desc *rx;526 struct virtchnl2_singleq_rx_buf_desc *single_buf;527 528 void *desc_ring;529 };530 union {531 struct {532 struct idpf_bufq_set *bufq_sets;533 struct napi_struct *napi;534 };535 struct {536 struct libeth_fqe *rx_buf;537 struct page_pool *pp;538 };539 };540 struct net_device *netdev;541 void __iomem *tail;542 543 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS);544 u16 idx;545 u16 desc_count;546 547 u32 rxdids;548 const struct libeth_rx_pt *rx_ptype_lkup;549 __cacheline_group_end_aligned(read_mostly);550 551 __cacheline_group_begin_aligned(read_write);552 u16 next_to_use;553 u16 next_to_clean;554 u16 next_to_alloc;555 556 struct sk_buff *skb;557 u32 truesize;558 559 struct u64_stats_sync stats_sync;560 struct idpf_rx_queue_stats q_stats;561 __cacheline_group_end_aligned(read_write);562 563 __cacheline_group_begin_aligned(cold);564 u32 q_id;565 u32 size;566 dma_addr_t dma;567 568 struct idpf_q_vector *q_vector;569 570 u16 rx_buffer_low_watermark;571 u16 rx_hbuf_size;572 u16 rx_buf_size;573 u16 rx_max_pkt_size;574 __cacheline_group_end_aligned(cold);575};576libeth_cacheline_set_assert(struct idpf_rx_queue, 64,577 80 + sizeof(struct u64_stats_sync),578 32);579 580/**581 * struct idpf_tx_queue - software structure representing a transmit queue582 * @base_tx: base Tx descriptor array583 * @base_ctx: base Tx context descriptor array584 * @flex_tx: flex Tx descriptor array585 * @flex_ctx: flex Tx context descriptor array586 * @desc_ring: virtual descriptor ring address587 * @tx_buf: See struct idpf_tx_buf588 * @txq_grp: See struct idpf_txq_group589 * @dev: Device back pointer for DMA mapping590 * @tail: Tail offset. Used for both queue models single and split591 * @flags: See enum idpf_queue_flags_t592 * @idx: For TX queue, it is used as index to map between TX queue group and593 * hot path TX pointers stored in vport. Used in both singleq/splitq.594 * @desc_count: Number of descriptors595 * @tx_min_pkt_len: Min supported packet length596 * @compl_tag_gen_s: Completion tag generation bit597 * The format of the completion tag will change based on the TXQ598 * descriptor ring size so that we can maintain roughly the same level599 * of "uniqueness" across all descriptor sizes. For example, if the600 * TXQ descriptor ring size is 64 (the minimum size supported), the601 * completion tag will be formatted as below:602 * 15 6 5 0603 * --------------------------------604 * | GEN=0-1023 |IDX = 0-63|605 * --------------------------------606 *607 * This gives us 64*1024 = 65536 possible unique values. Similarly, if608 * the TXQ descriptor ring size is 8160 (the maximum size supported),609 * the completion tag will be formatted as below:610 * 15 13 12 0611 * --------------------------------612 * |GEN | IDX = 0-8159 |613 * --------------------------------614 *615 * This gives us 8*8160 = 65280 possible unique values.616 * @netdev: &net_device corresponding to this queue617 * @next_to_use: Next descriptor to use618 * @next_to_clean: Next descriptor to clean619 * @cleaned_bytes: Splitq only, TXQ only: When a TX completion is received on620 * the TX completion queue, it can be for any TXQ associated621 * with that completion queue. This means we can clean up to622 * N TXQs during a single call to clean the completion queue.623 * cleaned_bytes|pkts tracks the clean stats per TXQ during624 * that single call to clean the completion queue. By doing so,625 * we can update BQL with aggregate cleaned stats for each TXQ626 * only once at the end of the cleaning routine.627 * @clean_budget: singleq only, queue cleaning budget628 * @cleaned_pkts: Number of packets cleaned for the above said case629 * @tx_max_bufs: Max buffers that can be transmitted with scatter-gather630 * @stash: Tx buffer stash for Flow-based scheduling mode631 * @compl_tag_bufid_m: Completion tag buffer id mask632 * @compl_tag_cur_gen: Used to keep track of current completion tag generation633 * @compl_tag_gen_max: To determine when compl_tag_cur_gen should be reset634 * @stats_sync: See struct u64_stats_sync635 * @q_stats: See union idpf_tx_queue_stats636 * @q_id: Queue id637 * @size: Length of descriptor ring in bytes638 * @dma: Physical address of ring639 * @q_vector: Backreference to associated vector640 */641struct idpf_tx_queue {642 __cacheline_group_begin_aligned(read_mostly);643 union {644 struct idpf_base_tx_desc *base_tx;645 struct idpf_base_tx_ctx_desc *base_ctx;646 union idpf_tx_flex_desc *flex_tx;647 struct idpf_flex_tx_ctx_desc *flex_ctx;648 649 void *desc_ring;650 };651 struct libeth_sqe *tx_buf;652 struct idpf_txq_group *txq_grp;653 struct device *dev;654 void __iomem *tail;655 656 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS);657 u16 idx;658 u16 desc_count;659 660 u16 tx_min_pkt_len;661 u16 compl_tag_gen_s;662 663 struct net_device *netdev;664 __cacheline_group_end_aligned(read_mostly);665 666 __cacheline_group_begin_aligned(read_write);667 u16 next_to_use;668 u16 next_to_clean;669 670 union {671 u32 cleaned_bytes;672 u32 clean_budget;673 };674 u16 cleaned_pkts;675 676 u16 tx_max_bufs;677 struct idpf_txq_stash *stash;678 679 u16 compl_tag_bufid_m;680 u16 compl_tag_cur_gen;681 u16 compl_tag_gen_max;682 683 struct u64_stats_sync stats_sync;684 struct idpf_tx_queue_stats q_stats;685 __cacheline_group_end_aligned(read_write);686 687 __cacheline_group_begin_aligned(cold);688 u32 q_id;689 u32 size;690 dma_addr_t dma;691 692 struct idpf_q_vector *q_vector;693 __cacheline_group_end_aligned(cold);694};695libeth_cacheline_set_assert(struct idpf_tx_queue, 64,696 88 + sizeof(struct u64_stats_sync),697 24);698 699/**700 * struct idpf_buf_queue - software structure representing a buffer queue701 * @split_buf: buffer descriptor array702 * @hdr_buf: &libeth_fqe for header buffers703 * @hdr_pp: &page_pool for header buffers704 * @buf: &libeth_fqe for data buffers705 * @pp: &page_pool for data buffers706 * @tail: Tail offset707 * @flags: See enum idpf_queue_flags_t708 * @desc_count: Number of descriptors709 * @next_to_use: Next descriptor to use710 * @next_to_clean: Next descriptor to clean711 * @next_to_alloc: RX buffer to allocate at712 * @hdr_truesize: truesize for buffer headers713 * @truesize: truesize for data buffers714 * @q_id: Queue id715 * @size: Length of descriptor ring in bytes716 * @dma: Physical address of ring717 * @q_vector: Backreference to associated vector718 * @rx_buffer_low_watermark: RX buffer low watermark719 * @rx_hbuf_size: Header buffer size720 * @rx_buf_size: Buffer size721 */722struct idpf_buf_queue {723 __cacheline_group_begin_aligned(read_mostly);724 struct virtchnl2_splitq_rx_buf_desc *split_buf;725 struct libeth_fqe *hdr_buf;726 struct page_pool *hdr_pp;727 struct libeth_fqe *buf;728 struct page_pool *pp;729 void __iomem *tail;730 731 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS);732 u32 desc_count;733 __cacheline_group_end_aligned(read_mostly);734 735 __cacheline_group_begin_aligned(read_write);736 u32 next_to_use;737 u32 next_to_clean;738 u32 next_to_alloc;739 740 u32 hdr_truesize;741 u32 truesize;742 __cacheline_group_end_aligned(read_write);743 744 __cacheline_group_begin_aligned(cold);745 u32 q_id;746 u32 size;747 dma_addr_t dma;748 749 struct idpf_q_vector *q_vector;750 751 u16 rx_buffer_low_watermark;752 u16 rx_hbuf_size;753 u16 rx_buf_size;754 __cacheline_group_end_aligned(cold);755};756libeth_cacheline_set_assert(struct idpf_buf_queue, 64, 24, 32);757 758/**759 * struct idpf_compl_queue - software structure representing a completion queue760 * @comp: completion descriptor array761 * @txq_grp: See struct idpf_txq_group762 * @flags: See enum idpf_queue_flags_t763 * @desc_count: Number of descriptors764 * @clean_budget: queue cleaning budget765 * @netdev: &net_device corresponding to this queue766 * @next_to_use: Next descriptor to use. Relevant in both split & single txq767 * and bufq.768 * @next_to_clean: Next descriptor to clean769 * @num_completions: Only relevant for TX completion queue. It tracks the770 * number of completions received to compare against the771 * number of completions pending, as accumulated by the772 * TX queues.773 * @q_id: Queue id774 * @size: Length of descriptor ring in bytes775 * @dma: Physical address of ring776 * @q_vector: Backreference to associated vector777 */778struct idpf_compl_queue {779 __cacheline_group_begin_aligned(read_mostly);780 struct idpf_splitq_tx_compl_desc *comp;781 struct idpf_txq_group *txq_grp;782 783 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS);784 u32 desc_count;785 786 u32 clean_budget;787 struct net_device *netdev;788 __cacheline_group_end_aligned(read_mostly);789 790 __cacheline_group_begin_aligned(read_write);791 u32 next_to_use;792 u32 next_to_clean;793 794 aligned_u64 num_completions;795 __cacheline_group_end_aligned(read_write);796 797 __cacheline_group_begin_aligned(cold);798 u32 q_id;799 u32 size;800 dma_addr_t dma;801 802 struct idpf_q_vector *q_vector;803 __cacheline_group_end_aligned(cold);804};805libeth_cacheline_set_assert(struct idpf_compl_queue, 40, 16, 24);806 807/**808 * struct idpf_sw_queue809 * @ring: Pointer to the ring810 * @flags: See enum idpf_queue_flags_t811 * @desc_count: Descriptor count812 * @next_to_use: Buffer to allocate at813 * @next_to_clean: Next descriptor to clean814 *815 * Software queues are used in splitq mode to manage buffers between rxq816 * producer and the bufq consumer. These are required in order to maintain a817 * lockless buffer management system and are strictly software only constructs.818 */819struct idpf_sw_queue {820 __cacheline_group_begin_aligned(read_mostly);821 u32 *ring;822 823 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS);824 u32 desc_count;825 __cacheline_group_end_aligned(read_mostly);826 827 __cacheline_group_begin_aligned(read_write);828 u32 next_to_use;829 u32 next_to_clean;830 __cacheline_group_end_aligned(read_write);831};832libeth_cacheline_group_assert(struct idpf_sw_queue, read_mostly, 24);833libeth_cacheline_group_assert(struct idpf_sw_queue, read_write, 8);834libeth_cacheline_struct_assert(struct idpf_sw_queue, 24, 8);835 836/**837 * struct idpf_rxq_set838 * @rxq: RX queue839 * @refillq: pointers to refill queues840 *841 * Splitq only. idpf_rxq_set associates an rxq with at an array of refillqs.842 * Each rxq needs a refillq to return used buffers back to the respective bufq.843 * Bufqs then clean these refillqs for buffers to give to hardware.844 */845struct idpf_rxq_set {846 struct idpf_rx_queue rxq;847 struct idpf_sw_queue *refillq[IDPF_MAX_BUFQS_PER_RXQ_GRP];848};849 850/**851 * struct idpf_bufq_set852 * @bufq: Buffer queue853 * @num_refillqs: Number of refill queues. This is always equal to num_rxq_sets854 * in idpf_rxq_group.855 * @refillqs: Pointer to refill queues array.856 *857 * Splitq only. idpf_bufq_set associates a bufq to an array of refillqs.858 * In this bufq_set, there will be one refillq for each rxq in this rxq_group.859 * Used buffers received by rxqs will be put on refillqs which bufqs will860 * clean to return new buffers back to hardware.861 *862 * Buffers needed by some number of rxqs associated in this rxq_group are863 * managed by at most two bufqs (depending on performance configuration).864 */865struct idpf_bufq_set {866 struct idpf_buf_queue bufq;867 int num_refillqs;868 struct idpf_sw_queue *refillqs;869};870 871/**872 * struct idpf_rxq_group873 * @vport: Vport back pointer874 * @singleq: Struct with single queue related members875 * @singleq.num_rxq: Number of RX queues associated876 * @singleq.rxqs: Array of RX queue pointers877 * @splitq: Struct with split queue related members878 * @splitq.num_rxq_sets: Number of RX queue sets879 * @splitq.rxq_sets: Array of RX queue sets880 * @splitq.bufq_sets: Buffer queue set pointer881 *882 * In singleq mode, an rxq_group is simply an array of rxqs. In splitq, a883 * rxq_group contains all the rxqs, bufqs and refillqs needed to884 * manage buffers in splitq mode.885 */886struct idpf_rxq_group {887 struct idpf_vport *vport;888 889 union {890 struct {891 u16 num_rxq;892 struct idpf_rx_queue *rxqs[IDPF_LARGE_MAX_Q];893 } singleq;894 struct {895 u16 num_rxq_sets;896 struct idpf_rxq_set *rxq_sets[IDPF_LARGE_MAX_Q];897 struct idpf_bufq_set *bufq_sets;898 } splitq;899 };900};901 902/**903 * struct idpf_txq_group904 * @vport: Vport back pointer905 * @num_txq: Number of TX queues associated906 * @txqs: Array of TX queue pointers907 * @stashes: array of OOO stashes for the queues908 * @complq: Associated completion queue pointer, split queue only909 * @num_completions_pending: Total number of completions pending for the910 * completion queue, acculumated for all TX queues911 * associated with that completion queue.912 *913 * Between singleq and splitq, a txq_group is largely the same except for the914 * complq. In splitq a single complq is responsible for handling completions915 * for some number of txqs associated in this txq_group.916 */917struct idpf_txq_group {918 struct idpf_vport *vport;919 920 u16 num_txq;921 struct idpf_tx_queue *txqs[IDPF_LARGE_MAX_Q];922 struct idpf_txq_stash *stashes;923 924 struct idpf_compl_queue *complq;925 926 aligned_u64 num_completions_pending;927};928 929static inline int idpf_q_vector_to_mem(const struct idpf_q_vector *q_vector)930{931 u32 cpu;932 933 if (!q_vector)934 return NUMA_NO_NODE;935 936 cpu = cpumask_first(q_vector->affinity_mask);937 938 return cpu < nr_cpu_ids ? cpu_to_mem(cpu) : NUMA_NO_NODE;939}940 941/**942 * idpf_size_to_txd_count - Get number of descriptors needed for large Tx frag943 * @size: transmit request size in bytes944 *945 * In the case where a large frag (>= 16K) needs to be split across multiple946 * descriptors, we need to assume that we can have no more than 12K of data947 * per descriptor due to hardware alignment restrictions (4K alignment).948 */949static inline u32 idpf_size_to_txd_count(unsigned int size)950{951 return DIV_ROUND_UP(size, IDPF_TX_MAX_DESC_DATA_ALIGNED);952}953 954/**955 * idpf_tx_singleq_build_ctob - populate command tag offset and size956 * @td_cmd: Command to be filled in desc957 * @td_offset: Offset to be filled in desc958 * @size: Size of the buffer959 * @td_tag: td tag to be filled960 *961 * Returns the 64 bit value populated with the input parameters962 */963static inline __le64 idpf_tx_singleq_build_ctob(u64 td_cmd, u64 td_offset,964 unsigned int size, u64 td_tag)965{966 return cpu_to_le64(IDPF_TX_DESC_DTYPE_DATA |967 (td_cmd << IDPF_TXD_QW1_CMD_S) |968 (td_offset << IDPF_TXD_QW1_OFFSET_S) |969 ((u64)size << IDPF_TXD_QW1_TX_BUF_SZ_S) |970 (td_tag << IDPF_TXD_QW1_L2TAG1_S));971}972 973void idpf_tx_splitq_build_ctb(union idpf_tx_flex_desc *desc,974 struct idpf_tx_splitq_params *params,975 u16 td_cmd, u16 size);976void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc,977 struct idpf_tx_splitq_params *params,978 u16 td_cmd, u16 size);979/**980 * idpf_tx_splitq_build_desc - determine which type of data descriptor to build981 * @desc: descriptor to populate982 * @params: pointer to tx params struct983 * @td_cmd: command to be filled in desc984 * @size: size of buffer985 */986static inline void idpf_tx_splitq_build_desc(union idpf_tx_flex_desc *desc,987 struct idpf_tx_splitq_params *params,988 u16 td_cmd, u16 size)989{990 if (params->dtype == IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2)991 idpf_tx_splitq_build_ctb(desc, params, td_cmd, size);992 else993 idpf_tx_splitq_build_flow_desc(desc, params, td_cmd, size);994}995 996/**997 * idpf_vport_intr_set_wb_on_itr - enable descriptor writeback on disabled interrupts998 * @q_vector: pointer to queue vector struct999 */1000static inline void idpf_vport_intr_set_wb_on_itr(struct idpf_q_vector *q_vector)1001{1002 struct idpf_intr_reg *reg;1003 1004 if (q_vector->wb_on_itr)1005 return;1006 1007 q_vector->wb_on_itr = true;1008 reg = &q_vector->intr_reg;1009 1010 writel(reg->dyn_ctl_wb_on_itr_m | reg->dyn_ctl_intena_msk_m |1011 (IDPF_NO_ITR_UPDATE_IDX << reg->dyn_ctl_itridx_s),1012 reg->dyn_ctl);1013}1014 1015int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget);1016void idpf_vport_init_num_qs(struct idpf_vport *vport,1017 struct virtchnl2_create_vport *vport_msg);1018void idpf_vport_calc_num_q_desc(struct idpf_vport *vport);1019int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_index,1020 struct virtchnl2_create_vport *vport_msg,1021 struct idpf_vport_max_q *max_q);1022void idpf_vport_calc_num_q_groups(struct idpf_vport *vport);1023int idpf_vport_queues_alloc(struct idpf_vport *vport);1024void idpf_vport_queues_rel(struct idpf_vport *vport);1025void idpf_vport_intr_rel(struct idpf_vport *vport);1026int idpf_vport_intr_alloc(struct idpf_vport *vport);1027void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector);1028void idpf_vport_intr_deinit(struct idpf_vport *vport);1029int idpf_vport_intr_init(struct idpf_vport *vport);1030void idpf_vport_intr_ena(struct idpf_vport *vport);1031int idpf_config_rss(struct idpf_vport *vport);1032int idpf_init_rss(struct idpf_vport *vport);1033void idpf_deinit_rss(struct idpf_vport *vport);1034int idpf_rx_bufs_init_all(struct idpf_vport *vport);1035void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb,1036 unsigned int size);1037struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size);1038void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val,1039 bool xmit_more);1040unsigned int idpf_size_to_txd_count(unsigned int size);1041netdev_tx_t idpf_tx_drop_skb(struct idpf_tx_queue *tx_q, struct sk_buff *skb);1042void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb,1043 struct idpf_tx_buf *first, u16 ring_idx);1044unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq,1045 struct sk_buff *skb);1046void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue);1047netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb,1048 struct idpf_tx_queue *tx_q);1049netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev);1050bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq,1051 u16 cleaned_count);1052int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off);1053 1054static inline bool idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q,1055 u32 needed)1056{1057 return !netif_subqueue_maybe_stop(tx_q->netdev, tx_q->idx,1058 IDPF_DESC_UNUSED(tx_q),1059 needed, needed);1060}1061 1062#endif /* !_IDPF_TXRX_H_ */1063