267 lines · c
1/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */2 3#ifndef _FUNETH_TXRX_H4#define _FUNETH_TXRX_H5 6#include <linux/netdevice.h>7#include <linux/u64_stats_sync.h>8#include <net/xdp.h>9 10/* Tx descriptor size */11#define FUNETH_SQE_SIZE 64U12 13/* Size of device headers per Tx packet */14#define FUNETH_FUNOS_HDR_SZ (sizeof(struct fun_eth_tx_req))15 16/* Number of gather list entries per Tx descriptor */17#define FUNETH_GLE_PER_DESC (FUNETH_SQE_SIZE / sizeof(struct fun_dataop_gl))18 19/* Max gather list size in bytes for an sk_buff. */20#define FUNETH_MAX_GL_SZ ((MAX_SKB_FRAGS + 1) * sizeof(struct fun_dataop_gl))21 22#if IS_ENABLED(CONFIG_TLS_DEVICE)23# define FUNETH_TLS_SZ sizeof(struct fun_eth_tls)24#else25# define FUNETH_TLS_SZ 026#endif27 28/* Max number of Tx descriptors for an sk_buff using a gather list. */29#define FUNETH_MAX_GL_DESC \30 DIV_ROUND_UP((FUNETH_FUNOS_HDR_SZ + FUNETH_MAX_GL_SZ + FUNETH_TLS_SZ), \31 FUNETH_SQE_SIZE)32 33/* Max number of Tx descriptors for any packet. */34#define FUNETH_MAX_PKT_DESC FUNETH_MAX_GL_DESC35 36/* Rx CQ descriptor size. */37#define FUNETH_CQE_SIZE 64U38 39/* Offset of cqe_info within a CQE. */40#define FUNETH_CQE_INFO_OFFSET (FUNETH_CQE_SIZE - sizeof(struct fun_cqe_info))41 42/* Construct the IRQ portion of a CQ doorbell. The resulting value arms the43 * interrupt with the supplied time delay and packet count moderation settings.44 */45#define FUN_IRQ_CQ_DB(usec, pkts) \46 (FUN_DB_IRQ_ARM_F | ((usec) << FUN_DB_INTCOAL_USEC_S) | \47 ((pkts) << FUN_DB_INTCOAL_ENTRIES_S))48 49/* As above for SQ doorbells. */50#define FUN_IRQ_SQ_DB(usec, pkts) \51 (FUN_DB_IRQ_ARM_F | \52 ((usec) << FUN_DB_INTCOAL_USEC_S) | \53 ((pkts) << FUN_DB_INTCOAL_ENTRIES_S))54 55/* Per packet tailroom. Present only for 1-frag packets. */56#define FUN_RX_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info))57 58/* Per packet headroom for XDP. Preferred over XDP_PACKET_HEADROOM to59 * accommodate two packets per buffer for 4K pages and 1500B MTUs.60 */61#define FUN_XDP_HEADROOM 19262 63/* Initialization state of a queue. */64enum {65 FUN_QSTATE_DESTROYED, /* what queue? */66 FUN_QSTATE_INIT_SW, /* exists in SW, not on the device */67 FUN_QSTATE_INIT_FULL, /* exists both in SW and on device */68};69 70/* Initialization state of an interrupt. */71enum {72 FUN_IRQ_INIT, /* initialized and in the XArray but inactive */73 FUN_IRQ_REQUESTED, /* request_irq() done */74 FUN_IRQ_ENABLED, /* processing enabled */75 FUN_IRQ_DISABLED, /* processing disabled */76};77 78struct bpf_prog;79 80struct funeth_txq_stats { /* per Tx queue SW counters */81 u64 tx_pkts; /* # of Tx packets */82 u64 tx_bytes; /* total bytes of Tx packets */83 u64 tx_cso; /* # of packets with checksum offload */84 u64 tx_tso; /* # of non-encapsulated TSO super-packets */85 u64 tx_encap_tso; /* # of encapsulated TSO super-packets */86 u64 tx_uso; /* # of non-encapsulated UDP LSO super-packets */87 u64 tx_more; /* # of DBs elided due to xmit_more */88 u64 tx_nstops; /* # of times the queue has stopped */89 u64 tx_nrestarts; /* # of times the queue has restarted */90 u64 tx_map_err; /* # of packets dropped due to DMA mapping errors */91 u64 tx_xdp_full; /* # of XDP packets that could not be enqueued */92 u64 tx_tls_pkts; /* # of Tx TLS packets offloaded to HW */93 u64 tx_tls_bytes; /* Tx bytes of HW-handled TLS payload */94 u64 tx_tls_fallback; /* attempted Tx TLS offloads punted to SW */95 u64 tx_tls_drops; /* attempted Tx TLS offloads dropped */96};97 98struct funeth_tx_info { /* per Tx descriptor state */99 union {100 struct sk_buff *skb; /* associated packet (sk_buff path) */101 struct xdp_frame *xdpf; /* associated XDP frame (XDP path) */102 };103};104 105struct funeth_txq {106 /* RO cacheline of frequently accessed data */107 u32 mask; /* queue depth - 1 */108 u32 hw_qid; /* device ID of the queue */109 void *desc; /* base address of descriptor ring */110 struct funeth_tx_info *info;111 struct device *dma_dev; /* device for DMA mappings */112 volatile __be64 *hw_wb; /* HW write-back location */113 u32 __iomem *db; /* SQ doorbell register address */114 struct netdev_queue *ndq;115 dma_addr_t dma_addr; /* DMA address of descriptor ring */116 /* producer R/W cacheline */117 u16 qidx; /* queue index within net_device */118 u16 ethid;119 u32 prod_cnt; /* producer counter */120 struct funeth_txq_stats stats;121 /* shared R/W cacheline, primarily accessed by consumer */122 u32 irq_db_val; /* value written to IRQ doorbell */123 u32 cons_cnt; /* consumer (cleanup) counter */124 struct net_device *netdev;125 struct fun_irq *irq;126 int numa_node;127 u8 init_state; /* queue initialization state */128 struct u64_stats_sync syncp;129};130 131struct funeth_rxq_stats { /* per Rx queue SW counters */132 u64 rx_pkts; /* # of received packets, including SW drops */133 u64 rx_bytes; /* total size of received packets */134 u64 rx_cso; /* # of packets with checksum offload */135 u64 rx_bufs; /* total # of Rx buffers provided to device */136 u64 gro_pkts; /* # of GRO superpackets */137 u64 gro_merged; /* # of pkts merged into existing GRO superpackets */138 u64 rx_page_alloc; /* # of page allocations for Rx buffers */139 u64 rx_budget; /* NAPI iterations that exhausted their budget */140 u64 rx_mem_drops; /* # of packets dropped due to memory shortage */141 u64 rx_map_err; /* # of page DMA mapping errors */142 u64 xdp_drops; /* XDP_DROPped packets */143 u64 xdp_tx; /* successful XDP transmits */144 u64 xdp_redir; /* successful XDP redirects */145 u64 xdp_err; /* packets dropped due to XDP errors */146};147 148struct funeth_rxbuf { /* per Rx buffer state */149 struct page *page; /* associated page */150 dma_addr_t dma_addr; /* DMA address of page start */151 int pg_refs; /* page refs held by driver */152 int node; /* page node, or -1 if it is PF_MEMALLOC */153};154 155struct funeth_rx_cache { /* cache of DMA-mapped previously used buffers */156 struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */157 unsigned int prod_cnt; /* producer counter */158 unsigned int cons_cnt; /* consumer counter */159 unsigned int mask; /* depth - 1 */160};161 162/* An Rx queue consists of a CQ and an SQ used to provide Rx buffers. */163struct funeth_rxq {164 struct net_device *netdev;165 struct napi_struct *napi;166 struct device *dma_dev; /* device for DMA mappings */167 void *cqes; /* base of CQ descriptor ring */168 const void *next_cqe_info; /* fun_cqe_info of next CQE */169 u32 __iomem *cq_db; /* CQ doorbell register address */170 unsigned int cq_head; /* CQ head index */171 unsigned int cq_mask; /* CQ depth - 1 */172 u16 phase; /* CQ phase tag */173 u16 qidx; /* queue index within net_device */174 unsigned int irq_db_val; /* IRQ info for CQ doorbell */175 struct fun_eprq_rqbuf *rqes; /* base of RQ descriptor ring */176 struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */177 struct funeth_rxbuf *cur_buf; /* currently active buffer */178 u32 __iomem *rq_db; /* RQ doorbell register address */179 unsigned int rq_cons; /* RQ consumer counter */180 unsigned int rq_mask; /* RQ depth - 1 */181 unsigned int buf_offset; /* offset of next pkt in head buffer */182 u8 xdp_flush; /* XDP flush types needed at NAPI end */183 u8 init_state; /* queue initialization state */184 u16 headroom; /* per packet headroom */185 unsigned int rq_cons_db; /* value of rq_cons at last RQ db */186 unsigned int rq_db_thres; /* # of new buffers needed to write RQ db */187 struct funeth_rxbuf spare_buf; /* spare for next buffer replacement */188 struct funeth_rx_cache cache; /* used buffer cache */189 struct bpf_prog *xdp_prog; /* optional XDP BPF program */190 struct funeth_rxq_stats stats;191 dma_addr_t cq_dma_addr; /* DMA address of CQE ring */192 dma_addr_t rq_dma_addr; /* DMA address of RQE ring */193 u16 irq_cnt;194 u32 hw_cqid; /* device ID of the queue's CQ */195 u32 hw_sqid; /* device ID of the queue's SQ */196 int numa_node;197 struct u64_stats_sync syncp;198 struct xdp_rxq_info xdp_rxq;199};200 201#define FUN_QSTAT_INC(q, counter) \202 do { \203 u64_stats_update_begin(&(q)->syncp); \204 (q)->stats.counter++; \205 u64_stats_update_end(&(q)->syncp); \206 } while (0)207 208#define FUN_QSTAT_READ(q, seq, stats_copy) \209 do { \210 seq = u64_stats_fetch_begin(&(q)->syncp); \211 stats_copy = (q)->stats; \212 } while (u64_stats_fetch_retry(&(q)->syncp, (seq)))213 214#define FUN_INT_NAME_LEN (IFNAMSIZ + 16)215 216struct fun_irq {217 struct napi_struct napi;218 struct funeth_txq *txq;219 struct funeth_rxq *rxq;220 u8 state;221 u16 irq_idx; /* index of MSI-X interrupt */222 int irq; /* Linux IRQ vector */223 cpumask_t affinity_mask; /* IRQ affinity */224 struct irq_affinity_notify aff_notify;225 char name[FUN_INT_NAME_LEN];226} ____cacheline_internodealigned_in_smp;227 228/* Return the start address of the idx-th Tx descriptor. */229static inline void *fun_tx_desc_addr(const struct funeth_txq *q,230 unsigned int idx)231{232 return q->desc + idx * FUNETH_SQE_SIZE;233}234 235static inline void fun_txq_wr_db(const struct funeth_txq *q)236{237 unsigned int tail = q->prod_cnt & q->mask;238 239 writel(tail, q->db);240}241 242static inline int fun_irq_node(const struct fun_irq *p)243{244 return cpu_to_mem(cpumask_first(&p->affinity_mask));245}246 247int fun_rxq_napi_poll(struct napi_struct *napi, int budget);248int fun_txq_napi_poll(struct napi_struct *napi, int budget);249netdev_tx_t fun_start_xmit(struct sk_buff *skb, struct net_device *netdev);250bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf);251int fun_xdp_xmit_frames(struct net_device *dev, int n,252 struct xdp_frame **frames, u32 flags);253 254int funeth_txq_create(struct net_device *dev, unsigned int qidx,255 unsigned int ndesc, struct fun_irq *irq, int state,256 struct funeth_txq **qp);257int fun_txq_create_dev(struct funeth_txq *q, struct fun_irq *irq);258struct funeth_txq *funeth_txq_free(struct funeth_txq *q, int state);259int funeth_rxq_create(struct net_device *dev, unsigned int qidx,260 unsigned int ncqe, unsigned int nrqe, struct fun_irq *irq,261 int state, struct funeth_rxq **qp);262int fun_rxq_create_dev(struct funeth_rxq *q, struct fun_irq *irq);263struct funeth_rxq *funeth_rxq_free(struct funeth_rxq *q, int state);264int fun_rxq_set_bpf(struct funeth_rxq *q, struct bpf_prog *prog);265 266#endif /* _FUNETH_TXRX_H */267