219 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/* Atlantic Network Driver3 *4 * Copyright (C) 2014-2019 aQuantia Corporation5 * Copyright (C) 2019-2020 Marvell International Ltd.6 */7 8/* File aq_ring.h: Declaration of functions for Rx/Tx rings. */9 10#ifndef AQ_RING_H11#define AQ_RING_H12 13#include "aq_common.h"14#include "aq_vec.h"15 16#define AQ_XDP_HEADROOM ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8)17#define AQ_XDP_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info))18 19struct page;20struct aq_nic_cfg_s;21 22struct aq_rxpage {23 struct page *page;24 dma_addr_t daddr;25 unsigned int order;26 unsigned int pg_off;27};28 29/* TxC SOP DX EOP30 * +----------+----------+----------+-----------31 * 8bytes|len l3,l4 | pa | pa | pa32 * +----------+----------+----------+-----------33 * 4/8bytes|len pkt |len pkt | | skb34 * +----------+----------+----------+-----------35 * 4/8bytes|is_gso |len,flags |len |len,is_eop36 * +----------+----------+----------+-----------37 *38 * This aq_ring_buff_s doesn't have endianness dependency.39 * It is __packed for cache line optimizations.40 */41struct __packed aq_ring_buff_s {42 union {43 /* RX/TX */44 dma_addr_t pa;45 /* RX */46 struct {47 u32 rss_hash;48 u16 next;49 u8 is_hash_l4;50 u8 rsvd1;51 struct aq_rxpage rxdata;52 u16 vlan_rx_tag;53 };54 /* EOP */55 struct {56 dma_addr_t pa_eop;57 struct sk_buff *skb;58 struct xdp_frame *xdpf;59 };60 /* TxC */61 struct {62 u32 mss;63 u8 len_l2;64 u8 len_l3;65 u8 len_l4;66 u8 is_ipv6:1;67 u8 rsvd2:7;68 u32 len_pkt;69 u16 vlan_tx_tag;70 };71 };72 union {73 struct {74 u32 len:16;75 u32 is_ip_cso:1;76 u32 is_udp_cso:1;77 u32 is_tcp_cso:1;78 u32 is_cso_err:1;79 u32 is_sop:1;80 u32 is_eop:1;81 u32 is_gso_tcp:1;82 u32 is_gso_udp:1;83 u32 is_mapped:1;84 u32 is_cleaned:1;85 u32 is_error:1;86 u32 is_vlan:1;87 u32 is_lro:1;88 u32 rsvd3:3;89 u16 eop_index;90 u16 rsvd4;91 };92 u64 flags;93 };94};95 96struct aq_ring_stats_rx_s {97 struct u64_stats_sync syncp; /* must be first */98 u64 errors;99 u64 packets;100 u64 bytes;101 u64 lro_packets;102 u64 jumbo_packets;103 u64 alloc_fails;104 u64 skb_alloc_fails;105 u64 polls;106 u64 pg_losts;107 u64 pg_flips;108 u64 pg_reuses;109 u64 xdp_aborted;110 u64 xdp_drop;111 u64 xdp_pass;112 u64 xdp_tx;113 u64 xdp_invalid;114 u64 xdp_redirect;115};116 117struct aq_ring_stats_tx_s {118 struct u64_stats_sync syncp; /* must be first */119 u64 errors;120 u64 packets;121 u64 bytes;122 u64 queue_restarts;123};124 125union aq_ring_stats_s {126 struct aq_ring_stats_rx_s rx;127 struct aq_ring_stats_tx_s tx;128};129 130enum atl_ring_type {131 ATL_RING_TX,132 ATL_RING_RX,133};134 135struct aq_ring_s {136 struct aq_ring_buff_s *buff_ring;137 u8 *dx_ring; /* descriptors ring, dma shared mem */138 struct aq_nic_s *aq_nic;139 unsigned int idx; /* for HW layer registers operations */140 unsigned int hw_head;141 unsigned int sw_head;142 unsigned int sw_tail;143 unsigned int size; /* descriptors number */144 unsigned int dx_size; /* TX or RX descriptor size, */145 /* stored here for fater math */146 u16 page_order;147 u16 page_offset;148 u16 frame_max;149 u16 tail_size;150 union aq_ring_stats_s stats;151 dma_addr_t dx_ring_pa;152 struct bpf_prog *xdp_prog;153 enum atl_ring_type ring_type;154 struct xdp_rxq_info xdp_rxq;155};156 157struct aq_ring_param_s {158 unsigned int vec_idx;159 unsigned int cpu;160 cpumask_t affinity_mask;161};162 163static inline void *aq_buf_vaddr(struct aq_rxpage *rxpage)164{165 return page_to_virt(rxpage->page) + rxpage->pg_off;166}167 168static inline dma_addr_t aq_buf_daddr(struct aq_rxpage *rxpage)169{170 return rxpage->daddr + rxpage->pg_off;171}172 173static inline unsigned int aq_ring_next_dx(struct aq_ring_s *self,174 unsigned int dx)175{176 return (++dx >= self->size) ? 0U : dx;177}178 179static inline unsigned int aq_ring_avail_dx(struct aq_ring_s *self)180{181 return (((self->sw_tail >= self->sw_head)) ?182 (self->size - 1) - self->sw_tail + self->sw_head :183 self->sw_head - self->sw_tail - 1);184}185 186int aq_ring_tx_alloc(struct aq_ring_s *self,187 struct aq_nic_s *aq_nic,188 unsigned int idx,189 struct aq_nic_cfg_s *aq_nic_cfg);190int aq_ring_rx_alloc(struct aq_ring_s *self,191 struct aq_nic_s *aq_nic,192 unsigned int idx,193 struct aq_nic_cfg_s *aq_nic_cfg);194 195int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type);196void aq_ring_rx_deinit(struct aq_ring_s *self);197void aq_ring_free(struct aq_ring_s *self);198void aq_ring_update_queue_state(struct aq_ring_s *ring);199void aq_ring_queue_wake(struct aq_ring_s *ring);200void aq_ring_queue_stop(struct aq_ring_s *ring);201bool aq_ring_tx_clean(struct aq_ring_s *self);202int aq_xdp_xmit(struct net_device *dev, int num_frames,203 struct xdp_frame **frames, u32 flags);204int aq_ring_rx_clean(struct aq_ring_s *self,205 struct napi_struct *napi,206 int *work_done,207 int budget);208int aq_ring_rx_fill(struct aq_ring_s *self);209 210int aq_ring_hwts_rx_alloc(struct aq_ring_s *self,211 struct aq_nic_s *aq_nic, unsigned int idx,212 unsigned int size, unsigned int dx_size);213void aq_ring_hwts_rx_free(struct aq_ring_s *self);214void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic);215 216unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data);217 218#endif /* AQ_RING_H */219