260 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (C) 2024 Intel Corporation */3 4#include <net/libeth/rx.h>5 6/* Rx buffer management */7 8/**9 * libeth_rx_hw_len_mtu - get the actual buffer size to be passed to HW10 * @pp: &page_pool_params of the netdev to calculate the size for11 * @max_len: maximum buffer size for a single descriptor12 *13 * Return: HW-writeable length per one buffer to pass it to the HW accounting:14 * MTU the @dev has, HW required alignment, minimum and maximum allowed values,15 * and system's page size.16 */17static u32 libeth_rx_hw_len_mtu(const struct page_pool_params *pp, u32 max_len)18{19 u32 len;20 21 len = READ_ONCE(pp->netdev->mtu) + LIBETH_RX_LL_LEN;22 len = ALIGN(len, LIBETH_RX_BUF_STRIDE);23 len = min3(len, ALIGN_DOWN(max_len ? : U32_MAX, LIBETH_RX_BUF_STRIDE),24 pp->max_len);25 26 return len;27}28 29/**30 * libeth_rx_hw_len_truesize - get the short buffer size to be passed to HW31 * @pp: &page_pool_params of the netdev to calculate the size for32 * @max_len: maximum buffer size for a single descriptor33 * @truesize: desired truesize for the buffers34 *35 * Return: HW-writeable length per one buffer to pass it to the HW ignoring the36 * MTU and closest to the passed truesize. Can be used for "short" buffer37 * queues to fragment pages more efficiently.38 */39static u32 libeth_rx_hw_len_truesize(const struct page_pool_params *pp,40 u32 max_len, u32 truesize)41{42 u32 min, len;43 44 min = SKB_HEAD_ALIGN(pp->offset + LIBETH_RX_BUF_STRIDE);45 truesize = clamp(roundup_pow_of_two(truesize), roundup_pow_of_two(min),46 PAGE_SIZE << LIBETH_RX_PAGE_ORDER);47 48 len = SKB_WITH_OVERHEAD(truesize - pp->offset);49 len = ALIGN_DOWN(len, LIBETH_RX_BUF_STRIDE) ? : LIBETH_RX_BUF_STRIDE;50 len = min3(len, ALIGN_DOWN(max_len ? : U32_MAX, LIBETH_RX_BUF_STRIDE),51 pp->max_len);52 53 return len;54}55 56/**57 * libeth_rx_page_pool_params - calculate params with the stack overhead58 * @fq: buffer queue to calculate the size for59 * @pp: &page_pool_params of the netdev60 *61 * Set the PP params to will all needed stack overhead (headroom, tailroom) and62 * both the HW buffer length and the truesize for all types of buffers. For63 * "short" buffers, truesize never exceeds the "wanted" one; for the rest,64 * it can be up to the page size.65 *66 * Return: true on success, false on invalid input params.67 */68static bool libeth_rx_page_pool_params(struct libeth_fq *fq,69 struct page_pool_params *pp)70{71 pp->offset = LIBETH_SKB_HEADROOM;72 /* HW-writeable / syncable length per one page */73 pp->max_len = LIBETH_RX_PAGE_LEN(pp->offset);74 75 /* HW-writeable length per buffer */76 switch (fq->type) {77 case LIBETH_FQE_MTU:78 fq->buf_len = libeth_rx_hw_len_mtu(pp, fq->buf_len);79 break;80 case LIBETH_FQE_SHORT:81 fq->buf_len = libeth_rx_hw_len_truesize(pp, fq->buf_len,82 fq->truesize);83 break;84 case LIBETH_FQE_HDR:85 fq->buf_len = ALIGN(LIBETH_MAX_HEAD, LIBETH_RX_BUF_STRIDE);86 break;87 default:88 return false;89 }90 91 /* Buffer size to allocate */92 fq->truesize = roundup_pow_of_two(SKB_HEAD_ALIGN(pp->offset +93 fq->buf_len));94 95 return true;96}97 98/**99 * libeth_rx_page_pool_params_zc - calculate params without the stack overhead100 * @fq: buffer queue to calculate the size for101 * @pp: &page_pool_params of the netdev102 *103 * Set the PP params to exclude the stack overhead and both the buffer length104 * and the truesize, which are equal for the data buffers. Note that this105 * requires separate header buffers to be always active and account the106 * overhead.107 * With the MTU == ``PAGE_SIZE``, this allows the kernel to enable the zerocopy108 * mode.109 *110 * Return: true on success, false on invalid input params.111 */112static bool libeth_rx_page_pool_params_zc(struct libeth_fq *fq,113 struct page_pool_params *pp)114{115 u32 mtu, max;116 117 pp->offset = 0;118 pp->max_len = PAGE_SIZE << LIBETH_RX_PAGE_ORDER;119 120 switch (fq->type) {121 case LIBETH_FQE_MTU:122 mtu = READ_ONCE(pp->netdev->mtu);123 break;124 case LIBETH_FQE_SHORT:125 mtu = fq->truesize;126 break;127 default:128 return false;129 }130 131 mtu = roundup_pow_of_two(mtu);132 max = min(rounddown_pow_of_two(fq->buf_len ? : U32_MAX),133 pp->max_len);134 135 fq->buf_len = clamp(mtu, LIBETH_RX_BUF_STRIDE, max);136 fq->truesize = fq->buf_len;137 138 return true;139}140 141/**142 * libeth_rx_fq_create - create a PP with the default libeth settings143 * @fq: buffer queue struct to fill144 * @napi: &napi_struct covering this PP (no usage outside its poll loops)145 *146 * Return: %0 on success, -%errno on failure.147 */148int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi)149{150 struct page_pool_params pp = {151 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,152 .order = LIBETH_RX_PAGE_ORDER,153 .pool_size = fq->count,154 .nid = fq->nid,155 .dev = napi->dev->dev.parent,156 .netdev = napi->dev,157 .napi = napi,158 .dma_dir = DMA_FROM_DEVICE,159 };160 struct libeth_fqe *fqes;161 struct page_pool *pool;162 bool ret;163 164 if (!fq->hsplit)165 ret = libeth_rx_page_pool_params(fq, &pp);166 else167 ret = libeth_rx_page_pool_params_zc(fq, &pp);168 if (!ret)169 return -EINVAL;170 171 pool = page_pool_create(&pp);172 if (IS_ERR(pool))173 return PTR_ERR(pool);174 175 fqes = kvcalloc_node(fq->count, sizeof(*fqes), GFP_KERNEL, fq->nid);176 if (!fqes)177 goto err_buf;178 179 fq->fqes = fqes;180 fq->pp = pool;181 182 return 0;183 184err_buf:185 page_pool_destroy(pool);186 187 return -ENOMEM;188}189EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_create, LIBETH);190 191/**192 * libeth_rx_fq_destroy - destroy a &page_pool created by libeth193 * @fq: buffer queue to process194 */195void libeth_rx_fq_destroy(struct libeth_fq *fq)196{197 kvfree(fq->fqes);198 page_pool_destroy(fq->pp);199}200EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_destroy, LIBETH);201 202/**203 * libeth_rx_recycle_slow - recycle a libeth page from the NAPI context204 * @page: page to recycle205 *206 * To be used on exceptions or rare cases not requiring fast inline recycling.207 */208void libeth_rx_recycle_slow(struct page *page)209{210 page_pool_recycle_direct(page->pp, page);211}212EXPORT_SYMBOL_NS_GPL(libeth_rx_recycle_slow, LIBETH);213 214/* Converting abstract packet type numbers into a software structure with215 * the packet parameters to do O(1) lookup on Rx.216 */217 218static const u16 libeth_rx_pt_xdp_oip[] = {219 [LIBETH_RX_PT_OUTER_L2] = XDP_RSS_TYPE_NONE,220 [LIBETH_RX_PT_OUTER_IPV4] = XDP_RSS_L3_IPV4,221 [LIBETH_RX_PT_OUTER_IPV6] = XDP_RSS_L3_IPV6,222};223 224static const u16 libeth_rx_pt_xdp_iprot[] = {225 [LIBETH_RX_PT_INNER_NONE] = XDP_RSS_TYPE_NONE,226 [LIBETH_RX_PT_INNER_UDP] = XDP_RSS_L4_UDP,227 [LIBETH_RX_PT_INNER_TCP] = XDP_RSS_L4_TCP,228 [LIBETH_RX_PT_INNER_SCTP] = XDP_RSS_L4_SCTP,229 [LIBETH_RX_PT_INNER_ICMP] = XDP_RSS_L4_ICMP,230 [LIBETH_RX_PT_INNER_TIMESYNC] = XDP_RSS_TYPE_NONE,231};232 233static const u16 libeth_rx_pt_xdp_pl[] = {234 [LIBETH_RX_PT_PAYLOAD_NONE] = XDP_RSS_TYPE_NONE,235 [LIBETH_RX_PT_PAYLOAD_L2] = XDP_RSS_TYPE_NONE,236 [LIBETH_RX_PT_PAYLOAD_L3] = XDP_RSS_TYPE_NONE,237 [LIBETH_RX_PT_PAYLOAD_L4] = XDP_RSS_L4,238};239 240/**241 * libeth_rx_pt_gen_hash_type - generate an XDP RSS hash type for a PT242 * @pt: PT structure to evaluate243 *244 * Generates ```hash_type``` field with XDP RSS type values from the parsed245 * packet parameters if they're obtained dynamically at runtime.246 */247void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt)248{249 pt->hash_type = 0;250 pt->hash_type |= libeth_rx_pt_xdp_oip[pt->outer_ip];251 pt->hash_type |= libeth_rx_pt_xdp_iprot[pt->inner_prot];252 pt->hash_type |= libeth_rx_pt_xdp_pl[pt->payload_layer];253}254EXPORT_SYMBOL_NS_GPL(libeth_rx_pt_gen_hash_type, LIBETH);255 256/* Module */257 258MODULE_DESCRIPTION("Common Ethernet library");259MODULE_LICENSE("GPL");260