470 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.4 */5 6#include "ena_xdp.h"7 8static int validate_xdp_req_id(struct ena_ring *tx_ring, u16 req_id)9{10 struct ena_tx_buffer *tx_info;11 12 tx_info = &tx_ring->tx_buffer_info[req_id];13 if (likely(tx_info->xdpf))14 return 0;15 16 return handle_invalid_req_id(tx_ring, req_id, tx_info, true);17}18 19static int ena_xdp_tx_map_frame(struct ena_ring *tx_ring,20 struct ena_tx_buffer *tx_info,21 struct xdp_frame *xdpf,22 struct ena_com_tx_ctx *ena_tx_ctx)23{24 struct ena_adapter *adapter = tx_ring->adapter;25 struct ena_com_buf *ena_buf;26 int push_len = 0;27 dma_addr_t dma;28 void *data;29 u32 size;30 31 tx_info->xdpf = xdpf;32 data = tx_info->xdpf->data;33 size = tx_info->xdpf->len;34 35 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {36 /* Designate part of the packet for LLQ */37 push_len = min_t(u32, size, tx_ring->tx_max_header_size);38 39 ena_tx_ctx->push_header = data;40 41 size -= push_len;42 data += push_len;43 }44 45 ena_tx_ctx->header_len = push_len;46 47 if (size > 0) {48 dma = dma_map_single(tx_ring->dev,49 data,50 size,51 DMA_TO_DEVICE);52 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))53 goto error_report_dma_error;54 55 tx_info->map_linear_data = 0;56 57 ena_buf = tx_info->bufs;58 ena_buf->paddr = dma;59 ena_buf->len = size;60 61 ena_tx_ctx->ena_bufs = ena_buf;62 ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1;63 }64 65 return 0;66 67error_report_dma_error:68 ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,69 &tx_ring->syncp);70 netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n");71 72 return -EINVAL;73}74 75int ena_xdp_xmit_frame(struct ena_ring *tx_ring,76 struct ena_adapter *adapter,77 struct xdp_frame *xdpf,78 int flags)79{80 struct ena_com_tx_ctx ena_tx_ctx = {};81 struct ena_tx_buffer *tx_info;82 u16 next_to_use, req_id;83 int rc;84 85 next_to_use = tx_ring->next_to_use;86 req_id = tx_ring->free_ids[next_to_use];87 tx_info = &tx_ring->tx_buffer_info[req_id];88 tx_info->num_of_bufs = 0;89 90 rc = ena_xdp_tx_map_frame(tx_ring, tx_info, xdpf, &ena_tx_ctx);91 if (unlikely(rc))92 goto err;93 94 ena_tx_ctx.req_id = req_id;95 96 rc = ena_xmit_common(adapter,97 tx_ring,98 tx_info,99 &ena_tx_ctx,100 next_to_use,101 xdpf->len);102 if (rc)103 goto error_unmap_dma;104 105 /* trigger the dma engine. ena_ring_tx_doorbell()106 * calls a memory barrier inside it.107 */108 if (flags & XDP_XMIT_FLUSH)109 ena_ring_tx_doorbell(tx_ring);110 111 return rc;112 113error_unmap_dma:114 ena_unmap_tx_buff(tx_ring, tx_info);115err:116 tx_info->xdpf = NULL;117 118 return rc;119}120 121int ena_xdp_xmit(struct net_device *dev, int n,122 struct xdp_frame **frames, u32 flags)123{124 struct ena_adapter *adapter = netdev_priv(dev);125 struct ena_ring *tx_ring;126 int qid, i, nxmit = 0;127 128 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))129 return -EINVAL;130 131 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))132 return -ENETDOWN;133 134 /* We assume that all rings have the same XDP program */135 if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog))136 return -ENXIO;137 138 qid = smp_processor_id() % adapter->xdp_num_queues;139 qid += adapter->xdp_first_ring;140 tx_ring = &adapter->tx_ring[qid];141 142 /* Other CPU ids might try to send thorugh this queue */143 spin_lock(&tx_ring->xdp_tx_lock);144 145 for (i = 0; i < n; i++) {146 if (ena_xdp_xmit_frame(tx_ring, adapter, frames[i], 0))147 break;148 nxmit++;149 }150 151 /* Ring doorbell to make device aware of the packets */152 if (flags & XDP_XMIT_FLUSH)153 ena_ring_tx_doorbell(tx_ring);154 155 spin_unlock(&tx_ring->xdp_tx_lock);156 157 /* Return number of packets sent */158 return nxmit;159}160 161static void ena_init_all_xdp_queues(struct ena_adapter *adapter)162{163 adapter->xdp_first_ring = adapter->num_io_queues;164 adapter->xdp_num_queues = adapter->num_io_queues;165 166 ena_init_io_rings(adapter,167 adapter->xdp_first_ring,168 adapter->xdp_num_queues);169}170 171int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter)172{173 u32 xdp_first_ring = adapter->xdp_first_ring;174 u32 xdp_num_queues = adapter->xdp_num_queues;175 int rc = 0;176 177 rc = ena_setup_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues);178 if (rc)179 goto setup_err;180 181 rc = ena_create_io_tx_queues_in_range(adapter, xdp_first_ring, xdp_num_queues);182 if (rc)183 goto create_err;184 185 return 0;186 187create_err:188 ena_free_all_io_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues);189setup_err:190 return rc;191}192 193/* Provides a way for both kernel and bpf-prog to know194 * more about the RX-queue a given XDP frame arrived on.195 */196int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)197{198 int rc;199 200 rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0);201 202 netif_dbg(rx_ring->adapter, ifup, rx_ring->netdev, "Registering RX info for queue %d",203 rx_ring->qid);204 if (rc) {205 netif_err(rx_ring->adapter, ifup, rx_ring->netdev,206 "Failed to register xdp rx queue info. RX queue num %d rc: %d\n",207 rx_ring->qid, rc);208 goto err;209 }210 211 rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL);212 213 if (rc) {214 netif_err(rx_ring->adapter, ifup, rx_ring->netdev,215 "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n",216 rx_ring->qid, rc);217 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);218 }219 220err:221 return rc;222}223 224void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring)225{226 netif_dbg(rx_ring->adapter, ifdown, rx_ring->netdev,227 "Unregistering RX info for queue %d",228 rx_ring->qid);229 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq);230 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);231}232 233void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,234 struct bpf_prog *prog,235 int first, int count)236{237 struct bpf_prog *old_bpf_prog;238 struct ena_ring *rx_ring;239 int i = 0;240 241 for (i = first; i < count; i++) {242 rx_ring = &adapter->rx_ring[i];243 old_bpf_prog = xchg(&rx_ring->xdp_bpf_prog, prog);244 245 if (!old_bpf_prog && prog) {246 rx_ring->rx_headroom = XDP_PACKET_HEADROOM;247 } else if (old_bpf_prog && !prog) {248 rx_ring->rx_headroom = NET_SKB_PAD;249 }250 }251}252 253static void ena_xdp_exchange_program(struct ena_adapter *adapter,254 struct bpf_prog *prog)255{256 struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog);257 258 ena_xdp_exchange_program_rx_in_range(adapter,259 prog,260 0,261 adapter->num_io_queues);262 263 if (old_bpf_prog)264 bpf_prog_put(old_bpf_prog);265}266 267static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter)268{269 bool was_up;270 int rc;271 272 was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);273 274 if (was_up)275 ena_down(adapter);276 277 adapter->xdp_first_ring = 0;278 adapter->xdp_num_queues = 0;279 ena_xdp_exchange_program(adapter, NULL);280 if (was_up) {281 rc = ena_up(adapter);282 if (rc)283 return rc;284 }285 return 0;286}287 288static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf)289{290 struct ena_adapter *adapter = netdev_priv(netdev);291 struct bpf_prog *prog = bpf->prog;292 struct bpf_prog *old_bpf_prog;293 int rc, prev_mtu;294 bool is_up;295 296 is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);297 rc = ena_xdp_allowed(adapter);298 if (rc == ENA_XDP_ALLOWED) {299 old_bpf_prog = adapter->xdp_bpf_prog;300 if (prog) {301 if (!is_up) {302 ena_init_all_xdp_queues(adapter);303 } else if (!old_bpf_prog) {304 ena_down(adapter);305 ena_init_all_xdp_queues(adapter);306 }307 ena_xdp_exchange_program(adapter, prog);308 309 netif_dbg(adapter, drv, adapter->netdev, "Set a new XDP program\n");310 311 if (is_up && !old_bpf_prog) {312 rc = ena_up(adapter);313 if (rc)314 return rc;315 }316 xdp_features_set_redirect_target(netdev, false);317 } else if (old_bpf_prog) {318 xdp_features_clear_redirect_target(netdev);319 netif_dbg(adapter, drv, adapter->netdev, "Removing XDP program\n");320 321 rc = ena_destroy_and_free_all_xdp_queues(adapter);322 if (rc)323 return rc;324 }325 326 prev_mtu = netdev->max_mtu;327 netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu;328 329 if (!old_bpf_prog)330 netif_info(adapter, drv, adapter->netdev,331 "XDP program is set, changing the max_mtu from %d to %d",332 prev_mtu, netdev->max_mtu);333 334 } else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) {335 netif_err(adapter, drv, adapter->netdev,336 "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on",337 netdev->mtu, ENA_XDP_MAX_MTU);338 NL_SET_ERR_MSG_MOD(bpf->extack,339 "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info");340 return -EINVAL;341 } else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) {342 netif_err(adapter, drv, adapter->netdev,343 "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n",344 adapter->num_io_queues, adapter->max_num_io_queues);345 NL_SET_ERR_MSG_MOD(bpf->extack,346 "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info");347 return -EINVAL;348 }349 350 return 0;351}352 353/* This is the main xdp callback, it's used by the kernel to set/unset the xdp354 * program as well as to query the current xdp program id.355 */356int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf)357{358 switch (bpf->command) {359 case XDP_SETUP_PROG:360 return ena_xdp_set(netdev, bpf);361 default:362 return -EINVAL;363 }364 return 0;365}366 367static int ena_clean_xdp_irq(struct ena_ring *tx_ring, u32 budget)368{369 u32 total_done = 0;370 u16 next_to_clean;371 int tx_pkts = 0;372 u16 req_id;373 int rc;374 375 if (unlikely(!tx_ring))376 return 0;377 next_to_clean = tx_ring->next_to_clean;378 379 while (tx_pkts < budget) {380 struct ena_tx_buffer *tx_info;381 struct xdp_frame *xdpf;382 383 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,384 &req_id);385 if (rc) {386 if (unlikely(rc == -EINVAL))387 handle_invalid_req_id(tx_ring, req_id, NULL, true);388 break;389 }390 391 /* validate that the request id points to a valid xdp_frame */392 rc = validate_xdp_req_id(tx_ring, req_id);393 if (rc)394 break;395 396 tx_info = &tx_ring->tx_buffer_info[req_id];397 398 tx_info->last_jiffies = 0;399 400 xdpf = tx_info->xdpf;401 tx_info->xdpf = NULL;402 ena_unmap_tx_buff(tx_ring, tx_info);403 xdp_return_frame(xdpf);404 405 tx_pkts++;406 total_done += tx_info->tx_descs;407 tx_ring->free_ids[next_to_clean] = req_id;408 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,409 tx_ring->ring_size);410 411 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,412 "tx_poll: q %d pkt #%d req_id %d\n", tx_ring->qid, tx_pkts, req_id);413 }414 415 tx_ring->next_to_clean = next_to_clean;416 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);417 418 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,419 "tx_poll: q %d done. total pkts: %d\n",420 tx_ring->qid, tx_pkts);421 422 return tx_pkts;423}424 425/* This is the XDP napi callback. XDP queues use a separate napi callback426 * than Rx/Tx queues.427 */428int ena_xdp_io_poll(struct napi_struct *napi, int budget)429{430 struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);431 struct ena_ring *tx_ring;432 u32 work_done;433 int ret;434 435 tx_ring = ena_napi->tx_ring;436 437 if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||438 test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {439 napi_complete_done(napi, 0);440 return 0;441 }442 443 work_done = ena_clean_xdp_irq(tx_ring, budget);444 445 /* If the device is about to reset or down, avoid unmask446 * the interrupt and return 0 so NAPI won't reschedule447 */448 if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags))) {449 napi_complete_done(napi, 0);450 ret = 0;451 } else if (budget > work_done) {452 ena_increase_stat(&tx_ring->tx_stats.napi_comp, 1,453 &tx_ring->syncp);454 if (napi_complete_done(napi, work_done))455 ena_unmask_interrupt(tx_ring, NULL);456 457 ena_update_ring_numa_node(tx_ring, NULL);458 ret = work_done;459 } else {460 ret = budget;461 }462 463 u64_stats_update_begin(&tx_ring->syncp);464 tx_ring->tx_stats.tx_poll++;465 u64_stats_update_end(&tx_ring->syncp);466 tx_ring->tx_stats.last_napi_jiffies = jiffies;467 468 return ret;469}470