1045 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/****************************************************************************3 * Driver for Solarflare network controllers and boards4 * Copyright 2018 Solarflare Communications Inc.5 *6 * This program is free software; you can redistribute it and/or modify it7 * under the terms of the GNU General Public License version 2 as published8 * by the Free Software Foundation, incorporated herein by reference.9 */10 11#include "net_driver.h"12#include <linux/module.h>13#include <linux/iommu.h>14#include <net/rps.h>15#include "efx.h"16#include "nic.h"17#include "rx_common.h"18 19/* This is the percentage fill level below which new RX descriptors20 * will be added to the RX descriptor ring.21 */22static unsigned int rx_refill_threshold;23module_param(rx_refill_threshold, uint, 0444);24MODULE_PARM_DESC(rx_refill_threshold,25 "RX descriptor ring refill threshold (%)");26 27/* RX maximum head room required.28 *29 * This must be at least 1 to prevent overflow, plus one packet-worth30 * to allow pipelined receives.31 */32#define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS)33 34/* Check the RX page recycle ring for a page that can be reused. */35static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)36{37 struct efx_nic *efx = rx_queue->efx;38 struct efx_rx_page_state *state;39 unsigned int index;40 struct page *page;41 42 if (unlikely(!rx_queue->page_ring))43 return NULL;44 index = rx_queue->page_remove & rx_queue->page_ptr_mask;45 page = rx_queue->page_ring[index];46 if (page == NULL)47 return NULL;48 49 rx_queue->page_ring[index] = NULL;50 /* page_remove cannot exceed page_add. */51 if (rx_queue->page_remove != rx_queue->page_add)52 ++rx_queue->page_remove;53 54 /* If page_count is 1 then we hold the only reference to this page. */55 if (page_count(page) == 1) {56 ++rx_queue->page_recycle_count;57 return page;58 } else {59 state = page_address(page);60 dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,61 PAGE_SIZE << efx->rx_buffer_order,62 DMA_FROM_DEVICE);63 put_page(page);64 ++rx_queue->page_recycle_failed;65 }66 67 return NULL;68}69 70/* Attempt to recycle the page if there is an RX recycle ring; the page can71 * only be added if this is the final RX buffer, to prevent pages being used in72 * the descriptor ring and appearing in the recycle ring simultaneously.73 */74static void efx_recycle_rx_page(struct efx_channel *channel,75 struct efx_rx_buffer *rx_buf)76{77 struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);78 struct efx_nic *efx = rx_queue->efx;79 struct page *page = rx_buf->page;80 unsigned int index;81 82 /* Only recycle the page after processing the final buffer. */83 if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE))84 return;85 86 index = rx_queue->page_add & rx_queue->page_ptr_mask;87 if (rx_queue->page_ring[index] == NULL) {88 unsigned int read_index = rx_queue->page_remove &89 rx_queue->page_ptr_mask;90 91 /* The next slot in the recycle ring is available, but92 * increment page_remove if the read pointer currently93 * points here.94 */95 if (read_index == index)96 ++rx_queue->page_remove;97 rx_queue->page_ring[index] = page;98 ++rx_queue->page_add;99 return;100 }101 ++rx_queue->page_recycle_full;102 efx_unmap_rx_buffer(efx, rx_buf);103 put_page(rx_buf->page);104}105 106/* Recycle the pages that are used by buffers that have just been received. */107void efx_recycle_rx_pages(struct efx_channel *channel,108 struct efx_rx_buffer *rx_buf,109 unsigned int n_frags)110{111 struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);112 113 if (unlikely(!rx_queue->page_ring))114 return;115 116 do {117 efx_recycle_rx_page(channel, rx_buf);118 rx_buf = efx_rx_buf_next(rx_queue, rx_buf);119 } while (--n_frags);120}121 122void efx_discard_rx_packet(struct efx_channel *channel,123 struct efx_rx_buffer *rx_buf,124 unsigned int n_frags)125{126 struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);127 128 efx_recycle_rx_pages(channel, rx_buf, n_frags);129 130 efx_free_rx_buffers(rx_queue, rx_buf, n_frags);131}132 133static void efx_init_rx_recycle_ring(struct efx_rx_queue *rx_queue)134{135 unsigned int bufs_in_recycle_ring, page_ring_size;136 struct efx_nic *efx = rx_queue->efx;137 138 bufs_in_recycle_ring = efx_rx_recycle_ring_size(efx);139 page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring /140 efx->rx_bufs_per_page);141 rx_queue->page_ring = kcalloc(page_ring_size,142 sizeof(*rx_queue->page_ring), GFP_KERNEL);143 if (!rx_queue->page_ring)144 rx_queue->page_ptr_mask = 0;145 else146 rx_queue->page_ptr_mask = page_ring_size - 1;147}148 149static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue)150{151 struct efx_nic *efx = rx_queue->efx;152 int i;153 154 if (unlikely(!rx_queue->page_ring))155 return;156 157 /* Unmap and release the pages in the recycle ring. Remove the ring. */158 for (i = 0; i <= rx_queue->page_ptr_mask; i++) {159 struct page *page = rx_queue->page_ring[i];160 struct efx_rx_page_state *state;161 162 if (page == NULL)163 continue;164 165 state = page_address(page);166 dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,167 PAGE_SIZE << efx->rx_buffer_order,168 DMA_FROM_DEVICE);169 put_page(page);170 }171 kfree(rx_queue->page_ring);172 rx_queue->page_ring = NULL;173}174 175static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,176 struct efx_rx_buffer *rx_buf)177{178 /* Release the page reference we hold for the buffer. */179 if (rx_buf->page)180 put_page(rx_buf->page);181 182 /* If this is the last buffer in a page, unmap and free it. */183 if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) {184 efx_unmap_rx_buffer(rx_queue->efx, rx_buf);185 efx_free_rx_buffers(rx_queue, rx_buf, 1);186 }187 rx_buf->page = NULL;188}189 190int efx_probe_rx_queue(struct efx_rx_queue *rx_queue)191{192 struct efx_nic *efx = rx_queue->efx;193 unsigned int entries;194 int rc;195 196 /* Create the smallest power-of-two aligned ring */197 entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE);198 EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);199 rx_queue->ptr_mask = entries - 1;200 201 netif_dbg(efx, probe, efx->net_dev,202 "creating RX queue %d size %#x mask %#x\n",203 efx_rx_queue_index(rx_queue), efx->rxq_entries,204 rx_queue->ptr_mask);205 206 /* Allocate RX buffers */207 rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),208 GFP_KERNEL);209 if (!rx_queue->buffer)210 return -ENOMEM;211 212 rc = efx_nic_probe_rx(rx_queue);213 if (rc) {214 kfree(rx_queue->buffer);215 rx_queue->buffer = NULL;216 }217 218 return rc;219}220 221void efx_init_rx_queue(struct efx_rx_queue *rx_queue)222{223 unsigned int max_fill, trigger, max_trigger;224 struct efx_nic *efx = rx_queue->efx;225 int rc = 0;226 227 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,228 "initialising RX queue %d\n", efx_rx_queue_index(rx_queue));229 230 /* Initialise ptr fields */231 rx_queue->added_count = 0;232 rx_queue->notified_count = 0;233 rx_queue->granted_count = 0;234 rx_queue->removed_count = 0;235 rx_queue->min_fill = -1U;236 efx_init_rx_recycle_ring(rx_queue);237 238 rx_queue->page_remove = 0;239 rx_queue->page_add = rx_queue->page_ptr_mask + 1;240 rx_queue->page_recycle_count = 0;241 rx_queue->page_recycle_failed = 0;242 rx_queue->page_recycle_full = 0;243 244 /* Initialise limit fields */245 max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;246 max_trigger =247 max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page;248 if (rx_refill_threshold != 0) {249 trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;250 if (trigger > max_trigger)251 trigger = max_trigger;252 } else {253 trigger = max_trigger;254 }255 256 rx_queue->max_fill = max_fill;257 rx_queue->fast_fill_trigger = trigger;258 rx_queue->refill_enabled = true;259 260 /* Initialise XDP queue information */261 rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev,262 rx_queue->core_index, 0);263 264 if (rc) {265 netif_err(efx, rx_err, efx->net_dev,266 "Failure to initialise XDP queue information rc=%d\n",267 rc);268 efx->xdp_rxq_info_failed = true;269 } else {270 rx_queue->xdp_rxq_info_valid = true;271 }272 273 /* Set up RX descriptor ring */274 efx_nic_init_rx(rx_queue);275}276 277void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)278{279 struct efx_rx_buffer *rx_buf;280 int i;281 282 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,283 "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));284 285 del_timer_sync(&rx_queue->slow_fill);286 if (rx_queue->grant_credits)287 flush_work(&rx_queue->grant_work);288 289 /* Release RX buffers from the current read ptr to the write ptr */290 if (rx_queue->buffer) {291 for (i = rx_queue->removed_count; i < rx_queue->added_count;292 i++) {293 unsigned int index = i & rx_queue->ptr_mask;294 295 rx_buf = efx_rx_buffer(rx_queue, index);296 efx_fini_rx_buffer(rx_queue, rx_buf);297 }298 }299 300 efx_fini_rx_recycle_ring(rx_queue);301 302 if (rx_queue->xdp_rxq_info_valid)303 xdp_rxq_info_unreg(&rx_queue->xdp_rxq_info);304 305 rx_queue->xdp_rxq_info_valid = false;306}307 308void efx_remove_rx_queue(struct efx_rx_queue *rx_queue)309{310 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,311 "destroying RX queue %d\n", efx_rx_queue_index(rx_queue));312 313 efx_nic_remove_rx(rx_queue);314 315 kfree(rx_queue->buffer);316 rx_queue->buffer = NULL;317}318 319/* Unmap a DMA-mapped page. This function is only called for the final RX320 * buffer in a page.321 */322void efx_unmap_rx_buffer(struct efx_nic *efx,323 struct efx_rx_buffer *rx_buf)324{325 struct page *page = rx_buf->page;326 327 if (page) {328 struct efx_rx_page_state *state = page_address(page);329 330 dma_unmap_page(&efx->pci_dev->dev,331 state->dma_addr,332 PAGE_SIZE << efx->rx_buffer_order,333 DMA_FROM_DEVICE);334 }335}336 337void efx_free_rx_buffers(struct efx_rx_queue *rx_queue,338 struct efx_rx_buffer *rx_buf,339 unsigned int num_bufs)340{341 do {342 if (rx_buf->page) {343 put_page(rx_buf->page);344 rx_buf->page = NULL;345 }346 rx_buf = efx_rx_buf_next(rx_queue, rx_buf);347 } while (--num_bufs);348}349 350void efx_rx_slow_fill(struct timer_list *t)351{352 struct efx_rx_queue *rx_queue = from_timer(rx_queue, t, slow_fill);353 354 /* Post an event to cause NAPI to run and refill the queue */355 efx_nic_generate_fill_event(rx_queue);356 ++rx_queue->slow_fill_count;357}358 359void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)360{361 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10));362}363 364/* efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers365 *366 * @rx_queue: Efx RX queue367 *368 * This allocates a batch of pages, maps them for DMA, and populates369 * struct efx_rx_buffers for each one. Return a negative error code or370 * 0 on success. If a single page can be used for multiple buffers,371 * then the page will either be inserted fully, or not at all.372 */373static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue, bool atomic)374{375 unsigned int page_offset, index, count;376 struct efx_nic *efx = rx_queue->efx;377 struct efx_rx_page_state *state;378 struct efx_rx_buffer *rx_buf;379 dma_addr_t dma_addr;380 struct page *page;381 382 count = 0;383 do {384 page = efx_reuse_page(rx_queue);385 if (page == NULL) {386 page = alloc_pages(__GFP_COMP |387 (atomic ? GFP_ATOMIC : GFP_KERNEL),388 efx->rx_buffer_order);389 if (unlikely(page == NULL))390 return -ENOMEM;391 dma_addr =392 dma_map_page(&efx->pci_dev->dev, page, 0,393 PAGE_SIZE << efx->rx_buffer_order,394 DMA_FROM_DEVICE);395 if (unlikely(dma_mapping_error(&efx->pci_dev->dev,396 dma_addr))) {397 __free_pages(page, efx->rx_buffer_order);398 return -EIO;399 }400 state = page_address(page);401 state->dma_addr = dma_addr;402 } else {403 state = page_address(page);404 dma_addr = state->dma_addr;405 }406 407 dma_addr += sizeof(struct efx_rx_page_state);408 page_offset = sizeof(struct efx_rx_page_state);409 410 do {411 index = rx_queue->added_count & rx_queue->ptr_mask;412 rx_buf = efx_rx_buffer(rx_queue, index);413 rx_buf->dma_addr = dma_addr + efx->rx_ip_align +414 EFX_XDP_HEADROOM;415 rx_buf->page = page;416 rx_buf->page_offset = page_offset + efx->rx_ip_align +417 EFX_XDP_HEADROOM;418 rx_buf->len = efx->rx_dma_len;419 rx_buf->flags = 0;420 ++rx_queue->added_count;421 get_page(page);422 dma_addr += efx->rx_page_buf_step;423 page_offset += efx->rx_page_buf_step;424 } while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE);425 426 rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE;427 } while (++count < efx->rx_pages_per_batch);428 429 return 0;430}431 432void efx_rx_config_page_split(struct efx_nic *efx)433{434 efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + efx->rx_ip_align +435 EFX_XDP_HEADROOM + EFX_XDP_TAILROOM,436 EFX_RX_BUF_ALIGNMENT);437 efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 :438 ((PAGE_SIZE - sizeof(struct efx_rx_page_state)) /439 efx->rx_page_buf_step);440 efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) /441 efx->rx_bufs_per_page;442 efx->rx_pages_per_batch = DIV_ROUND_UP(EFX_RX_PREFERRED_BATCH,443 efx->rx_bufs_per_page);444}445 446/* efx_fast_push_rx_descriptors - push new RX descriptors quickly447 * @rx_queue: RX descriptor queue448 *449 * This will aim to fill the RX descriptor queue up to450 * @rx_queue->@max_fill. If there is insufficient atomic451 * memory to do so, a slow fill will be scheduled.452 *453 * The caller must provide serialisation (none is used here). In practise,454 * this means this function must run from the NAPI handler, or be called455 * when NAPI is disabled.456 */457void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue, bool atomic)458{459 struct efx_nic *efx = rx_queue->efx;460 unsigned int fill_level, batch_size;461 int space, rc = 0;462 463 if (!rx_queue->refill_enabled)464 return;465 466 /* Calculate current fill level, and exit if we don't need to fill */467 fill_level = (rx_queue->added_count - rx_queue->removed_count);468 EFX_WARN_ON_ONCE_PARANOID(fill_level > rx_queue->efx->rxq_entries);469 if (fill_level >= rx_queue->fast_fill_trigger)470 goto out;471 472 /* Record minimum fill level */473 if (unlikely(fill_level < rx_queue->min_fill)) {474 if (fill_level)475 rx_queue->min_fill = fill_level;476 }477 478 batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page;479 space = rx_queue->max_fill - fill_level;480 EFX_WARN_ON_ONCE_PARANOID(space < batch_size);481 482 netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,483 "RX queue %d fast-filling descriptor ring from"484 " level %d to level %d\n",485 efx_rx_queue_index(rx_queue), fill_level,486 rx_queue->max_fill);487 488 do {489 rc = efx_init_rx_buffers(rx_queue, atomic);490 if (unlikely(rc)) {491 /* Ensure that we don't leave the rx queue empty */492 efx_schedule_slow_fill(rx_queue);493 goto out;494 }495 } while ((space -= batch_size) >= batch_size);496 497 netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,498 "RX queue %d fast-filled descriptor ring "499 "to level %d\n", efx_rx_queue_index(rx_queue),500 rx_queue->added_count - rx_queue->removed_count);501 502 out:503 if (rx_queue->notified_count != rx_queue->added_count)504 efx_nic_notify_rx_desc(rx_queue);505}506 507/* Pass a received packet up through GRO. GRO can handle pages508 * regardless of checksum state and skbs with a good checksum.509 */510void511efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf,512 unsigned int n_frags, u8 *eh, __wsum csum)513{514 struct napi_struct *napi = &channel->napi_str;515 struct efx_nic *efx = channel->efx;516 struct sk_buff *skb;517 518 skb = napi_get_frags(napi);519 if (unlikely(!skb)) {520 struct efx_rx_queue *rx_queue;521 522 rx_queue = efx_channel_get_rx_queue(channel);523 efx_free_rx_buffers(rx_queue, rx_buf, n_frags);524 return;525 }526 527 if (efx->net_dev->features & NETIF_F_RXHASH &&528 efx_rx_buf_hash_valid(efx, eh))529 skb_set_hash(skb, efx_rx_buf_hash(efx, eh),530 PKT_HASH_TYPE_L3);531 if (csum) {532 skb->csum = csum;533 skb->ip_summed = CHECKSUM_COMPLETE;534 } else {535 skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ?536 CHECKSUM_UNNECESSARY : CHECKSUM_NONE);537 }538 skb->csum_level = !!(rx_buf->flags & EFX_RX_PKT_CSUM_LEVEL);539 540 for (;;) {541 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,542 rx_buf->page, rx_buf->page_offset,543 rx_buf->len);544 rx_buf->page = NULL;545 skb->len += rx_buf->len;546 if (skb_shinfo(skb)->nr_frags == n_frags)547 break;548 549 rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);550 }551 552 skb->data_len = skb->len;553 skb->truesize += n_frags * efx->rx_buffer_truesize;554 555 skb_record_rx_queue(skb, channel->rx_queue.core_index);556 557 napi_gro_frags(napi);558}559 560struct efx_rss_context_priv *efx_find_rss_context_entry(struct efx_nic *efx,561 u32 id)562{563 struct ethtool_rxfh_context *ctx;564 565 WARN_ON(!mutex_is_locked(&efx->net_dev->ethtool->rss_lock));566 567 ctx = xa_load(&efx->net_dev->ethtool->rss_ctx, id);568 if (!ctx)569 return NULL;570 return ethtool_rxfh_context_priv(ctx);571}572 573void efx_set_default_rx_indir_table(struct efx_nic *efx, u32 *indir)574{575 size_t i;576 577 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); i++)578 indir[i] = ethtool_rxfh_indir_default(i, efx->rss_spread);579}580 581/**582 * efx_filter_is_mc_recipient - test whether spec is a multicast recipient583 * @spec: Specification to test584 *585 * Return: %true if the specification is a non-drop RX filter that586 * matches a local MAC address I/G bit value of 1 or matches a local587 * IPv4 or IPv6 address value in the respective multicast address588 * range. Otherwise %false.589 */590bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec)591{592 if (!(spec->flags & EFX_FILTER_FLAG_RX) ||593 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP)594 return false;595 596 if (spec->match_flags &597 (EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_MAC_IG) &&598 is_multicast_ether_addr(spec->loc_mac))599 return true;600 601 if ((spec->match_flags &602 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==603 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {604 if (spec->ether_type == htons(ETH_P_IP) &&605 ipv4_is_multicast(spec->loc_host[0]))606 return true;607 if (spec->ether_type == htons(ETH_P_IPV6) &&608 ((const u8 *)spec->loc_host)[0] == 0xff)609 return true;610 }611 612 return false;613}614 615bool efx_filter_spec_equal(const struct efx_filter_spec *left,616 const struct efx_filter_spec *right)617{618 if ((left->match_flags ^ right->match_flags) |619 ((left->flags ^ right->flags) &620 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))621 return false;622 623 return memcmp(&left->vport_id, &right->vport_id,624 sizeof(struct efx_filter_spec) -625 offsetof(struct efx_filter_spec, vport_id)) == 0;626}627 628u32 efx_filter_spec_hash(const struct efx_filter_spec *spec)629{630 BUILD_BUG_ON(offsetof(struct efx_filter_spec, vport_id) & 3);631 return jhash2((const u32 *)&spec->vport_id,632 (sizeof(struct efx_filter_spec) -633 offsetof(struct efx_filter_spec, vport_id)) / 4,634 0);635}636 637#ifdef CONFIG_RFS_ACCEL638bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx,639 bool *force)640{641 if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) {642 /* ARFS is currently updating this entry, leave it */643 return false;644 }645 if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) {646 /* ARFS tried and failed to update this, so it's probably out647 * of date. Remove the filter and the ARFS rule entry.648 */649 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;650 *force = true;651 return true;652 } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */653 /* ARFS has moved on, so old filter is not needed. Since we did654 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will655 * not be removed by efx_rps_hash_del() subsequently.656 */657 *force = true;658 return true;659 }660 /* Remove it iff ARFS wants to. */661 return true;662}663 664static665struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx,666 const struct efx_filter_spec *spec)667{668 u32 hash = efx_filter_spec_hash(spec);669 670 lockdep_assert_held(&efx->rps_hash_lock);671 if (!efx->rps_hash_table)672 return NULL;673 return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE];674}675 676struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx,677 const struct efx_filter_spec *spec)678{679 struct efx_arfs_rule *rule;680 struct hlist_head *head;681 struct hlist_node *node;682 683 head = efx_rps_hash_bucket(efx, spec);684 if (!head)685 return NULL;686 hlist_for_each(node, head) {687 rule = container_of(node, struct efx_arfs_rule, node);688 if (efx_filter_spec_equal(spec, &rule->spec))689 return rule;690 }691 return NULL;692}693 694struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,695 const struct efx_filter_spec *spec,696 bool *new)697{698 struct efx_arfs_rule *rule;699 struct hlist_head *head;700 struct hlist_node *node;701 702 head = efx_rps_hash_bucket(efx, spec);703 if (!head)704 return NULL;705 hlist_for_each(node, head) {706 rule = container_of(node, struct efx_arfs_rule, node);707 if (efx_filter_spec_equal(spec, &rule->spec)) {708 *new = false;709 return rule;710 }711 }712 rule = kmalloc(sizeof(*rule), GFP_ATOMIC);713 *new = true;714 if (rule) {715 memcpy(&rule->spec, spec, sizeof(rule->spec));716 hlist_add_head(&rule->node, head);717 }718 return rule;719}720 721void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec)722{723 struct efx_arfs_rule *rule;724 struct hlist_head *head;725 struct hlist_node *node;726 727 head = efx_rps_hash_bucket(efx, spec);728 if (WARN_ON(!head))729 return;730 hlist_for_each(node, head) {731 rule = container_of(node, struct efx_arfs_rule, node);732 if (efx_filter_spec_equal(spec, &rule->spec)) {733 /* Someone already reused the entry. We know that if734 * this check doesn't fire (i.e. filter_id == REMOVING)735 * then the REMOVING mark was put there by our caller,736 * because caller is holding a lock on filter table and737 * only holders of that lock set REMOVING.738 */739 if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING)740 return;741 hlist_del(node);742 kfree(rule);743 return;744 }745 }746 /* We didn't find it. */747 WARN_ON(1);748}749#endif750 751int efx_probe_filters(struct efx_nic *efx)752{753 int rc;754 755 mutex_lock(&efx->mac_lock);756 rc = efx->type->filter_table_probe(efx);757 if (rc)758 goto out_unlock;759 760#ifdef CONFIG_RFS_ACCEL761 if (efx->type->offload_features & NETIF_F_NTUPLE) {762 struct efx_channel *channel;763 int i, success = 1;764 765 efx_for_each_channel(channel, efx) {766 channel->rps_flow_id =767 kcalloc(efx->type->max_rx_ip_filters,768 sizeof(*channel->rps_flow_id),769 GFP_KERNEL);770 if (!channel->rps_flow_id)771 success = 0;772 else773 for (i = 0;774 i < efx->type->max_rx_ip_filters;775 ++i)776 channel->rps_flow_id[i] =777 RPS_FLOW_ID_INVALID;778 channel->rfs_expire_index = 0;779 channel->rfs_filter_count = 0;780 }781 782 if (!success) {783 efx_for_each_channel(channel, efx) {784 kfree(channel->rps_flow_id);785 channel->rps_flow_id = NULL;786 }787 efx->type->filter_table_remove(efx);788 rc = -ENOMEM;789 goto out_unlock;790 }791 }792#endif793out_unlock:794 mutex_unlock(&efx->mac_lock);795 return rc;796}797 798void efx_remove_filters(struct efx_nic *efx)799{800#ifdef CONFIG_RFS_ACCEL801 struct efx_channel *channel;802 803 efx_for_each_channel(channel, efx) {804 cancel_delayed_work_sync(&channel->filter_work);805 kfree(channel->rps_flow_id);806 channel->rps_flow_id = NULL;807 }808#endif809 efx->type->filter_table_remove(efx);810}811 812#ifdef CONFIG_RFS_ACCEL813 814static void efx_filter_rfs_work(struct work_struct *data)815{816 struct efx_async_filter_insertion *req = container_of(data, struct efx_async_filter_insertion,817 work);818 struct efx_nic *efx = efx_netdev_priv(req->net_dev);819 struct efx_channel *channel = efx_get_channel(efx, req->rxq_index);820 int slot_idx = req - efx->rps_slot;821 struct efx_arfs_rule *rule;822 u16 arfs_id = 0;823 int rc;824 825 rc = efx->type->filter_insert(efx, &req->spec, true);826 if (rc >= 0)827 /* Discard 'priority' part of EF10+ filter ID (mcdi_filters) */828 rc %= efx->type->max_rx_ip_filters;829 if (efx->rps_hash_table) {830 spin_lock_bh(&efx->rps_hash_lock);831 rule = efx_rps_hash_find(efx, &req->spec);832 /* The rule might have already gone, if someone else's request833 * for the same spec was already worked and then expired before834 * we got around to our work. In that case we have nothing835 * tying us to an arfs_id, meaning that as soon as the filter836 * is considered for expiry it will be removed.837 */838 if (rule) {839 if (rc < 0)840 rule->filter_id = EFX_ARFS_FILTER_ID_ERROR;841 else842 rule->filter_id = rc;843 arfs_id = rule->arfs_id;844 }845 spin_unlock_bh(&efx->rps_hash_lock);846 }847 if (rc >= 0) {848 /* Remember this so we can check whether to expire the filter849 * later.850 */851 mutex_lock(&efx->rps_mutex);852 if (channel->rps_flow_id[rc] == RPS_FLOW_ID_INVALID)853 channel->rfs_filter_count++;854 channel->rps_flow_id[rc] = req->flow_id;855 mutex_unlock(&efx->rps_mutex);856 857 if (req->spec.ether_type == htons(ETH_P_IP))858 netif_info(efx, rx_status, efx->net_dev,859 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d id %u]\n",860 (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",861 req->spec.rem_host, ntohs(req->spec.rem_port),862 req->spec.loc_host, ntohs(req->spec.loc_port),863 req->rxq_index, req->flow_id, rc, arfs_id);864 else865 netif_info(efx, rx_status, efx->net_dev,866 "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d id %u]\n",867 (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",868 req->spec.rem_host, ntohs(req->spec.rem_port),869 req->spec.loc_host, ntohs(req->spec.loc_port),870 req->rxq_index, req->flow_id, rc, arfs_id);871 channel->n_rfs_succeeded++;872 } else {873 if (req->spec.ether_type == htons(ETH_P_IP))874 netif_dbg(efx, rx_status, efx->net_dev,875 "failed to steer %s %pI4:%u:%pI4:%u to queue %u [flow %u rc %d id %u]\n",876 (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",877 req->spec.rem_host, ntohs(req->spec.rem_port),878 req->spec.loc_host, ntohs(req->spec.loc_port),879 req->rxq_index, req->flow_id, rc, arfs_id);880 else881 netif_dbg(efx, rx_status, efx->net_dev,882 "failed to steer %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u rc %d id %u]\n",883 (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",884 req->spec.rem_host, ntohs(req->spec.rem_port),885 req->spec.loc_host, ntohs(req->spec.loc_port),886 req->rxq_index, req->flow_id, rc, arfs_id);887 channel->n_rfs_failed++;888 /* We're overloading the NIC's filter tables, so let's do a889 * chunk of extra expiry work.890 */891 __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count,892 100u));893 }894 895 /* Release references */896 clear_bit(slot_idx, &efx->rps_slot_map);897 dev_put(req->net_dev);898}899 900int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,901 u16 rxq_index, u32 flow_id)902{903 struct efx_nic *efx = efx_netdev_priv(net_dev);904 struct efx_async_filter_insertion *req;905 struct efx_arfs_rule *rule;906 struct flow_keys fk;907 int slot_idx;908 bool new;909 int rc;910 911 /* find a free slot */912 for (slot_idx = 0; slot_idx < EFX_RPS_MAX_IN_FLIGHT; slot_idx++)913 if (!test_and_set_bit(slot_idx, &efx->rps_slot_map))914 break;915 if (slot_idx >= EFX_RPS_MAX_IN_FLIGHT)916 return -EBUSY;917 918 if (flow_id == RPS_FLOW_ID_INVALID) {919 rc = -EINVAL;920 goto out_clear;921 }922 923 if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) {924 rc = -EPROTONOSUPPORT;925 goto out_clear;926 }927 928 if (fk.basic.n_proto != htons(ETH_P_IP) && fk.basic.n_proto != htons(ETH_P_IPV6)) {929 rc = -EPROTONOSUPPORT;930 goto out_clear;931 }932 if (fk.control.flags & FLOW_DIS_IS_FRAGMENT) {933 rc = -EPROTONOSUPPORT;934 goto out_clear;935 }936 937 req = efx->rps_slot + slot_idx;938 efx_filter_init_rx(&req->spec, EFX_FILTER_PRI_HINT,939 efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0,940 rxq_index);941 req->spec.match_flags =942 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |943 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |944 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;945 req->spec.ether_type = fk.basic.n_proto;946 req->spec.ip_proto = fk.basic.ip_proto;947 948 if (fk.basic.n_proto == htons(ETH_P_IP)) {949 req->spec.rem_host[0] = fk.addrs.v4addrs.src;950 req->spec.loc_host[0] = fk.addrs.v4addrs.dst;951 } else {952 memcpy(req->spec.rem_host, &fk.addrs.v6addrs.src,953 sizeof(struct in6_addr));954 memcpy(req->spec.loc_host, &fk.addrs.v6addrs.dst,955 sizeof(struct in6_addr));956 }957 958 req->spec.rem_port = fk.ports.src;959 req->spec.loc_port = fk.ports.dst;960 961 if (efx->rps_hash_table) {962 /* Add it to ARFS hash table */963 spin_lock(&efx->rps_hash_lock);964 rule = efx_rps_hash_add(efx, &req->spec, &new);965 if (!rule) {966 rc = -ENOMEM;967 goto out_unlock;968 }969 if (new)970 rule->arfs_id = efx->rps_next_id++ % RPS_NO_FILTER;971 rc = rule->arfs_id;972 /* Skip if existing or pending filter already does the right thing */973 if (!new && rule->rxq_index == rxq_index &&974 rule->filter_id >= EFX_ARFS_FILTER_ID_PENDING)975 goto out_unlock;976 rule->rxq_index = rxq_index;977 rule->filter_id = EFX_ARFS_FILTER_ID_PENDING;978 spin_unlock(&efx->rps_hash_lock);979 } else {980 /* Without an ARFS hash table, we just use arfs_id 0 for all981 * filters. This means if multiple flows hash to the same982 * flow_id, all but the most recently touched will be eligible983 * for expiry.984 */985 rc = 0;986 }987 988 /* Queue the request */989 dev_hold(req->net_dev = net_dev);990 INIT_WORK(&req->work, efx_filter_rfs_work);991 req->rxq_index = rxq_index;992 req->flow_id = flow_id;993 schedule_work(&req->work);994 return rc;995out_unlock:996 spin_unlock(&efx->rps_hash_lock);997out_clear:998 clear_bit(slot_idx, &efx->rps_slot_map);999 return rc;1000}1001 1002bool __efx_filter_rfs_expire(struct efx_channel *channel, unsigned int quota)1003{1004 bool (*expire_one)(struct efx_nic *efx, u32 flow_id, unsigned int index);1005 struct efx_nic *efx = channel->efx;1006 unsigned int index, size, start;1007 u32 flow_id;1008 1009 if (!mutex_trylock(&efx->rps_mutex))1010 return false;1011 expire_one = efx->type->filter_rfs_expire_one;1012 index = channel->rfs_expire_index;1013 start = index;1014 size = efx->type->max_rx_ip_filters;1015 while (quota) {1016 flow_id = channel->rps_flow_id[index];1017 1018 if (flow_id != RPS_FLOW_ID_INVALID) {1019 quota--;1020 if (expire_one(efx, flow_id, index)) {1021 netif_info(efx, rx_status, efx->net_dev,1022 "expired filter %d [channel %u flow %u]\n",1023 index, channel->channel, flow_id);1024 channel->rps_flow_id[index] = RPS_FLOW_ID_INVALID;1025 channel->rfs_filter_count--;1026 }1027 }1028 if (++index == size)1029 index = 0;1030 /* If we were called with a quota that exceeds the total number1031 * of filters in the table (which shouldn't happen, but could1032 * if two callers race), ensure that we don't loop forever -1033 * stop when we've examined every row of the table.1034 */1035 if (index == start)1036 break;1037 }1038 1039 channel->rfs_expire_index = index;1040 mutex_unlock(&efx->rps_mutex);1041 return true;1042}1043 1044#endif /* CONFIG_RFS_ACCEL */1045