512 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/****************************************************************************3 * Driver for Solarflare network controllers and boards4 * Copyright 2018 Solarflare Communications Inc.5 * Copyright 2019-2020 Xilinx Inc.6 *7 * This program is free software; you can redistribute it and/or modify it8 * under the terms of the GNU General Public License version 2 as published9 * by the Free Software Foundation, incorporated herein by reference.10 */11 12#include <net/ip6_checksum.h>13 14#include "net_driver.h"15#include "tx_common.h"16#include "nic_common.h"17#include "mcdi_functions.h"18#include "ef100_regs.h"19#include "io.h"20#include "ef100_tx.h"21#include "ef100_nic.h"22 23int ef100_tx_probe(struct efx_tx_queue *tx_queue)24{25 /* Allocate an extra descriptor for the QMDA status completion entry */26 return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd,27 (tx_queue->ptr_mask + 2) *28 sizeof(efx_oword_t),29 GFP_KERNEL);30}31 32void ef100_tx_init(struct efx_tx_queue *tx_queue)33{34 /* must be the inverse of lookup in efx_get_tx_channel */35 tx_queue->core_txq =36 netdev_get_tx_queue(tx_queue->efx->net_dev,37 tx_queue->channel->channel -38 tx_queue->efx->tx_channel_offset);39 40 /* This value is purely documentational; as EF100 never passes through41 * the switch statement in tx.c:__efx_enqueue_skb(), that switch does42 * not handle case 3. EF100's TSOv3 descriptors are generated by43 * ef100_make_tso_desc().44 * Meanwhile, all efx_mcdi_tx_init() cares about is that it's not 2.45 */46 tx_queue->tso_version = 3;47 if (efx_mcdi_tx_init(tx_queue))48 netdev_WARN(tx_queue->efx->net_dev,49 "failed to initialise TXQ %d\n", tx_queue->queue);50}51 52static bool ef100_tx_can_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb)53{54 struct efx_nic *efx = tx_queue->efx;55 struct ef100_nic_data *nic_data;56 struct efx_tx_buffer *buffer;57 size_t header_len;58 u32 mss;59 60 nic_data = efx->nic_data;61 62 if (!skb_is_gso_tcp(skb))63 return false;64 if (!(efx->net_dev->features & NETIF_F_TSO))65 return false;66 67 mss = skb_shinfo(skb)->gso_size;68 if (unlikely(mss < 4)) {69 WARN_ONCE(1, "MSS of %u is too small for TSO\n", mss);70 return false;71 }72 73 header_len = efx_tx_tso_header_length(skb);74 if (header_len > nic_data->tso_max_hdr_len)75 return false;76 77 if (skb_shinfo(skb)->gso_segs > nic_data->tso_max_payload_num_segs) {78 /* net_dev->gso_max_segs should've caught this */79 WARN_ON_ONCE(1);80 return false;81 }82 83 if (skb->data_len / mss > nic_data->tso_max_frames)84 return false;85 86 /* net_dev->gso_max_size should've caught this */87 if (WARN_ON_ONCE(skb->data_len > nic_data->tso_max_payload_len))88 return false;89 90 /* Reserve an empty buffer for the TSO V3 descriptor.91 * Convey the length of the header since we already know it.92 */93 buffer = efx_tx_queue_get_insert_buffer(tx_queue);94 buffer->flags = EFX_TX_BUF_TSO_V3 | EFX_TX_BUF_CONT;95 buffer->len = header_len;96 buffer->unmap_len = 0;97 buffer->skb = skb;98 ++tx_queue->insert_count;99 return true;100}101 102static efx_oword_t *ef100_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)103{104 if (likely(tx_queue->txd.addr))105 return ((efx_oword_t *)tx_queue->txd.addr) + index;106 else107 return NULL;108}109 110static void ef100_notify_tx_desc(struct efx_tx_queue *tx_queue)111{112 unsigned int write_ptr;113 efx_dword_t reg;114 115 tx_queue->xmit_pending = false;116 117 if (unlikely(tx_queue->notify_count == tx_queue->write_count))118 return;119 120 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;121 /* The write pointer goes into the high word */122 EFX_POPULATE_DWORD_1(reg, ERF_GZ_TX_RING_PIDX, write_ptr);123 efx_writed_page(tx_queue->efx, ®,124 ER_GZ_TX_RING_DOORBELL, tx_queue->queue);125 tx_queue->notify_count = tx_queue->write_count;126}127 128static void ef100_tx_push_buffers(struct efx_tx_queue *tx_queue)129{130 ef100_notify_tx_desc(tx_queue);131 ++tx_queue->pushes;132}133 134static void ef100_set_tx_csum_partial(const struct sk_buff *skb,135 struct efx_tx_buffer *buffer, efx_oword_t *txd)136{137 efx_oword_t csum;138 int csum_start;139 140 if (!skb || skb->ip_summed != CHECKSUM_PARTIAL)141 return;142 143 /* skb->csum_start has the offset from head, but we need the offset144 * from data.145 */146 csum_start = skb_checksum_start_offset(skb);147 EFX_POPULATE_OWORD_3(csum,148 ESF_GZ_TX_SEND_CSO_PARTIAL_EN, 1,149 ESF_GZ_TX_SEND_CSO_PARTIAL_START_W,150 csum_start >> 1,151 ESF_GZ_TX_SEND_CSO_PARTIAL_CSUM_W,152 skb->csum_offset >> 1);153 EFX_OR_OWORD(*txd, *txd, csum);154}155 156static void ef100_set_tx_hw_vlan(const struct sk_buff *skb, efx_oword_t *txd)157{158 u16 vlan_tci = skb_vlan_tag_get(skb);159 efx_oword_t vlan;160 161 EFX_POPULATE_OWORD_2(vlan,162 ESF_GZ_TX_SEND_VLAN_INSERT_EN, 1,163 ESF_GZ_TX_SEND_VLAN_INSERT_TCI, vlan_tci);164 EFX_OR_OWORD(*txd, *txd, vlan);165}166 167static void ef100_make_send_desc(struct efx_nic *efx,168 const struct sk_buff *skb,169 struct efx_tx_buffer *buffer, efx_oword_t *txd,170 unsigned int segment_count)171{172 /* TX send descriptor */173 EFX_POPULATE_OWORD_3(*txd,174 ESF_GZ_TX_SEND_NUM_SEGS, segment_count,175 ESF_GZ_TX_SEND_LEN, buffer->len,176 ESF_GZ_TX_SEND_ADDR, buffer->dma_addr);177 178 if (likely(efx->net_dev->features & NETIF_F_HW_CSUM))179 ef100_set_tx_csum_partial(skb, buffer, txd);180 if (efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX &&181 skb && skb_vlan_tag_present(skb))182 ef100_set_tx_hw_vlan(skb, txd);183}184 185static void ef100_make_tso_desc(struct efx_nic *efx,186 const struct sk_buff *skb,187 struct efx_tx_buffer *buffer, efx_oword_t *txd,188 unsigned int segment_count)189{190 bool gso_partial = skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL;191 unsigned int len, ip_offset, tcp_offset, payload_segs;192 u32 mangleid = ESE_GZ_TX_DESC_IP4_ID_INC_MOD16;193 unsigned int outer_ip_offset, outer_l4_offset;194 u16 vlan_tci = skb_vlan_tag_get(skb);195 u32 mss = skb_shinfo(skb)->gso_size;196 bool encap = skb->encapsulation;197 bool udp_encap = false;198 u16 vlan_enable = 0;199 struct tcphdr *tcp;200 bool outer_csum;201 u32 paylen;202 203 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_FIXEDID)204 mangleid = ESE_GZ_TX_DESC_IP4_ID_NO_OP;205 if (efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX)206 vlan_enable = skb_vlan_tag_present(skb);207 208 len = skb->len - buffer->len;209 /* We use 1 for the TSO descriptor and 1 for the header */210 payload_segs = segment_count - 2;211 if (encap) {212 outer_ip_offset = skb_network_offset(skb);213 outer_l4_offset = skb_transport_offset(skb);214 ip_offset = skb_inner_network_offset(skb);215 tcp_offset = skb_inner_transport_offset(skb);216 if (skb_shinfo(skb)->gso_type &217 (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM))218 udp_encap = true;219 } else {220 ip_offset = skb_network_offset(skb);221 tcp_offset = skb_transport_offset(skb);222 outer_ip_offset = outer_l4_offset = 0;223 }224 outer_csum = skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM;225 226 /* subtract TCP payload length from inner checksum */227 tcp = (void *)skb->data + tcp_offset;228 paylen = skb->len - tcp_offset;229 csum_replace_by_diff(&tcp->check, (__force __wsum)htonl(paylen));230 231 EFX_POPULATE_OWORD_19(*txd,232 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_TSO,233 ESF_GZ_TX_TSO_MSS, mss,234 ESF_GZ_TX_TSO_HDR_NUM_SEGS, 1,235 ESF_GZ_TX_TSO_PAYLOAD_NUM_SEGS, payload_segs,236 ESF_GZ_TX_TSO_HDR_LEN_W, buffer->len >> 1,237 ESF_GZ_TX_TSO_PAYLOAD_LEN, len,238 ESF_GZ_TX_TSO_CSO_OUTER_L4, outer_csum,239 ESF_GZ_TX_TSO_CSO_INNER_L4, 1,240 ESF_GZ_TX_TSO_INNER_L3_OFF_W, ip_offset >> 1,241 ESF_GZ_TX_TSO_INNER_L4_OFF_W, tcp_offset >> 1,242 ESF_GZ_TX_TSO_ED_INNER_IP4_ID, mangleid,243 ESF_GZ_TX_TSO_ED_INNER_IP_LEN, 1,244 ESF_GZ_TX_TSO_OUTER_L3_OFF_W, outer_ip_offset >> 1,245 ESF_GZ_TX_TSO_OUTER_L4_OFF_W, outer_l4_offset >> 1,246 ESF_GZ_TX_TSO_ED_OUTER_UDP_LEN, udp_encap && !gso_partial,247 ESF_GZ_TX_TSO_ED_OUTER_IP_LEN, encap && !gso_partial,248 ESF_GZ_TX_TSO_ED_OUTER_IP4_ID, encap ? mangleid :249 ESE_GZ_TX_DESC_IP4_ID_NO_OP,250 ESF_GZ_TX_TSO_VLAN_INSERT_EN, vlan_enable,251 ESF_GZ_TX_TSO_VLAN_INSERT_TCI, vlan_tci252 );253}254 255static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue,256 const struct sk_buff *skb,257 unsigned int segment_count,258 struct efx_rep *efv)259{260 unsigned int old_write_count = tx_queue->write_count;261 unsigned int new_write_count = old_write_count;262 struct efx_tx_buffer *buffer;263 unsigned int next_desc_type;264 unsigned int write_ptr;265 efx_oword_t *txd;266 unsigned int nr_descs = tx_queue->insert_count - old_write_count;267 268 if (unlikely(nr_descs == 0))269 return;270 271 if (segment_count)272 next_desc_type = ESE_GZ_TX_DESC_TYPE_TSO;273 else274 next_desc_type = ESE_GZ_TX_DESC_TYPE_SEND;275 276 if (unlikely(efv)) {277 /* Create TX override descriptor */278 write_ptr = new_write_count & tx_queue->ptr_mask;279 txd = ef100_tx_desc(tx_queue, write_ptr);280 ++new_write_count;281 282 tx_queue->packet_write_count = new_write_count;283 EFX_POPULATE_OWORD_3(*txd,284 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_PREFIX,285 ESF_GZ_TX_PREFIX_EGRESS_MPORT, efv->mport,286 ESF_GZ_TX_PREFIX_EGRESS_MPORT_EN, 1);287 nr_descs--;288 }289 290 /* if it's a raw write (such as XDP) then always SEND single frames */291 if (!skb)292 nr_descs = 1;293 294 do {295 write_ptr = new_write_count & tx_queue->ptr_mask;296 buffer = &tx_queue->buffer[write_ptr];297 txd = ef100_tx_desc(tx_queue, write_ptr);298 ++new_write_count;299 300 /* Create TX descriptor ring entry */301 tx_queue->packet_write_count = new_write_count;302 303 switch (next_desc_type) {304 case ESE_GZ_TX_DESC_TYPE_SEND:305 ef100_make_send_desc(tx_queue->efx, skb,306 buffer, txd, nr_descs);307 break;308 case ESE_GZ_TX_DESC_TYPE_TSO:309 /* TX TSO descriptor */310 WARN_ON_ONCE(!(buffer->flags & EFX_TX_BUF_TSO_V3));311 ef100_make_tso_desc(tx_queue->efx, skb,312 buffer, txd, nr_descs);313 break;314 default:315 /* TX segment descriptor */316 EFX_POPULATE_OWORD_3(*txd,317 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEG,318 ESF_GZ_TX_SEG_LEN, buffer->len,319 ESF_GZ_TX_SEG_ADDR, buffer->dma_addr);320 }321 /* if it's a raw write (such as XDP) then always SEND */322 next_desc_type = skb ? ESE_GZ_TX_DESC_TYPE_SEG :323 ESE_GZ_TX_DESC_TYPE_SEND;324 /* mark as an EFV buffer if applicable */325 if (unlikely(efv))326 buffer->flags |= EFX_TX_BUF_EFV;327 328 } while (new_write_count != tx_queue->insert_count);329 330 wmb(); /* Ensure descriptors are written before they are fetched */331 332 tx_queue->write_count = new_write_count;333 334 /* The write_count above must be updated before reading335 * channel->holdoff_doorbell to avoid a race with the336 * completion path, so ensure these operations are not337 * re-ordered. This also flushes the update of write_count338 * back into the cache.339 */340 smp_mb();341}342 343void ef100_tx_write(struct efx_tx_queue *tx_queue)344{345 ef100_tx_make_descriptors(tx_queue, NULL, 0, NULL);346 ef100_tx_push_buffers(tx_queue);347}348 349int ef100_ev_tx(struct efx_channel *channel, const efx_qword_t *p_event)350{351 unsigned int tx_done =352 EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_NUM_DESC);353 unsigned int qlabel =354 EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_Q_LABEL);355 struct efx_tx_queue *tx_queue =356 efx_channel_get_tx_queue(channel, qlabel);357 unsigned int tx_index = (tx_queue->read_count + tx_done - 1) &358 tx_queue->ptr_mask;359 360 return efx_xmit_done(tx_queue, tx_index);361}362 363/* Add a socket buffer to a TX queue364 *365 * You must hold netif_tx_lock() to call this function.366 *367 * Returns 0 on success, error code otherwise. In case of an error this368 * function will free the SKB.369 */370netdev_tx_t ef100_enqueue_skb(struct efx_tx_queue *tx_queue,371 struct sk_buff *skb)372{373 return __ef100_enqueue_skb(tx_queue, skb, NULL);374}375 376int __ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb,377 struct efx_rep *efv)378{379 unsigned int old_insert_count = tx_queue->insert_count;380 struct efx_nic *efx = tx_queue->efx;381 bool xmit_more = netdev_xmit_more();382 unsigned int fill_level;383 unsigned int segments;384 int rc;385 386 if (!tx_queue->buffer || !tx_queue->ptr_mask) {387 netif_stop_queue(efx->net_dev);388 dev_kfree_skb_any(skb);389 return -ENODEV;390 }391 392 segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0;393 if (segments == 1)394 segments = 0; /* Don't use TSO/GSO for a single segment. */395 if (segments && !ef100_tx_can_tso(tx_queue, skb)) {396 rc = efx_tx_tso_fallback(tx_queue, skb);397 tx_queue->tso_fallbacks++;398 if (rc)399 goto err;400 else401 return 0;402 }403 404 if (unlikely(efv)) {405 struct efx_tx_buffer *buffer = __efx_tx_queue_get_insert_buffer(tx_queue);406 407 /* Drop representor packets if the queue is stopped.408 * We currently don't assert backoff to representors so this is409 * to make sure representor traffic can't starve the main410 * net device.411 * And, of course, if there are no TX descriptors left.412 */413 if (netif_tx_queue_stopped(tx_queue->core_txq) ||414 unlikely(efx_tx_buffer_in_use(buffer))) {415 atomic64_inc(&efv->stats.tx_errors);416 rc = -ENOSPC;417 goto err;418 }419 420 /* Also drop representor traffic if it could cause us to421 * stop the queue. If we assert backoff and we haven't422 * received traffic on the main net device recently then the423 * TX watchdog can go off erroneously.424 */425 fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);426 fill_level += efx_tx_max_skb_descs(efx);427 if (fill_level > efx->txq_stop_thresh) {428 struct efx_tx_queue *txq2;429 430 /* Refresh cached fill level and re-check */431 efx_for_each_channel_tx_queue(txq2, tx_queue->channel)432 txq2->old_read_count = READ_ONCE(txq2->read_count);433 434 fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);435 fill_level += efx_tx_max_skb_descs(efx);436 if (fill_level > efx->txq_stop_thresh) {437 atomic64_inc(&efv->stats.tx_errors);438 rc = -ENOSPC;439 goto err;440 }441 }442 443 buffer->flags = EFX_TX_BUF_OPTION | EFX_TX_BUF_EFV;444 tx_queue->insert_count++;445 }446 447 /* Map for DMA and create descriptors */448 rc = efx_tx_map_data(tx_queue, skb, segments);449 if (rc)450 goto err;451 ef100_tx_make_descriptors(tx_queue, skb, segments, efv);452 453 fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);454 if (fill_level > efx->txq_stop_thresh) {455 struct efx_tx_queue *txq2;456 457 /* Because of checks above, representor traffic should458 * not be able to stop the queue.459 */460 WARN_ON(efv);461 462 netif_tx_stop_queue(tx_queue->core_txq);463 /* Re-read after a memory barrier in case we've raced with464 * the completion path. Otherwise there's a danger we'll never465 * restart the queue if all completions have just happened.466 */467 smp_mb();468 efx_for_each_channel_tx_queue(txq2, tx_queue->channel)469 txq2->old_read_count = READ_ONCE(txq2->read_count);470 fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);471 if (fill_level < efx->txq_stop_thresh)472 netif_tx_start_queue(tx_queue->core_txq);473 }474 475 tx_queue->xmit_pending = true;476 477 /* If xmit_more then we don't need to push the doorbell, unless there478 * are 256 descriptors already queued in which case we have to push to479 * ensure we never push more than 256 at once.480 *481 * Always push for representor traffic, and don't account it to parent482 * PF netdevice's BQL.483 */484 if (unlikely(efv) ||485 __netdev_tx_sent_queue(tx_queue->core_txq, skb->len, xmit_more) ||486 tx_queue->write_count - tx_queue->notify_count > 255)487 ef100_tx_push_buffers(tx_queue);488 489 if (segments) {490 tx_queue->tso_bursts++;491 tx_queue->tso_packets += segments;492 tx_queue->tx_packets += segments;493 } else {494 tx_queue->tx_packets++;495 }496 return 0;497 498err:499 efx_enqueue_unwind(tx_queue, old_insert_count);500 if (!IS_ERR_OR_NULL(skb))501 dev_kfree_skb_any(skb);502 503 /* If we're not expecting another transmit and we had something to push504 * on this queue then we need to push here to get the previous packets505 * out. We only enter this branch from before the xmit_more handling506 * above, so xmit_pending still refers to the old state.507 */508 if (tx_queue->xmit_pending && !xmit_more)509 ef100_tx_push_buffers(tx_queue);510 return rc;511}512