365 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause2 3/* Packet receive logic for Mellanox Gigabit Ethernet driver4 *5 * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES6 */7 8#include <linux/etherdevice.h>9#include <linux/skbuff.h>10 11#include "mlxbf_gige.h"12#include "mlxbf_gige_regs.h"13 14void mlxbf_gige_enable_multicast_rx(struct mlxbf_gige *priv)15{16 void __iomem *base = priv->base;17 u64 data;18 19 data = readq(base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);20 data |= MLXBF_GIGE_RX_MAC_FILTER_EN_MULTICAST;21 writeq(data, base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);22}23 24void mlxbf_gige_disable_multicast_rx(struct mlxbf_gige *priv)25{26 void __iomem *base = priv->base;27 u64 data;28 29 data = readq(base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);30 data &= ~MLXBF_GIGE_RX_MAC_FILTER_EN_MULTICAST;31 writeq(data, base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);32}33 34void mlxbf_gige_enable_mac_rx_filter(struct mlxbf_gige *priv,35 unsigned int index)36{37 void __iomem *base = priv->base;38 u64 control;39 40 /* Enable MAC receive filter mask for specified index */41 control = readq(base + MLXBF_GIGE_CONTROL);42 control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);43 writeq(control, base + MLXBF_GIGE_CONTROL);44}45 46void mlxbf_gige_disable_mac_rx_filter(struct mlxbf_gige *priv,47 unsigned int index)48{49 void __iomem *base = priv->base;50 u64 control;51 52 /* Disable MAC receive filter mask for specified index */53 control = readq(base + MLXBF_GIGE_CONTROL);54 control &= ~(MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);55 writeq(control, base + MLXBF_GIGE_CONTROL);56}57 58void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,59 unsigned int index, u64 dmac)60{61 void __iomem *base = priv->base;62 63 /* Write destination MAC to specified MAC RX filter */64 writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER +65 (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));66}67 68void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,69 unsigned int index, u64 *dmac)70{71 void __iomem *base = priv->base;72 73 /* Read destination MAC from specified MAC RX filter */74 *dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER +75 (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));76}77 78void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv)79{80 void __iomem *base = priv->base;81 u64 control;82 u64 end_mac;83 84 /* Enable MAC_ID_RANGE match functionality */85 control = readq(base + MLXBF_GIGE_CONTROL);86 control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;87 writeq(control, base + MLXBF_GIGE_CONTROL);88 89 /* Set start of destination MAC range check to 0 */90 writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START);91 92 /* Set end of destination MAC range check to all FFs */93 end_mac = BCAST_MAC_ADDR;94 writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END);95}96 97void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv)98{99 void __iomem *base = priv->base;100 u64 control;101 102 /* Disable MAC_ID_RANGE match functionality */103 control = readq(base + MLXBF_GIGE_CONTROL);104 control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;105 writeq(control, base + MLXBF_GIGE_CONTROL);106 107 /* NOTE: no need to change DMAC_RANGE_START or END;108 * those values are ignored since MAC_ID_RANGE_EN=0109 */110}111 112/* Receive Initialization113 * 1) Configures RX MAC filters via MMIO registers114 * 2) Allocates RX WQE array using coherent DMA mapping115 * 3) Initializes each element of RX WQE array with a receive116 * buffer pointer (also using coherent DMA mapping)117 * 4) Allocates RX CQE array using coherent DMA mapping118 * 5) Completes other misc receive initialization119 */120int mlxbf_gige_rx_init(struct mlxbf_gige *priv)121{122 size_t wq_size, cq_size;123 dma_addr_t *rx_wqe_ptr;124 dma_addr_t rx_buf_dma;125 u64 data;126 int i, j;127 128 /* Configure MAC RX filter #0 to allow RX of broadcast pkts */129 mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX,130 BCAST_MAC_ADDR);131 132 wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;133 priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size,134 &priv->rx_wqe_base_dma,135 GFP_KERNEL);136 if (!priv->rx_wqe_base)137 return -ENOMEM;138 139 /* Initialize 'rx_wqe_ptr' to point to first RX WQE in array140 * Each RX WQE is simply a receive buffer pointer, so walk141 * the entire array, allocating a 2KB buffer for each element142 */143 rx_wqe_ptr = priv->rx_wqe_base;144 145 for (i = 0; i < priv->rx_q_entries; i++) {146 priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,147 &rx_buf_dma, DMA_FROM_DEVICE);148 if (!priv->rx_skb[i])149 goto free_wqe_and_skb;150 *rx_wqe_ptr++ = rx_buf_dma;151 }152 153 /* Write RX WQE base address into MMIO reg */154 writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE);155 156 cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;157 priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size,158 &priv->rx_cqe_base_dma,159 GFP_KERNEL);160 if (!priv->rx_cqe_base)161 goto free_wqe_and_skb;162 163 for (i = 0; i < priv->rx_q_entries; i++)164 priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK;165 166 /* Write RX CQE base address into MMIO reg */167 writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE);168 169 /* Write RX_WQE_PI with current number of replenished buffers */170 writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI);171 172 /* Enable removal of CRC during RX */173 data = readq(priv->base + MLXBF_GIGE_RX);174 data |= MLXBF_GIGE_RX_STRIP_CRC_EN;175 writeq(data, priv->base + MLXBF_GIGE_RX);176 177 /* Enable RX MAC filter pass and discard counters */178 writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN,179 priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC);180 writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN,181 priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS);182 183 writeq(ilog2(priv->rx_q_entries),184 priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2);185 186 /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to187 * indicate readiness to receive interrupts188 */189 data = readq(priv->base + MLXBF_GIGE_INT_MASK);190 data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;191 writeq(data, priv->base + MLXBF_GIGE_INT_MASK);192 193 /* Enable RX DMA to write new packets to memory */194 data = readq(priv->base + MLXBF_GIGE_RX_DMA);195 data |= MLXBF_GIGE_RX_DMA_EN;196 writeq(data, priv->base + MLXBF_GIGE_RX_DMA);197 198 return 0;199 200free_wqe_and_skb:201 rx_wqe_ptr = priv->rx_wqe_base;202 for (j = 0; j < i; j++) {203 dma_unmap_single(priv->dev, *rx_wqe_ptr,204 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);205 dev_kfree_skb(priv->rx_skb[j]);206 rx_wqe_ptr++;207 }208 dma_free_coherent(priv->dev, wq_size,209 priv->rx_wqe_base, priv->rx_wqe_base_dma);210 return -ENOMEM;211}212 213/* Receive Deinitialization214 * This routine will free allocations done by mlxbf_gige_rx_init(),215 * namely the RX WQE and RX CQE arrays, as well as all RX buffers216 */217void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv)218{219 dma_addr_t *rx_wqe_ptr;220 size_t size;221 u64 data;222 int i;223 224 /* Disable RX DMA to prevent packet transfers to memory */225 data = readq(priv->base + MLXBF_GIGE_RX_DMA);226 data &= ~MLXBF_GIGE_RX_DMA_EN;227 writeq(data, priv->base + MLXBF_GIGE_RX_DMA);228 229 rx_wqe_ptr = priv->rx_wqe_base;230 231 for (i = 0; i < priv->rx_q_entries; i++) {232 dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ,233 DMA_FROM_DEVICE);234 dev_kfree_skb(priv->rx_skb[i]);235 rx_wqe_ptr++;236 }237 238 size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;239 dma_free_coherent(priv->dev, size,240 priv->rx_wqe_base, priv->rx_wqe_base_dma);241 242 size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;243 dma_free_coherent(priv->dev, size,244 priv->rx_cqe_base, priv->rx_cqe_base_dma);245 246 priv->rx_wqe_base = NULL;247 priv->rx_wqe_base_dma = 0;248 priv->rx_cqe_base = NULL;249 priv->rx_cqe_base_dma = 0;250 writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE);251 writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE);252}253 254static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts)255{256 struct net_device *netdev = priv->netdev;257 struct sk_buff *skb = NULL, *rx_skb;258 u16 rx_pi_rem, rx_ci_rem;259 dma_addr_t *rx_wqe_addr;260 dma_addr_t rx_buf_dma;261 u64 *rx_cqe_addr;262 u64 datalen;263 u64 rx_cqe;264 u16 rx_ci;265 u16 rx_pi;266 267 /* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */268 rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI);269 rx_pi_rem = rx_pi % priv->rx_q_entries;270 271 rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem;272 rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem;273 rx_cqe = *rx_cqe_addr;274 275 if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity)276 return false;277 278 if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) {279 /* Packet is OK, increment stats */280 datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK;281 netdev->stats.rx_packets++;282 netdev->stats.rx_bytes += datalen;283 284 skb = priv->rx_skb[rx_pi_rem];285 286 /* Alloc another RX SKB for this same index */287 rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,288 &rx_buf_dma, DMA_FROM_DEVICE);289 if (!rx_skb)290 return false;291 priv->rx_skb[rx_pi_rem] = rx_skb;292 dma_unmap_single(priv->dev, *rx_wqe_addr,293 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);294 295 skb_put(skb, datalen);296 297 skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */298 299 skb->protocol = eth_type_trans(skb, netdev);300 301 *rx_wqe_addr = rx_buf_dma;302 } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) {303 priv->stats.rx_mac_errors++;304 } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) {305 priv->stats.rx_truncate_errors++;306 }307 308 /* Read receive consumer index before replenish so that this routine309 * returns accurate return value even if packet is received into310 * just-replenished buffer prior to exiting this routine.311 */312 rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI);313 rx_ci_rem = rx_ci % priv->rx_q_entries;314 315 /* Let hardware know we've replenished one buffer */316 rx_pi++;317 318 /* Ensure completion of all writes before notifying HW of replenish */319 wmb();320 writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI);321 322 (*rx_pkts)++;323 324 rx_pi_rem = rx_pi % priv->rx_q_entries;325 if (rx_pi_rem == 0)326 priv->valid_polarity ^= 1;327 328 if (skb)329 netif_receive_skb(skb);330 331 return rx_pi_rem != rx_ci_rem;332}333 334/* Driver poll() function called by NAPI infrastructure */335int mlxbf_gige_poll(struct napi_struct *napi, int budget)336{337 struct mlxbf_gige *priv;338 bool remaining_pkts;339 int work_done = 0;340 u64 data;341 342 priv = container_of(napi, struct mlxbf_gige, napi);343 344 mlxbf_gige_handle_tx_complete(priv);345 346 do {347 remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done);348 } while (remaining_pkts && work_done < budget);349 350 /* If amount of work done < budget, turn off NAPI polling351 * via napi_complete_done(napi, work_done) and then352 * re-enable interrupts.353 */354 if (work_done < budget && napi_complete_done(napi, work_done)) {355 /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to356 * indicate receive readiness357 */358 data = readq(priv->base + MLXBF_GIGE_INT_MASK);359 data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;360 writeq(data, priv->base + MLXBF_GIGE_INT_MASK);361 }362 363 return work_done;364}365