525 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/****************************************************************************3 * Driver for Solarflare network controllers and boards4 * Copyright 2005-2006 Fen Systems Ltd.5 * Copyright 2006-2013 Solarflare Communications Inc.6 */7 8#include <linux/bitops.h>9#include <linux/delay.h>10#include <linux/interrupt.h>11#include <linux/pci.h>12#include <linux/module.h>13#include <linux/seq_file.h>14#include <linux/cpu_rmap.h>15#include "net_driver.h"16#include "bitfield.h"17#include "efx.h"18#include "nic.h"19#include "farch_regs.h"20#include "io.h"21#include "workarounds.h"22 23/**************************************************************************24 *25 * Generic buffer handling26 * These buffers are used for interrupt status, MAC stats, etc.27 *28 **************************************************************************/29 30int ef4_nic_alloc_buffer(struct ef4_nic *efx, struct ef4_buffer *buffer,31 unsigned int len, gfp_t gfp_flags)32{33 buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,34 &buffer->dma_addr, gfp_flags);35 if (!buffer->addr)36 return -ENOMEM;37 buffer->len = len;38 return 0;39}40 41void ef4_nic_free_buffer(struct ef4_nic *efx, struct ef4_buffer *buffer)42{43 if (buffer->addr) {44 dma_free_coherent(&efx->pci_dev->dev, buffer->len,45 buffer->addr, buffer->dma_addr);46 buffer->addr = NULL;47 }48}49 50/* Check whether an event is present in the eventq at the current51 * read pointer. Only useful for self-test.52 */53bool ef4_nic_event_present(struct ef4_channel *channel)54{55 return ef4_event_present(ef4_event(channel, channel->eventq_read_ptr));56}57 58void ef4_nic_event_test_start(struct ef4_channel *channel)59{60 channel->event_test_cpu = -1;61 smp_wmb();62 channel->efx->type->ev_test_generate(channel);63}64 65int ef4_nic_irq_test_start(struct ef4_nic *efx)66{67 efx->last_irq_cpu = -1;68 smp_wmb();69 return efx->type->irq_test_generate(efx);70}71 72/* Hook interrupt handler(s)73 * Try MSI and then legacy interrupts.74 */75int ef4_nic_init_interrupt(struct ef4_nic *efx)76{77 struct ef4_channel *channel;78 unsigned int n_irqs;79 int rc;80 81 if (!EF4_INT_MODE_USE_MSI(efx)) {82 rc = request_irq(efx->legacy_irq,83 efx->type->irq_handle_legacy, IRQF_SHARED,84 efx->name, efx);85 if (rc) {86 netif_err(efx, drv, efx->net_dev,87 "failed to hook legacy IRQ %d\n",88 efx->pci_dev->irq);89 goto fail1;90 }91 return 0;92 }93 94#ifdef CONFIG_RFS_ACCEL95 if (efx->interrupt_mode == EF4_INT_MODE_MSIX) {96 efx->net_dev->rx_cpu_rmap =97 alloc_irq_cpu_rmap(efx->n_rx_channels);98 if (!efx->net_dev->rx_cpu_rmap) {99 rc = -ENOMEM;100 goto fail1;101 }102 }103#endif104 105 /* Hook MSI or MSI-X interrupt */106 n_irqs = 0;107 ef4_for_each_channel(channel, efx) {108 rc = request_irq(channel->irq, efx->type->irq_handle_msi,109 IRQF_PROBE_SHARED, /* Not shared */110 efx->msi_context[channel->channel].name,111 &efx->msi_context[channel->channel]);112 if (rc) {113 netif_err(efx, drv, efx->net_dev,114 "failed to hook IRQ %d\n", channel->irq);115 goto fail2;116 }117 ++n_irqs;118 119#ifdef CONFIG_RFS_ACCEL120 if (efx->interrupt_mode == EF4_INT_MODE_MSIX &&121 channel->channel < efx->n_rx_channels) {122 rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,123 channel->irq);124 if (rc)125 goto fail2;126 }127#endif128 }129 130 return 0;131 132 fail2:133#ifdef CONFIG_RFS_ACCEL134 free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);135 efx->net_dev->rx_cpu_rmap = NULL;136#endif137 ef4_for_each_channel(channel, efx) {138 if (n_irqs-- == 0)139 break;140 free_irq(channel->irq, &efx->msi_context[channel->channel]);141 }142 fail1:143 return rc;144}145 146void ef4_nic_fini_interrupt(struct ef4_nic *efx)147{148 struct ef4_channel *channel;149 150#ifdef CONFIG_RFS_ACCEL151 free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);152 efx->net_dev->rx_cpu_rmap = NULL;153#endif154 155 if (EF4_INT_MODE_USE_MSI(efx)) {156 /* Disable MSI/MSI-X interrupts */157 ef4_for_each_channel(channel, efx)158 free_irq(channel->irq,159 &efx->msi_context[channel->channel]);160 } else {161 /* Disable legacy interrupt */162 free_irq(efx->legacy_irq, efx);163 }164}165 166/* Register dump */167 168#define REGISTER_REVISION_FA 1169#define REGISTER_REVISION_FB 2170#define REGISTER_REVISION_FC 3171#define REGISTER_REVISION_FZ 3 /* last Falcon arch revision */172#define REGISTER_REVISION_ED 4173#define REGISTER_REVISION_EZ 4 /* latest EF10 revision */174 175struct ef4_nic_reg {176 u32 offset:24;177 u32 min_revision:3, max_revision:3;178};179 180#define REGISTER(name, arch, min_rev, max_rev) { \181 arch ## R_ ## min_rev ## max_rev ## _ ## name, \182 REGISTER_REVISION_ ## arch ## min_rev, \183 REGISTER_REVISION_ ## arch ## max_rev \184}185#define REGISTER_AA(name) REGISTER(name, F, A, A)186#define REGISTER_AB(name) REGISTER(name, F, A, B)187#define REGISTER_AZ(name) REGISTER(name, F, A, Z)188#define REGISTER_BB(name) REGISTER(name, F, B, B)189#define REGISTER_BZ(name) REGISTER(name, F, B, Z)190#define REGISTER_CZ(name) REGISTER(name, F, C, Z)191 192static const struct ef4_nic_reg ef4_nic_regs[] = {193 REGISTER_AZ(ADR_REGION),194 REGISTER_AZ(INT_EN_KER),195 REGISTER_BZ(INT_EN_CHAR),196 REGISTER_AZ(INT_ADR_KER),197 REGISTER_BZ(INT_ADR_CHAR),198 /* INT_ACK_KER is WO */199 /* INT_ISR0 is RC */200 REGISTER_AZ(HW_INIT),201 REGISTER_CZ(USR_EV_CFG),202 REGISTER_AB(EE_SPI_HCMD),203 REGISTER_AB(EE_SPI_HADR),204 REGISTER_AB(EE_SPI_HDATA),205 REGISTER_AB(EE_BASE_PAGE),206 REGISTER_AB(EE_VPD_CFG0),207 /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */208 /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */209 /* PCIE_CORE_INDIRECT is indirect */210 REGISTER_AB(NIC_STAT),211 REGISTER_AB(GPIO_CTL),212 REGISTER_AB(GLB_CTL),213 /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */214 REGISTER_BZ(DP_CTRL),215 REGISTER_AZ(MEM_STAT),216 REGISTER_AZ(CS_DEBUG),217 REGISTER_AZ(ALTERA_BUILD),218 REGISTER_AZ(CSR_SPARE),219 REGISTER_AB(PCIE_SD_CTL0123),220 REGISTER_AB(PCIE_SD_CTL45),221 REGISTER_AB(PCIE_PCS_CTL_STAT),222 /* DEBUG_DATA_OUT is not used */223 /* DRV_EV is WO */224 REGISTER_AZ(EVQ_CTL),225 REGISTER_AZ(EVQ_CNT1),226 REGISTER_AZ(EVQ_CNT2),227 REGISTER_AZ(BUF_TBL_CFG),228 REGISTER_AZ(SRM_RX_DC_CFG),229 REGISTER_AZ(SRM_TX_DC_CFG),230 REGISTER_AZ(SRM_CFG),231 /* BUF_TBL_UPD is WO */232 REGISTER_AZ(SRM_UPD_EVQ),233 REGISTER_AZ(SRAM_PARITY),234 REGISTER_AZ(RX_CFG),235 REGISTER_BZ(RX_FILTER_CTL),236 /* RX_FLUSH_DESCQ is WO */237 REGISTER_AZ(RX_DC_CFG),238 REGISTER_AZ(RX_DC_PF_WM),239 REGISTER_BZ(RX_RSS_TKEY),240 /* RX_NODESC_DROP is RC */241 REGISTER_AA(RX_SELF_RST),242 /* RX_DEBUG, RX_PUSH_DROP are not used */243 REGISTER_CZ(RX_RSS_IPV6_REG1),244 REGISTER_CZ(RX_RSS_IPV6_REG2),245 REGISTER_CZ(RX_RSS_IPV6_REG3),246 /* TX_FLUSH_DESCQ is WO */247 REGISTER_AZ(TX_DC_CFG),248 REGISTER_AA(TX_CHKSM_CFG),249 REGISTER_AZ(TX_CFG),250 /* TX_PUSH_DROP is not used */251 REGISTER_AZ(TX_RESERVED),252 REGISTER_BZ(TX_PACE),253 /* TX_PACE_DROP_QID is RC */254 REGISTER_BB(TX_VLAN),255 REGISTER_BZ(TX_IPFIL_PORTEN),256 REGISTER_AB(MD_TXD),257 REGISTER_AB(MD_RXD),258 REGISTER_AB(MD_CS),259 REGISTER_AB(MD_PHY_ADR),260 REGISTER_AB(MD_ID),261 /* MD_STAT is RC */262 REGISTER_AB(MAC_STAT_DMA),263 REGISTER_AB(MAC_CTRL),264 REGISTER_BB(GEN_MODE),265 REGISTER_AB(MAC_MC_HASH_REG0),266 REGISTER_AB(MAC_MC_HASH_REG1),267 REGISTER_AB(GM_CFG1),268 REGISTER_AB(GM_CFG2),269 /* GM_IPG and GM_HD are not used */270 REGISTER_AB(GM_MAX_FLEN),271 /* GM_TEST is not used */272 REGISTER_AB(GM_ADR1),273 REGISTER_AB(GM_ADR2),274 REGISTER_AB(GMF_CFG0),275 REGISTER_AB(GMF_CFG1),276 REGISTER_AB(GMF_CFG2),277 REGISTER_AB(GMF_CFG3),278 REGISTER_AB(GMF_CFG4),279 REGISTER_AB(GMF_CFG5),280 REGISTER_BB(TX_SRC_MAC_CTL),281 REGISTER_AB(XM_ADR_LO),282 REGISTER_AB(XM_ADR_HI),283 REGISTER_AB(XM_GLB_CFG),284 REGISTER_AB(XM_TX_CFG),285 REGISTER_AB(XM_RX_CFG),286 REGISTER_AB(XM_MGT_INT_MASK),287 REGISTER_AB(XM_FC),288 REGISTER_AB(XM_PAUSE_TIME),289 REGISTER_AB(XM_TX_PARAM),290 REGISTER_AB(XM_RX_PARAM),291 /* XM_MGT_INT_MSK (note no 'A') is RC */292 REGISTER_AB(XX_PWR_RST),293 REGISTER_AB(XX_SD_CTL),294 REGISTER_AB(XX_TXDRV_CTL),295 /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */296 /* XX_CORE_STAT is partly RC */297};298 299struct ef4_nic_reg_table {300 u32 offset:24;301 u32 min_revision:3, max_revision:3;302 u32 step:6, rows:21;303};304 305#define REGISTER_TABLE_DIMENSIONS(_, offset, arch, min_rev, max_rev, step, rows) { \306 offset, \307 REGISTER_REVISION_ ## arch ## min_rev, \308 REGISTER_REVISION_ ## arch ## max_rev, \309 step, rows \310}311#define REGISTER_TABLE(name, arch, min_rev, max_rev) \312 REGISTER_TABLE_DIMENSIONS( \313 name, arch ## R_ ## min_rev ## max_rev ## _ ## name, \314 arch, min_rev, max_rev, \315 arch ## R_ ## min_rev ## max_rev ## _ ## name ## _STEP, \316 arch ## R_ ## min_rev ## max_rev ## _ ## name ## _ROWS)317#define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, F, A, A)318#define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, F, A, Z)319#define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, F, B, B)320#define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, F, B, Z)321#define REGISTER_TABLE_BB_CZ(name) \322 REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, B, B, \323 FR_BZ_ ## name ## _STEP, \324 FR_BB_ ## name ## _ROWS), \325 REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, F, C, Z, \326 FR_BZ_ ## name ## _STEP, \327 FR_CZ_ ## name ## _ROWS)328#define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, F, C, Z)329 330static const struct ef4_nic_reg_table ef4_nic_reg_tables[] = {331 /* DRIVER is not used */332 /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */333 REGISTER_TABLE_BB(TX_IPFIL_TBL),334 REGISTER_TABLE_BB(TX_SRC_MAC_TBL),335 REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),336 REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),337 REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),338 REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),339 REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),340 REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),341 /* We can't reasonably read all of the buffer table (up to 8MB!).342 * However this driver will only use a few entries. Reading343 * 1K entries allows for some expansion of queue count and344 * size before we need to change the version. */345 REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,346 F, A, A, 8, 1024),347 REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,348 F, B, Z, 8, 1024),349 REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),350 REGISTER_TABLE_BB_CZ(TIMER_TBL),351 REGISTER_TABLE_BB_CZ(TX_PACE_TBL),352 REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),353 /* TX_FILTER_TBL0 is huge and not used by this driver */354 REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),355 REGISTER_TABLE_CZ(MC_TREG_SMEM),356 /* MSIX_PBA_TABLE is not mapped */357 /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */358 REGISTER_TABLE_BZ(RX_FILTER_TBL0),359};360 361size_t ef4_nic_get_regs_len(struct ef4_nic *efx)362{363 const struct ef4_nic_reg *reg;364 const struct ef4_nic_reg_table *table;365 size_t len = 0;366 367 for (reg = ef4_nic_regs;368 reg < ef4_nic_regs + ARRAY_SIZE(ef4_nic_regs);369 reg++)370 if (efx->type->revision >= reg->min_revision &&371 efx->type->revision <= reg->max_revision)372 len += sizeof(ef4_oword_t);373 374 for (table = ef4_nic_reg_tables;375 table < ef4_nic_reg_tables + ARRAY_SIZE(ef4_nic_reg_tables);376 table++)377 if (efx->type->revision >= table->min_revision &&378 efx->type->revision <= table->max_revision)379 len += table->rows * min_t(size_t, table->step, 16);380 381 return len;382}383 384void ef4_nic_get_regs(struct ef4_nic *efx, void *buf)385{386 const struct ef4_nic_reg *reg;387 const struct ef4_nic_reg_table *table;388 389 for (reg = ef4_nic_regs;390 reg < ef4_nic_regs + ARRAY_SIZE(ef4_nic_regs);391 reg++) {392 if (efx->type->revision >= reg->min_revision &&393 efx->type->revision <= reg->max_revision) {394 ef4_reado(efx, (ef4_oword_t *)buf, reg->offset);395 buf += sizeof(ef4_oword_t);396 }397 }398 399 for (table = ef4_nic_reg_tables;400 table < ef4_nic_reg_tables + ARRAY_SIZE(ef4_nic_reg_tables);401 table++) {402 size_t size, i;403 404 if (!(efx->type->revision >= table->min_revision &&405 efx->type->revision <= table->max_revision))406 continue;407 408 size = min_t(size_t, table->step, 16);409 410 for (i = 0; i < table->rows; i++) {411 switch (table->step) {412 case 4: /* 32-bit SRAM */413 ef4_readd(efx, buf, table->offset + 4 * i);414 break;415 case 8: /* 64-bit SRAM */416 ef4_sram_readq(efx,417 efx->membase + table->offset,418 buf, i);419 break;420 case 16: /* 128-bit-readable register */421 ef4_reado_table(efx, buf, table->offset, i);422 break;423 case 32: /* 128-bit register, interleaved */424 ef4_reado_table(efx, buf, table->offset, 2 * i);425 break;426 default:427 WARN_ON(1);428 return;429 }430 buf += size;431 }432 }433}434 435/**436 * ef4_nic_describe_stats - Describe supported statistics for ethtool437 * @desc: Array of &struct ef4_hw_stat_desc describing the statistics438 * @count: Length of the @desc array439 * @mask: Bitmask of which elements of @desc are enabled440 * @names: Buffer to copy names to, or %NULL. The names are copied441 * starting at intervals of %ETH_GSTRING_LEN bytes.442 *443 * Returns the number of visible statistics, i.e. the number of set444 * bits in the first @count bits of @mask for which a name is defined.445 */446size_t ef4_nic_describe_stats(const struct ef4_hw_stat_desc *desc, size_t count,447 const unsigned long *mask, u8 *names)448{449 size_t visible = 0;450 size_t index;451 452 for_each_set_bit(index, mask, count) {453 if (desc[index].name) {454 if (names) {455 strscpy(names, desc[index].name,456 ETH_GSTRING_LEN);457 names += ETH_GSTRING_LEN;458 }459 ++visible;460 }461 }462 463 return visible;464}465 466/**467 * ef4_nic_update_stats - Convert statistics DMA buffer to array of u64468 * @desc: Array of &struct ef4_hw_stat_desc describing the DMA buffer469 * layout. DMA widths of 0, 16, 32 and 64 are supported; where470 * the width is specified as 0 the corresponding element of471 * @stats is not updated.472 * @count: Length of the @desc array473 * @mask: Bitmask of which elements of @desc are enabled474 * @stats: Buffer to update with the converted statistics. The length475 * of this array must be at least @count.476 * @dma_buf: DMA buffer containing hardware statistics477 * @accumulate: If set, the converted values will be added rather than478 * directly stored to the corresponding elements of @stats479 */480void ef4_nic_update_stats(const struct ef4_hw_stat_desc *desc, size_t count,481 const unsigned long *mask,482 u64 *stats, const void *dma_buf, bool accumulate)483{484 size_t index;485 486 for_each_set_bit(index, mask, count) {487 if (desc[index].dma_width) {488 const void *addr = dma_buf + desc[index].offset;489 u64 val;490 491 switch (desc[index].dma_width) {492 case 16:493 val = le16_to_cpup((__le16 *)addr);494 break;495 case 32:496 val = le32_to_cpup((__le32 *)addr);497 break;498 case 64:499 val = le64_to_cpup((__le64 *)addr);500 break;501 default:502 WARN_ON(1);503 val = 0;504 break;505 }506 507 if (accumulate)508 stats[index] += val;509 else510 stats[index] = val;511 }512 }513}514 515void ef4_nic_fix_nodesc_drop_stat(struct ef4_nic *efx, u64 *rx_nodesc_drops)516{517 /* if down, or this is the first update after coming up */518 if (!(efx->net_dev->flags & IFF_UP) || !efx->rx_nodesc_drops_prev_state)519 efx->rx_nodesc_drops_while_down +=520 *rx_nodesc_drops - efx->rx_nodesc_drops_total;521 efx->rx_nodesc_drops_total = *rx_nodesc_drops;522 efx->rx_nodesc_drops_prev_state = !!(efx->net_dev->flags & IFF_UP);523 *rx_nodesc_drops -= efx->rx_nodesc_drops_while_down;524}525