1351 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/****************************************************************************3 * Driver for Solarflare network controllers and boards4 * Copyright 2005-2006 Fen Systems Ltd.5 * Copyright 2005-2013 Solarflare Communications Inc.6 */7 8#include <linux/filter.h>9#include <linux/module.h>10#include <linux/pci.h>11#include <linux/netdevice.h>12#include <linux/etherdevice.h>13#include <linux/delay.h>14#include <linux/notifier.h>15#include <linux/ip.h>16#include <linux/tcp.h>17#include <linux/in.h>18#include <linux/ethtool.h>19#include <linux/topology.h>20#include <linux/gfp.h>21#include <linux/interrupt.h>22#include "net_driver.h"23#include <net/gre.h>24#include <net/udp_tunnel.h>25#include "efx.h"26#include "efx_common.h"27#include "efx_channels.h"28#include "rx_common.h"29#include "tx_common.h"30#include "nic.h"31#include "io.h"32#include "selftest.h"33#include "sriov.h"34#ifdef CONFIG_SFC_SIENA_SRIOV35#include "siena_sriov.h"36#endif37 38#include "mcdi_port_common.h"39#include "mcdi_pcol.h"40#include "workarounds.h"41 42/**************************************************************************43 *44 * Configurable values45 *46 *************************************************************************/47 48module_param_named(interrupt_mode, efx_siena_interrupt_mode, uint, 0444);49MODULE_PARM_DESC(interrupt_mode,50 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");51 52module_param_named(rss_cpus, efx_siena_rss_cpus, uint, 0444);53MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");54 55/*56 * Use separate channels for TX and RX events57 *58 * Set this to 1 to use separate channels for TX and RX. It allows us59 * to control interrupt affinity separately for TX and RX.60 *61 * This is only used in MSI-X interrupt mode62 */63bool efx_siena_separate_tx_channels;64module_param_named(efx_separate_tx_channels, efx_siena_separate_tx_channels,65 bool, 0444);66MODULE_PARM_DESC(efx_separate_tx_channels,67 "Use separate channels for TX and RX");68 69/* Initial interrupt moderation settings. They can be modified after70 * module load with ethtool.71 *72 * The default for RX should strike a balance between increasing the73 * round-trip latency and reducing overhead.74 */75static unsigned int rx_irq_mod_usec = 60;76 77/* Initial interrupt moderation settings. They can be modified after78 * module load with ethtool.79 *80 * This default is chosen to ensure that a 10G link does not go idle81 * while a TX queue is stopped after it has become full. A queue is82 * restarted when it drops below half full. The time this takes (assuming83 * worst case 3 descriptors per packet and 1024 descriptors) is84 * 512 / 3 * 1.2 = 205 usec.85 */86static unsigned int tx_irq_mod_usec = 150;87 88static bool phy_flash_cfg;89module_param(phy_flash_cfg, bool, 0644);90MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");91 92static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |93 NETIF_MSG_LINK | NETIF_MSG_IFDOWN |94 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |95 NETIF_MSG_TX_ERR | NETIF_MSG_HW);96module_param(debug, uint, 0);97MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");98 99/**************************************************************************100 *101 * Utility functions and prototypes102 *103 *************************************************************************/104 105static void efx_remove_port(struct efx_nic *efx);106static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog);107static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp);108static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs,109 u32 flags);110 111#define EFX_ASSERT_RESET_SERIALISED(efx) \112 do { \113 if ((efx->state == STATE_READY) || \114 (efx->state == STATE_RECOVERY) || \115 (efx->state == STATE_DISABLED)) \116 ASSERT_RTNL(); \117 } while (0)118 119/**************************************************************************120 *121 * Port handling122 *123 **************************************************************************/124 125static void efx_fini_port(struct efx_nic *efx);126 127static int efx_probe_port(struct efx_nic *efx)128{129 int rc;130 131 netif_dbg(efx, probe, efx->net_dev, "create port\n");132 133 if (phy_flash_cfg)134 efx->phy_mode = PHY_MODE_SPECIAL;135 136 /* Connect up MAC/PHY operations table */137 rc = efx->type->probe_port(efx);138 if (rc)139 return rc;140 141 /* Initialise MAC address to permanent address */142 eth_hw_addr_set(efx->net_dev, efx->net_dev->perm_addr);143 144 return 0;145}146 147static int efx_init_port(struct efx_nic *efx)148{149 int rc;150 151 netif_dbg(efx, drv, efx->net_dev, "init port\n");152 153 mutex_lock(&efx->mac_lock);154 155 efx->port_initialized = true;156 157 /* Ensure the PHY advertises the correct flow control settings */158 rc = efx_siena_mcdi_port_reconfigure(efx);159 if (rc && rc != -EPERM)160 goto fail;161 162 mutex_unlock(&efx->mac_lock);163 return 0;164 165fail:166 mutex_unlock(&efx->mac_lock);167 return rc;168}169 170static void efx_fini_port(struct efx_nic *efx)171{172 netif_dbg(efx, drv, efx->net_dev, "shut down port\n");173 174 if (!efx->port_initialized)175 return;176 177 efx->port_initialized = false;178 179 efx->link_state.up = false;180 efx_siena_link_status_changed(efx);181}182 183static void efx_remove_port(struct efx_nic *efx)184{185 netif_dbg(efx, drv, efx->net_dev, "destroying port\n");186 187 efx->type->remove_port(efx);188}189 190/**************************************************************************191 *192 * NIC handling193 *194 **************************************************************************/195 196static LIST_HEAD(efx_primary_list);197static LIST_HEAD(efx_unassociated_list);198 199static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right)200{201 return left->type == right->type &&202 left->vpd_sn && right->vpd_sn &&203 !strcmp(left->vpd_sn, right->vpd_sn);204}205 206static void efx_associate(struct efx_nic *efx)207{208 struct efx_nic *other, *next;209 210 if (efx->primary == efx) {211 /* Adding primary function; look for secondaries */212 213 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n");214 list_add_tail(&efx->node, &efx_primary_list);215 216 list_for_each_entry_safe(other, next, &efx_unassociated_list,217 node) {218 if (efx_same_controller(efx, other)) {219 list_del(&other->node);220 netif_dbg(other, probe, other->net_dev,221 "moving to secondary list of %s %s\n",222 pci_name(efx->pci_dev),223 efx->net_dev->name);224 list_add_tail(&other->node,225 &efx->secondary_list);226 other->primary = efx;227 }228 }229 } else {230 /* Adding secondary function; look for primary */231 232 list_for_each_entry(other, &efx_primary_list, node) {233 if (efx_same_controller(efx, other)) {234 netif_dbg(efx, probe, efx->net_dev,235 "adding to secondary list of %s %s\n",236 pci_name(other->pci_dev),237 other->net_dev->name);238 list_add_tail(&efx->node,239 &other->secondary_list);240 efx->primary = other;241 return;242 }243 }244 245 netif_dbg(efx, probe, efx->net_dev,246 "adding to unassociated list\n");247 list_add_tail(&efx->node, &efx_unassociated_list);248 }249}250 251static void efx_dissociate(struct efx_nic *efx)252{253 struct efx_nic *other, *next;254 255 list_del(&efx->node);256 efx->primary = NULL;257 258 list_for_each_entry_safe(other, next, &efx->secondary_list, node) {259 list_del(&other->node);260 netif_dbg(other, probe, other->net_dev,261 "moving to unassociated list\n");262 list_add_tail(&other->node, &efx_unassociated_list);263 other->primary = NULL;264 }265}266 267static int efx_probe_nic(struct efx_nic *efx)268{269 int rc;270 271 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");272 273 /* Carry out hardware-type specific initialisation */274 rc = efx->type->probe(efx);275 if (rc)276 return rc;277 278 do {279 if (!efx->max_channels || !efx->max_tx_channels) {280 netif_err(efx, drv, efx->net_dev,281 "Insufficient resources to allocate"282 " any channels\n");283 rc = -ENOSPC;284 goto fail1;285 }286 287 /* Determine the number of channels and queues by trying288 * to hook in MSI-X interrupts.289 */290 rc = efx_siena_probe_interrupts(efx);291 if (rc)292 goto fail1;293 294 rc = efx_siena_set_channels(efx);295 if (rc)296 goto fail1;297 298 /* dimension_resources can fail with EAGAIN */299 rc = efx->type->dimension_resources(efx);300 if (rc != 0 && rc != -EAGAIN)301 goto fail2;302 303 if (rc == -EAGAIN)304 /* try again with new max_channels */305 efx_siena_remove_interrupts(efx);306 307 } while (rc == -EAGAIN);308 309 if (efx->n_channels > 1)310 netdev_rss_key_fill(efx->rss_context.rx_hash_key,311 sizeof(efx->rss_context.rx_hash_key));312 efx_siena_set_default_rx_indir_table(efx, &efx->rss_context);313 314 /* Initialise the interrupt moderation settings */315 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000);316 efx_siena_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec,317 true, true);318 319 return 0;320 321fail2:322 efx_siena_remove_interrupts(efx);323fail1:324 efx->type->remove(efx);325 return rc;326}327 328static void efx_remove_nic(struct efx_nic *efx)329{330 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");331 332 efx_siena_remove_interrupts(efx);333 efx->type->remove(efx);334}335 336/**************************************************************************337 *338 * NIC startup/shutdown339 *340 *************************************************************************/341 342static int efx_probe_all(struct efx_nic *efx)343{344 int rc;345 346 rc = efx_probe_nic(efx);347 if (rc) {348 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");349 goto fail1;350 }351 352 rc = efx_probe_port(efx);353 if (rc) {354 netif_err(efx, probe, efx->net_dev, "failed to create port\n");355 goto fail2;356 }357 358 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT);359 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) {360 rc = -EINVAL;361 goto fail3;362 }363 364#ifdef CONFIG_SFC_SIENA_SRIOV365 rc = efx->type->vswitching_probe(efx);366 if (rc) /* not fatal; the PF will still work fine */367 netif_warn(efx, probe, efx->net_dev,368 "failed to setup vswitching rc=%d;"369 " VFs may not function\n", rc);370#endif371 372 rc = efx_siena_probe_filters(efx);373 if (rc) {374 netif_err(efx, probe, efx->net_dev,375 "failed to create filter tables\n");376 goto fail4;377 }378 379 rc = efx_siena_probe_channels(efx);380 if (rc)381 goto fail5;382 383 return 0;384 385 fail5:386 efx_siena_remove_filters(efx);387 fail4:388#ifdef CONFIG_SFC_SIENA_SRIOV389 efx->type->vswitching_remove(efx);390#endif391 fail3:392 efx_remove_port(efx);393 fail2:394 efx_remove_nic(efx);395 fail1:396 return rc;397}398 399static void efx_remove_all(struct efx_nic *efx)400{401 rtnl_lock();402 efx_xdp_setup_prog(efx, NULL);403 rtnl_unlock();404 405 efx_siena_remove_channels(efx);406 efx_siena_remove_filters(efx);407#ifdef CONFIG_SFC_SIENA_SRIOV408 efx->type->vswitching_remove(efx);409#endif410 efx_remove_port(efx);411 efx_remove_nic(efx);412}413 414/**************************************************************************415 *416 * Interrupt moderation417 *418 **************************************************************************/419unsigned int efx_siena_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs)420{421 if (usecs == 0)422 return 0;423 if (usecs * 1000 < efx->timer_quantum_ns)424 return 1; /* never round down to 0 */425 return usecs * 1000 / efx->timer_quantum_ns;426}427 428/* Set interrupt moderation parameters */429int efx_siena_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,430 unsigned int rx_usecs, bool rx_adaptive,431 bool rx_may_override_tx)432{433 struct efx_channel *channel;434 unsigned int timer_max_us;435 436 EFX_ASSERT_RESET_SERIALISED(efx);437 438 timer_max_us = efx->timer_max_ns / 1000;439 440 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us)441 return -EINVAL;442 443 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 &&444 !rx_may_override_tx) {445 netif_err(efx, drv, efx->net_dev, "Channels are shared. "446 "RX and TX IRQ moderation must be equal\n");447 return -EINVAL;448 }449 450 efx->irq_rx_adaptive = rx_adaptive;451 efx->irq_rx_moderation_us = rx_usecs;452 efx_for_each_channel(channel, efx) {453 if (efx_channel_has_rx_queue(channel))454 channel->irq_moderation_us = rx_usecs;455 else if (efx_channel_has_tx_queues(channel))456 channel->irq_moderation_us = tx_usecs;457 else if (efx_channel_is_xdp_tx(channel))458 channel->irq_moderation_us = tx_usecs;459 }460 461 return 0;462}463 464void efx_siena_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,465 unsigned int *rx_usecs, bool *rx_adaptive)466{467 *rx_adaptive = efx->irq_rx_adaptive;468 *rx_usecs = efx->irq_rx_moderation_us;469 470 /* If channels are shared between RX and TX, so is IRQ471 * moderation. Otherwise, IRQ moderation is the same for all472 * TX channels and is not adaptive.473 */474 if (efx->tx_channel_offset == 0) {475 *tx_usecs = *rx_usecs;476 } else {477 struct efx_channel *tx_channel;478 479 tx_channel = efx->channel[efx->tx_channel_offset];480 *tx_usecs = tx_channel->irq_moderation_us;481 }482}483 484/**************************************************************************485 *486 * ioctls487 *488 *************************************************************************/489 490/* Net device ioctl491 * Context: process, rtnl_lock() held.492 */493static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)494{495 struct efx_nic *efx = netdev_priv(net_dev);496 struct mii_ioctl_data *data = if_mii(ifr);497 498 /* Convert phy_id from older PRTAD/DEVAD format */499 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&500 (data->phy_id & 0xfc00) == 0x0400)501 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;502 503 return mdio_mii_ioctl(&efx->mdio, data, cmd);504}505 506/**************************************************************************507 *508 * Kernel net device interface509 *510 *************************************************************************/511 512/* Context: process, rtnl_lock() held. */513static int efx_net_open(struct net_device *net_dev)514{515 struct efx_nic *efx = netdev_priv(net_dev);516 int rc;517 518 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",519 raw_smp_processor_id());520 521 rc = efx_check_disabled(efx);522 if (rc)523 return rc;524 if (efx->phy_mode & PHY_MODE_SPECIAL)525 return -EBUSY;526 if (efx_siena_mcdi_poll_reboot(efx) && efx_siena_reset(efx, RESET_TYPE_ALL))527 return -EIO;528 529 /* Notify the kernel of the link state polled during driver load,530 * before the monitor starts running */531 efx_siena_link_status_changed(efx);532 533 efx_siena_start_all(efx);534 if (efx->state == STATE_DISABLED || efx->reset_pending)535 netif_device_detach(efx->net_dev);536 efx_siena_selftest_async_start(efx);537 return 0;538}539 540/* Context: process, rtnl_lock() held.541 * Note that the kernel will ignore our return code; this method542 * should really be a void.543 */544static int efx_net_stop(struct net_device *net_dev)545{546 struct efx_nic *efx = netdev_priv(net_dev);547 548 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",549 raw_smp_processor_id());550 551 /* Stop the device and flush all the channels */552 efx_siena_stop_all(efx);553 554 return 0;555}556 557static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid)558{559 struct efx_nic *efx = netdev_priv(net_dev);560 561 if (efx->type->vlan_rx_add_vid)562 return efx->type->vlan_rx_add_vid(efx, proto, vid);563 else564 return -EOPNOTSUPP;565}566 567static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid)568{569 struct efx_nic *efx = netdev_priv(net_dev);570 571 if (efx->type->vlan_rx_kill_vid)572 return efx->type->vlan_rx_kill_vid(efx, proto, vid);573 else574 return -EOPNOTSUPP;575}576 577static int efx_siena_hwtstamp_set(struct net_device *net_dev,578 struct kernel_hwtstamp_config *config,579 struct netlink_ext_ack *extack)580{581 struct efx_nic *efx = netdev_priv(net_dev);582 583 return efx_siena_ptp_set_ts_config(efx, config, extack);584}585 586static int efx_siena_hwtstamp_get(struct net_device *net_dev,587 struct kernel_hwtstamp_config *config)588{589 struct efx_nic *efx = netdev_priv(net_dev);590 591 return efx_siena_ptp_get_ts_config(efx, config);592}593 594static const struct net_device_ops efx_netdev_ops = {595 .ndo_open = efx_net_open,596 .ndo_stop = efx_net_stop,597 .ndo_get_stats64 = efx_siena_net_stats,598 .ndo_tx_timeout = efx_siena_watchdog,599 .ndo_start_xmit = efx_siena_hard_start_xmit,600 .ndo_validate_addr = eth_validate_addr,601 .ndo_eth_ioctl = efx_ioctl,602 .ndo_change_mtu = efx_siena_change_mtu,603 .ndo_set_mac_address = efx_siena_set_mac_address,604 .ndo_set_rx_mode = efx_siena_set_rx_mode,605 .ndo_set_features = efx_siena_set_features,606 .ndo_features_check = efx_siena_features_check,607 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid,608 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid,609 .ndo_hwtstamp_set = efx_siena_hwtstamp_set,610 .ndo_hwtstamp_get = efx_siena_hwtstamp_get,611#ifdef CONFIG_SFC_SIENA_SRIOV612 .ndo_set_vf_mac = efx_sriov_set_vf_mac,613 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan,614 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk,615 .ndo_get_vf_config = efx_sriov_get_vf_config,616 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state,617#endif618 .ndo_get_phys_port_id = efx_siena_get_phys_port_id,619 .ndo_get_phys_port_name = efx_siena_get_phys_port_name,620 .ndo_setup_tc = efx_siena_setup_tc,621#ifdef CONFIG_RFS_ACCEL622 .ndo_rx_flow_steer = efx_siena_filter_rfs,623#endif624 .ndo_xdp_xmit = efx_xdp_xmit,625 .ndo_bpf = efx_xdp626};627 628static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog)629{630 struct bpf_prog *old_prog;631 632 if (efx->xdp_rxq_info_failed) {633 netif_err(efx, drv, efx->net_dev,634 "Unable to bind XDP program due to previous failure of rxq_info\n");635 return -EINVAL;636 }637 638 if (prog && efx->net_dev->mtu > efx_siena_xdp_max_mtu(efx)) {639 netif_err(efx, drv, efx->net_dev,640 "Unable to configure XDP with MTU of %d (max: %d)\n",641 efx->net_dev->mtu, efx_siena_xdp_max_mtu(efx));642 return -EINVAL;643 }644 645 old_prog = rtnl_dereference(efx->xdp_prog);646 rcu_assign_pointer(efx->xdp_prog, prog);647 /* Release the reference that was originally passed by the caller. */648 if (old_prog)649 bpf_prog_put(old_prog);650 651 return 0;652}653 654/* Context: process, rtnl_lock() held. */655static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp)656{657 struct efx_nic *efx = netdev_priv(dev);658 659 switch (xdp->command) {660 case XDP_SETUP_PROG:661 return efx_xdp_setup_prog(efx, xdp->prog);662 default:663 return -EINVAL;664 }665}666 667static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs,668 u32 flags)669{670 struct efx_nic *efx = netdev_priv(dev);671 672 if (!netif_running(dev))673 return -EINVAL;674 675 return efx_siena_xdp_tx_buffers(efx, n, xdpfs, flags & XDP_XMIT_FLUSH);676}677 678static void efx_update_name(struct efx_nic *efx)679{680 strcpy(efx->name, efx->net_dev->name);681 efx_siena_mtd_rename(efx);682 efx_siena_set_channel_names(efx);683}684 685static int efx_netdev_event(struct notifier_block *this,686 unsigned long event, void *ptr)687{688 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr);689 690 if ((net_dev->netdev_ops == &efx_netdev_ops) &&691 event == NETDEV_CHANGENAME)692 efx_update_name(netdev_priv(net_dev));693 694 return NOTIFY_DONE;695}696 697static struct notifier_block efx_netdev_notifier = {698 .notifier_call = efx_netdev_event,699};700 701static ssize_t phy_type_show(struct device *dev,702 struct device_attribute *attr, char *buf)703{704 struct efx_nic *efx = dev_get_drvdata(dev);705 return sprintf(buf, "%d\n", efx->phy_type);706}707static DEVICE_ATTR_RO(phy_type);708 709static int efx_register_netdev(struct efx_nic *efx)710{711 struct net_device *net_dev = efx->net_dev;712 struct efx_channel *channel;713 int rc;714 715 net_dev->watchdog_timeo = 5 * HZ;716 net_dev->irq = efx->pci_dev->irq;717 net_dev->netdev_ops = &efx_netdev_ops;718 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)719 net_dev->priv_flags |= IFF_UNICAST_FLT;720 net_dev->ethtool_ops = &efx_siena_ethtool_ops;721 netif_set_tso_max_segs(net_dev, EFX_TSO_MAX_SEGS);722 net_dev->min_mtu = EFX_MIN_MTU;723 net_dev->max_mtu = EFX_MAX_MTU;724 725 rtnl_lock();726 727 /* Enable resets to be scheduled and check whether any were728 * already requested. If so, the NIC is probably hosed so we729 * abort.730 */731 efx->state = STATE_READY;732 smp_mb(); /* ensure we change state before checking reset_pending */733 if (efx->reset_pending) {734 pci_err(efx->pci_dev, "aborting probe due to scheduled reset\n");735 rc = -EIO;736 goto fail_locked;737 }738 739 rc = dev_alloc_name(net_dev, net_dev->name);740 if (rc < 0)741 goto fail_locked;742 efx_update_name(efx);743 744 /* Always start with carrier off; PHY events will detect the link */745 netif_carrier_off(net_dev);746 747 rc = register_netdevice(net_dev);748 if (rc)749 goto fail_locked;750 751 efx_for_each_channel(channel, efx) {752 struct efx_tx_queue *tx_queue;753 efx_for_each_channel_tx_queue(tx_queue, channel)754 efx_siena_init_tx_queue_core_txq(tx_queue);755 }756 757 efx_associate(efx);758 759 rtnl_unlock();760 761 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);762 if (rc) {763 netif_err(efx, drv, efx->net_dev,764 "failed to init net dev attributes\n");765 goto fail_registered;766 }767 768 efx_siena_init_mcdi_logging(efx);769 770 return 0;771 772fail_registered:773 rtnl_lock();774 efx_dissociate(efx);775 unregister_netdevice(net_dev);776fail_locked:777 efx->state = STATE_UNINIT;778 rtnl_unlock();779 netif_err(efx, drv, efx->net_dev, "could not register net dev\n");780 return rc;781}782 783static void efx_unregister_netdev(struct efx_nic *efx)784{785 if (!efx->net_dev)786 return;787 788 BUG_ON(netdev_priv(efx->net_dev) != efx);789 790 if (efx_dev_registered(efx)) {791 strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));792 efx_siena_fini_mcdi_logging(efx);793 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);794 unregister_netdev(efx->net_dev);795 }796}797 798/**************************************************************************799 *800 * List of NICs we support801 *802 **************************************************************************/803 804/* PCI device ID table */805static const struct pci_device_id efx_pci_table[] = {806 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */807 .driver_data = (unsigned long)&siena_a0_nic_type},808 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */809 .driver_data = (unsigned long)&siena_a0_nic_type},810 {0} /* end of list */811};812 813/**************************************************************************814 *815 * Data housekeeping816 *817 **************************************************************************/818 819void efx_siena_update_sw_stats(struct efx_nic *efx, u64 *stats)820{821 u64 n_rx_nodesc_trunc = 0;822 struct efx_channel *channel;823 824 efx_for_each_channel(channel, efx)825 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;826 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc;827 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);828}829 830/**************************************************************************831 *832 * PCI interface833 *834 **************************************************************************/835 836/* Main body of final NIC shutdown code837 * This is called only at module unload (or hotplug removal).838 */839static void efx_pci_remove_main(struct efx_nic *efx)840{841 /* Flush reset_work. It can no longer be scheduled since we842 * are not READY.843 */844 BUG_ON(efx->state == STATE_READY);845 efx_siena_flush_reset_workqueue(efx);846 847 efx_siena_disable_interrupts(efx);848 efx_siena_clear_interrupt_affinity(efx);849 efx_siena_fini_interrupt(efx);850 efx_fini_port(efx);851 efx->type->fini(efx);852 efx_siena_fini_napi(efx);853 efx_remove_all(efx);854}855 856/* Final NIC shutdown857 * This is called only at module unload (or hotplug removal). A PF can call858 * this on its VFs to ensure they are unbound first.859 */860static void efx_pci_remove(struct pci_dev *pci_dev)861{862 struct efx_nic *efx;863 864 efx = pci_get_drvdata(pci_dev);865 if (!efx)866 return;867 868 /* Mark the NIC as fini, then stop the interface */869 rtnl_lock();870 efx_dissociate(efx);871 dev_close(efx->net_dev);872 efx_siena_disable_interrupts(efx);873 efx->state = STATE_UNINIT;874 rtnl_unlock();875 876 if (efx->type->sriov_fini)877 efx->type->sriov_fini(efx);878 879 efx_unregister_netdev(efx);880 881 efx_siena_mtd_remove(efx);882 883 efx_pci_remove_main(efx);884 885 efx_siena_fini_io(efx);886 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");887 888 efx_siena_fini_struct(efx);889 free_netdev(efx->net_dev);890};891 892/* NIC VPD information893 * Called during probe to display the part number of the894 * installed NIC.895 */896static void efx_probe_vpd_strings(struct efx_nic *efx)897{898 struct pci_dev *dev = efx->pci_dev;899 unsigned int vpd_size, kw_len;900 u8 *vpd_data;901 int start;902 903 vpd_data = pci_vpd_alloc(dev, &vpd_size);904 if (IS_ERR(vpd_data)) {905 pci_warn(dev, "Unable to read VPD\n");906 return;907 }908 909 start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,910 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);911 if (start < 0)912 pci_err(dev, "Part number not found or incomplete\n");913 else914 pci_info(dev, "Part Number : %.*s\n", kw_len, vpd_data + start);915 916 start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,917 PCI_VPD_RO_KEYWORD_SERIALNO, &kw_len);918 if (start < 0)919 pci_err(dev, "Serial number not found or incomplete\n");920 else921 efx->vpd_sn = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL);922 923 kfree(vpd_data);924}925 926 927/* Main body of NIC initialisation928 * This is called at module load (or hotplug insertion, theoretically).929 */930static int efx_pci_probe_main(struct efx_nic *efx)931{932 int rc;933 934 /* Do start-of-day initialisation */935 rc = efx_probe_all(efx);936 if (rc)937 goto fail1;938 939 efx_siena_init_napi(efx);940 941 down_write(&efx->filter_sem);942 rc = efx->type->init(efx);943 up_write(&efx->filter_sem);944 if (rc) {945 pci_err(efx->pci_dev, "failed to initialise NIC\n");946 goto fail3;947 }948 949 rc = efx_init_port(efx);950 if (rc) {951 netif_err(efx, probe, efx->net_dev,952 "failed to initialise port\n");953 goto fail4;954 }955 956 rc = efx_siena_init_interrupt(efx);957 if (rc)958 goto fail5;959 960 efx_siena_set_interrupt_affinity(efx);961 rc = efx_siena_enable_interrupts(efx);962 if (rc)963 goto fail6;964 965 return 0;966 967 fail6:968 efx_siena_clear_interrupt_affinity(efx);969 efx_siena_fini_interrupt(efx);970 fail5:971 efx_fini_port(efx);972 fail4:973 efx->type->fini(efx);974 fail3:975 efx_siena_fini_napi(efx);976 efx_remove_all(efx);977 fail1:978 return rc;979}980 981static int efx_pci_probe_post_io(struct efx_nic *efx)982{983 struct net_device *net_dev = efx->net_dev;984 int rc = efx_pci_probe_main(efx);985 986 if (rc)987 return rc;988 989 if (efx->type->sriov_init) {990 rc = efx->type->sriov_init(efx);991 if (rc)992 pci_err(efx->pci_dev, "SR-IOV can't be enabled rc %d\n",993 rc);994 }995 996 /* Determine netdevice features */997 net_dev->features |= (efx->type->offload_features | NETIF_F_SG |998 NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL);999 if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))1000 net_dev->features |= NETIF_F_TSO6;1001 /* Check whether device supports TSO */1002 if (!efx->type->tso_versions || !efx->type->tso_versions(efx))1003 net_dev->features &= ~NETIF_F_ALL_TSO;1004 /* Mask for features that also apply to VLAN devices */1005 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG |1006 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |1007 NETIF_F_RXCSUM);1008 1009 net_dev->hw_features |= net_dev->features & ~efx->fixed_features;1010 1011 /* Disable receiving frames with bad FCS, by default. */1012 net_dev->features &= ~NETIF_F_RXALL;1013 1014 /* Disable VLAN filtering by default. It may be enforced if1015 * the feature is fixed (i.e. VLAN filters are required to1016 * receive VLAN tagged packets due to vPort restrictions).1017 */1018 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;1019 net_dev->features |= efx->fixed_features;1020 1021 net_dev->xdp_features = NETDEV_XDP_ACT_BASIC |1022 NETDEV_XDP_ACT_REDIRECT |1023 NETDEV_XDP_ACT_NDO_XMIT;1024 1025 rc = efx_register_netdev(efx);1026 if (!rc)1027 return 0;1028 1029 efx_pci_remove_main(efx);1030 return rc;1031}1032 1033/* NIC initialisation1034 *1035 * This is called at module load (or hotplug insertion,1036 * theoretically). It sets up PCI mappings, resets the NIC,1037 * sets up and registers the network devices with the kernel and hooks1038 * the interrupt service routine. It does not prepare the device for1039 * transmission; this is left to the first time one of the network1040 * interfaces is brought up (i.e. efx_net_open).1041 */1042static int efx_pci_probe(struct pci_dev *pci_dev,1043 const struct pci_device_id *entry)1044{1045 struct net_device *net_dev;1046 struct efx_nic *efx;1047 int rc;1048 1049 /* Allocate and initialise a struct net_device and struct efx_nic */1050 net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,1051 EFX_MAX_RX_QUEUES);1052 if (!net_dev)1053 return -ENOMEM;1054 efx = netdev_priv(net_dev);1055 efx->type = (const struct efx_nic_type *) entry->driver_data;1056 efx->fixed_features |= NETIF_F_HIGHDMA;1057 1058 pci_set_drvdata(pci_dev, efx);1059 SET_NETDEV_DEV(net_dev, &pci_dev->dev);1060 rc = efx_siena_init_struct(efx, pci_dev, net_dev);1061 if (rc)1062 goto fail1;1063 1064 pci_info(pci_dev, "Solarflare NIC detected\n");1065 1066 if (!efx->type->is_vf)1067 efx_probe_vpd_strings(efx);1068 1069 /* Set up basic I/O (BAR mappings etc) */1070 rc = efx_siena_init_io(efx, efx->type->mem_bar(efx),1071 efx->type->max_dma_mask,1072 efx->type->mem_map_size(efx));1073 if (rc)1074 goto fail2;1075 1076 rc = efx_pci_probe_post_io(efx);1077 if (rc) {1078 /* On failure, retry once immediately.1079 * If we aborted probe due to a scheduled reset, dismiss it.1080 */1081 efx->reset_pending = 0;1082 rc = efx_pci_probe_post_io(efx);1083 if (rc) {1084 /* On another failure, retry once more1085 * after a 50-305ms delay.1086 */1087 unsigned char r;1088 1089 get_random_bytes(&r, 1);1090 msleep((unsigned int)r + 50);1091 efx->reset_pending = 0;1092 rc = efx_pci_probe_post_io(efx);1093 }1094 }1095 if (rc)1096 goto fail3;1097 1098 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");1099 1100 /* Try to create MTDs, but allow this to fail */1101 rtnl_lock();1102 rc = efx_mtd_probe(efx);1103 rtnl_unlock();1104 if (rc && rc != -EPERM)1105 netif_warn(efx, probe, efx->net_dev,1106 "failed to create MTDs (%d)\n", rc);1107 1108 if (efx->type->udp_tnl_push_ports)1109 efx->type->udp_tnl_push_ports(efx);1110 1111 return 0;1112 1113 fail3:1114 efx_siena_fini_io(efx);1115 fail2:1116 efx_siena_fini_struct(efx);1117 fail1:1118 WARN_ON(rc > 0);1119 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);1120 free_netdev(net_dev);1121 return rc;1122}1123 1124/* efx_pci_sriov_configure returns the actual number of Virtual Functions1125 * enabled on success1126 */1127#ifdef CONFIG_SFC_SIENA_SRIOV1128static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs)1129{1130 int rc;1131 struct efx_nic *efx = pci_get_drvdata(dev);1132 1133 if (efx->type->sriov_configure) {1134 rc = efx->type->sriov_configure(efx, num_vfs);1135 if (rc)1136 return rc;1137 else1138 return num_vfs;1139 } else1140 return -EOPNOTSUPP;1141}1142#endif1143 1144static int efx_pm_freeze(struct device *dev)1145{1146 struct efx_nic *efx = dev_get_drvdata(dev);1147 1148 rtnl_lock();1149 1150 if (efx->state != STATE_DISABLED) {1151 efx->state = STATE_UNINIT;1152 1153 efx_device_detach_sync(efx);1154 1155 efx_siena_stop_all(efx);1156 efx_siena_disable_interrupts(efx);1157 }1158 1159 rtnl_unlock();1160 1161 return 0;1162}1163 1164static void efx_pci_shutdown(struct pci_dev *pci_dev)1165{1166 struct efx_nic *efx = pci_get_drvdata(pci_dev);1167 1168 if (!efx)1169 return;1170 1171 efx_pm_freeze(&pci_dev->dev);1172 pci_disable_device(pci_dev);1173}1174 1175static int efx_pm_thaw(struct device *dev)1176{1177 int rc;1178 struct efx_nic *efx = dev_get_drvdata(dev);1179 1180 rtnl_lock();1181 1182 if (efx->state != STATE_DISABLED) {1183 rc = efx_siena_enable_interrupts(efx);1184 if (rc)1185 goto fail;1186 1187 mutex_lock(&efx->mac_lock);1188 efx_siena_mcdi_port_reconfigure(efx);1189 mutex_unlock(&efx->mac_lock);1190 1191 efx_siena_start_all(efx);1192 1193 efx_device_attach_if_not_resetting(efx);1194 1195 efx->state = STATE_READY;1196 1197 efx->type->resume_wol(efx);1198 }1199 1200 rtnl_unlock();1201 1202 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */1203 efx_siena_queue_reset_work(efx);1204 1205 return 0;1206 1207fail:1208 rtnl_unlock();1209 1210 return rc;1211}1212 1213static int efx_pm_poweroff(struct device *dev)1214{1215 struct pci_dev *pci_dev = to_pci_dev(dev);1216 struct efx_nic *efx = pci_get_drvdata(pci_dev);1217 1218 efx->type->fini(efx);1219 1220 efx->reset_pending = 0;1221 1222 pci_save_state(pci_dev);1223 return pci_set_power_state(pci_dev, PCI_D3hot);1224}1225 1226/* Used for both resume and restore */1227static int efx_pm_resume(struct device *dev)1228{1229 struct pci_dev *pci_dev = to_pci_dev(dev);1230 struct efx_nic *efx = pci_get_drvdata(pci_dev);1231 int rc;1232 1233 rc = pci_set_power_state(pci_dev, PCI_D0);1234 if (rc)1235 return rc;1236 pci_restore_state(pci_dev);1237 rc = pci_enable_device(pci_dev);1238 if (rc)1239 return rc;1240 pci_set_master(efx->pci_dev);1241 rc = efx->type->reset(efx, RESET_TYPE_ALL);1242 if (rc)1243 return rc;1244 down_write(&efx->filter_sem);1245 rc = efx->type->init(efx);1246 up_write(&efx->filter_sem);1247 if (rc)1248 return rc;1249 rc = efx_pm_thaw(dev);1250 return rc;1251}1252 1253static int efx_pm_suspend(struct device *dev)1254{1255 int rc;1256 1257 efx_pm_freeze(dev);1258 rc = efx_pm_poweroff(dev);1259 if (rc)1260 efx_pm_resume(dev);1261 return rc;1262}1263 1264static const struct dev_pm_ops efx_pm_ops = {1265 .suspend = efx_pm_suspend,1266 .resume = efx_pm_resume,1267 .freeze = efx_pm_freeze,1268 .thaw = efx_pm_thaw,1269 .poweroff = efx_pm_poweroff,1270 .restore = efx_pm_resume,1271};1272 1273static struct pci_driver efx_pci_driver = {1274 .name = KBUILD_MODNAME,1275 .id_table = efx_pci_table,1276 .probe = efx_pci_probe,1277 .remove = efx_pci_remove,1278 .driver.pm = &efx_pm_ops,1279 .shutdown = efx_pci_shutdown,1280 .err_handler = &efx_siena_err_handlers,1281#ifdef CONFIG_SFC_SIENA_SRIOV1282 .sriov_configure = efx_pci_sriov_configure,1283#endif1284};1285 1286/**************************************************************************1287 *1288 * Kernel module interface1289 *1290 *************************************************************************/1291 1292static int __init efx_init_module(void)1293{1294 int rc;1295 1296 pr_info("Solarflare Siena driver\n");1297 1298 rc = register_netdevice_notifier(&efx_netdev_notifier);1299 if (rc)1300 goto err_notifier;1301 1302#ifdef CONFIG_SFC_SIENA_SRIOV1303 rc = efx_init_sriov();1304 if (rc)1305 goto err_sriov;1306#endif1307 1308 rc = efx_siena_create_reset_workqueue();1309 if (rc)1310 goto err_reset;1311 1312 rc = pci_register_driver(&efx_pci_driver);1313 if (rc < 0)1314 goto err_pci;1315 1316 return 0;1317 1318 err_pci:1319 efx_siena_destroy_reset_workqueue();1320 err_reset:1321#ifdef CONFIG_SFC_SIENA_SRIOV1322 efx_fini_sriov();1323 err_sriov:1324#endif1325 unregister_netdevice_notifier(&efx_netdev_notifier);1326 err_notifier:1327 return rc;1328}1329 1330static void __exit efx_exit_module(void)1331{1332 pr_info("Solarflare Siena driver unloading\n");1333 1334 pci_unregister_driver(&efx_pci_driver);1335 efx_siena_destroy_reset_workqueue();1336#ifdef CONFIG_SFC_SIENA_SRIOV1337 efx_fini_sriov();1338#endif1339 unregister_netdevice_notifier(&efx_netdev_notifier);1340 1341}1342 1343module_init(efx_init_module);1344module_exit(efx_exit_module);1345 1346MODULE_AUTHOR("Solarflare Communications and "1347 "Michael Brown <mbrown@fensystems.co.uk>");1348MODULE_DESCRIPTION("Solarflare Siena network driver");1349MODULE_LICENSE("GPL");1350MODULE_DEVICE_TABLE(pci, efx_pci_table);1351