1688 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/****************************************************************************3 * Driver for Solarflare network controllers and boards4 * Copyright 2010-2012 Solarflare Communications Inc.5 */6#include <linux/pci.h>7#include <linux/module.h>8#include "net_driver.h"9#include "efx.h"10#include "efx_channels.h"11#include "nic.h"12#include "io.h"13#include "mcdi.h"14#include "filter.h"15#include "mcdi_pcol.h"16#include "farch_regs.h"17#include "siena_sriov.h"18#include "vfdi.h"19 20/* Number of longs required to track all the VIs in a VF */21#define VI_MASK_LENGTH BITS_TO_LONGS(1 << EFX_VI_SCALE_MAX)22 23/* Maximum number of RX queues supported */24#define VF_MAX_RX_QUEUES 6325 26/**27 * enum efx_vf_tx_filter_mode - TX MAC filtering behaviour28 * @VF_TX_FILTER_OFF: Disabled29 * @VF_TX_FILTER_AUTO: Enabled if MAC address assigned to VF and only30 * 2 TX queues allowed per VF.31 * @VF_TX_FILTER_ON: Enabled32 */33enum efx_vf_tx_filter_mode {34 VF_TX_FILTER_OFF,35 VF_TX_FILTER_AUTO,36 VF_TX_FILTER_ON,37};38 39/**40 * struct siena_vf - Back-end resource and protocol state for a PCI VF41 * @efx: The Efx NIC owning this VF42 * @pci_rid: The PCI requester ID for this VF43 * @pci_name: The PCI name (formatted address) of this VF44 * @index: Index of VF within its port and PF.45 * @req: VFDI incoming request work item. Incoming USR_EV events are received46 * by the NAPI handler, but must be handled by executing MCDI requests47 * inside a work item.48 * @req_addr: VFDI incoming request DMA address (in VF's PCI address space).49 * @req_type: Expected next incoming (from VF) %VFDI_EV_TYPE member.50 * @req_seqno: Expected next incoming (from VF) %VFDI_EV_SEQ member.51 * @msg_seqno: Next %VFDI_EV_SEQ member to reply to VF. Protected by52 * @status_lock53 * @busy: VFDI request queued to be processed or being processed. Receiving54 * a VFDI request when @busy is set is an error condition.55 * @buf: Incoming VFDI requests are DMA from the VF into this buffer.56 * @buftbl_base: Buffer table entries for this VF start at this index.57 * @rx_filtering: Receive filtering has been requested by the VF driver.58 * @rx_filter_flags: The flags sent in the %VFDI_OP_INSERT_FILTER request.59 * @rx_filter_qid: VF relative qid for RX filter requested by VF.60 * @rx_filter_id: Receive MAC filter ID. Only one filter per VF is supported.61 * @tx_filter_mode: Transmit MAC filtering mode.62 * @tx_filter_id: Transmit MAC filter ID.63 * @addr: The MAC address and outer vlan tag of the VF.64 * @status_addr: VF DMA address of page for &struct vfdi_status updates.65 * @status_lock: Mutex protecting @msg_seqno, @status_addr, @addr,66 * @peer_page_addrs and @peer_page_count from simultaneous67 * updates by the VM and consumption by68 * efx_siena_sriov_update_vf_addr()69 * @peer_page_addrs: Pointer to an array of guest pages for local addresses.70 * @peer_page_count: Number of entries in @peer_page_count.71 * @evq0_addrs: Array of guest pages backing evq0.72 * @evq0_count: Number of entries in @evq0_addrs.73 * @flush_waitq: wait queue used by %VFDI_OP_FINI_ALL_QUEUES handler74 * to wait for flush completions.75 * @txq_lock: Mutex for TX queue allocation.76 * @txq_mask: Mask of initialized transmit queues.77 * @txq_count: Number of initialized transmit queues.78 * @rxq_mask: Mask of initialized receive queues.79 * @rxq_count: Number of initialized receive queues.80 * @rxq_retry_mask: Mask or receive queues that need to be flushed again81 * due to flush failure.82 * @rxq_retry_count: Number of receive queues in @rxq_retry_mask.83 * @reset_work: Work item to schedule a VF reset.84 */85struct siena_vf {86 struct efx_nic *efx;87 unsigned int pci_rid;88 char pci_name[13]; /* dddd:bb:dd.f */89 unsigned int index;90 struct work_struct req;91 u64 req_addr;92 int req_type;93 unsigned req_seqno;94 unsigned msg_seqno;95 bool busy;96 struct efx_buffer buf;97 unsigned buftbl_base;98 bool rx_filtering;99 enum efx_filter_flags rx_filter_flags;100 unsigned rx_filter_qid;101 int rx_filter_id;102 enum efx_vf_tx_filter_mode tx_filter_mode;103 int tx_filter_id;104 struct vfdi_endpoint addr;105 u64 status_addr;106 struct mutex status_lock;107 u64 *peer_page_addrs;108 unsigned peer_page_count;109 u64 evq0_addrs[EFX_MAX_VF_EVQ_SIZE * sizeof(efx_qword_t) /110 EFX_BUF_SIZE];111 unsigned evq0_count;112 wait_queue_head_t flush_waitq;113 struct mutex txq_lock;114 unsigned long txq_mask[VI_MASK_LENGTH];115 unsigned txq_count;116 unsigned long rxq_mask[VI_MASK_LENGTH];117 unsigned rxq_count;118 unsigned long rxq_retry_mask[VI_MASK_LENGTH];119 atomic_t rxq_retry_count;120 struct work_struct reset_work;121};122 123struct efx_memcpy_req {124 unsigned int from_rid;125 void *from_buf;126 u64 from_addr;127 unsigned int to_rid;128 u64 to_addr;129 unsigned length;130};131 132/**133 * struct efx_local_addr - A MAC address on the vswitch without a VF.134 *135 * Siena does not have a switch, so VFs can't transmit data to each136 * other. Instead the VFs must be made aware of the local addresses137 * on the vswitch, so that they can arrange for an alternative138 * software datapath to be used.139 *140 * @link: List head for insertion into efx->local_addr_list.141 * @addr: Ethernet address142 */143struct efx_local_addr {144 struct list_head link;145 u8 addr[ETH_ALEN];146};147 148/**149 * struct efx_endpoint_page - Page of vfdi_endpoint structures150 *151 * @link: List head for insertion into efx->local_page_list.152 * @ptr: Pointer to page.153 * @addr: DMA address of page.154 */155struct efx_endpoint_page {156 struct list_head link;157 void *ptr;158 dma_addr_t addr;159};160 161/* Buffer table entries are reserved txq0,rxq0,evq0,txq1,rxq1,evq1 */162#define EFX_BUFTBL_TXQ_BASE(_vf, _qid) \163 ((_vf)->buftbl_base + EFX_VF_BUFTBL_PER_VI * (_qid))164#define EFX_BUFTBL_RXQ_BASE(_vf, _qid) \165 (EFX_BUFTBL_TXQ_BASE(_vf, _qid) + \166 (EFX_MAX_DMAQ_SIZE * sizeof(efx_qword_t) / EFX_BUF_SIZE))167#define EFX_BUFTBL_EVQ_BASE(_vf, _qid) \168 (EFX_BUFTBL_TXQ_BASE(_vf, _qid) + \169 (2 * EFX_MAX_DMAQ_SIZE * sizeof(efx_qword_t) / EFX_BUF_SIZE))170 171#define EFX_FIELD_MASK(_field) \172 ((1 << _field ## _WIDTH) - 1)173 174/* VFs can only use this many transmit channels */175static unsigned int vf_max_tx_channels = 2;176module_param(vf_max_tx_channels, uint, 0444);177MODULE_PARM_DESC(vf_max_tx_channels,178 "Limit the number of TX channels VFs can use");179 180static int max_vfs = -1;181module_param(max_vfs, int, 0444);182MODULE_PARM_DESC(max_vfs,183 "Reduce the number of VFs initialized by the driver");184 185/* Workqueue used by VFDI communication. We can't use the global186 * workqueue because it may be running the VF driver's probe()187 * routine, which will be blocked there waiting for a VFDI response.188 */189static struct workqueue_struct *vfdi_workqueue;190 191static unsigned abs_index(struct siena_vf *vf, unsigned index)192{193 return EFX_VI_BASE + vf->index * efx_vf_size(vf->efx) + index;194}195 196static int efx_siena_sriov_cmd(struct efx_nic *efx, bool enable,197 unsigned *vi_scale_out, unsigned *vf_total_out)198{199 MCDI_DECLARE_BUF(inbuf, MC_CMD_SRIOV_IN_LEN);200 MCDI_DECLARE_BUF(outbuf, MC_CMD_SRIOV_OUT_LEN);201 unsigned vi_scale, vf_total;202 size_t outlen;203 int rc;204 205 MCDI_SET_DWORD(inbuf, SRIOV_IN_ENABLE, enable ? 1 : 0);206 MCDI_SET_DWORD(inbuf, SRIOV_IN_VI_BASE, EFX_VI_BASE);207 MCDI_SET_DWORD(inbuf, SRIOV_IN_VF_COUNT, efx->vf_count);208 209 rc = efx_siena_mcdi_rpc_quiet(efx, MC_CMD_SRIOV, inbuf,210 MC_CMD_SRIOV_IN_LEN, outbuf,211 MC_CMD_SRIOV_OUT_LEN, &outlen);212 if (rc)213 return rc;214 if (outlen < MC_CMD_SRIOV_OUT_LEN)215 return -EIO;216 217 vf_total = MCDI_DWORD(outbuf, SRIOV_OUT_VF_TOTAL);218 vi_scale = MCDI_DWORD(outbuf, SRIOV_OUT_VI_SCALE);219 if (vi_scale > EFX_VI_SCALE_MAX)220 return -EOPNOTSUPP;221 222 if (vi_scale_out)223 *vi_scale_out = vi_scale;224 if (vf_total_out)225 *vf_total_out = vf_total;226 227 return 0;228}229 230static void efx_siena_sriov_usrev(struct efx_nic *efx, bool enabled)231{232 struct siena_nic_data *nic_data = efx->nic_data;233 efx_oword_t reg;234 235 EFX_POPULATE_OWORD_2(reg,236 FRF_CZ_USREV_DIS, enabled ? 0 : 1,237 FRF_CZ_DFLT_EVQ, nic_data->vfdi_channel->channel);238 efx_writeo(efx, ®, FR_CZ_USR_EV_CFG);239}240 241static int efx_siena_sriov_memcpy(struct efx_nic *efx,242 struct efx_memcpy_req *req,243 unsigned int count)244{245 MCDI_DECLARE_BUF(inbuf, MCDI_CTL_SDU_LEN_MAX_V1);246 MCDI_DECLARE_STRUCT_PTR(record);247 unsigned int index, used;248 u64 from_addr;249 u32 from_rid;250 int rc;251 252 mb(); /* Finish writing source/reading dest before DMA starts */253 254 if (WARN_ON(count > MC_CMD_MEMCPY_IN_RECORD_MAXNUM))255 return -ENOBUFS;256 used = MC_CMD_MEMCPY_IN_LEN(count);257 258 for (index = 0; index < count; index++) {259 record = MCDI_ARRAY_STRUCT_PTR(inbuf, MEMCPY_IN_RECORD, index);260 MCDI_SET_DWORD(record, MEMCPY_RECORD_TYPEDEF_NUM_RECORDS,261 count);262 MCDI_SET_DWORD(record, MEMCPY_RECORD_TYPEDEF_TO_RID,263 req->to_rid);264 MCDI_SET_QWORD(record, MEMCPY_RECORD_TYPEDEF_TO_ADDR,265 req->to_addr);266 if (req->from_buf == NULL) {267 from_rid = req->from_rid;268 from_addr = req->from_addr;269 } else {270 if (WARN_ON(used + req->length >271 MCDI_CTL_SDU_LEN_MAX_V1)) {272 rc = -ENOBUFS;273 goto out;274 }275 276 from_rid = MC_CMD_MEMCPY_RECORD_TYPEDEF_RID_INLINE;277 from_addr = used;278 memcpy(_MCDI_PTR(inbuf, used), req->from_buf,279 req->length);280 used += req->length;281 }282 283 MCDI_SET_DWORD(record, MEMCPY_RECORD_TYPEDEF_FROM_RID, from_rid);284 MCDI_SET_QWORD(record, MEMCPY_RECORD_TYPEDEF_FROM_ADDR,285 from_addr);286 MCDI_SET_DWORD(record, MEMCPY_RECORD_TYPEDEF_LENGTH,287 req->length);288 289 ++req;290 }291 292 rc = efx_siena_mcdi_rpc(efx, MC_CMD_MEMCPY, inbuf, used, NULL, 0, NULL);293out:294 mb(); /* Don't write source/read dest before DMA is complete */295 296 return rc;297}298 299/* The TX filter is entirely controlled by this driver, and is modified300 * underneath the feet of the VF301 */302static void efx_siena_sriov_reset_tx_filter(struct siena_vf *vf)303{304 struct efx_nic *efx = vf->efx;305 struct efx_filter_spec filter;306 u16 vlan;307 int rc;308 309 if (vf->tx_filter_id != -1) {310 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,311 vf->tx_filter_id);312 netif_dbg(efx, hw, efx->net_dev, "Removed vf %s tx filter %d\n",313 vf->pci_name, vf->tx_filter_id);314 vf->tx_filter_id = -1;315 }316 317 if (is_zero_ether_addr(vf->addr.mac_addr))318 return;319 320 /* Turn on TX filtering automatically if not explicitly321 * enabled or disabled.322 */323 if (vf->tx_filter_mode == VF_TX_FILTER_AUTO && vf_max_tx_channels <= 2)324 vf->tx_filter_mode = VF_TX_FILTER_ON;325 326 vlan = ntohs(vf->addr.tci) & VLAN_VID_MASK;327 efx_filter_init_tx(&filter, abs_index(vf, 0));328 rc = efx_filter_set_eth_local(&filter,329 vlan ? vlan : EFX_FILTER_VID_UNSPEC,330 vf->addr.mac_addr);331 BUG_ON(rc);332 333 rc = efx_filter_insert_filter(efx, &filter, true);334 if (rc < 0) {335 netif_warn(efx, hw, efx->net_dev,336 "Unable to migrate tx filter for vf %s\n",337 vf->pci_name);338 } else {339 netif_dbg(efx, hw, efx->net_dev, "Inserted vf %s tx filter %d\n",340 vf->pci_name, rc);341 vf->tx_filter_id = rc;342 }343}344 345/* The RX filter is managed here on behalf of the VF driver */346static void efx_siena_sriov_reset_rx_filter(struct siena_vf *vf)347{348 struct efx_nic *efx = vf->efx;349 struct efx_filter_spec filter;350 u16 vlan;351 int rc;352 353 if (vf->rx_filter_id != -1) {354 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,355 vf->rx_filter_id);356 netif_dbg(efx, hw, efx->net_dev, "Removed vf %s rx filter %d\n",357 vf->pci_name, vf->rx_filter_id);358 vf->rx_filter_id = -1;359 }360 361 if (!vf->rx_filtering || is_zero_ether_addr(vf->addr.mac_addr))362 return;363 364 vlan = ntohs(vf->addr.tci) & VLAN_VID_MASK;365 efx_filter_init_rx(&filter, EFX_FILTER_PRI_REQUIRED,366 vf->rx_filter_flags,367 abs_index(vf, vf->rx_filter_qid));368 rc = efx_filter_set_eth_local(&filter,369 vlan ? vlan : EFX_FILTER_VID_UNSPEC,370 vf->addr.mac_addr);371 BUG_ON(rc);372 373 rc = efx_filter_insert_filter(efx, &filter, true);374 if (rc < 0) {375 netif_warn(efx, hw, efx->net_dev,376 "Unable to insert rx filter for vf %s\n",377 vf->pci_name);378 } else {379 netif_dbg(efx, hw, efx->net_dev, "Inserted vf %s rx filter %d\n",380 vf->pci_name, rc);381 vf->rx_filter_id = rc;382 }383}384 385static void __efx_siena_sriov_update_vf_addr(struct siena_vf *vf)386{387 struct efx_nic *efx = vf->efx;388 struct siena_nic_data *nic_data = efx->nic_data;389 390 efx_siena_sriov_reset_tx_filter(vf);391 efx_siena_sriov_reset_rx_filter(vf);392 queue_work(vfdi_workqueue, &nic_data->peer_work);393}394 395/* Push the peer list to this VF. The caller must hold status_lock to interlock396 * with VFDI requests, and they must be serialised against manipulation of397 * local_page_list, either by acquiring local_lock or by running from398 * efx_siena_sriov_peer_work()399 */400static void __efx_siena_sriov_push_vf_status(struct siena_vf *vf)401{402 struct efx_nic *efx = vf->efx;403 struct siena_nic_data *nic_data = efx->nic_data;404 struct vfdi_status *status = nic_data->vfdi_status.addr;405 struct efx_memcpy_req copy[4];406 struct efx_endpoint_page *epp;407 unsigned int pos, count;408 unsigned data_offset;409 efx_qword_t event;410 411 WARN_ON(!mutex_is_locked(&vf->status_lock));412 WARN_ON(!vf->status_addr);413 414 status->local = vf->addr;415 status->generation_end = ++status->generation_start;416 417 memset(copy, '\0', sizeof(copy));418 /* Write generation_start */419 copy[0].from_buf = &status->generation_start;420 copy[0].to_rid = vf->pci_rid;421 copy[0].to_addr = vf->status_addr + offsetof(struct vfdi_status,422 generation_start);423 copy[0].length = sizeof(status->generation_start);424 /* DMA the rest of the structure (excluding the generations). This425 * assumes that the non-generation portion of vfdi_status is in426 * one chunk starting at the version member.427 */428 data_offset = offsetof(struct vfdi_status, version);429 copy[1].from_rid = efx->pci_dev->devfn;430 copy[1].from_addr = nic_data->vfdi_status.dma_addr + data_offset;431 copy[1].to_rid = vf->pci_rid;432 copy[1].to_addr = vf->status_addr + data_offset;433 copy[1].length = status->length - data_offset;434 435 /* Copy the peer pages */436 pos = 2;437 count = 0;438 list_for_each_entry(epp, &nic_data->local_page_list, link) {439 if (count == vf->peer_page_count) {440 /* The VF driver will know they need to provide more441 * pages because peer_addr_count is too large.442 */443 break;444 }445 copy[pos].from_buf = NULL;446 copy[pos].from_rid = efx->pci_dev->devfn;447 copy[pos].from_addr = epp->addr;448 copy[pos].to_rid = vf->pci_rid;449 copy[pos].to_addr = vf->peer_page_addrs[count];450 copy[pos].length = EFX_PAGE_SIZE;451 452 if (++pos == ARRAY_SIZE(copy)) {453 efx_siena_sriov_memcpy(efx, copy, ARRAY_SIZE(copy));454 pos = 0;455 }456 ++count;457 }458 459 /* Write generation_end */460 copy[pos].from_buf = &status->generation_end;461 copy[pos].to_rid = vf->pci_rid;462 copy[pos].to_addr = vf->status_addr + offsetof(struct vfdi_status,463 generation_end);464 copy[pos].length = sizeof(status->generation_end);465 efx_siena_sriov_memcpy(efx, copy, pos + 1);466 467 /* Notify the guest */468 EFX_POPULATE_QWORD_3(event,469 FSF_AZ_EV_CODE, FSE_CZ_EV_CODE_USER_EV,470 VFDI_EV_SEQ, (vf->msg_seqno & 0xff),471 VFDI_EV_TYPE, VFDI_EV_TYPE_STATUS);472 ++vf->msg_seqno;473 efx_farch_generate_event(efx,474 EFX_VI_BASE + vf->index * efx_vf_size(efx),475 &event);476}477 478static void efx_siena_sriov_bufs(struct efx_nic *efx, unsigned offset,479 u64 *addr, unsigned count)480{481 efx_qword_t buf;482 unsigned pos;483 484 for (pos = 0; pos < count; ++pos) {485 EFX_POPULATE_QWORD_3(buf,486 FRF_AZ_BUF_ADR_REGION, 0,487 FRF_AZ_BUF_ADR_FBUF,488 addr ? addr[pos] >> 12 : 0,489 FRF_AZ_BUF_OWNER_ID_FBUF, 0);490 efx_sram_writeq(efx, efx->membase + FR_BZ_BUF_FULL_TBL,491 &buf, offset + pos);492 }493}494 495static bool bad_vf_index(struct efx_nic *efx, unsigned index)496{497 return index >= efx_vf_size(efx);498}499 500static bool bad_buf_count(unsigned buf_count, unsigned max_entry_count)501{502 unsigned max_buf_count = max_entry_count *503 sizeof(efx_qword_t) / EFX_BUF_SIZE;504 505 return ((buf_count & (buf_count - 1)) || buf_count > max_buf_count);506}507 508/* Check that VI specified by per-port index belongs to a VF.509 * Optionally set VF index and VI index within the VF.510 */511static bool map_vi_index(struct efx_nic *efx, unsigned abs_index,512 struct siena_vf **vf_out, unsigned *rel_index_out)513{514 struct siena_nic_data *nic_data = efx->nic_data;515 unsigned vf_i;516 517 if (abs_index < EFX_VI_BASE)518 return true;519 vf_i = (abs_index - EFX_VI_BASE) / efx_vf_size(efx);520 if (vf_i >= efx->vf_init_count)521 return true;522 523 if (vf_out)524 *vf_out = nic_data->vf + vf_i;525 if (rel_index_out)526 *rel_index_out = abs_index % efx_vf_size(efx);527 return false;528}529 530static int efx_vfdi_init_evq(struct siena_vf *vf)531{532 struct efx_nic *efx = vf->efx;533 struct vfdi_req *req = vf->buf.addr;534 unsigned vf_evq = req->u.init_evq.index;535 unsigned buf_count = req->u.init_evq.buf_count;536 unsigned abs_evq = abs_index(vf, vf_evq);537 unsigned buftbl = EFX_BUFTBL_EVQ_BASE(vf, vf_evq);538 efx_oword_t reg;539 540 if (bad_vf_index(efx, vf_evq) ||541 bad_buf_count(buf_count, EFX_MAX_VF_EVQ_SIZE)) {542 if (net_ratelimit())543 netif_err(efx, hw, efx->net_dev,544 "ERROR: Invalid INIT_EVQ from %s: evq %d bufs %d\n",545 vf->pci_name, vf_evq, buf_count);546 return VFDI_RC_EINVAL;547 }548 549 efx_siena_sriov_bufs(efx, buftbl, req->u.init_evq.addr, buf_count);550 551 EFX_POPULATE_OWORD_3(reg,552 FRF_CZ_TIMER_Q_EN, 1,553 FRF_CZ_HOST_NOTIFY_MODE, 0,554 FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);555 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, abs_evq);556 EFX_POPULATE_OWORD_3(reg,557 FRF_AZ_EVQ_EN, 1,558 FRF_AZ_EVQ_SIZE, __ffs(buf_count),559 FRF_AZ_EVQ_BUF_BASE_ID, buftbl);560 efx_writeo_table(efx, ®, FR_BZ_EVQ_PTR_TBL, abs_evq);561 562 if (vf_evq == 0) {563 memcpy(vf->evq0_addrs, req->u.init_evq.addr,564 buf_count * sizeof(u64));565 vf->evq0_count = buf_count;566 }567 568 return VFDI_RC_SUCCESS;569}570 571static int efx_vfdi_init_rxq(struct siena_vf *vf)572{573 struct efx_nic *efx = vf->efx;574 struct vfdi_req *req = vf->buf.addr;575 unsigned vf_rxq = req->u.init_rxq.index;576 unsigned vf_evq = req->u.init_rxq.evq;577 unsigned buf_count = req->u.init_rxq.buf_count;578 unsigned buftbl = EFX_BUFTBL_RXQ_BASE(vf, vf_rxq);579 unsigned label;580 efx_oword_t reg;581 582 if (bad_vf_index(efx, vf_evq) || bad_vf_index(efx, vf_rxq) ||583 vf_rxq >= VF_MAX_RX_QUEUES ||584 bad_buf_count(buf_count, EFX_MAX_DMAQ_SIZE)) {585 if (net_ratelimit())586 netif_err(efx, hw, efx->net_dev,587 "ERROR: Invalid INIT_RXQ from %s: rxq %d evq %d "588 "buf_count %d\n", vf->pci_name, vf_rxq,589 vf_evq, buf_count);590 return VFDI_RC_EINVAL;591 }592 if (__test_and_set_bit(req->u.init_rxq.index, vf->rxq_mask))593 ++vf->rxq_count;594 efx_siena_sriov_bufs(efx, buftbl, req->u.init_rxq.addr, buf_count);595 596 label = req->u.init_rxq.label & EFX_FIELD_MASK(FRF_AZ_RX_DESCQ_LABEL);597 EFX_POPULATE_OWORD_6(reg,598 FRF_AZ_RX_DESCQ_BUF_BASE_ID, buftbl,599 FRF_AZ_RX_DESCQ_EVQ_ID, abs_index(vf, vf_evq),600 FRF_AZ_RX_DESCQ_LABEL, label,601 FRF_AZ_RX_DESCQ_SIZE, __ffs(buf_count),602 FRF_AZ_RX_DESCQ_JUMBO,603 !!(req->u.init_rxq.flags &604 VFDI_RXQ_FLAG_SCATTER_EN),605 FRF_AZ_RX_DESCQ_EN, 1);606 efx_writeo_table(efx, ®, FR_BZ_RX_DESC_PTR_TBL,607 abs_index(vf, vf_rxq));608 609 return VFDI_RC_SUCCESS;610}611 612static int efx_vfdi_init_txq(struct siena_vf *vf)613{614 struct efx_nic *efx = vf->efx;615 struct vfdi_req *req = vf->buf.addr;616 unsigned vf_txq = req->u.init_txq.index;617 unsigned vf_evq = req->u.init_txq.evq;618 unsigned buf_count = req->u.init_txq.buf_count;619 unsigned buftbl = EFX_BUFTBL_TXQ_BASE(vf, vf_txq);620 unsigned label, eth_filt_en;621 efx_oword_t reg;622 623 if (bad_vf_index(efx, vf_evq) || bad_vf_index(efx, vf_txq) ||624 vf_txq >= vf_max_tx_channels ||625 bad_buf_count(buf_count, EFX_MAX_DMAQ_SIZE)) {626 if (net_ratelimit())627 netif_err(efx, hw, efx->net_dev,628 "ERROR: Invalid INIT_TXQ from %s: txq %d evq %d "629 "buf_count %d\n", vf->pci_name, vf_txq,630 vf_evq, buf_count);631 return VFDI_RC_EINVAL;632 }633 634 mutex_lock(&vf->txq_lock);635 if (__test_and_set_bit(req->u.init_txq.index, vf->txq_mask))636 ++vf->txq_count;637 mutex_unlock(&vf->txq_lock);638 efx_siena_sriov_bufs(efx, buftbl, req->u.init_txq.addr, buf_count);639 640 eth_filt_en = vf->tx_filter_mode == VF_TX_FILTER_ON;641 642 label = req->u.init_txq.label & EFX_FIELD_MASK(FRF_AZ_TX_DESCQ_LABEL);643 EFX_POPULATE_OWORD_8(reg,644 FRF_CZ_TX_DPT_Q_MASK_WIDTH, min(efx->vi_scale, 1U),645 FRF_CZ_TX_DPT_ETH_FILT_EN, eth_filt_en,646 FRF_AZ_TX_DESCQ_EN, 1,647 FRF_AZ_TX_DESCQ_BUF_BASE_ID, buftbl,648 FRF_AZ_TX_DESCQ_EVQ_ID, abs_index(vf, vf_evq),649 FRF_AZ_TX_DESCQ_LABEL, label,650 FRF_AZ_TX_DESCQ_SIZE, __ffs(buf_count),651 FRF_BZ_TX_NON_IP_DROP_DIS, 1);652 efx_writeo_table(efx, ®, FR_BZ_TX_DESC_PTR_TBL,653 abs_index(vf, vf_txq));654 655 return VFDI_RC_SUCCESS;656}657 658/* Returns true when efx_vfdi_fini_all_queues should wake */659static bool efx_vfdi_flush_wake(struct siena_vf *vf)660{661 /* Ensure that all updates are visible to efx_vfdi_fini_all_queues() */662 smp_mb();663 664 return (!vf->txq_count && !vf->rxq_count) ||665 atomic_read(&vf->rxq_retry_count);666}667 668static void efx_vfdi_flush_clear(struct siena_vf *vf)669{670 memset(vf->txq_mask, 0, sizeof(vf->txq_mask));671 vf->txq_count = 0;672 memset(vf->rxq_mask, 0, sizeof(vf->rxq_mask));673 vf->rxq_count = 0;674 memset(vf->rxq_retry_mask, 0, sizeof(vf->rxq_retry_mask));675 atomic_set(&vf->rxq_retry_count, 0);676}677 678static int efx_vfdi_fini_all_queues(struct siena_vf *vf)679{680 struct efx_nic *efx = vf->efx;681 efx_oword_t reg;682 unsigned count = efx_vf_size(efx);683 unsigned vf_offset = EFX_VI_BASE + vf->index * efx_vf_size(efx);684 unsigned timeout = HZ;685 unsigned index, rxqs_count;686 MCDI_DECLARE_BUF(inbuf, MC_CMD_FLUSH_RX_QUEUES_IN_LENMAX);687 int rc;688 689 BUILD_BUG_ON(VF_MAX_RX_QUEUES >690 MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_MAXNUM);691 692 rtnl_lock();693 efx_siena_prepare_flush(efx);694 rtnl_unlock();695 696 /* Flush all the initialized queues */697 rxqs_count = 0;698 for (index = 0; index < count; ++index) {699 if (test_bit(index, vf->txq_mask)) {700 EFX_POPULATE_OWORD_2(reg,701 FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,702 FRF_AZ_TX_FLUSH_DESCQ,703 vf_offset + index);704 efx_writeo(efx, ®, FR_AZ_TX_FLUSH_DESCQ);705 }706 if (test_bit(index, vf->rxq_mask)) {707 MCDI_SET_ARRAY_DWORD(708 inbuf, FLUSH_RX_QUEUES_IN_QID_OFST,709 rxqs_count, vf_offset + index);710 rxqs_count++;711 }712 }713 714 atomic_set(&vf->rxq_retry_count, 0);715 while (timeout && (vf->rxq_count || vf->txq_count)) {716 rc = efx_siena_mcdi_rpc(efx, MC_CMD_FLUSH_RX_QUEUES, inbuf,717 MC_CMD_FLUSH_RX_QUEUES_IN_LEN(rxqs_count),718 NULL, 0, NULL);719 WARN_ON(rc < 0);720 721 timeout = wait_event_timeout(vf->flush_waitq,722 efx_vfdi_flush_wake(vf),723 timeout);724 rxqs_count = 0;725 for (index = 0; index < count; ++index) {726 if (test_and_clear_bit(index, vf->rxq_retry_mask)) {727 atomic_dec(&vf->rxq_retry_count);728 MCDI_SET_ARRAY_DWORD(729 inbuf, FLUSH_RX_QUEUES_IN_QID_OFST,730 rxqs_count, vf_offset + index);731 rxqs_count++;732 }733 }734 }735 736 rtnl_lock();737 siena_finish_flush(efx);738 rtnl_unlock();739 740 /* Irrespective of success/failure, fini the queues */741 EFX_ZERO_OWORD(reg);742 for (index = 0; index < count; ++index) {743 efx_writeo_table(efx, ®, FR_BZ_RX_DESC_PTR_TBL,744 vf_offset + index);745 efx_writeo_table(efx, ®, FR_BZ_TX_DESC_PTR_TBL,746 vf_offset + index);747 efx_writeo_table(efx, ®, FR_BZ_EVQ_PTR_TBL,748 vf_offset + index);749 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL,750 vf_offset + index);751 }752 efx_siena_sriov_bufs(efx, vf->buftbl_base, NULL,753 EFX_VF_BUFTBL_PER_VI * efx_vf_size(efx));754 efx_vfdi_flush_clear(vf);755 756 vf->evq0_count = 0;757 758 return timeout ? 0 : VFDI_RC_ETIMEDOUT;759}760 761static int efx_vfdi_insert_filter(struct siena_vf *vf)762{763 struct efx_nic *efx = vf->efx;764 struct siena_nic_data *nic_data = efx->nic_data;765 struct vfdi_req *req = vf->buf.addr;766 unsigned vf_rxq = req->u.mac_filter.rxq;767 unsigned flags;768 769 if (bad_vf_index(efx, vf_rxq) || vf->rx_filtering) {770 if (net_ratelimit())771 netif_err(efx, hw, efx->net_dev,772 "ERROR: Invalid INSERT_FILTER from %s: rxq %d "773 "flags 0x%x\n", vf->pci_name, vf_rxq,774 req->u.mac_filter.flags);775 return VFDI_RC_EINVAL;776 }777 778 flags = 0;779 if (req->u.mac_filter.flags & VFDI_MAC_FILTER_FLAG_RSS)780 flags |= EFX_FILTER_FLAG_RX_RSS;781 if (req->u.mac_filter.flags & VFDI_MAC_FILTER_FLAG_SCATTER)782 flags |= EFX_FILTER_FLAG_RX_SCATTER;783 vf->rx_filter_flags = flags;784 vf->rx_filter_qid = vf_rxq;785 vf->rx_filtering = true;786 787 efx_siena_sriov_reset_rx_filter(vf);788 queue_work(vfdi_workqueue, &nic_data->peer_work);789 790 return VFDI_RC_SUCCESS;791}792 793static int efx_vfdi_remove_all_filters(struct siena_vf *vf)794{795 struct efx_nic *efx = vf->efx;796 struct siena_nic_data *nic_data = efx->nic_data;797 798 vf->rx_filtering = false;799 efx_siena_sriov_reset_rx_filter(vf);800 queue_work(vfdi_workqueue, &nic_data->peer_work);801 802 return VFDI_RC_SUCCESS;803}804 805static int efx_vfdi_set_status_page(struct siena_vf *vf)806{807 struct efx_nic *efx = vf->efx;808 struct siena_nic_data *nic_data = efx->nic_data;809 struct vfdi_req *req = vf->buf.addr;810 u64 page_count = req->u.set_status_page.peer_page_count;811 u64 max_page_count =812 (EFX_PAGE_SIZE -813 offsetof(struct vfdi_req, u.set_status_page.peer_page_addr[0]))814 / sizeof(req->u.set_status_page.peer_page_addr[0]);815 816 if (!req->u.set_status_page.dma_addr || page_count > max_page_count) {817 if (net_ratelimit())818 netif_err(efx, hw, efx->net_dev,819 "ERROR: Invalid SET_STATUS_PAGE from %s\n",820 vf->pci_name);821 return VFDI_RC_EINVAL;822 }823 824 mutex_lock(&nic_data->local_lock);825 mutex_lock(&vf->status_lock);826 vf->status_addr = req->u.set_status_page.dma_addr;827 828 kfree(vf->peer_page_addrs);829 vf->peer_page_addrs = NULL;830 vf->peer_page_count = 0;831 832 if (page_count) {833 vf->peer_page_addrs = kcalloc(page_count, sizeof(u64),834 GFP_KERNEL);835 if (vf->peer_page_addrs) {836 memcpy(vf->peer_page_addrs,837 req->u.set_status_page.peer_page_addr,838 page_count * sizeof(u64));839 vf->peer_page_count = page_count;840 }841 }842 843 __efx_siena_sriov_push_vf_status(vf);844 mutex_unlock(&vf->status_lock);845 mutex_unlock(&nic_data->local_lock);846 847 return VFDI_RC_SUCCESS;848}849 850static int efx_vfdi_clear_status_page(struct siena_vf *vf)851{852 mutex_lock(&vf->status_lock);853 vf->status_addr = 0;854 mutex_unlock(&vf->status_lock);855 856 return VFDI_RC_SUCCESS;857}858 859typedef int (*efx_vfdi_op_t)(struct siena_vf *vf);860 861static const efx_vfdi_op_t vfdi_ops[VFDI_OP_LIMIT] = {862 [VFDI_OP_INIT_EVQ] = efx_vfdi_init_evq,863 [VFDI_OP_INIT_TXQ] = efx_vfdi_init_txq,864 [VFDI_OP_INIT_RXQ] = efx_vfdi_init_rxq,865 [VFDI_OP_FINI_ALL_QUEUES] = efx_vfdi_fini_all_queues,866 [VFDI_OP_INSERT_FILTER] = efx_vfdi_insert_filter,867 [VFDI_OP_REMOVE_ALL_FILTERS] = efx_vfdi_remove_all_filters,868 [VFDI_OP_SET_STATUS_PAGE] = efx_vfdi_set_status_page,869 [VFDI_OP_CLEAR_STATUS_PAGE] = efx_vfdi_clear_status_page,870};871 872static void efx_siena_sriov_vfdi(struct work_struct *work)873{874 struct siena_vf *vf = container_of(work, struct siena_vf, req);875 struct efx_nic *efx = vf->efx;876 struct vfdi_req *req = vf->buf.addr;877 struct efx_memcpy_req copy[2];878 int rc;879 880 /* Copy this page into the local address space */881 memset(copy, '\0', sizeof(copy));882 copy[0].from_rid = vf->pci_rid;883 copy[0].from_addr = vf->req_addr;884 copy[0].to_rid = efx->pci_dev->devfn;885 copy[0].to_addr = vf->buf.dma_addr;886 copy[0].length = EFX_PAGE_SIZE;887 rc = efx_siena_sriov_memcpy(efx, copy, 1);888 if (rc) {889 /* If we can't get the request, we can't reply to the caller */890 if (net_ratelimit())891 netif_err(efx, hw, efx->net_dev,892 "ERROR: Unable to fetch VFDI request from %s rc %d\n",893 vf->pci_name, -rc);894 vf->busy = false;895 return;896 }897 898 if (req->op < VFDI_OP_LIMIT && vfdi_ops[req->op] != NULL) {899 rc = vfdi_ops[req->op](vf);900 if (rc == 0) {901 netif_dbg(efx, hw, efx->net_dev,902 "vfdi request %d from %s ok\n",903 req->op, vf->pci_name);904 }905 } else {906 netif_dbg(efx, hw, efx->net_dev,907 "ERROR: Unrecognised request %d from VF %s addr "908 "%llx\n", req->op, vf->pci_name,909 (unsigned long long)vf->req_addr);910 rc = VFDI_RC_EOPNOTSUPP;911 }912 913 /* Allow subsequent VF requests */914 vf->busy = false;915 smp_wmb();916 917 /* Respond to the request */918 req->rc = rc;919 req->op = VFDI_OP_RESPONSE;920 921 memset(copy, '\0', sizeof(copy));922 copy[0].from_buf = &req->rc;923 copy[0].to_rid = vf->pci_rid;924 copy[0].to_addr = vf->req_addr + offsetof(struct vfdi_req, rc);925 copy[0].length = sizeof(req->rc);926 copy[1].from_buf = &req->op;927 copy[1].to_rid = vf->pci_rid;928 copy[1].to_addr = vf->req_addr + offsetof(struct vfdi_req, op);929 copy[1].length = sizeof(req->op);930 931 (void)efx_siena_sriov_memcpy(efx, copy, ARRAY_SIZE(copy));932}933 934 935 936/* After a reset the event queues inside the guests no longer exist. Fill the937 * event ring in guest memory with VFDI reset events, then (re-initialise) the938 * event queue to raise an interrupt. The guest driver will then recover.939 */940 941static void efx_siena_sriov_reset_vf(struct siena_vf *vf,942 struct efx_buffer *buffer)943{944 struct efx_nic *efx = vf->efx;945 struct efx_memcpy_req copy_req[4];946 efx_qword_t event;947 unsigned int pos, count, k, buftbl, abs_evq;948 efx_oword_t reg;949 efx_dword_t ptr;950 int rc;951 952 BUG_ON(buffer->len != EFX_PAGE_SIZE);953 954 if (!vf->evq0_count)955 return;956 BUG_ON(vf->evq0_count & (vf->evq0_count - 1));957 958 mutex_lock(&vf->status_lock);959 EFX_POPULATE_QWORD_3(event,960 FSF_AZ_EV_CODE, FSE_CZ_EV_CODE_USER_EV,961 VFDI_EV_SEQ, vf->msg_seqno,962 VFDI_EV_TYPE, VFDI_EV_TYPE_RESET);963 vf->msg_seqno++;964 for (pos = 0; pos < EFX_PAGE_SIZE; pos += sizeof(event))965 memcpy(buffer->addr + pos, &event, sizeof(event));966 967 for (pos = 0; pos < vf->evq0_count; pos += count) {968 count = min_t(unsigned, vf->evq0_count - pos,969 ARRAY_SIZE(copy_req));970 for (k = 0; k < count; k++) {971 copy_req[k].from_buf = NULL;972 copy_req[k].from_rid = efx->pci_dev->devfn;973 copy_req[k].from_addr = buffer->dma_addr;974 copy_req[k].to_rid = vf->pci_rid;975 copy_req[k].to_addr = vf->evq0_addrs[pos + k];976 copy_req[k].length = EFX_PAGE_SIZE;977 }978 rc = efx_siena_sriov_memcpy(efx, copy_req, count);979 if (rc) {980 if (net_ratelimit())981 netif_err(efx, hw, efx->net_dev,982 "ERROR: Unable to notify %s of reset"983 ": %d\n", vf->pci_name, -rc);984 break;985 }986 }987 988 /* Reinitialise, arm and trigger evq0 */989 abs_evq = abs_index(vf, 0);990 buftbl = EFX_BUFTBL_EVQ_BASE(vf, 0);991 efx_siena_sriov_bufs(efx, buftbl, vf->evq0_addrs, vf->evq0_count);992 993 EFX_POPULATE_OWORD_3(reg,994 FRF_CZ_TIMER_Q_EN, 1,995 FRF_CZ_HOST_NOTIFY_MODE, 0,996 FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);997 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, abs_evq);998 EFX_POPULATE_OWORD_3(reg,999 FRF_AZ_EVQ_EN, 1,1000 FRF_AZ_EVQ_SIZE, __ffs(vf->evq0_count),1001 FRF_AZ_EVQ_BUF_BASE_ID, buftbl);1002 efx_writeo_table(efx, ®, FR_BZ_EVQ_PTR_TBL, abs_evq);1003 EFX_POPULATE_DWORD_1(ptr, FRF_AZ_EVQ_RPTR, 0);1004 efx_writed(efx, &ptr, FR_BZ_EVQ_RPTR + FR_BZ_EVQ_RPTR_STEP * abs_evq);1005 1006 mutex_unlock(&vf->status_lock);1007}1008 1009static void efx_siena_sriov_reset_vf_work(struct work_struct *work)1010{1011 struct siena_vf *vf = container_of(work, struct siena_vf, req);1012 struct efx_nic *efx = vf->efx;1013 struct efx_buffer buf;1014 1015 if (!efx_siena_alloc_buffer(efx, &buf, EFX_PAGE_SIZE, GFP_NOIO)) {1016 efx_siena_sriov_reset_vf(vf, &buf);1017 efx_siena_free_buffer(efx, &buf);1018 }1019}1020 1021static void efx_siena_sriov_handle_no_channel(struct efx_nic *efx)1022{1023 netif_err(efx, drv, efx->net_dev,1024 "ERROR: IOV requires MSI-X and 1 additional interrupt"1025 "vector. IOV disabled\n");1026 efx->vf_count = 0;1027}1028 1029static int efx_siena_sriov_probe_channel(struct efx_channel *channel)1030{1031 struct siena_nic_data *nic_data = channel->efx->nic_data;1032 nic_data->vfdi_channel = channel;1033 1034 return 0;1035}1036 1037static void1038efx_siena_sriov_get_channel_name(struct efx_channel *channel,1039 char *buf, size_t len)1040{1041 snprintf(buf, len, "%s-iov", channel->efx->name);1042}1043 1044static const struct efx_channel_type efx_siena_sriov_channel_type = {1045 .handle_no_channel = efx_siena_sriov_handle_no_channel,1046 .pre_probe = efx_siena_sriov_probe_channel,1047 .post_remove = efx_siena_channel_dummy_op_void,1048 .get_name = efx_siena_sriov_get_channel_name,1049 /* no copy operation; channel must not be reallocated */1050 .keep_eventq = true,1051};1052 1053void efx_siena_sriov_probe(struct efx_nic *efx)1054{1055 unsigned count;1056 1057 if (!max_vfs)1058 return;1059 1060 if (efx_siena_sriov_cmd(efx, false, &efx->vi_scale, &count)) {1061 pci_info(efx->pci_dev, "no SR-IOV VFs probed\n");1062 return;1063 }1064 if (count > 0 && count > max_vfs)1065 count = max_vfs;1066 1067 /* efx_nic_dimension_resources() will reduce vf_count as appopriate */1068 efx->vf_count = count;1069 1070 efx->extra_channel_type[EFX_EXTRA_CHANNEL_IOV] = &efx_siena_sriov_channel_type;1071}1072 1073/* Copy the list of individual addresses into the vfdi_status.peers1074 * array and auxiliary pages, protected by %local_lock. Drop that lock1075 * and then broadcast the address list to every VF.1076 */1077static void efx_siena_sriov_peer_work(struct work_struct *data)1078{1079 struct siena_nic_data *nic_data = container_of(data,1080 struct siena_nic_data,1081 peer_work);1082 struct efx_nic *efx = nic_data->efx;1083 struct vfdi_status *vfdi_status = nic_data->vfdi_status.addr;1084 struct siena_vf *vf;1085 struct efx_local_addr *local_addr;1086 struct vfdi_endpoint *peer;1087 struct efx_endpoint_page *epp;1088 struct list_head pages;1089 unsigned int peer_space;1090 unsigned int peer_count;1091 unsigned int pos;1092 1093 mutex_lock(&nic_data->local_lock);1094 1095 /* Move the existing peer pages off %local_page_list */1096 INIT_LIST_HEAD(&pages);1097 list_splice_tail_init(&nic_data->local_page_list, &pages);1098 1099 /* Populate the VF addresses starting from entry 1 (entry 0 is1100 * the PF address)1101 */1102 peer = vfdi_status->peers + 1;1103 peer_space = ARRAY_SIZE(vfdi_status->peers) - 1;1104 peer_count = 1;1105 for (pos = 0; pos < efx->vf_count; ++pos) {1106 vf = nic_data->vf + pos;1107 1108 mutex_lock(&vf->status_lock);1109 if (vf->rx_filtering && !is_zero_ether_addr(vf->addr.mac_addr)) {1110 *peer++ = vf->addr;1111 ++peer_count;1112 --peer_space;1113 BUG_ON(peer_space == 0);1114 }1115 mutex_unlock(&vf->status_lock);1116 }1117 1118 /* Fill the remaining addresses */1119 list_for_each_entry(local_addr, &nic_data->local_addr_list, link) {1120 ether_addr_copy(peer->mac_addr, local_addr->addr);1121 peer->tci = 0;1122 ++peer;1123 ++peer_count;1124 if (--peer_space == 0) {1125 if (list_empty(&pages)) {1126 epp = kmalloc(sizeof(*epp), GFP_KERNEL);1127 if (!epp)1128 break;1129 epp->ptr = dma_alloc_coherent(1130 &efx->pci_dev->dev, EFX_PAGE_SIZE,1131 &epp->addr, GFP_KERNEL);1132 if (!epp->ptr) {1133 kfree(epp);1134 break;1135 }1136 } else {1137 epp = list_first_entry(1138 &pages, struct efx_endpoint_page, link);1139 list_del(&epp->link);1140 }1141 1142 list_add_tail(&epp->link, &nic_data->local_page_list);1143 peer = (struct vfdi_endpoint *)epp->ptr;1144 peer_space = EFX_PAGE_SIZE / sizeof(struct vfdi_endpoint);1145 }1146 }1147 vfdi_status->peer_count = peer_count;1148 mutex_unlock(&nic_data->local_lock);1149 1150 /* Free any now unused endpoint pages */1151 while (!list_empty(&pages)) {1152 epp = list_first_entry(1153 &pages, struct efx_endpoint_page, link);1154 list_del(&epp->link);1155 dma_free_coherent(&efx->pci_dev->dev, EFX_PAGE_SIZE,1156 epp->ptr, epp->addr);1157 kfree(epp);1158 }1159 1160 /* Finally, push the pages */1161 for (pos = 0; pos < efx->vf_count; ++pos) {1162 vf = nic_data->vf + pos;1163 1164 mutex_lock(&vf->status_lock);1165 if (vf->status_addr)1166 __efx_siena_sriov_push_vf_status(vf);1167 mutex_unlock(&vf->status_lock);1168 }1169}1170 1171static void efx_siena_sriov_free_local(struct efx_nic *efx)1172{1173 struct siena_nic_data *nic_data = efx->nic_data;1174 struct efx_local_addr *local_addr;1175 struct efx_endpoint_page *epp;1176 1177 while (!list_empty(&nic_data->local_addr_list)) {1178 local_addr = list_first_entry(&nic_data->local_addr_list,1179 struct efx_local_addr, link);1180 list_del(&local_addr->link);1181 kfree(local_addr);1182 }1183 1184 while (!list_empty(&nic_data->local_page_list)) {1185 epp = list_first_entry(&nic_data->local_page_list,1186 struct efx_endpoint_page, link);1187 list_del(&epp->link);1188 dma_free_coherent(&efx->pci_dev->dev, EFX_PAGE_SIZE,1189 epp->ptr, epp->addr);1190 kfree(epp);1191 }1192}1193 1194static int efx_siena_sriov_vf_alloc(struct efx_nic *efx)1195{1196 unsigned index;1197 struct siena_vf *vf;1198 struct siena_nic_data *nic_data = efx->nic_data;1199 1200 nic_data->vf = kcalloc(efx->vf_count, sizeof(*nic_data->vf),1201 GFP_KERNEL);1202 if (!nic_data->vf)1203 return -ENOMEM;1204 1205 for (index = 0; index < efx->vf_count; ++index) {1206 vf = nic_data->vf + index;1207 1208 vf->efx = efx;1209 vf->index = index;1210 vf->rx_filter_id = -1;1211 vf->tx_filter_mode = VF_TX_FILTER_AUTO;1212 vf->tx_filter_id = -1;1213 INIT_WORK(&vf->req, efx_siena_sriov_vfdi);1214 INIT_WORK(&vf->reset_work, efx_siena_sriov_reset_vf_work);1215 init_waitqueue_head(&vf->flush_waitq);1216 mutex_init(&vf->status_lock);1217 mutex_init(&vf->txq_lock);1218 }1219 1220 return 0;1221}1222 1223static void efx_siena_sriov_vfs_fini(struct efx_nic *efx)1224{1225 struct siena_nic_data *nic_data = efx->nic_data;1226 struct siena_vf *vf;1227 unsigned int pos;1228 1229 for (pos = 0; pos < efx->vf_count; ++pos) {1230 vf = nic_data->vf + pos;1231 1232 efx_siena_free_buffer(efx, &vf->buf);1233 kfree(vf->peer_page_addrs);1234 vf->peer_page_addrs = NULL;1235 vf->peer_page_count = 0;1236 1237 vf->evq0_count = 0;1238 }1239}1240 1241static int efx_siena_sriov_vfs_init(struct efx_nic *efx)1242{1243 struct pci_dev *pci_dev = efx->pci_dev;1244 struct siena_nic_data *nic_data = efx->nic_data;1245 unsigned index, devfn, sriov, buftbl_base;1246 u16 offset, stride;1247 struct siena_vf *vf;1248 int rc;1249 1250 sriov = pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV);1251 if (!sriov)1252 return -ENOENT;1253 1254 pci_read_config_word(pci_dev, sriov + PCI_SRIOV_VF_OFFSET, &offset);1255 pci_read_config_word(pci_dev, sriov + PCI_SRIOV_VF_STRIDE, &stride);1256 1257 buftbl_base = nic_data->vf_buftbl_base;1258 devfn = pci_dev->devfn + offset;1259 for (index = 0; index < efx->vf_count; ++index) {1260 vf = nic_data->vf + index;1261 1262 /* Reserve buffer entries */1263 vf->buftbl_base = buftbl_base;1264 buftbl_base += EFX_VF_BUFTBL_PER_VI * efx_vf_size(efx);1265 1266 vf->pci_rid = devfn;1267 snprintf(vf->pci_name, sizeof(vf->pci_name),1268 "%04x:%02x:%02x.%d",1269 pci_domain_nr(pci_dev->bus), pci_dev->bus->number,1270 PCI_SLOT(devfn), PCI_FUNC(devfn));1271 1272 rc = efx_siena_alloc_buffer(efx, &vf->buf, EFX_PAGE_SIZE,1273 GFP_KERNEL);1274 if (rc)1275 goto fail;1276 1277 devfn += stride;1278 }1279 1280 return 0;1281 1282fail:1283 efx_siena_sriov_vfs_fini(efx);1284 return rc;1285}1286 1287int efx_siena_sriov_init(struct efx_nic *efx)1288{1289 struct net_device *net_dev = efx->net_dev;1290 struct siena_nic_data *nic_data = efx->nic_data;1291 struct vfdi_status *vfdi_status;1292 int rc;1293 1294 /* Ensure there's room for vf_channel */1295 BUILD_BUG_ON(EFX_MAX_CHANNELS + 1 >= EFX_VI_BASE);1296 /* Ensure that VI_BASE is aligned on VI_SCALE */1297 BUILD_BUG_ON(EFX_VI_BASE & ((1 << EFX_VI_SCALE_MAX) - 1));1298 1299 if (efx->vf_count == 0)1300 return 0;1301 1302 rc = efx_siena_sriov_cmd(efx, true, NULL, NULL);1303 if (rc)1304 goto fail_cmd;1305 1306 rc = efx_siena_alloc_buffer(efx, &nic_data->vfdi_status,1307 sizeof(*vfdi_status), GFP_KERNEL);1308 if (rc)1309 goto fail_status;1310 vfdi_status = nic_data->vfdi_status.addr;1311 memset(vfdi_status, 0, sizeof(*vfdi_status));1312 vfdi_status->version = 1;1313 vfdi_status->length = sizeof(*vfdi_status);1314 vfdi_status->max_tx_channels = vf_max_tx_channels;1315 vfdi_status->vi_scale = efx->vi_scale;1316 vfdi_status->rss_rxq_count = efx->rss_spread;1317 vfdi_status->peer_count = 1 + efx->vf_count;1318 vfdi_status->timer_quantum_ns = efx->timer_quantum_ns;1319 1320 rc = efx_siena_sriov_vf_alloc(efx);1321 if (rc)1322 goto fail_alloc;1323 1324 mutex_init(&nic_data->local_lock);1325 INIT_WORK(&nic_data->peer_work, efx_siena_sriov_peer_work);1326 INIT_LIST_HEAD(&nic_data->local_addr_list);1327 INIT_LIST_HEAD(&nic_data->local_page_list);1328 1329 rc = efx_siena_sriov_vfs_init(efx);1330 if (rc)1331 goto fail_vfs;1332 1333 rtnl_lock();1334 ether_addr_copy(vfdi_status->peers[0].mac_addr, net_dev->dev_addr);1335 efx->vf_init_count = efx->vf_count;1336 rtnl_unlock();1337 1338 efx_siena_sriov_usrev(efx, true);1339 1340 /* At this point we must be ready to accept VFDI requests */1341 1342 rc = pci_enable_sriov(efx->pci_dev, efx->vf_count);1343 if (rc)1344 goto fail_pci;1345 1346 netif_info(efx, probe, net_dev,1347 "enabled SR-IOV for %d VFs, %d VI per VF\n",1348 efx->vf_count, efx_vf_size(efx));1349 return 0;1350 1351fail_pci:1352 efx_siena_sriov_usrev(efx, false);1353 rtnl_lock();1354 efx->vf_init_count = 0;1355 rtnl_unlock();1356 efx_siena_sriov_vfs_fini(efx);1357fail_vfs:1358 cancel_work_sync(&nic_data->peer_work);1359 efx_siena_sriov_free_local(efx);1360 kfree(nic_data->vf);1361fail_alloc:1362 efx_siena_free_buffer(efx, &nic_data->vfdi_status);1363fail_status:1364 efx_siena_sriov_cmd(efx, false, NULL, NULL);1365fail_cmd:1366 return rc;1367}1368 1369void efx_siena_sriov_fini(struct efx_nic *efx)1370{1371 struct siena_vf *vf;1372 unsigned int pos;1373 struct siena_nic_data *nic_data = efx->nic_data;1374 1375 if (efx->vf_init_count == 0)1376 return;1377 1378 /* Disable all interfaces to reconfiguration */1379 BUG_ON(nic_data->vfdi_channel->enabled);1380 efx_siena_sriov_usrev(efx, false);1381 rtnl_lock();1382 efx->vf_init_count = 0;1383 rtnl_unlock();1384 1385 /* Flush all reconfiguration work */1386 for (pos = 0; pos < efx->vf_count; ++pos) {1387 vf = nic_data->vf + pos;1388 cancel_work_sync(&vf->req);1389 cancel_work_sync(&vf->reset_work);1390 }1391 cancel_work_sync(&nic_data->peer_work);1392 1393 pci_disable_sriov(efx->pci_dev);1394 1395 /* Tear down back-end state */1396 efx_siena_sriov_vfs_fini(efx);1397 efx_siena_sriov_free_local(efx);1398 kfree(nic_data->vf);1399 efx_siena_free_buffer(efx, &nic_data->vfdi_status);1400 efx_siena_sriov_cmd(efx, false, NULL, NULL);1401}1402 1403void efx_siena_sriov_event(struct efx_channel *channel, efx_qword_t *event)1404{1405 struct efx_nic *efx = channel->efx;1406 struct siena_vf *vf;1407 unsigned qid, seq, type, data;1408 1409 qid = EFX_QWORD_FIELD(*event, FSF_CZ_USER_QID);1410 1411 /* USR_EV_REG_VALUE is dword0, so access the VFDI_EV fields directly */1412 BUILD_BUG_ON(FSF_CZ_USER_EV_REG_VALUE_LBN != 0);1413 seq = EFX_QWORD_FIELD(*event, VFDI_EV_SEQ);1414 type = EFX_QWORD_FIELD(*event, VFDI_EV_TYPE);1415 data = EFX_QWORD_FIELD(*event, VFDI_EV_DATA);1416 1417 netif_vdbg(efx, hw, efx->net_dev,1418 "USR_EV event from qid %d seq 0x%x type %d data 0x%x\n",1419 qid, seq, type, data);1420 1421 if (map_vi_index(efx, qid, &vf, NULL))1422 return;1423 if (vf->busy)1424 goto error;1425 1426 if (type == VFDI_EV_TYPE_REQ_WORD0) {1427 /* Resynchronise */1428 vf->req_type = VFDI_EV_TYPE_REQ_WORD0;1429 vf->req_seqno = seq + 1;1430 vf->req_addr = 0;1431 } else if (seq != (vf->req_seqno++ & 0xff) || type != vf->req_type)1432 goto error;1433 1434 switch (vf->req_type) {1435 case VFDI_EV_TYPE_REQ_WORD0:1436 case VFDI_EV_TYPE_REQ_WORD1:1437 case VFDI_EV_TYPE_REQ_WORD2:1438 vf->req_addr |= (u64)data << (vf->req_type << 4);1439 ++vf->req_type;1440 return;1441 1442 case VFDI_EV_TYPE_REQ_WORD3:1443 vf->req_addr |= (u64)data << 48;1444 vf->req_type = VFDI_EV_TYPE_REQ_WORD0;1445 vf->busy = true;1446 queue_work(vfdi_workqueue, &vf->req);1447 return;1448 }1449 1450error:1451 if (net_ratelimit())1452 netif_err(efx, hw, efx->net_dev,1453 "ERROR: Screaming VFDI request from %s\n",1454 vf->pci_name);1455 /* Reset the request and sequence number */1456 vf->req_type = VFDI_EV_TYPE_REQ_WORD0;1457 vf->req_seqno = seq + 1;1458}1459 1460void efx_siena_sriov_flr(struct efx_nic *efx, unsigned vf_i)1461{1462 struct siena_nic_data *nic_data = efx->nic_data;1463 struct siena_vf *vf;1464 1465 if (vf_i > efx->vf_init_count)1466 return;1467 vf = nic_data->vf + vf_i;1468 netif_info(efx, hw, efx->net_dev,1469 "FLR on VF %s\n", vf->pci_name);1470 1471 vf->status_addr = 0;1472 efx_vfdi_remove_all_filters(vf);1473 efx_vfdi_flush_clear(vf);1474 1475 vf->evq0_count = 0;1476}1477 1478int efx_siena_sriov_mac_address_changed(struct efx_nic *efx)1479{1480 struct siena_nic_data *nic_data = efx->nic_data;1481 struct vfdi_status *vfdi_status = nic_data->vfdi_status.addr;1482 1483 if (!efx->vf_init_count)1484 return 0;1485 ether_addr_copy(vfdi_status->peers[0].mac_addr,1486 efx->net_dev->dev_addr);1487 queue_work(vfdi_workqueue, &nic_data->peer_work);1488 1489 return 0;1490}1491 1492void efx_siena_sriov_tx_flush_done(struct efx_nic *efx, efx_qword_t *event)1493{1494 struct siena_vf *vf;1495 unsigned queue, qid;1496 1497 queue = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);1498 if (map_vi_index(efx, queue, &vf, &qid))1499 return;1500 /* Ignore flush completions triggered by an FLR */1501 if (!test_bit(qid, vf->txq_mask))1502 return;1503 1504 __clear_bit(qid, vf->txq_mask);1505 --vf->txq_count;1506 1507 if (efx_vfdi_flush_wake(vf))1508 wake_up(&vf->flush_waitq);1509}1510 1511void efx_siena_sriov_rx_flush_done(struct efx_nic *efx, efx_qword_t *event)1512{1513 struct siena_vf *vf;1514 unsigned ev_failed, queue, qid;1515 1516 queue = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);1517 ev_failed = EFX_QWORD_FIELD(*event,1518 FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);1519 if (map_vi_index(efx, queue, &vf, &qid))1520 return;1521 if (!test_bit(qid, vf->rxq_mask))1522 return;1523 1524 if (ev_failed) {1525 set_bit(qid, vf->rxq_retry_mask);1526 atomic_inc(&vf->rxq_retry_count);1527 } else {1528 __clear_bit(qid, vf->rxq_mask);1529 --vf->rxq_count;1530 }1531 if (efx_vfdi_flush_wake(vf))1532 wake_up(&vf->flush_waitq);1533}1534 1535/* Called from napi. Schedule the reset work item */1536void efx_siena_sriov_desc_fetch_err(struct efx_nic *efx, unsigned dmaq)1537{1538 struct siena_vf *vf;1539 unsigned int rel;1540 1541 if (map_vi_index(efx, dmaq, &vf, &rel))1542 return;1543 1544 if (net_ratelimit())1545 netif_err(efx, hw, efx->net_dev,1546 "VF %d DMA Q %d reports descriptor fetch error.\n",1547 vf->index, rel);1548 queue_work(vfdi_workqueue, &vf->reset_work);1549}1550 1551/* Reset all VFs */1552void efx_siena_sriov_reset(struct efx_nic *efx)1553{1554 struct siena_nic_data *nic_data = efx->nic_data;1555 unsigned int vf_i;1556 struct efx_buffer buf;1557 struct siena_vf *vf;1558 1559 ASSERT_RTNL();1560 1561 if (efx->vf_init_count == 0)1562 return;1563 1564 efx_siena_sriov_usrev(efx, true);1565 (void)efx_siena_sriov_cmd(efx, true, NULL, NULL);1566 1567 if (efx_siena_alloc_buffer(efx, &buf, EFX_PAGE_SIZE, GFP_NOIO))1568 return;1569 1570 for (vf_i = 0; vf_i < efx->vf_init_count; ++vf_i) {1571 vf = nic_data->vf + vf_i;1572 efx_siena_sriov_reset_vf(vf, &buf);1573 }1574 1575 efx_siena_free_buffer(efx, &buf);1576}1577 1578int efx_init_sriov(void)1579{1580 /* A single threaded workqueue is sufficient. efx_siena_sriov_vfdi() and1581 * efx_siena_sriov_peer_work() spend almost all their time sleeping for1582 * MCDI to complete anyway1583 */1584 vfdi_workqueue = create_singlethread_workqueue("sfc_vfdi");1585 if (!vfdi_workqueue)1586 return -ENOMEM;1587 return 0;1588}1589 1590void efx_fini_sriov(void)1591{1592 destroy_workqueue(vfdi_workqueue);1593}1594 1595int efx_siena_sriov_set_vf_mac(struct efx_nic *efx, int vf_i, const u8 *mac)1596{1597 struct siena_nic_data *nic_data = efx->nic_data;1598 struct siena_vf *vf;1599 1600 if (vf_i >= efx->vf_init_count)1601 return -EINVAL;1602 vf = nic_data->vf + vf_i;1603 1604 mutex_lock(&vf->status_lock);1605 ether_addr_copy(vf->addr.mac_addr, mac);1606 __efx_siena_sriov_update_vf_addr(vf);1607 mutex_unlock(&vf->status_lock);1608 1609 return 0;1610}1611 1612int efx_siena_sriov_set_vf_vlan(struct efx_nic *efx, int vf_i,1613 u16 vlan, u8 qos)1614{1615 struct siena_nic_data *nic_data = efx->nic_data;1616 struct siena_vf *vf;1617 u16 tci;1618 1619 if (vf_i >= efx->vf_init_count)1620 return -EINVAL;1621 vf = nic_data->vf + vf_i;1622 1623 mutex_lock(&vf->status_lock);1624 tci = (vlan & VLAN_VID_MASK) | ((qos & 0x7) << VLAN_PRIO_SHIFT);1625 vf->addr.tci = htons(tci);1626 __efx_siena_sriov_update_vf_addr(vf);1627 mutex_unlock(&vf->status_lock);1628 1629 return 0;1630}1631 1632int efx_siena_sriov_set_vf_spoofchk(struct efx_nic *efx, int vf_i,1633 bool spoofchk)1634{1635 struct siena_nic_data *nic_data = efx->nic_data;1636 struct siena_vf *vf;1637 int rc;1638 1639 if (vf_i >= efx->vf_init_count)1640 return -EINVAL;1641 vf = nic_data->vf + vf_i;1642 1643 mutex_lock(&vf->txq_lock);1644 if (vf->txq_count == 0) {1645 vf->tx_filter_mode =1646 spoofchk ? VF_TX_FILTER_ON : VF_TX_FILTER_OFF;1647 rc = 0;1648 } else {1649 /* This cannot be changed while TX queues are running */1650 rc = -EBUSY;1651 }1652 mutex_unlock(&vf->txq_lock);1653 return rc;1654}1655 1656int efx_siena_sriov_get_vf_config(struct efx_nic *efx, int vf_i,1657 struct ifla_vf_info *ivi)1658{1659 struct siena_nic_data *nic_data = efx->nic_data;1660 struct siena_vf *vf;1661 u16 tci;1662 1663 if (vf_i >= efx->vf_init_count)1664 return -EINVAL;1665 vf = nic_data->vf + vf_i;1666 1667 ivi->vf = vf_i;1668 ether_addr_copy(ivi->mac, vf->addr.mac_addr);1669 ivi->max_tx_rate = 0;1670 ivi->min_tx_rate = 0;1671 tci = ntohs(vf->addr.tci);1672 ivi->vlan = tci & VLAN_VID_MASK;1673 ivi->qos = (tci >> VLAN_PRIO_SHIFT) & 0x7;1674 ivi->spoofchk = vf->tx_filter_mode == VF_TX_FILTER_ON;1675 1676 return 0;1677}1678 1679bool efx_siena_sriov_wanted(struct efx_nic *efx)1680{1681 return efx->vf_count != 0;1682}1683 1684int efx_siena_sriov_configure(struct efx_nic *efx, int num_vfs)1685{1686 return 0;1687}1688