1460 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality4 * Between PCI RC and EP5 *6 * Copyright (C) 2020 Texas Instruments7 * Copyright (C) 2022 NXP8 *9 * Based on pci-epf-ntb.c10 * Author: Frank Li <Frank.Li@nxp.com>11 * Author: Kishon Vijay Abraham I <kishon@ti.com>12 */13 14/*15 * +------------+ +---------------------------------------+16 * | | | |17 * +------------+ | +--------------+18 * | NTB | | | NTB |19 * | NetDev | | | NetDev |20 * +------------+ | +--------------+21 * | NTB | | | NTB |22 * | Transfer | | | Transfer |23 * +------------+ | +--------------+24 * | | | | |25 * | PCI NTB | | | |26 * | EPF | | | |27 * | Driver | | | PCI Virtual |28 * | | +---------------+ | NTB Driver |29 * | | | PCI EP NTB |<------>| |30 * | | | FN Driver | | |31 * +------------+ +---------------+ +--------------+32 * | | | | | |33 * | PCI Bus | <-----> | PCI EP Bus | | Virtual PCI |34 * | | PCI | | | Bus |35 * +------------+ +---------------+--------+--------------+36 * PCIe Root Port PCI EP37 */38 39#include <linux/delay.h>40#include <linux/io.h>41#include <linux/module.h>42#include <linux/slab.h>43 44#include <linux/pci-epc.h>45#include <linux/pci-epf.h>46#include <linux/ntb.h>47 48static struct workqueue_struct *kpcintb_workqueue;49 50#define COMMAND_CONFIGURE_DOORBELL 151#define COMMAND_TEARDOWN_DOORBELL 252#define COMMAND_CONFIGURE_MW 353#define COMMAND_TEARDOWN_MW 454#define COMMAND_LINK_UP 555#define COMMAND_LINK_DOWN 656 57#define COMMAND_STATUS_OK 158#define COMMAND_STATUS_ERROR 259 60#define LINK_STATUS_UP BIT(0)61 62#define SPAD_COUNT 6463#define DB_COUNT 464#define NTB_MW_OFFSET 265#define DB_COUNT_MASK GENMASK(15, 0)66#define MSIX_ENABLE BIT(16)67#define MAX_DB_COUNT 3268#define MAX_MW 469 70enum epf_ntb_bar {71 BAR_CONFIG,72 BAR_DB,73 BAR_MW0,74 BAR_MW1,75 BAR_MW2,76};77 78/*79 * +--------------------------------------------------+ Base80 * | |81 * | |82 * | |83 * | Common Control Register |84 * | |85 * | |86 * | |87 * +-----------------------+--------------------------+ Base+spad_offset88 * | | |89 * | Peer Spad Space | Spad Space |90 * | | |91 * | | |92 * +-----------------------+--------------------------+ Base+spad_offset93 * | | | +spad_count * 494 * | | |95 * | Spad Space | Peer Spad Space |96 * | | |97 * +-----------------------+--------------------------+98 * Virtual PCI PCIe Endpoint99 * NTB Driver NTB Driver100 */101struct epf_ntb_ctrl {102 u32 command;103 u32 argument;104 u16 command_status;105 u16 link_status;106 u32 topology;107 u64 addr;108 u64 size;109 u32 num_mws;110 u32 reserved;111 u32 spad_offset;112 u32 spad_count;113 u32 db_entry_size;114 u32 db_data[MAX_DB_COUNT];115 u32 db_offset[MAX_DB_COUNT];116} __packed;117 118struct epf_ntb {119 struct ntb_dev ntb;120 struct pci_epf *epf;121 struct config_group group;122 123 u32 num_mws;124 u32 db_count;125 u32 spad_count;126 u64 mws_size[MAX_MW];127 u64 db;128 u32 vbus_number;129 u16 vntb_pid;130 u16 vntb_vid;131 132 bool linkup;133 u32 spad_size;134 135 enum pci_barno epf_ntb_bar[6];136 137 struct epf_ntb_ctrl *reg;138 139 u32 *epf_db;140 141 phys_addr_t vpci_mw_phy[MAX_MW];142 void __iomem *vpci_mw_addr[MAX_MW];143 144 struct delayed_work cmd_handler;145};146 147#define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)148#define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb)149 150static struct pci_epf_header epf_ntb_header = {151 .vendorid = PCI_ANY_ID,152 .deviceid = PCI_ANY_ID,153 .baseclass_code = PCI_BASE_CLASS_MEMORY,154 .interrupt_pin = PCI_INTERRUPT_INTA,155};156 157/**158 * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host (VHOST)159 * @ntb: NTB device that facilitates communication between HOST and VHOST160 * @link_up: true or false indicating Link is UP or Down161 *162 * Once NTB function in HOST invoke ntb_link_enable(),163 * this NTB function driver will trigger a link event to VHOST.164 *165 * Returns: Zero for success, or an error code in case of failure166 */167static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)168{169 if (link_up)170 ntb->reg->link_status |= LINK_STATUS_UP;171 else172 ntb->reg->link_status &= ~LINK_STATUS_UP;173 174 ntb_link_event(&ntb->ntb);175 return 0;176}177 178/**179 * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST180 * to access the memory window of HOST181 * @ntb: NTB device that facilitates communication between HOST and VHOST182 * @mw: Index of the memory window (either 0, 1, 2 or 3)183 *184 * EP Outbound Window185 * +--------+ +-----------+186 * | | | |187 * | | | |188 * | | | |189 * | | | |190 * | | +-----------+191 * | Virtual| | Memory Win|192 * | NTB | -----------> | |193 * | Driver | | |194 * | | +-----------+195 * | | | |196 * | | | |197 * +--------+ +-----------+198 * VHOST PCI EP199 *200 * Returns: Zero for success, or an error code in case of failure201 */202static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw)203{204 phys_addr_t phys_addr;205 u8 func_no, vfunc_no;206 u64 addr, size;207 int ret = 0;208 209 phys_addr = ntb->vpci_mw_phy[mw];210 addr = ntb->reg->addr;211 size = ntb->reg->size;212 213 func_no = ntb->epf->func_no;214 vfunc_no = ntb->epf->vfunc_no;215 216 ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size);217 if (ret)218 dev_err(&ntb->epf->epc->dev,219 "Failed to map memory window %d address\n", mw);220 return ret;221}222 223/**224 * epf_ntb_teardown_mw() - Teardown the configured OB ATU225 * @ntb: NTB device that facilitates communication between HOST and VHOST226 * @mw: Index of the memory window (either 0, 1, 2 or 3)227 *228 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using229 * pci_epc_unmap_addr()230 */231static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)232{233 pci_epc_unmap_addr(ntb->epf->epc,234 ntb->epf->func_no,235 ntb->epf->vfunc_no,236 ntb->vpci_mw_phy[mw]);237}238 239/**240 * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST241 * @work: work_struct for the epf_ntb_epc242 *243 * Workqueue function that gets invoked for the two epf_ntb_epc244 * periodically (once every 5ms) to see if it has received any commands245 * from NTB HOST. The HOST can send commands to configure doorbell or246 * configure memory window or to update link status.247 */248static void epf_ntb_cmd_handler(struct work_struct *work)249{250 struct epf_ntb_ctrl *ctrl;251 u32 command, argument;252 struct epf_ntb *ntb;253 struct device *dev;254 int ret;255 int i;256 257 ntb = container_of(work, struct epf_ntb, cmd_handler.work);258 259 for (i = 1; i < ntb->db_count; i++) {260 if (ntb->epf_db[i]) {261 ntb->db |= 1 << (i - 1);262 ntb_db_event(&ntb->ntb, i);263 ntb->epf_db[i] = 0;264 }265 }266 267 ctrl = ntb->reg;268 command = ctrl->command;269 if (!command)270 goto reset_handler;271 argument = ctrl->argument;272 273 ctrl->command = 0;274 ctrl->argument = 0;275 276 ctrl = ntb->reg;277 dev = &ntb->epf->dev;278 279 switch (command) {280 case COMMAND_CONFIGURE_DOORBELL:281 ctrl->command_status = COMMAND_STATUS_OK;282 break;283 case COMMAND_TEARDOWN_DOORBELL:284 ctrl->command_status = COMMAND_STATUS_OK;285 break;286 case COMMAND_CONFIGURE_MW:287 ret = epf_ntb_configure_mw(ntb, argument);288 if (ret < 0)289 ctrl->command_status = COMMAND_STATUS_ERROR;290 else291 ctrl->command_status = COMMAND_STATUS_OK;292 break;293 case COMMAND_TEARDOWN_MW:294 epf_ntb_teardown_mw(ntb, argument);295 ctrl->command_status = COMMAND_STATUS_OK;296 break;297 case COMMAND_LINK_UP:298 ntb->linkup = true;299 ret = epf_ntb_link_up(ntb, true);300 if (ret < 0)301 ctrl->command_status = COMMAND_STATUS_ERROR;302 else303 ctrl->command_status = COMMAND_STATUS_OK;304 goto reset_handler;305 case COMMAND_LINK_DOWN:306 ntb->linkup = false;307 ret = epf_ntb_link_up(ntb, false);308 if (ret < 0)309 ctrl->command_status = COMMAND_STATUS_ERROR;310 else311 ctrl->command_status = COMMAND_STATUS_OK;312 break;313 default:314 dev_err(dev, "UNKNOWN command: %d\n", command);315 break;316 }317 318reset_handler:319 queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,320 msecs_to_jiffies(5));321}322 323/**324 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR325 * @ntb: EPC associated with one of the HOST which holds peer's outbound326 * address.327 *328 * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and329 * self scratchpad region (removes inbound ATU configuration). While BAR0 is330 * the default self scratchpad BAR, an NTB could have other BARs for self331 * scratchpad (because of reserved BARs). This function can get the exact BAR332 * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].333 *334 * Please note the self scratchpad region and config region is combined to335 * a single region and mapped using the same BAR. Also note VHOST's peer336 * scratchpad is HOST's self scratchpad.337 *338 * Returns: void339 */340static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb)341{342 struct pci_epf_bar *epf_bar;343 enum pci_barno barno;344 345 barno = ntb->epf_ntb_bar[BAR_CONFIG];346 epf_bar = &ntb->epf->bar[barno];347 348 pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);349}350 351/**352 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR353 * @ntb: NTB device that facilitates communication between HOST and VHOST354 *355 * Map BAR0 of EP CONTROLLER which contains the VHOST's config and356 * self scratchpad region.357 *358 * Please note the self scratchpad region and config region is combined to359 * a single region and mapped using the same BAR.360 *361 * Returns: Zero for success, or an error code in case of failure362 */363static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb)364{365 struct pci_epf_bar *epf_bar;366 enum pci_barno barno;367 u8 func_no, vfunc_no;368 struct device *dev;369 int ret;370 371 dev = &ntb->epf->dev;372 func_no = ntb->epf->func_no;373 vfunc_no = ntb->epf->vfunc_no;374 barno = ntb->epf_ntb_bar[BAR_CONFIG];375 epf_bar = &ntb->epf->bar[barno];376 377 ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar);378 if (ret) {379 dev_err(dev, "inft: Config/Status/SPAD BAR set failed\n");380 return ret;381 }382 return 0;383}384 385/**386 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with387 * config + scratchpad region388 * @ntb: NTB device that facilitates communication between HOST and VHOST389 */390static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)391{392 enum pci_barno barno;393 394 barno = ntb->epf_ntb_bar[BAR_CONFIG];395 pci_epf_free_space(ntb->epf, ntb->reg, barno, 0);396}397 398/**399 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad400 * region401 * @ntb: NTB device that facilitates communication between HOST and VHOST402 *403 * Allocate the Local Memory mentioned in the above diagram. The size of404 * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION405 * is obtained from "spad-count" configfs entry.406 *407 * Returns: Zero for success, or an error code in case of failure408 */409static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)410{411 size_t align;412 enum pci_barno barno;413 struct epf_ntb_ctrl *ctrl;414 u32 spad_size, ctrl_size;415 u64 size;416 struct pci_epf *epf = ntb->epf;417 struct device *dev = &epf->dev;418 u32 spad_count;419 void *base;420 int i;421 const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc,422 epf->func_no,423 epf->vfunc_no);424 barno = ntb->epf_ntb_bar[BAR_CONFIG];425 size = epc_features->bar[barno].fixed_size;426 align = epc_features->align;427 428 if ((!IS_ALIGNED(size, align)))429 return -EINVAL;430 431 spad_count = ntb->spad_count;432 433 ctrl_size = sizeof(struct epf_ntb_ctrl);434 spad_size = 2 * spad_count * sizeof(u32);435 436 if (!align) {437 ctrl_size = roundup_pow_of_two(ctrl_size);438 spad_size = roundup_pow_of_two(spad_size);439 } else {440 ctrl_size = ALIGN(ctrl_size, align);441 spad_size = ALIGN(spad_size, align);442 }443 444 if (!size)445 size = ctrl_size + spad_size;446 else if (size < ctrl_size + spad_size)447 return -EINVAL;448 449 base = pci_epf_alloc_space(epf, size, barno, epc_features, 0);450 if (!base) {451 dev_err(dev, "Config/Status/SPAD alloc region fail\n");452 return -ENOMEM;453 }454 455 ntb->reg = base;456 457 ctrl = ntb->reg;458 ctrl->spad_offset = ctrl_size;459 460 ctrl->spad_count = spad_count;461 ctrl->num_mws = ntb->num_mws;462 ntb->spad_size = spad_size;463 464 ctrl->db_entry_size = sizeof(u32);465 466 for (i = 0; i < ntb->db_count; i++) {467 ntb->reg->db_data[i] = 1 + i;468 ntb->reg->db_offset[i] = 0;469 }470 471 return 0;472}473 474/**475 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability476 * @ntb: NTB device that facilitates communication between HOST and VHOST477 *478 * Configure MSI/MSI-X capability for each interface with number of479 * interrupts equal to "db_count" configfs entry.480 *481 * Returns: Zero for success, or an error code in case of failure482 */483static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)484{485 const struct pci_epc_features *epc_features;486 struct device *dev;487 u32 db_count;488 int ret;489 490 dev = &ntb->epf->dev;491 492 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);493 494 if (!(epc_features->msix_capable || epc_features->msi_capable)) {495 dev_err(dev, "MSI or MSI-X is required for doorbell\n");496 return -EINVAL;497 }498 499 db_count = ntb->db_count;500 if (db_count > MAX_DB_COUNT) {501 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);502 return -EINVAL;503 }504 505 ntb->db_count = db_count;506 507 if (epc_features->msi_capable) {508 ret = pci_epc_set_msi(ntb->epf->epc,509 ntb->epf->func_no,510 ntb->epf->vfunc_no,511 16);512 if (ret) {513 dev_err(dev, "MSI configuration failed\n");514 return ret;515 }516 }517 518 return 0;519}520 521/**522 * epf_ntb_db_bar_init() - Configure Doorbell window BARs523 * @ntb: NTB device that facilitates communication between HOST and VHOST524 *525 * Returns: Zero for success, or an error code in case of failure526 */527static int epf_ntb_db_bar_init(struct epf_ntb *ntb)528{529 const struct pci_epc_features *epc_features;530 struct device *dev = &ntb->epf->dev;531 int ret;532 struct pci_epf_bar *epf_bar;533 void __iomem *mw_addr;534 enum pci_barno barno;535 size_t size = sizeof(u32) * ntb->db_count;536 537 epc_features = pci_epc_get_features(ntb->epf->epc,538 ntb->epf->func_no,539 ntb->epf->vfunc_no);540 barno = ntb->epf_ntb_bar[BAR_DB];541 542 mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0);543 if (!mw_addr) {544 dev_err(dev, "Failed to allocate OB address\n");545 return -ENOMEM;546 }547 548 ntb->epf_db = mw_addr;549 550 epf_bar = &ntb->epf->bar[barno];551 552 ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);553 if (ret) {554 dev_err(dev, "Doorbell BAR set failed\n");555 goto err_alloc_peer_mem;556 }557 return ret;558 559err_alloc_peer_mem:560 pci_epf_free_space(ntb->epf, mw_addr, barno, 0);561 return -1;562}563 564static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws);565 566/**567 * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory568 * allocated in peer's outbound address space569 * @ntb: NTB device that facilitates communication between HOST and VHOST570 */571static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)572{573 enum pci_barno barno;574 575 barno = ntb->epf_ntb_bar[BAR_DB];576 pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);577 pci_epc_clear_bar(ntb->epf->epc,578 ntb->epf->func_no,579 ntb->epf->vfunc_no,580 &ntb->epf->bar[barno]);581}582 583/**584 * epf_ntb_mw_bar_init() - Configure Memory window BARs585 * @ntb: NTB device that facilitates communication between HOST and VHOST586 *587 * Returns: Zero for success, or an error code in case of failure588 */589static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)590{591 int ret = 0;592 int i;593 u64 size;594 enum pci_barno barno;595 struct device *dev = &ntb->epf->dev;596 597 for (i = 0; i < ntb->num_mws; i++) {598 size = ntb->mws_size[i];599 barno = ntb->epf_ntb_bar[BAR_MW0 + i];600 601 ntb->epf->bar[barno].barno = barno;602 ntb->epf->bar[barno].size = size;603 ntb->epf->bar[barno].addr = NULL;604 ntb->epf->bar[barno].phys_addr = 0;605 ntb->epf->bar[barno].flags |= upper_32_bits(size) ?606 PCI_BASE_ADDRESS_MEM_TYPE_64 :607 PCI_BASE_ADDRESS_MEM_TYPE_32;608 609 ret = pci_epc_set_bar(ntb->epf->epc,610 ntb->epf->func_no,611 ntb->epf->vfunc_no,612 &ntb->epf->bar[barno]);613 if (ret) {614 dev_err(dev, "MW set failed\n");615 goto err_alloc_mem;616 }617 618 /* Allocate EPC outbound memory windows to vpci vntb device */619 ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc,620 &ntb->vpci_mw_phy[i],621 size);622 if (!ntb->vpci_mw_addr[i]) {623 ret = -ENOMEM;624 dev_err(dev, "Failed to allocate source address\n");625 goto err_set_bar;626 }627 }628 629 return ret;630 631err_set_bar:632 pci_epc_clear_bar(ntb->epf->epc,633 ntb->epf->func_no,634 ntb->epf->vfunc_no,635 &ntb->epf->bar[barno]);636err_alloc_mem:637 epf_ntb_mw_bar_clear(ntb, i);638 return ret;639}640 641/**642 * epf_ntb_mw_bar_clear() - Clear Memory window BARs643 * @ntb: NTB device that facilitates communication between HOST and VHOST644 * @num_mws: the number of Memory window BARs that to be cleared645 */646static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)647{648 enum pci_barno barno;649 int i;650 651 for (i = 0; i < num_mws; i++) {652 barno = ntb->epf_ntb_bar[BAR_MW0 + i];653 pci_epc_clear_bar(ntb->epf->epc,654 ntb->epf->func_no,655 ntb->epf->vfunc_no,656 &ntb->epf->bar[barno]);657 658 pci_epc_mem_free_addr(ntb->epf->epc,659 ntb->vpci_mw_phy[i],660 ntb->vpci_mw_addr[i],661 ntb->mws_size[i]);662 }663}664 665/**666 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface667 * @ntb: NTB device that facilitates communication between HOST and VHOST668 *669 * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces670 */671static void epf_ntb_epc_destroy(struct epf_ntb *ntb)672{673 pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0);674 pci_epc_put(ntb->epf->epc);675}676 677/**678 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB679 * constructs (scratchpad region, doorbell, memorywindow)680 * @ntb: NTB device that facilitates communication between HOST and VHOST681 *682 * Returns: Zero for success, or an error code in case of failure683 */684static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)685{686 const struct pci_epc_features *epc_features;687 enum pci_barno barno;688 enum epf_ntb_bar bar;689 struct device *dev;690 u32 num_mws;691 int i;692 693 barno = BAR_0;694 num_mws = ntb->num_mws;695 dev = &ntb->epf->dev;696 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);697 698 /* These are required BARs which are mandatory for NTB functionality */699 for (bar = BAR_CONFIG; bar <= BAR_MW0; bar++, barno++) {700 barno = pci_epc_get_next_free_bar(epc_features, barno);701 if (barno < 0) {702 dev_err(dev, "Fail to get NTB function BAR\n");703 return barno;704 }705 ntb->epf_ntb_bar[bar] = barno;706 }707 708 /* These are optional BARs which don't impact NTB functionality */709 for (bar = BAR_MW1, i = 1; i < num_mws; bar++, barno++, i++) {710 barno = pci_epc_get_next_free_bar(epc_features, barno);711 if (barno < 0) {712 ntb->num_mws = i;713 dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);714 }715 ntb->epf_ntb_bar[bar] = barno;716 }717 718 return 0;719}720 721/**722 * epf_ntb_epc_init() - Initialize NTB interface723 * @ntb: NTB device that facilitates communication between HOST and VHOST724 *725 * Wrapper to initialize a particular EPC interface and start the workqueue726 * to check for commands from HOST. This function will write to the727 * EP controller HW for configuring it.728 *729 * Returns: Zero for success, or an error code in case of failure730 */731static int epf_ntb_epc_init(struct epf_ntb *ntb)732{733 u8 func_no, vfunc_no;734 struct pci_epc *epc;735 struct pci_epf *epf;736 struct device *dev;737 int ret;738 739 epf = ntb->epf;740 dev = &epf->dev;741 epc = epf->epc;742 func_no = ntb->epf->func_no;743 vfunc_no = ntb->epf->vfunc_no;744 745 ret = epf_ntb_config_sspad_bar_set(ntb);746 if (ret) {747 dev_err(dev, "Config/self SPAD BAR init failed");748 return ret;749 }750 751 ret = epf_ntb_configure_interrupt(ntb);752 if (ret) {753 dev_err(dev, "Interrupt configuration failed\n");754 goto err_config_interrupt;755 }756 757 ret = epf_ntb_db_bar_init(ntb);758 if (ret) {759 dev_err(dev, "DB BAR init failed\n");760 goto err_db_bar_init;761 }762 763 ret = epf_ntb_mw_bar_init(ntb);764 if (ret) {765 dev_err(dev, "MW BAR init failed\n");766 goto err_mw_bar_init;767 }768 769 if (vfunc_no <= 1) {770 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);771 if (ret) {772 dev_err(dev, "Configuration header write failed\n");773 goto err_write_header;774 }775 }776 777 INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);778 queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);779 780 return 0;781 782err_write_header:783 epf_ntb_mw_bar_clear(ntb, ntb->num_mws);784err_mw_bar_init:785 epf_ntb_db_bar_clear(ntb);786err_db_bar_init:787err_config_interrupt:788 epf_ntb_config_sspad_bar_clear(ntb);789 790 return ret;791}792 793 794/**795 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces796 * @ntb: NTB device that facilitates communication between HOST and VHOST797 *798 * Wrapper to cleanup all NTB interfaces.799 */800static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)801{802 epf_ntb_mw_bar_clear(ntb, ntb->num_mws);803 epf_ntb_db_bar_clear(ntb);804 epf_ntb_config_sspad_bar_clear(ntb);805}806 807#define EPF_NTB_R(_name) \808static ssize_t epf_ntb_##_name##_show(struct config_item *item, \809 char *page) \810{ \811 struct config_group *group = to_config_group(item); \812 struct epf_ntb *ntb = to_epf_ntb(group); \813 \814 return sprintf(page, "%d\n", ntb->_name); \815}816 817#define EPF_NTB_W(_name) \818static ssize_t epf_ntb_##_name##_store(struct config_item *item, \819 const char *page, size_t len) \820{ \821 struct config_group *group = to_config_group(item); \822 struct epf_ntb *ntb = to_epf_ntb(group); \823 u32 val; \824 int ret; \825 \826 ret = kstrtou32(page, 0, &val); \827 if (ret) \828 return ret; \829 \830 ntb->_name = val; \831 \832 return len; \833}834 835#define EPF_NTB_MW_R(_name) \836static ssize_t epf_ntb_##_name##_show(struct config_item *item, \837 char *page) \838{ \839 struct config_group *group = to_config_group(item); \840 struct epf_ntb *ntb = to_epf_ntb(group); \841 struct device *dev = &ntb->epf->dev; \842 int win_no; \843 \844 if (sscanf(#_name, "mw%d", &win_no) != 1) \845 return -EINVAL; \846 \847 if (win_no <= 0 || win_no > ntb->num_mws) { \848 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \849 return -EINVAL; \850 } \851 \852 return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \853}854 855#define EPF_NTB_MW_W(_name) \856static ssize_t epf_ntb_##_name##_store(struct config_item *item, \857 const char *page, size_t len) \858{ \859 struct config_group *group = to_config_group(item); \860 struct epf_ntb *ntb = to_epf_ntb(group); \861 struct device *dev = &ntb->epf->dev; \862 int win_no; \863 u64 val; \864 int ret; \865 \866 ret = kstrtou64(page, 0, &val); \867 if (ret) \868 return ret; \869 \870 if (sscanf(#_name, "mw%d", &win_no) != 1) \871 return -EINVAL; \872 \873 if (win_no <= 0 || win_no > ntb->num_mws) { \874 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \875 return -EINVAL; \876 } \877 \878 ntb->mws_size[win_no - 1] = val; \879 \880 return len; \881}882 883static ssize_t epf_ntb_num_mws_store(struct config_item *item,884 const char *page, size_t len)885{886 struct config_group *group = to_config_group(item);887 struct epf_ntb *ntb = to_epf_ntb(group);888 u32 val;889 int ret;890 891 ret = kstrtou32(page, 0, &val);892 if (ret)893 return ret;894 895 if (val > MAX_MW)896 return -EINVAL;897 898 ntb->num_mws = val;899 900 return len;901}902 903EPF_NTB_R(spad_count)904EPF_NTB_W(spad_count)905EPF_NTB_R(db_count)906EPF_NTB_W(db_count)907EPF_NTB_R(num_mws)908EPF_NTB_R(vbus_number)909EPF_NTB_W(vbus_number)910EPF_NTB_R(vntb_pid)911EPF_NTB_W(vntb_pid)912EPF_NTB_R(vntb_vid)913EPF_NTB_W(vntb_vid)914EPF_NTB_MW_R(mw1)915EPF_NTB_MW_W(mw1)916EPF_NTB_MW_R(mw2)917EPF_NTB_MW_W(mw2)918EPF_NTB_MW_R(mw3)919EPF_NTB_MW_W(mw3)920EPF_NTB_MW_R(mw4)921EPF_NTB_MW_W(mw4)922 923CONFIGFS_ATTR(epf_ntb_, spad_count);924CONFIGFS_ATTR(epf_ntb_, db_count);925CONFIGFS_ATTR(epf_ntb_, num_mws);926CONFIGFS_ATTR(epf_ntb_, mw1);927CONFIGFS_ATTR(epf_ntb_, mw2);928CONFIGFS_ATTR(epf_ntb_, mw3);929CONFIGFS_ATTR(epf_ntb_, mw4);930CONFIGFS_ATTR(epf_ntb_, vbus_number);931CONFIGFS_ATTR(epf_ntb_, vntb_pid);932CONFIGFS_ATTR(epf_ntb_, vntb_vid);933 934static struct configfs_attribute *epf_ntb_attrs[] = {935 &epf_ntb_attr_spad_count,936 &epf_ntb_attr_db_count,937 &epf_ntb_attr_num_mws,938 &epf_ntb_attr_mw1,939 &epf_ntb_attr_mw2,940 &epf_ntb_attr_mw3,941 &epf_ntb_attr_mw4,942 &epf_ntb_attr_vbus_number,943 &epf_ntb_attr_vntb_pid,944 &epf_ntb_attr_vntb_vid,945 NULL,946};947 948static const struct config_item_type ntb_group_type = {949 .ct_attrs = epf_ntb_attrs,950 .ct_owner = THIS_MODULE,951};952 953/**954 * epf_ntb_add_cfs() - Add configfs directory specific to NTB955 * @epf: NTB endpoint function device956 * @group: A pointer to the config_group structure referencing a group of957 * config_items of a specific type that belong to a specific sub-system.958 *959 * Add configfs directory specific to NTB. This directory will hold960 * NTB specific properties like db_count, spad_count, num_mws etc.,961 *962 * Returns: Pointer to config_group963 */964static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,965 struct config_group *group)966{967 struct epf_ntb *ntb = epf_get_drvdata(epf);968 struct config_group *ntb_group = &ntb->group;969 struct device *dev = &epf->dev;970 971 config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);972 973 return ntb_group;974}975 976/*==== virtual PCI bus driver, which only load virtual NTB PCI driver ====*/977 978static u32 pci_space[] = {979 0xffffffff, /* Device ID, Vendor ID */980 0, /* Status, Command */981 0xffffffff, /* Base Class, Subclass, Prog Intf, Revision ID */982 0x40, /* BIST, Header Type, Latency Timer, Cache Line Size */983 0, /* BAR 0 */984 0, /* BAR 1 */985 0, /* BAR 2 */986 0, /* BAR 3 */987 0, /* BAR 4 */988 0, /* BAR 5 */989 0, /* Cardbus CIS Pointer */990 0, /* Subsystem ID, Subsystem Vendor ID */991 0, /* ROM Base Address */992 0, /* Reserved, Capabilities Pointer */993 0, /* Reserved */994 0, /* Max_Lat, Min_Gnt, Interrupt Pin, Interrupt Line */995};996 997static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val)998{999 if (devfn == 0) {1000 memcpy(val, ((u8 *)pci_space) + where, size);1001 return PCIBIOS_SUCCESSFUL;1002 }1003 return PCIBIOS_DEVICE_NOT_FOUND;1004}1005 1006static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val)1007{1008 return 0;1009}1010 1011static struct pci_ops vpci_ops = {1012 .read = pci_read,1013 .write = pci_write,1014};1015 1016static int vpci_scan_bus(void *sysdata)1017{1018 struct pci_bus *vpci_bus;1019 struct epf_ntb *ndev = sysdata;1020 1021 vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata);1022 if (!vpci_bus) {1023 pr_err("create pci bus failed\n");1024 return -EINVAL;1025 }1026 1027 pci_bus_add_devices(vpci_bus);1028 1029 return 0;1030}1031 1032/*==================== Virtual PCIe NTB driver ==========================*/1033 1034static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx)1035{1036 struct epf_ntb *ndev = ntb_ndev(ntb);1037 1038 return ndev->num_mws;1039}1040 1041static int vntb_epf_spad_count(struct ntb_dev *ntb)1042{1043 return ntb_ndev(ntb)->spad_count;1044}1045 1046static int vntb_epf_peer_mw_count(struct ntb_dev *ntb)1047{1048 return ntb_ndev(ntb)->num_mws;1049}1050 1051static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb)1052{1053 return BIT_ULL(ntb_ndev(ntb)->db_count) - 1;1054}1055 1056static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)1057{1058 return 0;1059}1060 1061static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,1062 dma_addr_t addr, resource_size_t size)1063{1064 struct epf_ntb *ntb = ntb_ndev(ndev);1065 struct pci_epf_bar *epf_bar;1066 enum pci_barno barno;1067 int ret;1068 struct device *dev;1069 1070 dev = &ntb->ntb.dev;1071 barno = ntb->epf_ntb_bar[BAR_MW0 + idx];1072 epf_bar = &ntb->epf->bar[barno];1073 epf_bar->phys_addr = addr;1074 epf_bar->barno = barno;1075 epf_bar->size = size;1076 1077 ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);1078 if (ret) {1079 dev_err(dev, "failure set mw trans\n");1080 return ret;1081 }1082 return 0;1083}1084 1085static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx)1086{1087 return 0;1088}1089 1090static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx,1091 phys_addr_t *base, resource_size_t *size)1092{1093 1094 struct epf_ntb *ntb = ntb_ndev(ndev);1095 1096 if (base)1097 *base = ntb->vpci_mw_phy[idx];1098 1099 if (size)1100 *size = ntb->mws_size[idx];1101 1102 return 0;1103}1104 1105static int vntb_epf_link_enable(struct ntb_dev *ntb,1106 enum ntb_speed max_speed,1107 enum ntb_width max_width)1108{1109 return 0;1110}1111 1112static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx)1113{1114 struct epf_ntb *ntb = ntb_ndev(ndev);1115 int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * sizeof(u32);1116 u32 val;1117 void __iomem *base = (void __iomem *)ntb->reg;1118 1119 val = readl(base + off + ct + idx * sizeof(u32));1120 return val;1121}1122 1123static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val)1124{1125 struct epf_ntb *ntb = ntb_ndev(ndev);1126 struct epf_ntb_ctrl *ctrl = ntb->reg;1127 int off = ctrl->spad_offset, ct = ctrl->spad_count * sizeof(u32);1128 void __iomem *base = (void __iomem *)ntb->reg;1129 1130 writel(val, base + off + ct + idx * sizeof(u32));1131 return 0;1132}1133 1134static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx)1135{1136 struct epf_ntb *ntb = ntb_ndev(ndev);1137 struct epf_ntb_ctrl *ctrl = ntb->reg;1138 int off = ctrl->spad_offset;1139 void __iomem *base = (void __iomem *)ntb->reg;1140 u32 val;1141 1142 val = readl(base + off + idx * sizeof(u32));1143 return val;1144}1145 1146static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val)1147{1148 struct epf_ntb *ntb = ntb_ndev(ndev);1149 struct epf_ntb_ctrl *ctrl = ntb->reg;1150 int off = ctrl->spad_offset;1151 void __iomem *base = (void __iomem *)ntb->reg;1152 1153 writel(val, base + off + idx * sizeof(u32));1154 return 0;1155}1156 1157static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits)1158{1159 u32 interrupt_num = ffs(db_bits) + 1;1160 struct epf_ntb *ntb = ntb_ndev(ndev);1161 u8 func_no, vfunc_no;1162 int ret;1163 1164 func_no = ntb->epf->func_no;1165 vfunc_no = ntb->epf->vfunc_no;1166 1167 ret = pci_epc_raise_irq(ntb->epf->epc, func_no, vfunc_no,1168 PCI_IRQ_MSI, interrupt_num + 1);1169 if (ret)1170 dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n");1171 1172 return ret;1173}1174 1175static u64 vntb_epf_db_read(struct ntb_dev *ndev)1176{1177 struct epf_ntb *ntb = ntb_ndev(ndev);1178 1179 return ntb->db;1180}1181 1182static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,1183 resource_size_t *addr_align,1184 resource_size_t *size_align,1185 resource_size_t *size_max)1186{1187 struct epf_ntb *ntb = ntb_ndev(ndev);1188 1189 if (addr_align)1190 *addr_align = SZ_4K;1191 1192 if (size_align)1193 *size_align = 1;1194 1195 if (size_max)1196 *size_max = ntb->mws_size[idx];1197 1198 return 0;1199}1200 1201static u64 vntb_epf_link_is_up(struct ntb_dev *ndev,1202 enum ntb_speed *speed,1203 enum ntb_width *width)1204{1205 struct epf_ntb *ntb = ntb_ndev(ndev);1206 1207 return ntb->reg->link_status;1208}1209 1210static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits)1211{1212 return 0;1213}1214 1215static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits)1216{1217 struct epf_ntb *ntb = ntb_ndev(ndev);1218 1219 ntb->db &= ~db_bits;1220 return 0;1221}1222 1223static int vntb_epf_link_disable(struct ntb_dev *ntb)1224{1225 return 0;1226}1227 1228static const struct ntb_dev_ops vntb_epf_ops = {1229 .mw_count = vntb_epf_mw_count,1230 .spad_count = vntb_epf_spad_count,1231 .peer_mw_count = vntb_epf_peer_mw_count,1232 .db_valid_mask = vntb_epf_db_valid_mask,1233 .db_set_mask = vntb_epf_db_set_mask,1234 .mw_set_trans = vntb_epf_mw_set_trans,1235 .mw_clear_trans = vntb_epf_mw_clear_trans,1236 .peer_mw_get_addr = vntb_epf_peer_mw_get_addr,1237 .link_enable = vntb_epf_link_enable,1238 .spad_read = vntb_epf_spad_read,1239 .spad_write = vntb_epf_spad_write,1240 .peer_spad_read = vntb_epf_peer_spad_read,1241 .peer_spad_write = vntb_epf_peer_spad_write,1242 .peer_db_set = vntb_epf_peer_db_set,1243 .db_read = vntb_epf_db_read,1244 .mw_get_align = vntb_epf_mw_get_align,1245 .link_is_up = vntb_epf_link_is_up,1246 .db_clear_mask = vntb_epf_db_clear_mask,1247 .db_clear = vntb_epf_db_clear,1248 .link_disable = vntb_epf_link_disable,1249};1250 1251static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id)1252{1253 int ret;1254 struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata;1255 struct device *dev = &pdev->dev;1256 1257 ndev->ntb.pdev = pdev;1258 ndev->ntb.topo = NTB_TOPO_NONE;1259 ndev->ntb.ops = &vntb_epf_ops;1260 1261 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));1262 if (ret) {1263 dev_err(dev, "Cannot set DMA mask\n");1264 return ret;1265 }1266 1267 ret = ntb_register_device(&ndev->ntb);1268 if (ret) {1269 dev_err(dev, "Failed to register NTB device\n");1270 return ret;1271 }1272 1273 dev_dbg(dev, "PCI Virtual NTB driver loaded\n");1274 return 0;1275}1276 1277static struct pci_device_id pci_vntb_table[] = {1278 {1279 PCI_DEVICE(0xffff, 0xffff),1280 },1281 {},1282};1283 1284static struct pci_driver vntb_pci_driver = {1285 .name = "pci-vntb",1286 .id_table = pci_vntb_table,1287 .probe = pci_vntb_probe,1288};1289 1290/* ============ PCIe EPF Driver Bind ====================*/1291 1292/**1293 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality1294 * @epf: NTB endpoint function device1295 *1296 * Initialize both the endpoint controllers associated with NTB function device.1297 * Invoked when a primary interface or secondary interface is bound to EPC1298 * device. This function will succeed only when EPC is bound to both the1299 * interfaces.1300 *1301 * Returns: Zero for success, or an error code in case of failure1302 */1303static int epf_ntb_bind(struct pci_epf *epf)1304{1305 struct epf_ntb *ntb = epf_get_drvdata(epf);1306 struct device *dev = &epf->dev;1307 int ret;1308 1309 if (!epf->epc) {1310 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");1311 return 0;1312 }1313 1314 ret = epf_ntb_init_epc_bar(ntb);1315 if (ret) {1316 dev_err(dev, "Failed to create NTB EPC\n");1317 goto err_bar_init;1318 }1319 1320 ret = epf_ntb_config_spad_bar_alloc(ntb);1321 if (ret) {1322 dev_err(dev, "Failed to allocate BAR memory\n");1323 goto err_bar_alloc;1324 }1325 1326 ret = epf_ntb_epc_init(ntb);1327 if (ret) {1328 dev_err(dev, "Failed to initialize EPC\n");1329 goto err_bar_alloc;1330 }1331 1332 epf_set_drvdata(epf, ntb);1333 1334 pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid;1335 pci_vntb_table[0].vendor = ntb->vntb_vid;1336 pci_vntb_table[0].device = ntb->vntb_pid;1337 1338 ret = pci_register_driver(&vntb_pci_driver);1339 if (ret) {1340 dev_err(dev, "failure register vntb pci driver\n");1341 goto err_epc_cleanup;1342 }1343 1344 ret = vpci_scan_bus(ntb);1345 if (ret)1346 goto err_unregister;1347 1348 return 0;1349 1350err_unregister:1351 pci_unregister_driver(&vntb_pci_driver);1352err_epc_cleanup:1353 epf_ntb_epc_cleanup(ntb);1354err_bar_alloc:1355 epf_ntb_config_spad_bar_free(ntb);1356 1357err_bar_init:1358 epf_ntb_epc_destroy(ntb);1359 1360 return ret;1361}1362 1363/**1364 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()1365 * @epf: NTB endpoint function device1366 *1367 * Cleanup the initialization from epf_ntb_bind()1368 */1369static void epf_ntb_unbind(struct pci_epf *epf)1370{1371 struct epf_ntb *ntb = epf_get_drvdata(epf);1372 1373 epf_ntb_epc_cleanup(ntb);1374 epf_ntb_config_spad_bar_free(ntb);1375 epf_ntb_epc_destroy(ntb);1376 1377 pci_unregister_driver(&vntb_pci_driver);1378}1379 1380// EPF driver probe1381static const struct pci_epf_ops epf_ntb_ops = {1382 .bind = epf_ntb_bind,1383 .unbind = epf_ntb_unbind,1384 .add_cfs = epf_ntb_add_cfs,1385};1386 1387/**1388 * epf_ntb_probe() - Probe NTB function driver1389 * @epf: NTB endpoint function device1390 * @id: NTB endpoint function device ID1391 *1392 * Probe NTB function driver when endpoint function bus detects a NTB1393 * endpoint function.1394 *1395 * Returns: Zero for success, or an error code in case of failure1396 */1397static int epf_ntb_probe(struct pci_epf *epf,1398 const struct pci_epf_device_id *id)1399{1400 struct epf_ntb *ntb;1401 struct device *dev;1402 1403 dev = &epf->dev;1404 1405 ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);1406 if (!ntb)1407 return -ENOMEM;1408 1409 epf->header = &epf_ntb_header;1410 ntb->epf = epf;1411 ntb->vbus_number = 0xff;1412 epf_set_drvdata(epf, ntb);1413 1414 dev_info(dev, "pci-ep epf driver loaded\n");1415 return 0;1416}1417 1418static const struct pci_epf_device_id epf_ntb_ids[] = {1419 {1420 .name = "pci_epf_vntb",1421 },1422 {},1423};1424 1425static struct pci_epf_driver epf_ntb_driver = {1426 .driver.name = "pci_epf_vntb",1427 .probe = epf_ntb_probe,1428 .id_table = epf_ntb_ids,1429 .ops = &epf_ntb_ops,1430 .owner = THIS_MODULE,1431};1432 1433static int __init epf_ntb_init(void)1434{1435 int ret;1436 1437 kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |1438 WQ_HIGHPRI, 0);1439 ret = pci_epf_register_driver(&epf_ntb_driver);1440 if (ret) {1441 destroy_workqueue(kpcintb_workqueue);1442 pr_err("Failed to register pci epf ntb driver --> %d\n", ret);1443 return ret;1444 }1445 1446 return 0;1447}1448module_init(epf_ntb_init);1449 1450static void __exit epf_ntb_exit(void)1451{1452 pci_epf_unregister_driver(&epf_ntb_driver);1453 destroy_workqueue(kpcintb_workqueue);1454}1455module_exit(epf_ntb_exit);1456 1457MODULE_DESCRIPTION("PCI EPF NTB DRIVER");1458MODULE_AUTHOR("Frank Li <Frank.li@nxp.com>");1459MODULE_LICENSE("GPL v2");1460