2150 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality4 *5 * Copyright (C) 2020 Texas Instruments6 * Author: Kishon Vijay Abraham I <kishon@ti.com>7 */8 9/*10 * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint11 * (EP) controller instances (see diagram below) in such a way that12 * transactions from one EP controller are routed to the other EP controller.13 * Once PCI NTB function driver configures the SoC with multiple EP instances,14 * HOST1 and HOST2 can communicate with each other using SoC as a bridge.15 *16 * +-------------+ +-------------+17 * | | | |18 * | HOST1 | | HOST2 |19 * | | | |20 * +------^------+ +------^------+21 * | |22 * | |23 * +---------|-------------------------------------------------|---------+24 * | +------v------+ +------v------+ |25 * | | | | | |26 * | | EP | | EP | |27 * | | CONTROLLER1 | | CONTROLLER2 | |28 * | | <-----------------------------------> | |29 * | | | | | |30 * | | | | | |31 * | | | SoC With Multiple EP Instances | | |32 * | | | (Configured using NTB Function) | | |33 * | +-------------+ +-------------+ |34 * +---------------------------------------------------------------------+35 */36 37#include <linux/delay.h>38#include <linux/io.h>39#include <linux/module.h>40#include <linux/slab.h>41 42#include <linux/pci-epc.h>43#include <linux/pci-epf.h>44 45static struct workqueue_struct *kpcintb_workqueue;46 47#define COMMAND_CONFIGURE_DOORBELL 148#define COMMAND_TEARDOWN_DOORBELL 249#define COMMAND_CONFIGURE_MW 350#define COMMAND_TEARDOWN_MW 451#define COMMAND_LINK_UP 552#define COMMAND_LINK_DOWN 653 54#define COMMAND_STATUS_OK 155#define COMMAND_STATUS_ERROR 256 57#define LINK_STATUS_UP BIT(0)58 59#define SPAD_COUNT 6460#define DB_COUNT 461#define NTB_MW_OFFSET 262#define DB_COUNT_MASK GENMASK(15, 0)63#define MSIX_ENABLE BIT(16)64#define MAX_DB_COUNT 3265#define MAX_MW 466 67enum epf_ntb_bar {68 BAR_CONFIG,69 BAR_PEER_SPAD,70 BAR_DB_MW1,71 BAR_MW2,72 BAR_MW3,73 BAR_MW4,74};75 76struct epf_ntb {77 u32 num_mws;78 u32 db_count;79 u32 spad_count;80 struct pci_epf *epf;81 u64 mws_size[MAX_MW];82 struct config_group group;83 struct epf_ntb_epc *epc[2];84};85 86#define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)87 88struct epf_ntb_epc {89 u8 func_no;90 u8 vfunc_no;91 bool linkup;92 bool is_msix;93 int msix_bar;94 u32 spad_size;95 struct pci_epc *epc;96 struct epf_ntb *epf_ntb;97 void __iomem *mw_addr[6];98 size_t msix_table_offset;99 struct epf_ntb_ctrl *reg;100 struct pci_epf_bar *epf_bar;101 enum pci_barno epf_ntb_bar[6];102 struct delayed_work cmd_handler;103 enum pci_epc_interface_type type;104 const struct pci_epc_features *epc_features;105};106 107struct epf_ntb_ctrl {108 u32 command;109 u32 argument;110 u16 command_status;111 u16 link_status;112 u32 topology;113 u64 addr;114 u64 size;115 u32 num_mws;116 u32 mw1_offset;117 u32 spad_offset;118 u32 spad_count;119 u32 db_entry_size;120 u32 db_data[MAX_DB_COUNT];121 u32 db_offset[MAX_DB_COUNT];122} __packed;123 124static struct pci_epf_header epf_ntb_header = {125 .vendorid = PCI_ANY_ID,126 .deviceid = PCI_ANY_ID,127 .baseclass_code = PCI_BASE_CLASS_MEMORY,128 .interrupt_pin = PCI_INTERRUPT_INTA,129};130 131/**132 * epf_ntb_link_up() - Raise link_up interrupt to both the hosts133 * @ntb: NTB device that facilitates communication between HOST1 and HOST2134 * @link_up: true or false indicating Link is UP or Down135 *136 * Once NTB function in HOST1 and the NTB function in HOST2 invoke137 * ntb_link_enable(), this NTB function driver will trigger a link event to138 * the NTB client in both the hosts.139 */140static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)141{142 enum pci_epc_interface_type type;143 struct epf_ntb_epc *ntb_epc;144 struct epf_ntb_ctrl *ctrl;145 unsigned int irq_type;146 struct pci_epc *epc;147 u8 func_no, vfunc_no;148 bool is_msix;149 int ret;150 151 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {152 ntb_epc = ntb->epc[type];153 epc = ntb_epc->epc;154 func_no = ntb_epc->func_no;155 vfunc_no = ntb_epc->vfunc_no;156 is_msix = ntb_epc->is_msix;157 ctrl = ntb_epc->reg;158 if (link_up)159 ctrl->link_status |= LINK_STATUS_UP;160 else161 ctrl->link_status &= ~LINK_STATUS_UP;162 irq_type = is_msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;163 ret = pci_epc_raise_irq(epc, func_no, vfunc_no, irq_type, 1);164 if (ret) {165 dev_err(&epc->dev,166 "%s intf: Failed to raise Link Up IRQ\n",167 pci_epc_interface_string(type));168 return ret;169 }170 }171 172 return 0;173}174 175/**176 * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host177 * to access the memory window of other host178 * @ntb: NTB device that facilitates communication between HOST1 and HOST2179 * @type: PRIMARY interface or SECONDARY interface180 * @mw: Index of the memory window (either 0, 1, 2 or 3)181 *182 * +-----------------+ +---->+----------------+-----------+-----------------+183 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |184 * +-----------------+ | +----------------+ +-----------------+185 * | BAR1 | | | Doorbell 2 +---------+ | |186 * +-----------------+----+ +----------------+ | | |187 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+188 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |189 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+190 * +-----------------+ | |----------------+ | | | |191 * | BAR4 | | | | | | +-----------------+192 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||193 * | BAR5 | | | | | | +-----------------+194 * +-----------------+ +---->-----------------+ | | | |195 * EP CONTROLLER 1 | | | | +-----------------+196 * | | | +---->+ MSI|X ADDRESS 4 |197 * +----------------+ | +-----------------+198 * (A) EP CONTROLLER 2 | | |199 * (OB SPACE) | | |200 * +-------> MW1 |201 * | |202 * | |203 * (B) +-----------------+204 * | |205 * | |206 * | |207 * | |208 * | |209 * +-----------------+210 * PCI Address Space211 * (Managed by HOST2)212 *213 * This function performs stage (B) in the above diagram (see MW1) i.e., map OB214 * address space of memory window to PCI address space.215 *216 * This operation requires 3 parameters217 * 1) Address in the outbound address space218 * 2) Address in the PCI Address space219 * 3) Size of the address region to be mapped220 *221 * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is222 * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3223 * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This224 * is populated in epf_ntb_alloc_peer_mem() in this driver.225 *226 * The address and size of the PCI address region that has to be mapped would227 * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is228 * connected to HOST2.229 *230 * Please note Memory window1 (MW1) and Doorbell registers together will be231 * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's232 * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1],233 * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2].234 */235static int epf_ntb_configure_mw(struct epf_ntb *ntb,236 enum pci_epc_interface_type type, u32 mw)237{238 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;239 struct pci_epf_bar *peer_epf_bar;240 enum pci_barno peer_barno;241 struct epf_ntb_ctrl *ctrl;242 phys_addr_t phys_addr;243 u8 func_no, vfunc_no;244 struct pci_epc *epc;245 u64 addr, size;246 int ret = 0;247 248 ntb_epc = ntb->epc[type];249 epc = ntb_epc->epc;250 251 peer_ntb_epc = ntb->epc[!type];252 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];253 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];254 255 phys_addr = peer_epf_bar->phys_addr;256 ctrl = ntb_epc->reg;257 addr = ctrl->addr;258 size = ctrl->size;259 if (mw + NTB_MW_OFFSET == BAR_DB_MW1)260 phys_addr += ctrl->mw1_offset;261 262 if (size > ntb->mws_size[mw]) {263 dev_err(&epc->dev,264 "%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n",265 pci_epc_interface_string(type), mw, size,266 ntb->mws_size[mw]);267 ret = -EINVAL;268 goto err_invalid_size;269 }270 271 func_no = ntb_epc->func_no;272 vfunc_no = ntb_epc->vfunc_no;273 274 ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, addr, size);275 if (ret)276 dev_err(&epc->dev,277 "%s intf: Failed to map memory window %d address\n",278 pci_epc_interface_string(type), mw);279 280err_invalid_size:281 282 return ret;283}284 285/**286 * epf_ntb_teardown_mw() - Teardown the configured OB ATU287 * @ntb: NTB device that facilitates communication between HOST1 and HOST2288 * @type: PRIMARY interface or SECONDARY interface289 * @mw: Index of the memory window (either 0, 1, 2 or 3)290 *291 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using292 * pci_epc_unmap_addr()293 */294static void epf_ntb_teardown_mw(struct epf_ntb *ntb,295 enum pci_epc_interface_type type, u32 mw)296{297 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;298 struct pci_epf_bar *peer_epf_bar;299 enum pci_barno peer_barno;300 struct epf_ntb_ctrl *ctrl;301 phys_addr_t phys_addr;302 u8 func_no, vfunc_no;303 struct pci_epc *epc;304 305 ntb_epc = ntb->epc[type];306 epc = ntb_epc->epc;307 308 peer_ntb_epc = ntb->epc[!type];309 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];310 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];311 312 phys_addr = peer_epf_bar->phys_addr;313 ctrl = ntb_epc->reg;314 if (mw + NTB_MW_OFFSET == BAR_DB_MW1)315 phys_addr += ctrl->mw1_offset;316 func_no = ntb_epc->func_no;317 vfunc_no = ntb_epc->vfunc_no;318 319 pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr);320}321 322/**323 * epf_ntb_configure_msi() - Map OB address space to MSI address324 * @ntb: NTB device that facilitates communication between HOST1 and HOST2325 * @type: PRIMARY interface or SECONDARY interface326 * @db_count: Number of doorbell interrupts to map327 *328 *+-----------------+ +----->+----------------+-----------+-----------------+329 *| BAR0 | | | Doorbell 1 +---+-------> MSI ADDRESS |330 *+-----------------+ | +----------------+ | +-----------------+331 *| BAR1 | | | Doorbell 2 +---+ | |332 *+-----------------+----+ +----------------+ | | |333 *| BAR2 | | Doorbell 3 +---+ | |334 *+-----------------+----+ +----------------+ | | |335 *| BAR3 | | | Doorbell 4 +---+ | |336 *+-----------------+ | |----------------+ | |337 *| BAR4 | | | | | |338 *+-----------------+ | | MW1 | | |339 *| BAR5 | | | | | |340 *+-----------------+ +----->-----------------+ | |341 * EP CONTROLLER 1 | | | |342 * | | | |343 * +----------------+ +-----------------+344 * (A) EP CONTROLLER 2 | |345 * (OB SPACE) | |346 * | MW1 |347 * | |348 * | |349 * (B) +-----------------+350 * | |351 * | |352 * | |353 * | |354 * | |355 * +-----------------+356 * PCI Address Space357 * (Managed by HOST2)358 *359 *360 * This function performs stage (B) in the above diagram (see Doorbell 1,361 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to362 * doorbell to MSI address in PCI address space.363 *364 * This operation requires 3 parameters365 * 1) Address reserved for doorbell in the outbound address space366 * 2) MSI-X address in the PCIe Address space367 * 3) Number of MSI-X interrupts that has to be configured368 *369 * The address in the outbound address space (for the Doorbell) is stored in370 * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to371 * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along372 * with address for MW1.373 *374 * pci_epc_map_msi_irq() takes the MSI address from MSI capability register375 * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI376 * address.377 *378 * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt379 * in db_data of the peer's control region. This helps the peer to raise380 * doorbell of the other host by writing db_data to the BAR corresponding to381 * BAR_DB_MW1.382 */383static int epf_ntb_configure_msi(struct epf_ntb *ntb,384 enum pci_epc_interface_type type, u16 db_count)385{386 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;387 u32 db_entry_size, db_data, db_offset;388 struct pci_epf_bar *peer_epf_bar;389 struct epf_ntb_ctrl *peer_ctrl;390 enum pci_barno peer_barno;391 phys_addr_t phys_addr;392 u8 func_no, vfunc_no;393 struct pci_epc *epc;394 int ret, i;395 396 ntb_epc = ntb->epc[type];397 epc = ntb_epc->epc;398 399 peer_ntb_epc = ntb->epc[!type];400 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];401 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];402 peer_ctrl = peer_ntb_epc->reg;403 db_entry_size = peer_ctrl->db_entry_size;404 405 phys_addr = peer_epf_bar->phys_addr;406 func_no = ntb_epc->func_no;407 vfunc_no = ntb_epc->vfunc_no;408 409 ret = pci_epc_map_msi_irq(epc, func_no, vfunc_no, phys_addr, db_count,410 db_entry_size, &db_data, &db_offset);411 if (ret) {412 dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n",413 pci_epc_interface_string(type));414 return ret;415 }416 417 for (i = 0; i < db_count; i++) {418 peer_ctrl->db_data[i] = db_data | i;419 peer_ctrl->db_offset[i] = db_offset;420 }421 422 return 0;423}424 425/**426 * epf_ntb_configure_msix() - Map OB address space to MSI-X address427 * @ntb: NTB device that facilitates communication between HOST1 and HOST2428 * @type: PRIMARY interface or SECONDARY interface429 * @db_count: Number of doorbell interrupts to map430 *431 *+-----------------+ +----->+----------------+-----------+-----------------+432 *| BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 |433 *+-----------------+ | +----------------+ +-----------------+434 *| BAR1 | | | Doorbell 2 +---------+ | |435 *+-----------------+----+ +----------------+ | | |436 *| BAR2 | | Doorbell 3 +-------+ | +-----------------+437 *+-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 |438 *| BAR3 | | | Doorbell 4 +-----+ | +-----------------+439 *+-----------------+ | |----------------+ | | | |440 *| BAR4 | | | | | | +-----------------+441 *+-----------------+ | | MW1 + | +-->+ MSI-X ADDRESS 3||442 *| BAR5 | | | | | +-----------------+443 *+-----------------+ +----->-----------------+ | | |444 * EP CONTROLLER 1 | | | +-----------------+445 * | | +---->+ MSI-X ADDRESS 4 |446 * +----------------+ +-----------------+447 * (A) EP CONTROLLER 2 | |448 * (OB SPACE) | |449 * | MW1 |450 * | |451 * | |452 * (B) +-----------------+453 * | |454 * | |455 * | |456 * | |457 * | |458 * +-----------------+459 * PCI Address Space460 * (Managed by HOST2)461 *462 * This function performs stage (B) in the above diagram (see Doorbell 1,463 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to464 * doorbell to MSI-X address in PCI address space.465 *466 * This operation requires 3 parameters467 * 1) Address reserved for doorbell in the outbound address space468 * 2) MSI-X address in the PCIe Address space469 * 3) Number of MSI-X interrupts that has to be configured470 *471 * The address in the outbound address space (for the Doorbell) is stored in472 * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to473 * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along474 * with address for MW1.475 *476 * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and477 * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected478 * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the479 * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix()480 * gets the MSI-X address and data.481 *482 * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt483 * in db_data of the peer's control region. This helps the peer to raise484 * doorbell of the other host by writing db_data to the BAR corresponding to485 * BAR_DB_MW1.486 */487static int epf_ntb_configure_msix(struct epf_ntb *ntb,488 enum pci_epc_interface_type type,489 u16 db_count)490{491 const struct pci_epc_features *epc_features;492 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;493 struct pci_epf_bar *peer_epf_bar, *epf_bar;494 struct pci_epf_msix_tbl *msix_tbl;495 struct epf_ntb_ctrl *peer_ctrl;496 u32 db_entry_size, msg_data;497 enum pci_barno peer_barno;498 phys_addr_t phys_addr;499 u8 func_no, vfunc_no;500 struct pci_epc *epc;501 size_t align;502 u64 msg_addr;503 int ret, i;504 505 ntb_epc = ntb->epc[type];506 epc = ntb_epc->epc;507 508 epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar];509 msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset;510 511 peer_ntb_epc = ntb->epc[!type];512 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];513 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];514 phys_addr = peer_epf_bar->phys_addr;515 peer_ctrl = peer_ntb_epc->reg;516 epc_features = ntb_epc->epc_features;517 align = epc_features->align;518 519 func_no = ntb_epc->func_no;520 vfunc_no = ntb_epc->vfunc_no;521 db_entry_size = peer_ctrl->db_entry_size;522 523 for (i = 0; i < db_count; i++) {524 msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align);525 msg_data = msix_tbl[i].msg_data;526 ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, msg_addr,527 db_entry_size);528 if (ret) {529 dev_err(&epc->dev,530 "%s intf: Failed to configure MSI-X IRQ\n",531 pci_epc_interface_string(type));532 return ret;533 }534 phys_addr = phys_addr + db_entry_size;535 peer_ctrl->db_data[i] = msg_data;536 peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1);537 }538 ntb_epc->is_msix = true;539 540 return 0;541}542 543/**544 * epf_ntb_configure_db() - Configure the Outbound Address Space for one host545 * to ring the doorbell of other host546 * @ntb: NTB device that facilitates communication between HOST1 and HOST2547 * @type: PRIMARY interface or SECONDARY interface548 * @db_count: Count of the number of doorbells that has to be configured549 * @msix: Indicates whether MSI-X or MSI should be used550 *551 * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for552 * one HOST to ring the doorbell of other HOST.553 */554static int epf_ntb_configure_db(struct epf_ntb *ntb,555 enum pci_epc_interface_type type,556 u16 db_count, bool msix)557{558 struct epf_ntb_epc *ntb_epc;559 struct pci_epc *epc;560 int ret;561 562 if (db_count > MAX_DB_COUNT)563 return -EINVAL;564 565 ntb_epc = ntb->epc[type];566 epc = ntb_epc->epc;567 568 if (msix)569 ret = epf_ntb_configure_msix(ntb, type, db_count);570 else571 ret = epf_ntb_configure_msi(ntb, type, db_count);572 573 if (ret)574 dev_err(&epc->dev, "%s intf: Failed to configure DB\n",575 pci_epc_interface_string(type));576 577 return ret;578}579 580/**581 * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X582 * address583 * @ntb: NTB device that facilitates communication between HOST1 and HOST2584 * @type: PRIMARY interface or SECONDARY interface585 *586 * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address.587 */588static void589epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type)590{591 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;592 struct pci_epf_bar *peer_epf_bar;593 enum pci_barno peer_barno;594 phys_addr_t phys_addr;595 u8 func_no, vfunc_no;596 struct pci_epc *epc;597 598 ntb_epc = ntb->epc[type];599 epc = ntb_epc->epc;600 601 peer_ntb_epc = ntb->epc[!type];602 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];603 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];604 phys_addr = peer_epf_bar->phys_addr;605 func_no = ntb_epc->func_no;606 vfunc_no = ntb_epc->vfunc_no;607 608 pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr);609}610 611/**612 * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host613 * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY)614 *615 * Workqueue function that gets invoked for the two epf_ntb_epc616 * periodically (once every 5ms) to see if it has received any commands617 * from NTB host. The host can send commands to configure doorbell or618 * configure memory window or to update link status.619 */620static void epf_ntb_cmd_handler(struct work_struct *work)621{622 enum pci_epc_interface_type type;623 struct epf_ntb_epc *ntb_epc;624 struct epf_ntb_ctrl *ctrl;625 u32 command, argument;626 struct epf_ntb *ntb;627 struct device *dev;628 u16 db_count;629 bool is_msix;630 int ret;631 632 ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work);633 ctrl = ntb_epc->reg;634 command = ctrl->command;635 if (!command)636 goto reset_handler;637 argument = ctrl->argument;638 639 ctrl->command = 0;640 ctrl->argument = 0;641 642 ctrl = ntb_epc->reg;643 type = ntb_epc->type;644 ntb = ntb_epc->epf_ntb;645 dev = &ntb->epf->dev;646 647 switch (command) {648 case COMMAND_CONFIGURE_DOORBELL:649 db_count = argument & DB_COUNT_MASK;650 is_msix = argument & MSIX_ENABLE;651 ret = epf_ntb_configure_db(ntb, type, db_count, is_msix);652 if (ret < 0)653 ctrl->command_status = COMMAND_STATUS_ERROR;654 else655 ctrl->command_status = COMMAND_STATUS_OK;656 break;657 case COMMAND_TEARDOWN_DOORBELL:658 epf_ntb_teardown_db(ntb, type);659 ctrl->command_status = COMMAND_STATUS_OK;660 break;661 case COMMAND_CONFIGURE_MW:662 ret = epf_ntb_configure_mw(ntb, type, argument);663 if (ret < 0)664 ctrl->command_status = COMMAND_STATUS_ERROR;665 else666 ctrl->command_status = COMMAND_STATUS_OK;667 break;668 case COMMAND_TEARDOWN_MW:669 epf_ntb_teardown_mw(ntb, type, argument);670 ctrl->command_status = COMMAND_STATUS_OK;671 break;672 case COMMAND_LINK_UP:673 ntb_epc->linkup = true;674 if (ntb->epc[PRIMARY_INTERFACE]->linkup &&675 ntb->epc[SECONDARY_INTERFACE]->linkup) {676 ret = epf_ntb_link_up(ntb, true);677 if (ret < 0)678 ctrl->command_status = COMMAND_STATUS_ERROR;679 else680 ctrl->command_status = COMMAND_STATUS_OK;681 goto reset_handler;682 }683 ctrl->command_status = COMMAND_STATUS_OK;684 break;685 case COMMAND_LINK_DOWN:686 ntb_epc->linkup = false;687 ret = epf_ntb_link_up(ntb, false);688 if (ret < 0)689 ctrl->command_status = COMMAND_STATUS_ERROR;690 else691 ctrl->command_status = COMMAND_STATUS_OK;692 break;693 default:694 dev_err(dev, "%s intf UNKNOWN command: %d\n",695 pci_epc_interface_string(type), command);696 break;697 }698 699reset_handler:700 queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler,701 msecs_to_jiffies(5));702}703 704/**705 * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR706 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound707 * address.708 *709 *+-----------------+------->+------------------+ +-----------------+710 *| BAR0 | | CONFIG REGION | | BAR0 |711 *+-----------------+----+ +------------------+<-------+-----------------+712 *| BAR1 | | |SCRATCHPAD REGION | | BAR1 |713 *+-----------------+ +-->+------------------+<-------+-----------------+714 *| BAR2 | Local Memory | BAR2 |715 *+-----------------+ +-----------------+716 *| BAR3 | | BAR3 |717 *+-----------------+ +-----------------+718 *| BAR4 | | BAR4 |719 *+-----------------+ +-----------------+720 *| BAR5 | | BAR5 |721 *+-----------------+ +-----------------+722 * EP CONTROLLER 1 EP CONTROLLER 2723 *724 * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad725 * region. While BAR1 is the default peer scratchpad BAR, an NTB could have726 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).727 * This function can get the exact BAR used for peer scratchpad from728 * epf_ntb_bar[BAR_PEER_SPAD].729 *730 * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function731 * gets the address of peer scratchpad from732 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].733 */734static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc)735{736 struct pci_epf_bar *epf_bar;737 enum pci_barno barno;738 u8 func_no, vfunc_no;739 struct pci_epc *epc;740 741 epc = ntb_epc->epc;742 func_no = ntb_epc->func_no;743 vfunc_no = ntb_epc->vfunc_no;744 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];745 epf_bar = &ntb_epc->epf_bar[barno];746 pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);747}748 749/**750 * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR751 * @ntb: NTB device that facilitates communication between HOST1 and HOST2752 * @type: PRIMARY interface or SECONDARY interface753 *754 *+-----------------+------->+------------------+ +-----------------+755 *| BAR0 | | CONFIG REGION | | BAR0 |756 *+-----------------+----+ +------------------+<-------+-----------------+757 *| BAR1 | | |SCRATCHPAD REGION | | BAR1 |758 *+-----------------+ +-->+------------------+<-------+-----------------+759 *| BAR2 | Local Memory | BAR2 |760 *+-----------------+ +-----------------+761 *| BAR3 | | BAR3 |762 *+-----------------+ +-----------------+763 *| BAR4 | | BAR4 |764 *+-----------------+ +-----------------+765 *| BAR5 | | BAR5 |766 *+-----------------+ +-----------------+767 * EP CONTROLLER 1 EP CONTROLLER 2768 *769 * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad770 * region. While BAR1 is the default peer scratchpad BAR, an NTB could have771 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).772 * This function can get the exact BAR used for peer scratchpad from773 * epf_ntb_bar[BAR_PEER_SPAD].774 *775 * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function776 * gets the address of peer scratchpad from777 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].778 */779static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb,780 enum pci_epc_interface_type type)781{782 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;783 struct pci_epf_bar *peer_epf_bar, *epf_bar;784 enum pci_barno peer_barno, barno;785 u32 peer_spad_offset;786 u8 func_no, vfunc_no;787 struct pci_epc *epc;788 struct device *dev;789 int ret;790 791 dev = &ntb->epf->dev;792 793 peer_ntb_epc = ntb->epc[!type];794 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG];795 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];796 797 ntb_epc = ntb->epc[type];798 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];799 epf_bar = &ntb_epc->epf_bar[barno];800 func_no = ntb_epc->func_no;801 vfunc_no = ntb_epc->vfunc_no;802 epc = ntb_epc->epc;803 804 peer_spad_offset = peer_ntb_epc->reg->spad_offset;805 epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset;806 epf_bar->size = peer_ntb_epc->spad_size;807 epf_bar->barno = barno;808 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;809 810 ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);811 if (ret) {812 dev_err(dev, "%s intf: peer SPAD BAR set failed\n",813 pci_epc_interface_string(type));814 return ret;815 }816 817 return 0;818}819 820/**821 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR822 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound823 * address.824 *825 * +-----------------+------->+------------------+ +-----------------+826 * | BAR0 | | CONFIG REGION | | BAR0 |827 * +-----------------+----+ +------------------+<-------+-----------------+828 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 |829 * +-----------------+ +-->+------------------+<-------+-----------------+830 * | BAR2 | Local Memory | BAR2 |831 * +-----------------+ +-----------------+832 * | BAR3 | | BAR3 |833 * +-----------------+ +-----------------+834 * | BAR4 | | BAR4 |835 * +-----------------+ +-----------------+836 * | BAR5 | | BAR5 |837 * +-----------------+ +-----------------+838 * EP CONTROLLER 1 EP CONTROLLER 2839 *840 * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and841 * self scratchpad region (removes inbound ATU configuration). While BAR0 is842 * the default self scratchpad BAR, an NTB could have other BARs for self843 * scratchpad (because of reserved BARs). This function can get the exact BAR844 * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].845 *846 * Please note the self scratchpad region and config region is combined to847 * a single region and mapped using the same BAR. Also note HOST2's peer848 * scratchpad is HOST1's self scratchpad.849 */850static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc)851{852 struct pci_epf_bar *epf_bar;853 enum pci_barno barno;854 u8 func_no, vfunc_no;855 struct pci_epc *epc;856 857 epc = ntb_epc->epc;858 func_no = ntb_epc->func_no;859 vfunc_no = ntb_epc->vfunc_no;860 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];861 epf_bar = &ntb_epc->epf_bar[barno];862 pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);863}864 865/**866 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR867 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound868 * address.869 *870 * +-----------------+------->+------------------+ +-----------------+871 * | BAR0 | | CONFIG REGION | | BAR0 |872 * +-----------------+----+ +------------------+<-------+-----------------+873 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 |874 * +-----------------+ +-->+------------------+<-------+-----------------+875 * | BAR2 | Local Memory | BAR2 |876 * +-----------------+ +-----------------+877 * | BAR3 | | BAR3 |878 * +-----------------+ +-----------------+879 * | BAR4 | | BAR4 |880 * +-----------------+ +-----------------+881 * | BAR5 | | BAR5 |882 * +-----------------+ +-----------------+883 * EP CONTROLLER 1 EP CONTROLLER 2884 *885 * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and886 * self scratchpad region. While BAR0 is the default self scratchpad BAR, an887 * NTB could have other BARs for self scratchpad (because of reserved BARs).888 * This function can get the exact BAR used for self scratchpad from889 * epf_ntb_bar[BAR_CONFIG].890 *891 * Please note the self scratchpad region and config region is combined to892 * a single region and mapped using the same BAR. Also note HOST2's peer893 * scratchpad is HOST1's self scratchpad.894 */895static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc)896{897 struct pci_epf_bar *epf_bar;898 enum pci_barno barno;899 u8 func_no, vfunc_no;900 struct epf_ntb *ntb;901 struct pci_epc *epc;902 struct device *dev;903 int ret;904 905 ntb = ntb_epc->epf_ntb;906 dev = &ntb->epf->dev;907 908 epc = ntb_epc->epc;909 func_no = ntb_epc->func_no;910 vfunc_no = ntb_epc->vfunc_no;911 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];912 epf_bar = &ntb_epc->epf_bar[barno];913 914 ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);915 if (ret) {916 dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n",917 pci_epc_interface_string(ntb_epc->type));918 return ret;919 }920 921 return 0;922}923 924/**925 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with926 * config + scratchpad region927 * @ntb: NTB device that facilitates communication between HOST1 and HOST2928 *929 * +-----------------+------->+------------------+ +-----------------+930 * | BAR0 | | CONFIG REGION | | BAR0 |931 * +-----------------+----+ +------------------+<-------+-----------------+932 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 |933 * +-----------------+ +-->+------------------+<-------+-----------------+934 * | BAR2 | Local Memory | BAR2 |935 * +-----------------+ +-----------------+936 * | BAR3 | | BAR3 |937 * +-----------------+ +-----------------+938 * | BAR4 | | BAR4 |939 * +-----------------+ +-----------------+940 * | BAR5 | | BAR5 |941 * +-----------------+ +-----------------+942 * EP CONTROLLER 1 EP CONTROLLER 2943 *944 * Free the Local Memory mentioned in the above diagram. After invoking this945 * function, any of config + self scratchpad region of HOST1 or peer scratchpad946 * region of HOST2 should not be accessed.947 */948static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)949{950 enum pci_epc_interface_type type;951 struct epf_ntb_epc *ntb_epc;952 enum pci_barno barno;953 struct pci_epf *epf;954 955 epf = ntb->epf;956 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {957 ntb_epc = ntb->epc[type];958 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];959 if (ntb_epc->reg)960 pci_epf_free_space(epf, ntb_epc->reg, barno, type);961 }962}963 964/**965 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad966 * region967 * @ntb: NTB device that facilitates communication between HOST1 and HOST2968 * @type: PRIMARY interface or SECONDARY interface969 *970 * +-----------------+------->+------------------+ +-----------------+971 * | BAR0 | | CONFIG REGION | | BAR0 |972 * +-----------------+----+ +------------------+<-------+-----------------+973 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 |974 * +-----------------+ +-->+------------------+<-------+-----------------+975 * | BAR2 | Local Memory | BAR2 |976 * +-----------------+ +-----------------+977 * | BAR3 | | BAR3 |978 * +-----------------+ +-----------------+979 * | BAR4 | | BAR4 |980 * +-----------------+ +-----------------+981 * | BAR5 | | BAR5 |982 * +-----------------+ +-----------------+983 * EP CONTROLLER 1 EP CONTROLLER 2984 *985 * Allocate the Local Memory mentioned in the above diagram. The size of986 * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION987 * is obtained from "spad-count" configfs entry.988 *989 * The size of both config region and scratchpad region has to be aligned,990 * since the scratchpad region will also be mapped as PEER SCRATCHPAD of991 * other host using a separate BAR.992 */993static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb,994 enum pci_epc_interface_type type)995{996 const struct pci_epc_features *peer_epc_features, *epc_features;997 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;998 size_t msix_table_size, pba_size, align;999 enum pci_barno peer_barno, barno;1000 struct epf_ntb_ctrl *ctrl;1001 u32 spad_size, ctrl_size;1002 u64 size, peer_size;1003 struct pci_epf *epf;1004 struct device *dev;1005 bool msix_capable;1006 u32 spad_count;1007 void *base;1008 1009 epf = ntb->epf;1010 dev = &epf->dev;1011 ntb_epc = ntb->epc[type];1012 1013 epc_features = ntb_epc->epc_features;1014 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];1015 size = epc_features->bar[barno].fixed_size;1016 align = epc_features->align;1017 1018 peer_ntb_epc = ntb->epc[!type];1019 peer_epc_features = peer_ntb_epc->epc_features;1020 peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];1021 peer_size = peer_epc_features->bar[peer_barno].fixed_size;1022 1023 /* Check if epc_features is populated incorrectly */1024 if ((!IS_ALIGNED(size, align)))1025 return -EINVAL;1026 1027 spad_count = ntb->spad_count;1028 1029 ctrl_size = sizeof(struct epf_ntb_ctrl);1030 spad_size = spad_count * 4;1031 1032 msix_capable = epc_features->msix_capable;1033 if (msix_capable) {1034 msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count;1035 ctrl_size = ALIGN(ctrl_size, 8);1036 ntb_epc->msix_table_offset = ctrl_size;1037 ntb_epc->msix_bar = barno;1038 /* Align to QWORD or 8 Bytes */1039 pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8);1040 ctrl_size = ctrl_size + msix_table_size + pba_size;1041 }1042 1043 if (!align) {1044 ctrl_size = roundup_pow_of_two(ctrl_size);1045 spad_size = roundup_pow_of_two(spad_size);1046 } else {1047 ctrl_size = ALIGN(ctrl_size, align);1048 spad_size = ALIGN(spad_size, align);1049 }1050 1051 if (peer_size) {1052 if (peer_size < spad_size)1053 spad_count = peer_size / 4;1054 spad_size = peer_size;1055 }1056 1057 /*1058 * In order to make sure SPAD offset is aligned to its size,1059 * expand control region size to the size of SPAD if SPAD size1060 * is greater than control region size.1061 */1062 if (spad_size > ctrl_size)1063 ctrl_size = spad_size;1064 1065 if (!size)1066 size = ctrl_size + spad_size;1067 else if (size < ctrl_size + spad_size)1068 return -EINVAL;1069 1070 base = pci_epf_alloc_space(epf, size, barno, epc_features, type);1071 if (!base) {1072 dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n",1073 pci_epc_interface_string(type));1074 return -ENOMEM;1075 }1076 1077 ntb_epc->reg = base;1078 1079 ctrl = ntb_epc->reg;1080 ctrl->spad_offset = ctrl_size;1081 ctrl->spad_count = spad_count;1082 ctrl->num_mws = ntb->num_mws;1083 ctrl->db_entry_size = align ? align : 4;1084 ntb_epc->spad_size = spad_size;1085 1086 return 0;1087}1088 1089/**1090 * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config +1091 * scratchpad region for each of PRIMARY and SECONDARY interface1092 * @ntb: NTB device that facilitates communication between HOST1 and HOST21093 *1094 * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for1095 * config + scratchpad region for a specific interface1096 */1097static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb)1098{1099 enum pci_epc_interface_type type;1100 struct device *dev;1101 int ret;1102 1103 dev = &ntb->epf->dev;1104 1105 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {1106 ret = epf_ntb_config_spad_bar_alloc(ntb, type);1107 if (ret) {1108 dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n",1109 pci_epc_interface_string(type));1110 return ret;1111 }1112 }1113 1114 return 0;1115}1116 1117/**1118 * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address1119 * space1120 * @ntb_epc: EPC associated with one of the HOST which holds peers outbound1121 * address regions1122 *1123 * +-----------------+ +---->+----------------+-----------+-----------------+1124 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |1125 * +-----------------+ | +----------------+ +-----------------+1126 * | BAR1 | | | Doorbell 2 +---------+ | |1127 * +-----------------+----+ +----------------+ | | |1128 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+1129 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |1130 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+1131 * +-----------------+ | |----------------+ | | | |1132 * | BAR4 | | | | | | +-----------------+1133 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||1134 * | BAR5 | | | | | | +-----------------+1135 * +-----------------+ +---->-----------------+ | | | |1136 * EP CONTROLLER 1 | | | | +-----------------+1137 * | | | +---->+ MSI|X ADDRESS 4 |1138 * +----------------+ | +-----------------+1139 * (A) EP CONTROLLER 2 | | |1140 * (OB SPACE) | | |1141 * +-------> MW1 |1142 * | |1143 * | |1144 * (B) +-----------------+1145 * | |1146 * | |1147 * | |1148 * | |1149 * | |1150 * +-----------------+1151 * PCI Address Space1152 * (Managed by HOST2)1153 *1154 * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram.1155 * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3,1156 * MW4).1157 */1158static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc)1159{1160 struct pci_epf_bar *epf_bar;1161 void __iomem *mw_addr;1162 phys_addr_t phys_addr;1163 enum epf_ntb_bar bar;1164 enum pci_barno barno;1165 struct pci_epc *epc;1166 size_t size;1167 1168 epc = ntb_epc->epc;1169 1170 for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {1171 barno = ntb_epc->epf_ntb_bar[bar];1172 mw_addr = ntb_epc->mw_addr[barno];1173 epf_bar = &ntb_epc->epf_bar[barno];1174 phys_addr = epf_bar->phys_addr;1175 size = epf_bar->size;1176 if (mw_addr) {1177 pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size);1178 ntb_epc->mw_addr[barno] = NULL;1179 }1180 }1181}1182 1183/**1184 * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR1185 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound1186 * address1187 *1188 * +-----------------+ +---->+----------------+-----------+-----------------+1189 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |1190 * +-----------------+ | +----------------+ +-----------------+1191 * | BAR1 | | | Doorbell 2 +---------+ | |1192 * +-----------------+----+ +----------------+ | | |1193 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+1194 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |1195 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+1196 * +-----------------+ | |----------------+ | | | |1197 * | BAR4 | | | | | | +-----------------+1198 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||1199 * | BAR5 | | | | | | +-----------------+1200 * +-----------------+ +---->-----------------+ | | | |1201 * EP CONTROLLER 1 | | | | +-----------------+1202 * | | | +---->+ MSI|X ADDRESS 4 |1203 * +----------------+ | +-----------------+1204 * (A) EP CONTROLLER 2 | | |1205 * (OB SPACE) | | |1206 * +-------> MW1 |1207 * | |1208 * | |1209 * (B) +-----------------+1210 * | |1211 * | |1212 * | |1213 * | |1214 * | |1215 * +-----------------+1216 * PCI Address Space1217 * (Managed by HOST2)1218 *1219 * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above1220 * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW21221 * BAR, MW3 BAR and MW4 BAR).1222 */1223static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc)1224{1225 struct pci_epf_bar *epf_bar;1226 enum epf_ntb_bar bar;1227 enum pci_barno barno;1228 u8 func_no, vfunc_no;1229 struct pci_epc *epc;1230 1231 epc = ntb_epc->epc;1232 1233 func_no = ntb_epc->func_no;1234 vfunc_no = ntb_epc->vfunc_no;1235 1236 for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {1237 barno = ntb_epc->epf_ntb_bar[bar];1238 epf_bar = &ntb_epc->epf_bar[barno];1239 pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);1240 }1241}1242 1243/**1244 * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory1245 * allocated in peers outbound address space1246 * @ntb: NTB device that facilitates communication between HOST1 and HOST21247 * @type: PRIMARY interface or SECONDARY interface1248 *1249 * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and1250 * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory.1251 */1252static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb,1253 enum pci_epc_interface_type type)1254{1255 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;1256 1257 ntb_epc = ntb->epc[type];1258 peer_ntb_epc = ntb->epc[!type];1259 1260 epf_ntb_db_mw_bar_clear(ntb_epc);1261 epf_ntb_free_peer_mem(peer_ntb_epc);1262}1263 1264/**1265 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability1266 * @ntb: NTB device that facilitates communication between HOST1 and HOST21267 * @type: PRIMARY interface or SECONDARY interface1268 *1269 * Configure MSI/MSI-X capability for each interface with number of1270 * interrupts equal to "db_count" configfs entry.1271 */1272static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,1273 enum pci_epc_interface_type type)1274{1275 const struct pci_epc_features *epc_features;1276 bool msix_capable, msi_capable;1277 struct epf_ntb_epc *ntb_epc;1278 u8 func_no, vfunc_no;1279 struct pci_epc *epc;1280 struct device *dev;1281 u32 db_count;1282 int ret;1283 1284 ntb_epc = ntb->epc[type];1285 dev = &ntb->epf->dev;1286 1287 epc_features = ntb_epc->epc_features;1288 msix_capable = epc_features->msix_capable;1289 msi_capable = epc_features->msi_capable;1290 1291 if (!(msix_capable || msi_capable)) {1292 dev_err(dev, "MSI or MSI-X is required for doorbell\n");1293 return -EINVAL;1294 }1295 1296 func_no = ntb_epc->func_no;1297 vfunc_no = ntb_epc->vfunc_no;1298 1299 db_count = ntb->db_count;1300 if (db_count > MAX_DB_COUNT) {1301 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);1302 return -EINVAL;1303 }1304 1305 ntb->db_count = db_count;1306 epc = ntb_epc->epc;1307 1308 if (msi_capable) {1309 ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count);1310 if (ret) {1311 dev_err(dev, "%s intf: MSI configuration failed\n",1312 pci_epc_interface_string(type));1313 return ret;1314 }1315 }1316 1317 if (msix_capable) {1318 ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count,1319 ntb_epc->msix_bar,1320 ntb_epc->msix_table_offset);1321 if (ret) {1322 dev_err(dev, "MSI configuration failed\n");1323 return ret;1324 }1325 }1326 1327 return 0;1328}1329 1330/**1331 * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space1332 * @dev: The PCI device.1333 * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound1334 * address1335 * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be1336 * BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4)1337 * @peer_ntb_epc: EPC associated with HOST whose outbound address space is1338 * used by @ntb_epc1339 * @size: Size of the address region that has to be allocated in peers OB SPACE1340 *1341 *1342 * +-----------------+ +---->+----------------+-----------+-----------------+1343 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |1344 * +-----------------+ | +----------------+ +-----------------+1345 * | BAR1 | | | Doorbell 2 +---------+ | |1346 * +-----------------+----+ +----------------+ | | |1347 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+1348 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |1349 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+1350 * +-----------------+ | |----------------+ | | | |1351 * | BAR4 | | | | | | +-----------------+1352 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||1353 * | BAR5 | | | | | | +-----------------+1354 * +-----------------+ +---->-----------------+ | | | |1355 * EP CONTROLLER 1 | | | | +-----------------+1356 * | | | +---->+ MSI|X ADDRESS 4 |1357 * +----------------+ | +-----------------+1358 * (A) EP CONTROLLER 2 | | |1359 * (OB SPACE) | | |1360 * +-------> MW1 |1361 * | |1362 * | |1363 * (B) +-----------------+1364 * | |1365 * | |1366 * | |1367 * | |1368 * | |1369 * +-----------------+1370 * PCI Address Space1371 * (Managed by HOST2)1372 *1373 * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate1374 * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4).1375 */1376static int epf_ntb_alloc_peer_mem(struct device *dev,1377 struct epf_ntb_epc *ntb_epc,1378 enum epf_ntb_bar bar,1379 struct epf_ntb_epc *peer_ntb_epc,1380 size_t size)1381{1382 const struct pci_epc_features *epc_features;1383 struct pci_epf_bar *epf_bar;1384 struct pci_epc *peer_epc;1385 phys_addr_t phys_addr;1386 void __iomem *mw_addr;1387 enum pci_barno barno;1388 size_t align;1389 1390 epc_features = ntb_epc->epc_features;1391 align = epc_features->align;1392 1393 if (size < 128)1394 size = 128;1395 1396 if (align)1397 size = ALIGN(size, align);1398 else1399 size = roundup_pow_of_two(size);1400 1401 peer_epc = peer_ntb_epc->epc;1402 mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size);1403 if (!mw_addr) {1404 dev_err(dev, "%s intf: Failed to allocate OB address\n",1405 pci_epc_interface_string(peer_ntb_epc->type));1406 return -ENOMEM;1407 }1408 1409 barno = ntb_epc->epf_ntb_bar[bar];1410 epf_bar = &ntb_epc->epf_bar[barno];1411 ntb_epc->mw_addr[barno] = mw_addr;1412 1413 epf_bar->phys_addr = phys_addr;1414 epf_bar->size = size;1415 epf_bar->barno = barno;1416 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;1417 1418 return 0;1419}1420 1421/**1422 * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs1423 * @ntb: NTB device that facilitates communication between HOST1 and HOST21424 * @type: PRIMARY interface or SECONDARY interface1425 *1426 * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates1427 * memory in OB address space of HOST2 and configures BAR of HOST11428 */1429static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb,1430 enum pci_epc_interface_type type)1431{1432 const struct pci_epc_features *epc_features;1433 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;1434 struct pci_epf_bar *epf_bar;1435 struct epf_ntb_ctrl *ctrl;1436 u32 num_mws, db_count;1437 enum epf_ntb_bar bar;1438 enum pci_barno barno;1439 u8 func_no, vfunc_no;1440 struct pci_epc *epc;1441 struct device *dev;1442 size_t align;1443 int ret, i;1444 u64 size;1445 1446 ntb_epc = ntb->epc[type];1447 peer_ntb_epc = ntb->epc[!type];1448 1449 dev = &ntb->epf->dev;1450 epc_features = ntb_epc->epc_features;1451 align = epc_features->align;1452 func_no = ntb_epc->func_no;1453 vfunc_no = ntb_epc->vfunc_no;1454 epc = ntb_epc->epc;1455 num_mws = ntb->num_mws;1456 db_count = ntb->db_count;1457 1458 for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) {1459 if (bar == BAR_DB_MW1) {1460 align = align ? align : 4;1461 size = db_count * align;1462 size = ALIGN(size, ntb->mws_size[i]);1463 ctrl = ntb_epc->reg;1464 ctrl->mw1_offset = size;1465 size += ntb->mws_size[i];1466 } else {1467 size = ntb->mws_size[i];1468 }1469 1470 ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar,1471 peer_ntb_epc, size);1472 if (ret) {1473 dev_err(dev, "%s intf: DoorBell mem alloc failed\n",1474 pci_epc_interface_string(type));1475 goto err_alloc_peer_mem;1476 }1477 1478 barno = ntb_epc->epf_ntb_bar[bar];1479 epf_bar = &ntb_epc->epf_bar[barno];1480 1481 ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);1482 if (ret) {1483 dev_err(dev, "%s intf: DoorBell BAR set failed\n",1484 pci_epc_interface_string(type));1485 goto err_alloc_peer_mem;1486 }1487 }1488 1489 return 0;1490 1491err_alloc_peer_mem:1492 epf_ntb_db_mw_bar_cleanup(ntb, type);1493 1494 return ret;1495}1496 1497/**1498 * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface1499 * @ntb: NTB device that facilitates communication between HOST1 and HOST21500 * @type: PRIMARY interface or SECONDARY interface1501 *1502 * Unbind NTB function device from EPC and relinquish reference to pci_epc1503 * for each of the interface.1504 */1505static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb,1506 enum pci_epc_interface_type type)1507{1508 struct epf_ntb_epc *ntb_epc;1509 struct pci_epc *epc;1510 struct pci_epf *epf;1511 1512 if (type < 0)1513 return;1514 1515 epf = ntb->epf;1516 ntb_epc = ntb->epc[type];1517 if (!ntb_epc)1518 return;1519 epc = ntb_epc->epc;1520 pci_epc_remove_epf(epc, epf, type);1521 pci_epc_put(epc);1522}1523 1524/**1525 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface1526 * @ntb: NTB device that facilitates communication between HOST1 and HOST21527 *1528 * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces1529 */1530static void epf_ntb_epc_destroy(struct epf_ntb *ntb)1531{1532 enum pci_epc_interface_type type;1533 1534 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)1535 epf_ntb_epc_destroy_interface(ntb, type);1536}1537 1538/**1539 * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface1540 * @ntb: NTB device that facilitates communication between HOST1 and HOST21541 * @epc: struct pci_epc to which a particular NTB interface should be associated1542 * @type: PRIMARY interface or SECONDARY interface1543 *1544 * Allocate memory for NTB EPC interface and initialize it.1545 */1546static int epf_ntb_epc_create_interface(struct epf_ntb *ntb,1547 struct pci_epc *epc,1548 enum pci_epc_interface_type type)1549{1550 const struct pci_epc_features *epc_features;1551 struct pci_epf_bar *epf_bar;1552 struct epf_ntb_epc *ntb_epc;1553 u8 func_no, vfunc_no;1554 struct pci_epf *epf;1555 struct device *dev;1556 1557 dev = &ntb->epf->dev;1558 1559 ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL);1560 if (!ntb_epc)1561 return -ENOMEM;1562 1563 epf = ntb->epf;1564 vfunc_no = epf->vfunc_no;1565 if (type == PRIMARY_INTERFACE) {1566 func_no = epf->func_no;1567 epf_bar = epf->bar;1568 } else {1569 func_no = epf->sec_epc_func_no;1570 epf_bar = epf->sec_epc_bar;1571 }1572 1573 ntb_epc->linkup = false;1574 ntb_epc->epc = epc;1575 ntb_epc->func_no = func_no;1576 ntb_epc->vfunc_no = vfunc_no;1577 ntb_epc->type = type;1578 ntb_epc->epf_bar = epf_bar;1579 ntb_epc->epf_ntb = ntb;1580 1581 epc_features = pci_epc_get_features(epc, func_no, vfunc_no);1582 if (!epc_features)1583 return -EINVAL;1584 ntb_epc->epc_features = epc_features;1585 1586 ntb->epc[type] = ntb_epc;1587 1588 return 0;1589}1590 1591/**1592 * epf_ntb_epc_create() - Create and initialize NTB EPC interface1593 * @ntb: NTB device that facilitates communication between HOST1 and HOST21594 *1595 * Get a reference to EPC device and bind NTB function device to that EPC1596 * for each of the interface. It is also a wrapper to1597 * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface1598 * and initialize it1599 */1600static int epf_ntb_epc_create(struct epf_ntb *ntb)1601{1602 struct pci_epf *epf;1603 struct device *dev;1604 int ret;1605 1606 epf = ntb->epf;1607 dev = &epf->dev;1608 1609 ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE);1610 if (ret) {1611 dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n");1612 return ret;1613 }1614 1615 ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc,1616 SECONDARY_INTERFACE);1617 if (ret) {1618 dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n");1619 goto err_epc_create;1620 }1621 1622 return 0;1623 1624err_epc_create:1625 epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE);1626 1627 return ret;1628}1629 1630/**1631 * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of1632 * the NTB constructs (scratchpad region, doorbell, memorywindow)1633 * @ntb: NTB device that facilitates communication between HOST1 and HOST21634 * @type: PRIMARY interface or SECONDARY interface1635 *1636 * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD,1637 * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4.1638 */1639static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb,1640 enum pci_epc_interface_type type)1641{1642 const struct pci_epc_features *epc_features;1643 struct epf_ntb_epc *ntb_epc;1644 enum pci_barno barno;1645 enum epf_ntb_bar bar;1646 struct device *dev;1647 u32 num_mws;1648 int i;1649 1650 barno = BAR_0;1651 ntb_epc = ntb->epc[type];1652 num_mws = ntb->num_mws;1653 dev = &ntb->epf->dev;1654 epc_features = ntb_epc->epc_features;1655 1656 /* These are required BARs which are mandatory for NTB functionality */1657 for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) {1658 barno = pci_epc_get_next_free_bar(epc_features, barno);1659 if (barno < 0) {1660 dev_err(dev, "%s intf: Fail to get NTB function BAR\n",1661 pci_epc_interface_string(type));1662 return barno;1663 }1664 ntb_epc->epf_ntb_bar[bar] = barno;1665 }1666 1667 /* These are optional BARs which don't impact NTB functionality */1668 for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) {1669 barno = pci_epc_get_next_free_bar(epc_features, barno);1670 if (barno < 0) {1671 ntb->num_mws = i;1672 dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);1673 }1674 ntb_epc->epf_ntb_bar[bar] = barno;1675 }1676 1677 return 0;1678}1679 1680/**1681 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB1682 * constructs (scratchpad region, doorbell, memorywindow)1683 * @ntb: NTB device that facilitates communication between HOST1 and HOST21684 *1685 * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs1686 * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2,1687 * BAR_MW3 and BAR_MW4 for all the interfaces.1688 */1689static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)1690{1691 enum pci_epc_interface_type type;1692 struct device *dev;1693 int ret;1694 1695 dev = &ntb->epf->dev;1696 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {1697 ret = epf_ntb_init_epc_bar_interface(ntb, type);1698 if (ret) {1699 dev_err(dev, "Fail to init EPC bar for %s interface\n",1700 pci_epc_interface_string(type));1701 return ret;1702 }1703 }1704 1705 return 0;1706}1707 1708/**1709 * epf_ntb_epc_init_interface() - Initialize NTB interface1710 * @ntb: NTB device that facilitates communication between HOST1 and HOST21711 * @type: PRIMARY interface or SECONDARY interface1712 *1713 * Wrapper to initialize a particular EPC interface and start the workqueue1714 * to check for commands from host. This function will write to the1715 * EP controller HW for configuring it.1716 */1717static int epf_ntb_epc_init_interface(struct epf_ntb *ntb,1718 enum pci_epc_interface_type type)1719{1720 struct epf_ntb_epc *ntb_epc;1721 u8 func_no, vfunc_no;1722 struct pci_epc *epc;1723 struct pci_epf *epf;1724 struct device *dev;1725 int ret;1726 1727 ntb_epc = ntb->epc[type];1728 epf = ntb->epf;1729 dev = &epf->dev;1730 epc = ntb_epc->epc;1731 func_no = ntb_epc->func_no;1732 vfunc_no = ntb_epc->vfunc_no;1733 1734 ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]);1735 if (ret) {1736 dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n",1737 pci_epc_interface_string(type));1738 return ret;1739 }1740 1741 ret = epf_ntb_peer_spad_bar_set(ntb, type);1742 if (ret) {1743 dev_err(dev, "%s intf: Peer SPAD BAR init failed\n",1744 pci_epc_interface_string(type));1745 goto err_peer_spad_bar_init;1746 }1747 1748 ret = epf_ntb_configure_interrupt(ntb, type);1749 if (ret) {1750 dev_err(dev, "%s intf: Interrupt configuration failed\n",1751 pci_epc_interface_string(type));1752 goto err_peer_spad_bar_init;1753 }1754 1755 ret = epf_ntb_db_mw_bar_init(ntb, type);1756 if (ret) {1757 dev_err(dev, "%s intf: DB/MW BAR init failed\n",1758 pci_epc_interface_string(type));1759 goto err_db_mw_bar_init;1760 }1761 1762 if (vfunc_no <= 1) {1763 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);1764 if (ret) {1765 dev_err(dev, "%s intf: Configuration header write failed\n",1766 pci_epc_interface_string(type));1767 goto err_write_header;1768 }1769 }1770 1771 INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler);1772 queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work);1773 1774 return 0;1775 1776err_write_header:1777 epf_ntb_db_mw_bar_cleanup(ntb, type);1778 1779err_db_mw_bar_init:1780 epf_ntb_peer_spad_bar_clear(ntb->epc[type]);1781 1782err_peer_spad_bar_init:1783 epf_ntb_config_sspad_bar_clear(ntb->epc[type]);1784 1785 return ret;1786}1787 1788/**1789 * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface1790 * @ntb: NTB device that facilitates communication between HOST1 and HOST21791 * @type: PRIMARY interface or SECONDARY interface1792 *1793 * Wrapper to cleanup a particular NTB interface.1794 */1795static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb,1796 enum pci_epc_interface_type type)1797{1798 struct epf_ntb_epc *ntb_epc;1799 1800 if (type < 0)1801 return;1802 1803 ntb_epc = ntb->epc[type];1804 cancel_delayed_work(&ntb_epc->cmd_handler);1805 epf_ntb_db_mw_bar_cleanup(ntb, type);1806 epf_ntb_peer_spad_bar_clear(ntb_epc);1807 epf_ntb_config_sspad_bar_clear(ntb_epc);1808}1809 1810/**1811 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces1812 * @ntb: NTB device that facilitates communication between HOST1 and HOST21813 *1814 * Wrapper to cleanup all NTB interfaces.1815 */1816static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)1817{1818 enum pci_epc_interface_type type;1819 1820 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)1821 epf_ntb_epc_cleanup_interface(ntb, type);1822}1823 1824/**1825 * epf_ntb_epc_init() - Initialize all NTB interfaces1826 * @ntb: NTB device that facilitates communication between HOST1 and HOST21827 *1828 * Wrapper to initialize all NTB interface and start the workqueue1829 * to check for commands from host.1830 */1831static int epf_ntb_epc_init(struct epf_ntb *ntb)1832{1833 enum pci_epc_interface_type type;1834 struct device *dev;1835 int ret;1836 1837 dev = &ntb->epf->dev;1838 1839 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {1840 ret = epf_ntb_epc_init_interface(ntb, type);1841 if (ret) {1842 dev_err(dev, "%s intf: Failed to initialize\n",1843 pci_epc_interface_string(type));1844 goto err_init_type;1845 }1846 }1847 1848 return 0;1849 1850err_init_type:1851 epf_ntb_epc_cleanup_interface(ntb, type - 1);1852 1853 return ret;1854}1855 1856/**1857 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality1858 * @epf: NTB endpoint function device1859 *1860 * Initialize both the endpoint controllers associated with NTB function device.1861 * Invoked when a primary interface or secondary interface is bound to EPC1862 * device. This function will succeed only when EPC is bound to both the1863 * interfaces.1864 */1865static int epf_ntb_bind(struct pci_epf *epf)1866{1867 struct epf_ntb *ntb = epf_get_drvdata(epf);1868 struct device *dev = &epf->dev;1869 int ret;1870 1871 if (!epf->epc) {1872 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");1873 return 0;1874 }1875 1876 if (!epf->sec_epc) {1877 dev_dbg(dev, "SECONDARY EPC interface not yet bound\n");1878 return 0;1879 }1880 1881 ret = epf_ntb_epc_create(ntb);1882 if (ret) {1883 dev_err(dev, "Failed to create NTB EPC\n");1884 return ret;1885 }1886 1887 ret = epf_ntb_init_epc_bar(ntb);1888 if (ret) {1889 dev_err(dev, "Failed to create NTB EPC\n");1890 goto err_bar_init;1891 }1892 1893 ret = epf_ntb_config_spad_bar_alloc_interface(ntb);1894 if (ret) {1895 dev_err(dev, "Failed to allocate BAR memory\n");1896 goto err_bar_alloc;1897 }1898 1899 ret = epf_ntb_epc_init(ntb);1900 if (ret) {1901 dev_err(dev, "Failed to initialize EPC\n");1902 goto err_bar_alloc;1903 }1904 1905 epf_set_drvdata(epf, ntb);1906 1907 return 0;1908 1909err_bar_alloc:1910 epf_ntb_config_spad_bar_free(ntb);1911 1912err_bar_init:1913 epf_ntb_epc_destroy(ntb);1914 1915 return ret;1916}1917 1918/**1919 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()1920 * @epf: NTB endpoint function device1921 *1922 * Cleanup the initialization from epf_ntb_bind()1923 */1924static void epf_ntb_unbind(struct pci_epf *epf)1925{1926 struct epf_ntb *ntb = epf_get_drvdata(epf);1927 1928 epf_ntb_epc_cleanup(ntb);1929 epf_ntb_config_spad_bar_free(ntb);1930 epf_ntb_epc_destroy(ntb);1931}1932 1933#define EPF_NTB_R(_name) \1934static ssize_t epf_ntb_##_name##_show(struct config_item *item, \1935 char *page) \1936{ \1937 struct config_group *group = to_config_group(item); \1938 struct epf_ntb *ntb = to_epf_ntb(group); \1939 \1940 return sysfs_emit(page, "%d\n", ntb->_name); \1941}1942 1943#define EPF_NTB_W(_name) \1944static ssize_t epf_ntb_##_name##_store(struct config_item *item, \1945 const char *page, size_t len) \1946{ \1947 struct config_group *group = to_config_group(item); \1948 struct epf_ntb *ntb = to_epf_ntb(group); \1949 u32 val; \1950 \1951 if (kstrtou32(page, 0, &val) < 0) \1952 return -EINVAL; \1953 \1954 ntb->_name = val; \1955 \1956 return len; \1957}1958 1959#define EPF_NTB_MW_R(_name) \1960static ssize_t epf_ntb_##_name##_show(struct config_item *item, \1961 char *page) \1962{ \1963 struct config_group *group = to_config_group(item); \1964 struct epf_ntb *ntb = to_epf_ntb(group); \1965 int win_no; \1966 \1967 sscanf(#_name, "mw%d", &win_no); \1968 \1969 return sysfs_emit(page, "%lld\n", ntb->mws_size[win_no - 1]); \1970}1971 1972#define EPF_NTB_MW_W(_name) \1973static ssize_t epf_ntb_##_name##_store(struct config_item *item, \1974 const char *page, size_t len) \1975{ \1976 struct config_group *group = to_config_group(item); \1977 struct epf_ntb *ntb = to_epf_ntb(group); \1978 struct device *dev = &ntb->epf->dev; \1979 int win_no; \1980 u64 val; \1981 \1982 if (kstrtou64(page, 0, &val) < 0) \1983 return -EINVAL; \1984 \1985 if (sscanf(#_name, "mw%d", &win_no) != 1) \1986 return -EINVAL; \1987 \1988 if (ntb->num_mws < win_no) { \1989 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \1990 return -EINVAL; \1991 } \1992 \1993 ntb->mws_size[win_no - 1] = val; \1994 \1995 return len; \1996}1997 1998static ssize_t epf_ntb_num_mws_store(struct config_item *item,1999 const char *page, size_t len)2000{2001 struct config_group *group = to_config_group(item);2002 struct epf_ntb *ntb = to_epf_ntb(group);2003 u32 val;2004 2005 if (kstrtou32(page, 0, &val) < 0)2006 return -EINVAL;2007 2008 if (val > MAX_MW)2009 return -EINVAL;2010 2011 ntb->num_mws = val;2012 2013 return len;2014}2015 2016EPF_NTB_R(spad_count)2017EPF_NTB_W(spad_count)2018EPF_NTB_R(db_count)2019EPF_NTB_W(db_count)2020EPF_NTB_R(num_mws)2021EPF_NTB_MW_R(mw1)2022EPF_NTB_MW_W(mw1)2023EPF_NTB_MW_R(mw2)2024EPF_NTB_MW_W(mw2)2025EPF_NTB_MW_R(mw3)2026EPF_NTB_MW_W(mw3)2027EPF_NTB_MW_R(mw4)2028EPF_NTB_MW_W(mw4)2029 2030CONFIGFS_ATTR(epf_ntb_, spad_count);2031CONFIGFS_ATTR(epf_ntb_, db_count);2032CONFIGFS_ATTR(epf_ntb_, num_mws);2033CONFIGFS_ATTR(epf_ntb_, mw1);2034CONFIGFS_ATTR(epf_ntb_, mw2);2035CONFIGFS_ATTR(epf_ntb_, mw3);2036CONFIGFS_ATTR(epf_ntb_, mw4);2037 2038static struct configfs_attribute *epf_ntb_attrs[] = {2039 &epf_ntb_attr_spad_count,2040 &epf_ntb_attr_db_count,2041 &epf_ntb_attr_num_mws,2042 &epf_ntb_attr_mw1,2043 &epf_ntb_attr_mw2,2044 &epf_ntb_attr_mw3,2045 &epf_ntb_attr_mw4,2046 NULL,2047};2048 2049static const struct config_item_type ntb_group_type = {2050 .ct_attrs = epf_ntb_attrs,2051 .ct_owner = THIS_MODULE,2052};2053 2054/**2055 * epf_ntb_add_cfs() - Add configfs directory specific to NTB2056 * @epf: NTB endpoint function device2057 * @group: A pointer to the config_group structure referencing a group of2058 * config_items of a specific type that belong to a specific sub-system.2059 *2060 * Add configfs directory specific to NTB. This directory will hold2061 * NTB specific properties like db_count, spad_count, num_mws etc.,2062 */2063static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,2064 struct config_group *group)2065{2066 struct epf_ntb *ntb = epf_get_drvdata(epf);2067 struct config_group *ntb_group = &ntb->group;2068 struct device *dev = &epf->dev;2069 2070 config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);2071 2072 return ntb_group;2073}2074 2075/**2076 * epf_ntb_probe() - Probe NTB function driver2077 * @epf: NTB endpoint function device2078 * @id: NTB endpoint function device ID2079 *2080 * Probe NTB function driver when endpoint function bus detects a NTB2081 * endpoint function.2082 */2083static int epf_ntb_probe(struct pci_epf *epf,2084 const struct pci_epf_device_id *id)2085{2086 struct epf_ntb *ntb;2087 struct device *dev;2088 2089 dev = &epf->dev;2090 2091 ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);2092 if (!ntb)2093 return -ENOMEM;2094 2095 epf->header = &epf_ntb_header;2096 ntb->epf = epf;2097 epf_set_drvdata(epf, ntb);2098 2099 return 0;2100}2101 2102static const struct pci_epf_ops epf_ntb_ops = {2103 .bind = epf_ntb_bind,2104 .unbind = epf_ntb_unbind,2105 .add_cfs = epf_ntb_add_cfs,2106};2107 2108static const struct pci_epf_device_id epf_ntb_ids[] = {2109 {2110 .name = "pci_epf_ntb",2111 },2112 {},2113};2114 2115static struct pci_epf_driver epf_ntb_driver = {2116 .driver.name = "pci_epf_ntb",2117 .probe = epf_ntb_probe,2118 .id_table = epf_ntb_ids,2119 .ops = &epf_ntb_ops,2120 .owner = THIS_MODULE,2121};2122 2123static int __init epf_ntb_init(void)2124{2125 int ret;2126 2127 kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |2128 WQ_HIGHPRI, 0);2129 ret = pci_epf_register_driver(&epf_ntb_driver);2130 if (ret) {2131 destroy_workqueue(kpcintb_workqueue);2132 pr_err("Failed to register pci epf ntb driver --> %d\n", ret);2133 return ret;2134 }2135 2136 return 0;2137}2138module_init(epf_ntb_init);2139 2140static void __exit epf_ntb_exit(void)2141{2142 pci_epf_unregister_driver(&epf_ntb_driver);2143 destroy_workqueue(kpcintb_workqueue);2144}2145module_exit(epf_ntb_exit);2146 2147MODULE_DESCRIPTION("PCI EPF NTB DRIVER");2148MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");2149MODULE_LICENSE("GPL v2");2150