834 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)2 3#include <linux/bitmap.h>4#include <linux/delay.h>5#include <linux/interrupt.h>6#include <linux/io.h>7#include <linux/io-64-nonatomic-lo-hi.h>8#include <linux/mm.h>9#include <linux/module.h>10#include <linux/nvme.h>11#include <linux/pci.h>12#include <linux/wait.h>13#include <linux/sched/signal.h>14 15#include "fun_queue.h"16#include "fun_dev.h"17 18#define FUN_ADMIN_CMD_TO_MS 300019 20enum {21 AQA_ASQS_SHIFT = 0,22 AQA_ACQS_SHIFT = 16,23 AQA_MIN_QUEUE_SIZE = 2,24 AQA_MAX_QUEUE_SIZE = 409625};26 27/* context for admin commands */28struct fun_cmd_ctx {29 fun_admin_callback_t cb; /* callback to invoke on completion */30 void *cb_data; /* user data provided to callback */31 int cpu; /* CPU where the cmd's tag was allocated */32};33 34/* Context for synchronous admin commands. */35struct fun_sync_cmd_ctx {36 struct completion compl;37 u8 *rsp_buf; /* caller provided response buffer */38 unsigned int rsp_len; /* response buffer size */39 u8 rsp_status; /* command response status */40};41 42/* Wait for the CSTS.RDY bit to match @enabled. */43static int fun_wait_ready(struct fun_dev *fdev, bool enabled)44{45 unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);46 u32 bit = enabled ? NVME_CSTS_RDY : 0;47 unsigned long deadline;48 49 deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */50 51 for (;;) {52 u32 csts = readl(fdev->bar + NVME_REG_CSTS);53 54 if (csts == ~0) {55 dev_err(fdev->dev, "CSTS register read %#x\n", csts);56 return -EIO;57 }58 59 if ((csts & NVME_CSTS_RDY) == bit)60 return 0;61 62 if (time_is_before_jiffies(deadline))63 break;64 65 msleep(100);66 }67 68 dev_err(fdev->dev,69 "Timed out waiting for device to indicate RDY %u; aborting %s\n",70 enabled, enabled ? "initialization" : "reset");71 return -ETIMEDOUT;72}73 74/* Check CSTS and return an error if it is unreadable or has unexpected75 * RDY value.76 */77static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)78{79 u32 csts = readl(fdev->bar + NVME_REG_CSTS);80 u32 actual_rdy = csts & NVME_CSTS_RDY;81 82 if (csts == ~0) {83 dev_err(fdev->dev, "CSTS register read %#x\n", csts);84 return -EIO;85 }86 if (actual_rdy != expected_rdy) {87 dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);88 return -EINVAL;89 }90 return 0;91}92 93/* Check that CSTS RDY has the expected value. Then write a new value to the CC94 * register and wait for CSTS RDY to match the new CC ENABLE state.95 */96static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)97{98 int rc = fun_check_csts_rdy(fdev, initial_rdy);99 100 if (rc)101 return rc;102 writel(fdev->cc_reg, fdev->bar + NVME_REG_CC);103 return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE));104}105 106static int fun_disable_ctrl(struct fun_dev *fdev)107{108 fdev->cc_reg &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE);109 return fun_update_cc_enable(fdev, 1);110}111 112static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,113 u32 admin_sqesz_log2)114{115 fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) |116 (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) |117 ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) |118 NVME_CC_ENABLE;119 120 return fun_update_cc_enable(fdev, 0);121}122 123static int fun_map_bars(struct fun_dev *fdev, const char *name)124{125 struct pci_dev *pdev = to_pci_dev(fdev->dev);126 int err;127 128 err = pci_request_mem_regions(pdev, name);129 if (err) {130 dev_err(&pdev->dev,131 "Couldn't get PCI memory resources, err %d\n", err);132 return err;133 }134 135 fdev->bar = pci_ioremap_bar(pdev, 0);136 if (!fdev->bar) {137 dev_err(&pdev->dev, "Couldn't map BAR 0\n");138 pci_release_mem_regions(pdev);139 return -ENOMEM;140 }141 142 return 0;143}144 145static void fun_unmap_bars(struct fun_dev *fdev)146{147 struct pci_dev *pdev = to_pci_dev(fdev->dev);148 149 if (fdev->bar) {150 iounmap(fdev->bar);151 fdev->bar = NULL;152 pci_release_mem_regions(pdev);153 }154}155 156static int fun_set_dma_masks(struct device *dev)157{158 int err;159 160 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));161 if (err)162 dev_err(dev, "DMA mask configuration failed, err %d\n", err);163 return err;164}165 166static irqreturn_t fun_admin_irq(int irq, void *data)167{168 struct fun_queue *funq = data;169 170 return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE;171}172 173static void fun_complete_admin_cmd(struct fun_queue *funq, void *data,174 void *entry, const struct fun_cqe_info *info)175{176 const struct fun_admin_rsp_common *rsp_common = entry;177 struct fun_dev *fdev = funq->fdev;178 struct fun_cmd_ctx *cmd_ctx;179 int cpu;180 u16 cid;181 182 if (info->sqhd == cpu_to_be16(0xffff)) {183 dev_dbg(fdev->dev, "adminq event");184 if (fdev->adminq_cb)185 fdev->adminq_cb(fdev, entry);186 return;187 }188 189 cid = be16_to_cpu(rsp_common->cid);190 dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,191 rsp_common->op, rsp_common->ret);192 193 cmd_ctx = &fdev->cmd_ctx[cid];194 if (cmd_ctx->cpu < 0) {195 dev_err(fdev->dev,196 "admin CQE with CID=%u, op=%u does not match a pending command\n",197 cid, rsp_common->op);198 return;199 }200 201 if (cmd_ctx->cb)202 cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));203 204 cpu = cmd_ctx->cpu;205 cmd_ctx->cpu = -1;206 sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu);207}208 209static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)210{211 unsigned int i;212 213 fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);214 if (!fdev->cmd_ctx)215 return -ENOMEM;216 217 for (i = 0; i < ntags; i++)218 fdev->cmd_ctx[i].cpu = -1;219 220 return 0;221}222 223/* Allocate and enable an admin queue and assign it the first IRQ vector. */224static int fun_enable_admin_queue(struct fun_dev *fdev,225 const struct fun_dev_params *areq)226{227 struct fun_queue_alloc_req qreq = {228 .cqe_size_log2 = areq->cqe_size_log2,229 .sqe_size_log2 = areq->sqe_size_log2,230 .cq_depth = areq->cq_depth,231 .sq_depth = areq->sq_depth,232 .rq_depth = areq->rq_depth,233 };234 unsigned int ntags = areq->sq_depth - 1;235 struct fun_queue *funq;236 int rc;237 238 if (fdev->admin_q)239 return -EEXIST;240 241 if (areq->sq_depth < AQA_MIN_QUEUE_SIZE ||242 areq->sq_depth > AQA_MAX_QUEUE_SIZE ||243 areq->cq_depth < AQA_MIN_QUEUE_SIZE ||244 areq->cq_depth > AQA_MAX_QUEUE_SIZE)245 return -EINVAL;246 247 fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq);248 if (!fdev->admin_q)249 return -ENOMEM;250 251 rc = fun_init_cmd_ctx(fdev, ntags);252 if (rc)253 goto free_q;254 255 rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false,256 GFP_KERNEL, dev_to_node(fdev->dev));257 if (rc)258 goto free_cmd_ctx;259 260 funq = fdev->admin_q;261 funq->cq_vector = 0;262 rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq);263 if (rc)264 goto free_sbq;265 266 fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL);267 fdev->adminq_cb = areq->event_cb;268 269 writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT |270 (funq->cq_depth - 1) << AQA_ACQS_SHIFT,271 fdev->bar + NVME_REG_AQA);272 273 writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ);274 writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ);275 276 rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2);277 if (rc)278 goto free_irq;279 280 if (areq->rq_depth) {281 rc = fun_create_rq(funq);282 if (rc)283 goto disable_ctrl;284 285 funq_rq_post(funq);286 }287 288 return 0;289 290disable_ctrl:291 fun_disable_ctrl(fdev);292free_irq:293 fun_free_irq(funq);294free_sbq:295 sbitmap_queue_free(&fdev->admin_sbq);296free_cmd_ctx:297 kvfree(fdev->cmd_ctx);298 fdev->cmd_ctx = NULL;299free_q:300 fun_free_queue(fdev->admin_q);301 fdev->admin_q = NULL;302 return rc;303}304 305static void fun_disable_admin_queue(struct fun_dev *fdev)306{307 struct fun_queue *admq = fdev->admin_q;308 309 if (!admq)310 return;311 312 fun_disable_ctrl(fdev);313 314 fun_free_irq(admq);315 __fun_process_cq(admq, 0);316 317 sbitmap_queue_free(&fdev->admin_sbq);318 319 kvfree(fdev->cmd_ctx);320 fdev->cmd_ctx = NULL;321 322 fun_free_queue(admq);323 fdev->admin_q = NULL;324}325 326/* Return %true if the admin queue has stopped servicing commands as can be327 * detected through registers. This isn't exhaustive and may provide false328 * negatives.329 */330static bool fun_adminq_stopped(struct fun_dev *fdev)331{332 u32 csts = readl(fdev->bar + NVME_REG_CSTS);333 334 return (csts & (NVME_CSTS_CFS | NVME_CSTS_RDY)) != NVME_CSTS_RDY;335}336 337static int fun_wait_for_tag(struct fun_dev *fdev, int *cpup)338{339 struct sbitmap_queue *sbq = &fdev->admin_sbq;340 struct sbq_wait_state *ws = &sbq->ws[0];341 DEFINE_SBQ_WAIT(wait);342 int tag;343 344 for (;;) {345 sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE);346 if (fdev->suppress_cmds) {347 tag = -ESHUTDOWN;348 break;349 }350 tag = sbitmap_queue_get(sbq, cpup);351 if (tag >= 0)352 break;353 schedule();354 }355 356 sbitmap_finish_wait(sbq, ws, &wait);357 return tag;358}359 360/* Submit an asynchronous admin command. Caller is responsible for implementing361 * any waiting or timeout. Upon command completion the callback @cb is called.362 */363int fun_submit_admin_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd,364 fun_admin_callback_t cb, void *cb_data, bool wait_ok)365{366 struct fun_queue *funq = fdev->admin_q;367 unsigned int cmdsize = cmd->len8 * 8;368 struct fun_cmd_ctx *cmd_ctx;369 int tag, cpu, rc = 0;370 371 if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))372 return -EMSGSIZE;373 374 tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu);375 if (tag < 0) {376 if (!wait_ok)377 return -EAGAIN;378 tag = fun_wait_for_tag(fdev, &cpu);379 if (tag < 0)380 return tag;381 }382 383 cmd->cid = cpu_to_be16(tag);384 385 cmd_ctx = &fdev->cmd_ctx[tag];386 cmd_ctx->cb = cb;387 cmd_ctx->cb_data = cb_data;388 389 spin_lock(&funq->sq_lock);390 391 if (unlikely(fdev->suppress_cmds)) {392 rc = -ESHUTDOWN;393 sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu);394 } else {395 cmd_ctx->cpu = cpu;396 memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);397 398 dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,399 cmd);400 401 if (++funq->sq_tail == funq->sq_depth)402 funq->sq_tail = 0;403 writel(funq->sq_tail, funq->sq_db);404 }405 spin_unlock(&funq->sq_lock);406 return rc;407}408 409/* Abandon a pending admin command by clearing the issuer's callback data.410 * Failure indicates that the command either has already completed or its411 * completion is racing with this call.412 */413static bool fun_abandon_admin_cmd(struct fun_dev *fd,414 const struct fun_admin_req_common *cmd,415 void *cb_data)416{417 u16 cid = be16_to_cpu(cmd->cid);418 struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];419 420 return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;421}422 423/* Stop submission of new admin commands and wake up any processes waiting for424 * tags. Already submitted commands are left to complete or time out.425 */426static void fun_admin_stop(struct fun_dev *fdev)427{428 spin_lock(&fdev->admin_q->sq_lock);429 fdev->suppress_cmds = true;430 spin_unlock(&fdev->admin_q->sq_lock);431 sbitmap_queue_wake_all(&fdev->admin_sbq);432}433 434/* The callback for synchronous execution of admin commands. It copies the435 * command response to the caller's buffer and signals completion.436 */437static void fun_admin_cmd_sync_cb(struct fun_dev *fd, void *rsp, void *cb_data)438{439 const struct fun_admin_rsp_common *rsp_common = rsp;440 struct fun_sync_cmd_ctx *ctx = cb_data;441 442 if (!ctx)443 return; /* command issuer timed out and left */444 if (ctx->rsp_buf) {445 unsigned int rsp_len = rsp_common->len8 * 8;446 447 if (unlikely(rsp_len > ctx->rsp_len)) {448 dev_err(fd->dev,449 "response for op %u is %uB > response buffer %uB\n",450 rsp_common->op, rsp_len, ctx->rsp_len);451 rsp_len = ctx->rsp_len;452 }453 memcpy(ctx->rsp_buf, rsp, rsp_len);454 }455 ctx->rsp_status = rsp_common->ret;456 complete(&ctx->compl);457}458 459/* Submit a synchronous admin command. */460int fun_submit_admin_sync_cmd(struct fun_dev *fdev,461 struct fun_admin_req_common *cmd, void *rsp,462 size_t rspsize, unsigned int timeout)463{464 struct fun_sync_cmd_ctx ctx = {465 .compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),466 .rsp_buf = rsp,467 .rsp_len = rspsize,468 };469 unsigned int cmdlen = cmd->len8 * 8;470 unsigned long jiffies_left;471 int ret;472 473 ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx,474 true);475 if (ret)476 return ret;477 478 if (!timeout)479 timeout = FUN_ADMIN_CMD_TO_MS;480 481 jiffies_left = wait_for_completion_timeout(&ctx.compl,482 msecs_to_jiffies(timeout));483 if (!jiffies_left) {484 /* The command timed out. Attempt to cancel it so we can return.485 * But if the command is in the process of completing we'll486 * wait for it.487 */488 if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) {489 dev_err(fdev->dev, "admin command timed out: %*ph\n",490 cmdlen, cmd);491 fun_admin_stop(fdev);492 /* see if the timeout was due to a queue failure */493 if (fun_adminq_stopped(fdev))494 dev_err(fdev->dev,495 "device does not accept admin commands\n");496 497 return -ETIMEDOUT;498 }499 wait_for_completion(&ctx.compl);500 }501 502 if (ctx.rsp_status) {503 dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",504 ctx.rsp_status, cmdlen, cmd);505 }506 507 return -ctx.rsp_status;508}509EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);510 511/* Return the number of device resources of the requested type. */512int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)513{514 union {515 struct fun_admin_res_count_req req;516 struct fun_admin_res_count_rsp rsp;517 } cmd;518 int rc;519 520 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));521 cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,522 0, 0);523 524 rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,525 sizeof(cmd), 0);526 return rc ? rc : be32_to_cpu(cmd.rsp.count.data);527}528EXPORT_SYMBOL_GPL(fun_get_res_count);529 530/* Request that the instance of resource @res with the given id be deleted. */531int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,532 unsigned int flags, u32 id)533{534 struct fun_admin_generic_destroy_req req = {535 .common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),536 .destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,537 flags, id)538 };539 540 return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);541}542EXPORT_SYMBOL_GPL(fun_res_destroy);543 544/* Bind two entities of the given types and IDs. */545int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,546 unsigned int id0, enum fun_admin_bind_type type1,547 unsigned int id1)548{549 DEFINE_RAW_FLEX(struct fun_admin_bind_req, cmd, entry, 2);550 551 cmd->common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,552 __struct_size(cmd));553 cmd->entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0);554 cmd->entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1);555 556 return fun_submit_admin_sync_cmd(fdev, &cmd->common, NULL, 0, 0);557}558EXPORT_SYMBOL_GPL(fun_bind);559 560static int fun_get_dev_limits(struct fun_dev *fdev)561{562 struct pci_dev *pdev = to_pci_dev(fdev->dev);563 unsigned int cq_count, sq_count, num_dbs;564 int rc;565 566 rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);567 if (rc < 0)568 return rc;569 cq_count = rc;570 571 rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);572 if (rc < 0)573 return rc;574 sq_count = rc;575 576 /* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the577 * device must provide additional queues.578 */579 if (cq_count < 2 || sq_count < 2 + !!fdev->admin_q->rq_depth)580 return -EINVAL;581 582 /* Calculate the max QID based on SQ/CQ/doorbell counts.583 * SQ/CQ doorbells alternate.584 */585 num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) >>586 (2 + NVME_CAP_STRIDE(fdev->cap_reg));587 fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;588 fdev->kern_end_qid = fdev->max_qid + 1;589 return 0;590}591 592/* Allocate all MSI-X vectors available on a function and at least @min_vecs. */593static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)594{595 int vecs, num_msix = pci_msix_vec_count(pdev);596 597 if (num_msix < 0)598 return num_msix;599 if (min_vecs > num_msix)600 return -ERANGE;601 602 vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX);603 if (vecs > 0) {604 dev_info(&pdev->dev,605 "Allocated %d IRQ vectors of %d requested\n",606 vecs, num_msix);607 } else {608 dev_err(&pdev->dev,609 "Unable to allocate at least %u IRQ vectors\n",610 min_vecs);611 }612 return vecs;613}614 615/* Allocate and initialize the IRQ manager state. */616static int fun_alloc_irq_mgr(struct fun_dev *fdev)617{618 fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL);619 if (!fdev->irq_map)620 return -ENOMEM;621 622 spin_lock_init(&fdev->irqmgr_lock);623 /* mark IRQ 0 allocated, it is used by the admin queue */624 __set_bit(0, fdev->irq_map);625 fdev->irqs_avail = fdev->num_irqs - 1;626 return 0;627}628 629/* Reserve @nirqs of the currently available IRQs and return their indices. */630int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices)631{632 unsigned int b, n = 0;633 int err = -ENOSPC;634 635 if (!nirqs)636 return 0;637 638 spin_lock(&fdev->irqmgr_lock);639 if (nirqs > fdev->irqs_avail)640 goto unlock;641 642 for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {643 __set_bit(b, fdev->irq_map);644 irq_indices[n++] = b;645 if (n >= nirqs)646 break;647 }648 649 WARN_ON(n < nirqs);650 fdev->irqs_avail -= n;651 err = n;652unlock:653 spin_unlock(&fdev->irqmgr_lock);654 return err;655}656EXPORT_SYMBOL(fun_reserve_irqs);657 658/* Release @nirqs previously allocated IRQS with the supplied indices. */659void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,660 u16 *irq_indices)661{662 unsigned int i;663 664 spin_lock(&fdev->irqmgr_lock);665 for (i = 0; i < nirqs; i++)666 __clear_bit(irq_indices[i], fdev->irq_map);667 fdev->irqs_avail += nirqs;668 spin_unlock(&fdev->irqmgr_lock);669}670EXPORT_SYMBOL(fun_release_irqs);671 672static void fun_serv_handler(struct work_struct *work)673{674 struct fun_dev *fd = container_of(work, struct fun_dev, service_task);675 676 if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))677 return;678 if (fd->serv_cb)679 fd->serv_cb(fd);680}681 682void fun_serv_stop(struct fun_dev *fd)683{684 set_bit(FUN_SERV_DISABLED, &fd->service_flags);685 cancel_work_sync(&fd->service_task);686}687EXPORT_SYMBOL_GPL(fun_serv_stop);688 689void fun_serv_restart(struct fun_dev *fd)690{691 clear_bit(FUN_SERV_DISABLED, &fd->service_flags);692 if (fd->service_flags)693 schedule_work(&fd->service_task);694}695EXPORT_SYMBOL_GPL(fun_serv_restart);696 697void fun_serv_sched(struct fun_dev *fd)698{699 if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))700 schedule_work(&fd->service_task);701}702EXPORT_SYMBOL_GPL(fun_serv_sched);703 704/* Check and try to get the device into a proper state for initialization,705 * i.e., CSTS.RDY = CC.EN = 0.706 */707static int sanitize_dev(struct fun_dev *fdev)708{709 int rc;710 711 fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP);712 fdev->cc_reg = readl(fdev->bar + NVME_REG_CC);713 714 /* First get RDY to agree with the current EN. Give RDY the opportunity715 * to complete a potential recent EN change.716 */717 rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE);718 if (rc)719 return rc;720 721 /* Next, reset the device if EN is currently 1. */722 if (fdev->cc_reg & NVME_CC_ENABLE)723 rc = fun_disable_ctrl(fdev);724 725 return rc;726}727 728/* Undo the device initialization of fun_dev_enable(). */729void fun_dev_disable(struct fun_dev *fdev)730{731 struct pci_dev *pdev = to_pci_dev(fdev->dev);732 733 pci_set_drvdata(pdev, NULL);734 735 if (fdev->fw_handle != FUN_HCI_ID_INVALID) {736 fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,737 fdev->fw_handle);738 fdev->fw_handle = FUN_HCI_ID_INVALID;739 }740 741 fun_disable_admin_queue(fdev);742 743 bitmap_free(fdev->irq_map);744 pci_free_irq_vectors(pdev);745 746 pci_disable_device(pdev);747 748 fun_unmap_bars(fdev);749}750EXPORT_SYMBOL(fun_dev_disable);751 752/* Perform basic initialization of a device, including753 * - PCI config space setup and BAR0 mapping754 * - interrupt management initialization755 * - 1 admin queue setup756 * - determination of some device limits, such as number of queues.757 */758int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,759 const struct fun_dev_params *areq, const char *name)760{761 int rc;762 763 fdev->dev = &pdev->dev;764 rc = fun_map_bars(fdev, name);765 if (rc)766 return rc;767 768 rc = fun_set_dma_masks(fdev->dev);769 if (rc)770 goto unmap;771 772 rc = pci_enable_device_mem(pdev);773 if (rc) {774 dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);775 goto unmap;776 }777 778 rc = sanitize_dev(fdev);779 if (rc)780 goto disable_dev;781 782 fdev->fw_handle = FUN_HCI_ID_INVALID;783 fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;784 fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);785 fdev->dbs = fdev->bar + NVME_REG_DBS;786 787 INIT_WORK(&fdev->service_task, fun_serv_handler);788 fdev->service_flags = FUN_SERV_DISABLED;789 fdev->serv_cb = areq->serv_cb;790 791 rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */792 if (rc < 0)793 goto disable_dev;794 fdev->num_irqs = rc;795 796 rc = fun_alloc_irq_mgr(fdev);797 if (rc)798 goto free_irqs;799 800 pci_set_master(pdev);801 rc = fun_enable_admin_queue(fdev, areq);802 if (rc)803 goto free_irq_mgr;804 805 rc = fun_get_dev_limits(fdev);806 if (rc < 0)807 goto disable_admin;808 809 pci_save_state(pdev);810 pci_set_drvdata(pdev, fdev);811 pcie_print_link_status(pdev);812 dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",813 fdev->q_depth, fdev->db_stride, fdev->max_qid,814 fdev->kern_end_qid);815 return 0;816 817disable_admin:818 fun_disable_admin_queue(fdev);819free_irq_mgr:820 bitmap_free(fdev->irq_map);821free_irqs:822 pci_free_irq_vectors(pdev);823disable_dev:824 pci_disable_device(pdev);825unmap:826 fun_unmap_bars(fdev);827 return rc;828}829EXPORT_SYMBOL(fun_dev_enable);830 831MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");832MODULE_DESCRIPTION("Core services driver for Fungible devices");833MODULE_LICENSE("Dual BSD/GPL");834