602 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)2 3#include <linux/dma-mapping.h>4#include <linux/interrupt.h>5#include <linux/log2.h>6#include <linux/mm.h>7#include <linux/netdevice.h>8#include <linux/pci.h>9#include <linux/slab.h>10 11#include "fun_dev.h"12#include "fun_queue.h"13 14/* Allocate memory for a queue. This includes the memory for the HW descriptor15 * ring, an optional 64b HW write-back area, and an optional SW state ring.16 * Returns the virtual and DMA addresses of the HW ring, the VA of the SW ring,17 * and the VA of the write-back area.18 */19void *fun_alloc_ring_mem(struct device *dma_dev, size_t depth,20 size_t hw_desc_sz, size_t sw_desc_sz, bool wb,21 int numa_node, dma_addr_t *dma_addr, void **sw_va,22 volatile __be64 **wb_va)23{24 int dev_node = dev_to_node(dma_dev);25 size_t dma_sz;26 void *va;27 28 if (numa_node == NUMA_NO_NODE)29 numa_node = dev_node;30 31 /* Place optional write-back area at end of descriptor ring. */32 dma_sz = hw_desc_sz * depth;33 if (wb)34 dma_sz += sizeof(u64);35 36 set_dev_node(dma_dev, numa_node);37 va = dma_alloc_coherent(dma_dev, dma_sz, dma_addr, GFP_KERNEL);38 set_dev_node(dma_dev, dev_node);39 if (!va)40 return NULL;41 42 if (sw_desc_sz) {43 *sw_va = kvzalloc_node(sw_desc_sz * depth, GFP_KERNEL,44 numa_node);45 if (!*sw_va) {46 dma_free_coherent(dma_dev, dma_sz, va, *dma_addr);47 return NULL;48 }49 }50 51 if (wb)52 *wb_va = va + dma_sz - sizeof(u64);53 return va;54}55EXPORT_SYMBOL_GPL(fun_alloc_ring_mem);56 57void fun_free_ring_mem(struct device *dma_dev, size_t depth, size_t hw_desc_sz,58 bool wb, void *hw_va, dma_addr_t dma_addr, void *sw_va)59{60 if (hw_va) {61 size_t sz = depth * hw_desc_sz;62 63 if (wb)64 sz += sizeof(u64);65 dma_free_coherent(dma_dev, sz, hw_va, dma_addr);66 }67 kvfree(sw_va);68}69EXPORT_SYMBOL_GPL(fun_free_ring_mem);70 71/* Prepare and issue an admin command to create an SQ on the device with the72 * provided parameters. If the queue ID is auto-allocated by the device it is73 * returned in *sqidp.74 */75int fun_sq_create(struct fun_dev *fdev, u16 flags, u32 sqid, u32 cqid,76 u8 sqe_size_log2, u32 sq_depth, dma_addr_t dma_addr,77 u8 coal_nentries, u8 coal_usec, u32 irq_num,78 u32 scan_start_id, u32 scan_end_id,79 u32 rq_buf_size_log2, u32 *sqidp, u32 __iomem **dbp)80{81 union {82 struct fun_admin_epsq_req req;83 struct fun_admin_generic_create_rsp rsp;84 } cmd;85 dma_addr_t wb_addr;86 u32 hw_qid;87 int rc;88 89 if (sq_depth > fdev->q_depth)90 return -EINVAL;91 if (flags & FUN_ADMIN_EPSQ_CREATE_FLAG_RQ)92 sqe_size_log2 = ilog2(sizeof(struct fun_eprq_rqbuf));93 94 wb_addr = dma_addr + (sq_depth << sqe_size_log2);95 96 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPSQ,97 sizeof(cmd.req));98 cmd.req.u.create =99 FUN_ADMIN_EPSQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,100 sqid, cqid, sqe_size_log2,101 sq_depth - 1, dma_addr, 0,102 coal_nentries, coal_usec,103 irq_num, scan_start_id,104 scan_end_id, 0,105 rq_buf_size_log2,106 ilog2(sizeof(u64)), wb_addr);107 108 rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,109 &cmd.rsp, sizeof(cmd.rsp), 0);110 if (rc)111 return rc;112 113 hw_qid = be32_to_cpu(cmd.rsp.id);114 *dbp = fun_sq_db_addr(fdev, hw_qid);115 if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)116 *sqidp = hw_qid;117 return rc;118}119EXPORT_SYMBOL_GPL(fun_sq_create);120 121/* Prepare and issue an admin command to create a CQ on the device with the122 * provided parameters. If the queue ID is auto-allocated by the device it is123 * returned in *cqidp.124 */125int fun_cq_create(struct fun_dev *fdev, u16 flags, u32 cqid, u32 rqid,126 u8 cqe_size_log2, u32 cq_depth, dma_addr_t dma_addr,127 u16 headroom, u16 tailroom, u8 coal_nentries, u8 coal_usec,128 u32 irq_num, u32 scan_start_id, u32 scan_end_id, u32 *cqidp,129 u32 __iomem **dbp)130{131 union {132 struct fun_admin_epcq_req req;133 struct fun_admin_generic_create_rsp rsp;134 } cmd;135 u32 hw_qid;136 int rc;137 138 if (cq_depth > fdev->q_depth)139 return -EINVAL;140 141 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPCQ,142 sizeof(cmd.req));143 cmd.req.u.create =144 FUN_ADMIN_EPCQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,145 cqid, rqid, cqe_size_log2,146 cq_depth - 1, dma_addr, tailroom,147 headroom / 2, 0, coal_nentries,148 coal_usec, irq_num,149 scan_start_id, scan_end_id, 0);150 151 rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,152 &cmd.rsp, sizeof(cmd.rsp), 0);153 if (rc)154 return rc;155 156 hw_qid = be32_to_cpu(cmd.rsp.id);157 *dbp = fun_cq_db_addr(fdev, hw_qid);158 if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)159 *cqidp = hw_qid;160 return rc;161}162EXPORT_SYMBOL_GPL(fun_cq_create);163 164static bool fun_sq_is_head_wb(const struct fun_queue *funq)165{166 return funq->sq_flags & FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS;167}168 169static void fun_clean_rq(struct fun_queue *funq)170{171 struct fun_dev *fdev = funq->fdev;172 struct fun_rq_info *rqinfo;173 unsigned int i;174 175 for (i = 0; i < funq->rq_depth; i++) {176 rqinfo = &funq->rq_info[i];177 if (rqinfo->page) {178 dma_unmap_page(fdev->dev, rqinfo->dma, PAGE_SIZE,179 DMA_FROM_DEVICE);180 put_page(rqinfo->page);181 rqinfo->page = NULL;182 }183 }184}185 186static int fun_fill_rq(struct fun_queue *funq)187{188 struct device *dev = funq->fdev->dev;189 int i, node = dev_to_node(dev);190 struct fun_rq_info *rqinfo;191 192 for (i = 0; i < funq->rq_depth; i++) {193 rqinfo = &funq->rq_info[i];194 rqinfo->page = alloc_pages_node(node, GFP_KERNEL, 0);195 if (unlikely(!rqinfo->page))196 return -ENOMEM;197 198 rqinfo->dma = dma_map_page(dev, rqinfo->page, 0,199 PAGE_SIZE, DMA_FROM_DEVICE);200 if (unlikely(dma_mapping_error(dev, rqinfo->dma))) {201 put_page(rqinfo->page);202 rqinfo->page = NULL;203 return -ENOMEM;204 }205 206 funq->rqes[i] = FUN_EPRQ_RQBUF_INIT(rqinfo->dma);207 }208 209 funq->rq_tail = funq->rq_depth - 1;210 return 0;211}212 213static void fun_rq_update_pos(struct fun_queue *funq, int buf_offset)214{215 if (buf_offset <= funq->rq_buf_offset) {216 struct fun_rq_info *rqinfo = &funq->rq_info[funq->rq_buf_idx];217 struct device *dev = funq->fdev->dev;218 219 dma_sync_single_for_device(dev, rqinfo->dma, PAGE_SIZE,220 DMA_FROM_DEVICE);221 funq->num_rqe_to_fill++;222 if (++funq->rq_buf_idx == funq->rq_depth)223 funq->rq_buf_idx = 0;224 }225 funq->rq_buf_offset = buf_offset;226}227 228/* Given a command response with data scattered across >= 1 RQ buffers return229 * a pointer to a contiguous buffer containing all the data. If the data is in230 * one RQ buffer the start address within that buffer is returned, otherwise a231 * new buffer is allocated and the data is gathered into it.232 */233static void *fun_data_from_rq(struct fun_queue *funq,234 const struct fun_rsp_common *rsp, bool *need_free)235{236 u32 bufoff, total_len, remaining, fragsize, dataoff;237 struct device *dma_dev = funq->fdev->dev;238 const struct fun_dataop_rqbuf *databuf;239 const struct fun_dataop_hdr *dataop;240 const struct fun_rq_info *rqinfo;241 void *data;242 243 dataop = (void *)rsp + rsp->suboff8 * 8;244 total_len = be32_to_cpu(dataop->total_len);245 246 if (likely(dataop->nsgl == 1)) {247 databuf = (struct fun_dataop_rqbuf *)dataop->imm;248 bufoff = be32_to_cpu(databuf->bufoff);249 fun_rq_update_pos(funq, bufoff);250 rqinfo = &funq->rq_info[funq->rq_buf_idx];251 dma_sync_single_for_cpu(dma_dev, rqinfo->dma + bufoff,252 total_len, DMA_FROM_DEVICE);253 *need_free = false;254 return page_address(rqinfo->page) + bufoff;255 }256 257 /* For scattered completions gather the fragments into one buffer. */258 259 data = kmalloc(total_len, GFP_ATOMIC);260 /* NULL is OK here. In case of failure we still need to consume the data261 * for proper buffer accounting but indicate an error in the response.262 */263 if (likely(data))264 *need_free = true;265 266 dataoff = 0;267 for (remaining = total_len; remaining; remaining -= fragsize) {268 fun_rq_update_pos(funq, 0);269 fragsize = min_t(unsigned int, PAGE_SIZE, remaining);270 if (data) {271 rqinfo = &funq->rq_info[funq->rq_buf_idx];272 dma_sync_single_for_cpu(dma_dev, rqinfo->dma, fragsize,273 DMA_FROM_DEVICE);274 memcpy(data + dataoff, page_address(rqinfo->page),275 fragsize);276 dataoff += fragsize;277 }278 }279 return data;280}281 282unsigned int __fun_process_cq(struct fun_queue *funq, unsigned int max)283{284 const struct fun_cqe_info *info;285 struct fun_rsp_common *rsp;286 unsigned int new_cqes;287 u16 sf_p, flags;288 bool need_free;289 void *cqe;290 291 if (!max)292 max = funq->cq_depth - 1;293 294 for (new_cqes = 0; new_cqes < max; new_cqes++) {295 cqe = funq->cqes + (funq->cq_head << funq->cqe_size_log2);296 info = funq_cqe_info(funq, cqe);297 sf_p = be16_to_cpu(info->sf_p);298 299 if ((sf_p & 1) != funq->cq_phase)300 break;301 302 /* ensure the phase tag is read before other CQE fields */303 dma_rmb();304 305 if (++funq->cq_head == funq->cq_depth) {306 funq->cq_head = 0;307 funq->cq_phase = !funq->cq_phase;308 }309 310 rsp = cqe;311 flags = be16_to_cpu(rsp->flags);312 313 need_free = false;314 if (unlikely(flags & FUN_REQ_COMMON_FLAG_CQE_IN_RQBUF)) {315 rsp = fun_data_from_rq(funq, rsp, &need_free);316 if (!rsp) {317 rsp = cqe;318 rsp->len8 = 1;319 if (rsp->ret == 0)320 rsp->ret = ENOMEM;321 }322 }323 324 if (funq->cq_cb)325 funq->cq_cb(funq, funq->cb_data, rsp, info);326 if (need_free)327 kfree(rsp);328 }329 330 dev_dbg(funq->fdev->dev, "CQ %u, new CQEs %u/%u, head %u, phase %u\n",331 funq->cqid, new_cqes, max, funq->cq_head, funq->cq_phase);332 return new_cqes;333}334 335unsigned int fun_process_cq(struct fun_queue *funq, unsigned int max)336{337 unsigned int processed;338 u32 db;339 340 processed = __fun_process_cq(funq, max);341 342 if (funq->num_rqe_to_fill) {343 funq->rq_tail = (funq->rq_tail + funq->num_rqe_to_fill) %344 funq->rq_depth;345 funq->num_rqe_to_fill = 0;346 writel(funq->rq_tail, funq->rq_db);347 }348 349 db = funq->cq_head | FUN_DB_IRQ_ARM_F;350 writel(db, funq->cq_db);351 return processed;352}353 354static int fun_alloc_sqes(struct fun_queue *funq)355{356 funq->sq_cmds = fun_alloc_ring_mem(funq->fdev->dev, funq->sq_depth,357 1 << funq->sqe_size_log2, 0,358 fun_sq_is_head_wb(funq),359 NUMA_NO_NODE, &funq->sq_dma_addr,360 NULL, &funq->sq_head);361 return funq->sq_cmds ? 0 : -ENOMEM;362}363 364static int fun_alloc_cqes(struct fun_queue *funq)365{366 funq->cqes = fun_alloc_ring_mem(funq->fdev->dev, funq->cq_depth,367 1 << funq->cqe_size_log2, 0, false,368 NUMA_NO_NODE, &funq->cq_dma_addr, NULL,369 NULL);370 return funq->cqes ? 0 : -ENOMEM;371}372 373static int fun_alloc_rqes(struct fun_queue *funq)374{375 funq->rqes = fun_alloc_ring_mem(funq->fdev->dev, funq->rq_depth,376 sizeof(*funq->rqes),377 sizeof(*funq->rq_info), false,378 NUMA_NO_NODE, &funq->rq_dma_addr,379 (void **)&funq->rq_info, NULL);380 return funq->rqes ? 0 : -ENOMEM;381}382 383/* Free a queue's structures. */384void fun_free_queue(struct fun_queue *funq)385{386 struct device *dev = funq->fdev->dev;387 388 fun_free_ring_mem(dev, funq->cq_depth, 1 << funq->cqe_size_log2, false,389 funq->cqes, funq->cq_dma_addr, NULL);390 fun_free_ring_mem(dev, funq->sq_depth, 1 << funq->sqe_size_log2,391 fun_sq_is_head_wb(funq), funq->sq_cmds,392 funq->sq_dma_addr, NULL);393 394 if (funq->rqes) {395 fun_clean_rq(funq);396 fun_free_ring_mem(dev, funq->rq_depth, sizeof(*funq->rqes),397 false, funq->rqes, funq->rq_dma_addr,398 funq->rq_info);399 }400 401 kfree(funq);402}403 404/* Allocate and initialize a funq's structures. */405struct fun_queue *fun_alloc_queue(struct fun_dev *fdev, int qid,406 const struct fun_queue_alloc_req *req)407{408 struct fun_queue *funq = kzalloc(sizeof(*funq), GFP_KERNEL);409 410 if (!funq)411 return NULL;412 413 funq->fdev = fdev;414 spin_lock_init(&funq->sq_lock);415 416 funq->qid = qid;417 418 /* Initial CQ/SQ/RQ ids */419 if (req->rq_depth) {420 funq->cqid = 2 * qid;421 if (funq->qid) {422 /* I/O Q: use rqid = cqid, sqid = +1 */423 funq->rqid = funq->cqid;424 funq->sqid = funq->rqid + 1;425 } else {426 /* Admin Q: sqid is always 0, use ID 1 for RQ */427 funq->sqid = 0;428 funq->rqid = 1;429 }430 } else {431 funq->cqid = qid;432 funq->sqid = qid;433 }434 435 funq->cq_flags = req->cq_flags;436 funq->sq_flags = req->sq_flags;437 438 funq->cqe_size_log2 = req->cqe_size_log2;439 funq->sqe_size_log2 = req->sqe_size_log2;440 441 funq->cq_depth = req->cq_depth;442 funq->sq_depth = req->sq_depth;443 444 funq->cq_intcoal_nentries = req->cq_intcoal_nentries;445 funq->cq_intcoal_usec = req->cq_intcoal_usec;446 447 funq->sq_intcoal_nentries = req->sq_intcoal_nentries;448 funq->sq_intcoal_usec = req->sq_intcoal_usec;449 450 if (fun_alloc_cqes(funq))451 goto free_funq;452 453 funq->cq_phase = 1;454 455 if (fun_alloc_sqes(funq))456 goto free_funq;457 458 if (req->rq_depth) {459 funq->rq_flags = req->rq_flags | FUN_ADMIN_EPSQ_CREATE_FLAG_RQ;460 funq->rq_depth = req->rq_depth;461 funq->rq_buf_offset = -1;462 463 if (fun_alloc_rqes(funq) || fun_fill_rq(funq))464 goto free_funq;465 }466 467 funq->cq_vector = -1;468 funq->cqe_info_offset = (1 << funq->cqe_size_log2) - sizeof(struct fun_cqe_info);469 470 /* SQ/CQ 0 are implicitly created, assign their doorbells now.471 * Other queues are assigned doorbells at their explicit creation.472 */473 if (funq->sqid == 0)474 funq->sq_db = fun_sq_db_addr(fdev, 0);475 if (funq->cqid == 0)476 funq->cq_db = fun_cq_db_addr(fdev, 0);477 478 return funq;479 480free_funq:481 fun_free_queue(funq);482 return NULL;483}484 485/* Create a funq's CQ on the device. */486static int fun_create_cq(struct fun_queue *funq)487{488 struct fun_dev *fdev = funq->fdev;489 unsigned int rqid;490 int rc;491 492 rqid = funq->cq_flags & FUN_ADMIN_EPCQ_CREATE_FLAG_RQ ?493 funq->rqid : FUN_HCI_ID_INVALID;494 rc = fun_cq_create(fdev, funq->cq_flags, funq->cqid, rqid,495 funq->cqe_size_log2, funq->cq_depth,496 funq->cq_dma_addr, 0, 0, funq->cq_intcoal_nentries,497 funq->cq_intcoal_usec, funq->cq_vector, 0, 0,498 &funq->cqid, &funq->cq_db);499 if (!rc)500 dev_dbg(fdev->dev, "created CQ %u\n", funq->cqid);501 502 return rc;503}504 505/* Create a funq's SQ on the device. */506static int fun_create_sq(struct fun_queue *funq)507{508 struct fun_dev *fdev = funq->fdev;509 int rc;510 511 rc = fun_sq_create(fdev, funq->sq_flags, funq->sqid, funq->cqid,512 funq->sqe_size_log2, funq->sq_depth,513 funq->sq_dma_addr, funq->sq_intcoal_nentries,514 funq->sq_intcoal_usec, funq->cq_vector, 0, 0,515 0, &funq->sqid, &funq->sq_db);516 if (!rc)517 dev_dbg(fdev->dev, "created SQ %u\n", funq->sqid);518 519 return rc;520}521 522/* Create a funq's RQ on the device. */523int fun_create_rq(struct fun_queue *funq)524{525 struct fun_dev *fdev = funq->fdev;526 int rc;527 528 rc = fun_sq_create(fdev, funq->rq_flags, funq->rqid, funq->cqid, 0,529 funq->rq_depth, funq->rq_dma_addr, 0, 0,530 funq->cq_vector, 0, 0, PAGE_SHIFT, &funq->rqid,531 &funq->rq_db);532 if (!rc)533 dev_dbg(fdev->dev, "created RQ %u\n", funq->rqid);534 535 return rc;536}537 538static unsigned int funq_irq(struct fun_queue *funq)539{540 return pci_irq_vector(to_pci_dev(funq->fdev->dev), funq->cq_vector);541}542 543int fun_request_irq(struct fun_queue *funq, const char *devname,544 irq_handler_t handler, void *data)545{546 int rc;547 548 if (funq->cq_vector < 0)549 return -EINVAL;550 551 funq->irq_handler = handler;552 funq->irq_data = data;553 554 snprintf(funq->irqname, sizeof(funq->irqname),555 funq->qid ? "%s-q[%d]" : "%s-adminq", devname, funq->qid);556 557 rc = request_irq(funq_irq(funq), handler, 0, funq->irqname, data);558 if (rc)559 funq->irq_handler = NULL;560 561 return rc;562}563 564/* Create all component queues of a funq on the device. */565int fun_create_queue(struct fun_queue *funq)566{567 int rc;568 569 rc = fun_create_cq(funq);570 if (rc)571 return rc;572 573 if (funq->rq_depth) {574 rc = fun_create_rq(funq);575 if (rc)576 goto release_cq;577 }578 579 rc = fun_create_sq(funq);580 if (rc)581 goto release_rq;582 583 return 0;584 585release_rq:586 fun_destroy_sq(funq->fdev, funq->rqid);587release_cq:588 fun_destroy_cq(funq->fdev, funq->cqid);589 return rc;590}591 592void fun_free_irq(struct fun_queue *funq)593{594 if (funq->irq_handler) {595 unsigned int vector = funq_irq(funq);596 597 free_irq(vector, funq->irq_data);598 funq->irq_handler = NULL;599 funq->irq_data = NULL;600 }601}602