576 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2016 Cavium, Inc.4 */5 6#include "cptvf.h"7#include "cptvf_algs.h"8#include "request_manager.h"9 10/**11 * get_free_pending_entry - get free entry from pending queue12 * @q: pending queue13 * @qlen: queue length14 */15static struct pending_entry *get_free_pending_entry(struct pending_queue *q,16 int qlen)17{18 struct pending_entry *ent = NULL;19 20 ent = &q->head[q->rear];21 if (unlikely(ent->busy)) {22 ent = NULL;23 goto no_free_entry;24 }25 26 q->rear++;27 if (unlikely(q->rear == qlen))28 q->rear = 0;29 30no_free_entry:31 return ent;32}33 34static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,35 int qno)36{37 struct pending_queue *queue = &pqinfo->queue[qno];38 39 queue->front++;40 if (unlikely(queue->front == pqinfo->qlen))41 queue->front = 0;42}43 44static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,45 int buf_count, u8 *buffer)46{47 int ret = 0, i, j;48 int components;49 struct sglist_component *sg_ptr = NULL;50 struct pci_dev *pdev = cptvf->pdev;51 52 if (unlikely(!list)) {53 dev_err(&pdev->dev, "Input List pointer is NULL\n");54 return -EFAULT;55 }56 57 for (i = 0; i < buf_count; i++) {58 if (likely(list[i].vptr)) {59 list[i].dma_addr = dma_map_single(&pdev->dev,60 list[i].vptr,61 list[i].size,62 DMA_BIDIRECTIONAL);63 if (unlikely(dma_mapping_error(&pdev->dev,64 list[i].dma_addr))) {65 dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",66 i);67 ret = -EIO;68 goto sg_cleanup;69 }70 }71 }72 73 components = buf_count / 4;74 sg_ptr = (struct sglist_component *)buffer;75 for (i = 0; i < components; i++) {76 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);77 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);78 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);79 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);80 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);81 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);82 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);83 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);84 sg_ptr++;85 }86 87 components = buf_count % 4;88 89 switch (components) {90 case 3:91 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);92 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);93 fallthrough;94 case 2:95 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);96 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);97 fallthrough;98 case 1:99 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);100 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);101 break;102 default:103 break;104 }105 106 return ret;107 108sg_cleanup:109 for (j = 0; j < i; j++) {110 if (list[j].dma_addr) {111 dma_unmap_single(&pdev->dev, list[i].dma_addr,112 list[i].size, DMA_BIDIRECTIONAL);113 }114 115 list[j].dma_addr = 0;116 }117 118 return ret;119}120 121static inline int setup_sgio_list(struct cpt_vf *cptvf,122 struct cpt_info_buffer *info,123 struct cpt_request_info *req)124{125 u16 g_sz_bytes = 0, s_sz_bytes = 0;126 int ret = 0;127 struct pci_dev *pdev = cptvf->pdev;128 129 if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {130 dev_err(&pdev->dev, "Request SG components are higher than supported\n");131 ret = -EINVAL;132 goto scatter_gather_clean;133 }134 135 /* Setup gather (input) components */136 g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);137 info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);138 if (!info->gather_components) {139 ret = -ENOMEM;140 goto scatter_gather_clean;141 }142 143 ret = setup_sgio_components(cptvf, req->in,144 req->incnt,145 info->gather_components);146 if (ret) {147 dev_err(&pdev->dev, "Failed to setup gather list\n");148 ret = -EFAULT;149 goto scatter_gather_clean;150 }151 152 /* Setup scatter (output) components */153 s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);154 info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);155 if (!info->scatter_components) {156 ret = -ENOMEM;157 goto scatter_gather_clean;158 }159 160 ret = setup_sgio_components(cptvf, req->out,161 req->outcnt,162 info->scatter_components);163 if (ret) {164 dev_err(&pdev->dev, "Failed to setup gather list\n");165 ret = -EFAULT;166 goto scatter_gather_clean;167 }168 169 /* Create and initialize DPTR */170 info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;171 info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);172 if (!info->in_buffer) {173 ret = -ENOMEM;174 goto scatter_gather_clean;175 }176 177 ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);178 ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);179 ((__be16 *)info->in_buffer)[2] = 0;180 ((__be16 *)info->in_buffer)[3] = 0;181 182 memcpy(&info->in_buffer[8], info->gather_components,183 g_sz_bytes);184 memcpy(&info->in_buffer[8 + g_sz_bytes],185 info->scatter_components, s_sz_bytes);186 187 info->dptr_baddr = dma_map_single(&pdev->dev,188 (void *)info->in_buffer,189 info->dlen,190 DMA_BIDIRECTIONAL);191 if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {192 dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);193 ret = -EIO;194 goto scatter_gather_clean;195 }196 197 /* Create and initialize RPTR */198 info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);199 if (!info->out_buffer) {200 ret = -ENOMEM;201 goto scatter_gather_clean;202 }203 204 *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);205 info->alternate_caddr = (u64 *)info->out_buffer;206 info->rptr_baddr = dma_map_single(&pdev->dev,207 (void *)info->out_buffer,208 COMPLETION_CODE_SIZE,209 DMA_BIDIRECTIONAL);210 if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {211 dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",212 COMPLETION_CODE_SIZE);213 ret = -EIO;214 goto scatter_gather_clean;215 }216 217 return 0;218 219scatter_gather_clean:220 return ret;221}222 223static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,224 u32 qno)225{226 struct pci_dev *pdev = cptvf->pdev;227 struct command_qinfo *qinfo = NULL;228 struct command_queue *queue;229 struct command_chunk *chunk;230 u8 *ent;231 int ret = 0;232 233 if (unlikely(qno >= cptvf->nr_queues)) {234 dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",235 qno, cptvf->nr_queues);236 return -EINVAL;237 }238 239 qinfo = &cptvf->cqinfo;240 queue = &qinfo->queue[qno];241 /* lock commad queue */242 spin_lock(&queue->lock);243 ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];244 memcpy(ent, (void *)cmd, qinfo->cmd_size);245 246 if (++queue->idx >= queue->qhead->size / 64) {247 hlist_for_each_entry(chunk, &queue->chead, nextchunk) {248 if (chunk == queue->qhead) {249 continue;250 } else {251 queue->qhead = chunk;252 break;253 }254 }255 queue->idx = 0;256 }257 /* make sure all memory stores are done before ringing doorbell */258 smp_wmb();259 cptvf_write_vq_doorbell(cptvf, 1);260 /* unlock command queue */261 spin_unlock(&queue->lock);262 263 return ret;264}265 266static void do_request_cleanup(struct cpt_vf *cptvf,267 struct cpt_info_buffer *info)268{269 int i;270 struct pci_dev *pdev = cptvf->pdev;271 struct cpt_request_info *req;272 273 if (info->dptr_baddr)274 dma_unmap_single(&pdev->dev, info->dptr_baddr,275 info->dlen, DMA_BIDIRECTIONAL);276 277 if (info->rptr_baddr)278 dma_unmap_single(&pdev->dev, info->rptr_baddr,279 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);280 281 if (info->comp_baddr)282 dma_unmap_single(&pdev->dev, info->comp_baddr,283 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);284 285 if (info->req) {286 req = info->req;287 for (i = 0; i < req->outcnt; i++) {288 if (req->out[i].dma_addr)289 dma_unmap_single(&pdev->dev,290 req->out[i].dma_addr,291 req->out[i].size,292 DMA_BIDIRECTIONAL);293 }294 295 for (i = 0; i < req->incnt; i++) {296 if (req->in[i].dma_addr)297 dma_unmap_single(&pdev->dev,298 req->in[i].dma_addr,299 req->in[i].size,300 DMA_BIDIRECTIONAL);301 }302 }303 304 kfree_sensitive(info->scatter_components);305 kfree_sensitive(info->gather_components);306 kfree_sensitive(info->out_buffer);307 kfree_sensitive(info->in_buffer);308 kfree_sensitive((void *)info->completion_addr);309 kfree_sensitive(info);310}311 312static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)313{314 struct pci_dev *pdev = cptvf->pdev;315 316 if (!info) {317 dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");318 return;319 }320 321 do_request_cleanup(cptvf, info);322}323 324static inline void process_pending_queue(struct cpt_vf *cptvf,325 struct pending_qinfo *pqinfo,326 int qno)327{328 struct pci_dev *pdev = cptvf->pdev;329 struct pending_queue *pqueue = &pqinfo->queue[qno];330 struct pending_entry *pentry = NULL;331 struct cpt_info_buffer *info = NULL;332 union cpt_res_s *status = NULL;333 unsigned char ccode;334 335 while (1) {336 spin_lock_bh(&pqueue->lock);337 pentry = &pqueue->head[pqueue->front];338 if (unlikely(!pentry->busy)) {339 spin_unlock_bh(&pqueue->lock);340 break;341 }342 343 info = (struct cpt_info_buffer *)pentry->post_arg;344 if (unlikely(!info)) {345 dev_err(&pdev->dev, "Pending Entry post arg NULL\n");346 pending_queue_inc_front(pqinfo, qno);347 spin_unlock_bh(&pqueue->lock);348 continue;349 }350 351 status = (union cpt_res_s *)pentry->completion_addr;352 ccode = status->s.compcode;353 if ((status->s.compcode == CPT_COMP_E_FAULT) ||354 (status->s.compcode == CPT_COMP_E_SWERR)) {355 dev_err(&pdev->dev, "Request failed with %s\n",356 (status->s.compcode == CPT_COMP_E_FAULT) ?357 "DMA Fault" : "Software error");358 pentry->completion_addr = NULL;359 pentry->busy = false;360 atomic64_dec((&pqueue->pending_count));361 pentry->post_arg = NULL;362 pending_queue_inc_front(pqinfo, qno);363 do_request_cleanup(cptvf, info);364 spin_unlock_bh(&pqueue->lock);365 break;366 } else if (status->s.compcode == COMPLETION_CODE_INIT) {367 /* check for timeout */368 if (time_after_eq(jiffies,369 (info->time_in +370 (CPT_COMMAND_TIMEOUT * HZ)))) {371 dev_err(&pdev->dev, "Request timed out");372 pentry->completion_addr = NULL;373 pentry->busy = false;374 atomic64_dec((&pqueue->pending_count));375 pentry->post_arg = NULL;376 pending_queue_inc_front(pqinfo, qno);377 do_request_cleanup(cptvf, info);378 spin_unlock_bh(&pqueue->lock);379 break;380 } else if ((*info->alternate_caddr ==381 (~COMPLETION_CODE_INIT)) &&382 (info->extra_time < TIME_IN_RESET_COUNT)) {383 info->time_in = jiffies;384 info->extra_time++;385 spin_unlock_bh(&pqueue->lock);386 break;387 }388 }389 390 pentry->completion_addr = NULL;391 pentry->busy = false;392 pentry->post_arg = NULL;393 atomic64_dec((&pqueue->pending_count));394 pending_queue_inc_front(pqinfo, qno);395 spin_unlock_bh(&pqueue->lock);396 397 do_post_process(info->cptvf, info);398 /*399 * Calling callback after we find400 * that the request has been serviced401 */402 pentry->callback(ccode, pentry->callback_arg);403 }404}405 406int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)407{408 int ret = 0, clear = 0, queue = 0;409 struct cpt_info_buffer *info = NULL;410 struct cptvf_request *cpt_req = NULL;411 union ctrl_info *ctrl = NULL;412 union cpt_res_s *result = NULL;413 struct pending_entry *pentry = NULL;414 struct pending_queue *pqueue = NULL;415 struct pci_dev *pdev = cptvf->pdev;416 u8 group = 0;417 struct cpt_vq_command vq_cmd;418 union cpt_inst_s cptinst;419 420 info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);421 if (unlikely(!info)) {422 dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");423 return -ENOMEM;424 }425 426 cpt_req = (struct cptvf_request *)&req->req;427 ctrl = (union ctrl_info *)&req->ctrl;428 429 info->cptvf = cptvf;430 group = ctrl->s.grp;431 ret = setup_sgio_list(cptvf, info, req);432 if (ret) {433 dev_err(&pdev->dev, "Setting up SG list failed");434 goto request_cleanup;435 }436 437 cpt_req->dlen = info->dlen;438 /*439 * Get buffer for union cpt_res_s response440 * structure and its physical address441 */442 info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);443 if (unlikely(!info->completion_addr)) {444 dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");445 ret = -ENOMEM;446 goto request_cleanup;447 }448 449 result = (union cpt_res_s *)info->completion_addr;450 result->s.compcode = COMPLETION_CODE_INIT;451 info->comp_baddr = dma_map_single(&pdev->dev,452 (void *)info->completion_addr,453 sizeof(union cpt_res_s),454 DMA_BIDIRECTIONAL);455 if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {456 dev_err(&pdev->dev, "mapping compptr Failed %lu\n",457 sizeof(union cpt_res_s));458 ret = -EFAULT;459 goto request_cleanup;460 }461 462 /* Fill the VQ command */463 vq_cmd.cmd.u64 = 0;464 vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);465 vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);466 vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);467 vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);468 469 vq_cmd.dptr = info->dptr_baddr;470 vq_cmd.rptr = info->rptr_baddr;471 vq_cmd.cptr.u64 = 0;472 vq_cmd.cptr.s.grp = group;473 /* Get Pending Entry to submit command */474 /* Always queue 0, because 1 queue per VF */475 queue = 0;476 pqueue = &cptvf->pqinfo.queue[queue];477 478 if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {479 dev_err(&pdev->dev, "pending threshold reached\n");480 process_pending_queue(cptvf, &cptvf->pqinfo, queue);481 }482 483get_pending_entry:484 spin_lock_bh(&pqueue->lock);485 pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);486 if (unlikely(!pentry)) {487 spin_unlock_bh(&pqueue->lock);488 if (clear == 0) {489 process_pending_queue(cptvf, &cptvf->pqinfo, queue);490 clear = 1;491 goto get_pending_entry;492 }493 dev_err(&pdev->dev, "Get free entry failed\n");494 dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",495 queue, pqueue->rear, pqueue->front);496 ret = -EFAULT;497 goto request_cleanup;498 }499 500 pentry->completion_addr = info->completion_addr;501 pentry->post_arg = (void *)info;502 pentry->callback = req->callback;503 pentry->callback_arg = req->callback_arg;504 info->pentry = pentry;505 pentry->busy = true;506 atomic64_inc(&pqueue->pending_count);507 508 /* Send CPT command */509 info->pentry = pentry;510 info->time_in = jiffies;511 info->req = req;512 513 /* Create the CPT_INST_S type command for HW intrepretation */514 cptinst.s.doneint = true;515 cptinst.s.res_addr = (u64)info->comp_baddr;516 cptinst.s.tag = 0;517 cptinst.s.grp = 0;518 cptinst.s.wq_ptr = 0;519 cptinst.s.ei0 = vq_cmd.cmd.u64;520 cptinst.s.ei1 = vq_cmd.dptr;521 cptinst.s.ei2 = vq_cmd.rptr;522 cptinst.s.ei3 = vq_cmd.cptr.u64;523 524 ret = send_cpt_command(cptvf, &cptinst, queue);525 spin_unlock_bh(&pqueue->lock);526 if (unlikely(ret)) {527 dev_err(&pdev->dev, "Send command failed for AE\n");528 ret = -EFAULT;529 goto request_cleanup;530 }531 532 return 0;533 534request_cleanup:535 dev_dbg(&pdev->dev, "Failed to submit CPT command\n");536 do_request_cleanup(cptvf, info);537 538 return ret;539}540 541void vq_post_process(struct cpt_vf *cptvf, u32 qno)542{543 struct pci_dev *pdev = cptvf->pdev;544 545 if (unlikely(qno > cptvf->nr_queues)) {546 dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",547 qno);548 return;549 }550 551 process_pending_queue(cptvf, &cptvf->pqinfo, qno);552}553 554int cptvf_do_request(void *vfdev, struct cpt_request_info *req)555{556 struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;557 struct pci_dev *pdev = cptvf->pdev;558 559 if (!cpt_device_ready(cptvf)) {560 dev_err(&pdev->dev, "CPT Device is not ready");561 return -ENODEV;562 }563 564 if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {565 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",566 cptvf->vfid);567 return -EINVAL;568 } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {569 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",570 cptvf->vfid);571 return -EINVAL;572 }573 574 return process_request(cptvf, req);575}576