399 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (C) 2020 Marvell. */3 4#include "otx2_cptvf.h"5#include "otx2_cpt_common.h"6 7/* Default timeout when waiting for free pending entry in us */8#define CPT_PENTRY_TIMEOUT 10009#define CPT_PENTRY_STEP 5010 11/* Default threshold for stopping and resuming sender requests */12#define CPT_IQ_STOP_MARGIN 12813#define CPT_IQ_RESUME_MARGIN 51214 15/* Default command timeout in seconds */16#define CPT_COMMAND_TIMEOUT 417#define CPT_TIME_IN_RESET_COUNT 518 19static void otx2_cpt_dump_sg_list(struct pci_dev *pdev,20 struct otx2_cpt_req_info *req)21{22 int i;23 24 pr_debug("Gather list size %d\n", req->in_cnt);25 for (i = 0; i < req->in_cnt; i++) {26 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i,27 req->in[i].size, req->in[i].vptr,28 req->in[i].dma_addr);29 pr_debug("Buffer hexdump (%d bytes)\n",30 req->in[i].size);31 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,32 req->in[i].vptr, req->in[i].size, false);33 }34 pr_debug("Scatter list size %d\n", req->out_cnt);35 for (i = 0; i < req->out_cnt; i++) {36 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i,37 req->out[i].size, req->out[i].vptr,38 req->out[i].dma_addr);39 pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);40 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,41 req->out[i].vptr, req->out[i].size, false);42 }43}44 45static inline struct otx2_cpt_pending_entry *get_free_pending_entry(46 struct otx2_cpt_pending_queue *q,47 int qlen)48{49 struct otx2_cpt_pending_entry *ent = NULL;50 51 ent = &q->head[q->rear];52 if (unlikely(ent->busy))53 return NULL;54 55 q->rear++;56 if (unlikely(q->rear == qlen))57 q->rear = 0;58 59 return ent;60}61 62static inline u32 modulo_inc(u32 index, u32 length, u32 inc)63{64 if (WARN_ON(inc > length))65 inc = length;66 67 index += inc;68 if (unlikely(index >= length))69 index -= length;70 71 return index;72}73 74static inline void free_pentry(struct otx2_cpt_pending_entry *pentry)75{76 pentry->completion_addr = NULL;77 pentry->info = NULL;78 pentry->callback = NULL;79 pentry->areq = NULL;80 pentry->resume_sender = false;81 pentry->busy = false;82}83 84static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,85 struct otx2_cpt_pending_queue *pqueue,86 struct otx2_cptlf_info *lf)87{88 struct otx2_cptvf_request *cpt_req = &req->req;89 struct otx2_cpt_pending_entry *pentry = NULL;90 union otx2_cpt_ctrl_info *ctrl = &req->ctrl;91 struct otx2_cpt_inst_info *info = NULL;92 union otx2_cpt_res_s *result = NULL;93 struct otx2_cpt_iq_command iq_cmd;94 union otx2_cpt_inst_s cptinst;95 int retry, ret = 0;96 u8 resume_sender;97 gfp_t gfp;98 99 gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :100 GFP_ATOMIC;101 if (unlikely(!otx2_cptlf_started(lf->lfs)))102 return -ENODEV;103 104 info = lf->lfs->ops->cpt_sg_info_create(pdev, req, gfp);105 if (unlikely(!info)) {106 dev_err(&pdev->dev, "Setting up cpt inst info failed");107 return -ENOMEM;108 }109 cpt_req->dlen = info->dlen;110 111 result = info->completion_addr;112 result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT;113 114 spin_lock_bh(&pqueue->lock);115 pentry = get_free_pending_entry(pqueue, pqueue->qlen);116 retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;117 while (unlikely(!pentry) && retry--) {118 spin_unlock_bh(&pqueue->lock);119 udelay(CPT_PENTRY_STEP);120 spin_lock_bh(&pqueue->lock);121 pentry = get_free_pending_entry(pqueue, pqueue->qlen);122 }123 124 if (unlikely(!pentry)) {125 ret = -ENOSPC;126 goto destroy_info;127 }128 129 /*130 * Check if we are close to filling in entire pending queue,131 * if so then tell the sender to stop/sleep by returning -EBUSY132 * We do it only for context which can sleep (GFP_KERNEL)133 */134 if (gfp == GFP_KERNEL &&135 pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {136 pentry->resume_sender = true;137 } else138 pentry->resume_sender = false;139 resume_sender = pentry->resume_sender;140 pqueue->pending_count++;141 142 pentry->completion_addr = info->completion_addr;143 pentry->info = info;144 pentry->callback = req->callback;145 pentry->areq = req->areq;146 pentry->busy = true;147 info->pentry = pentry;148 info->time_in = jiffies;149 info->req = req;150 151 /* Fill in the command */152 iq_cmd.cmd.u = 0;153 iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);154 iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);155 iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);156 iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);157 158 /* 64-bit swap for microcode data reads, not needed for addresses*/159 cpu_to_be64s(&iq_cmd.cmd.u);160 iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60;161 iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60;162 iq_cmd.cptr.s.cptr = cpt_req->cptr_dma;163 iq_cmd.cptr.s.grp = ctrl->s.grp;164 165 /* Fill in the CPT_INST_S type command for HW interpretation */166 otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr);167 168 /* Print debug info if enabled */169 otx2_cpt_dump_sg_list(pdev, req);170 pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE);171 print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false);172 pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);173 print_hex_dump_debug("", 0, 16, 1, info->in_buffer,174 cpt_req->dlen, false);175 176 /* Send CPT command */177 lf->lfs->ops->send_cmd(&cptinst, 1, lf);178 179 /*180 * We allocate and prepare pending queue entry in critical section181 * together with submitting CPT instruction to CPT instruction queue182 * to make sure that order of CPT requests is the same in both183 * pending and instruction queues184 */185 spin_unlock_bh(&pqueue->lock);186 187 ret = resume_sender ? -EBUSY : -EINPROGRESS;188 return ret;189 190destroy_info:191 spin_unlock_bh(&pqueue->lock);192 otx2_cpt_info_destroy(pdev, info);193 return ret;194}195 196int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,197 int cpu_num)198{199 struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);200 struct otx2_cptlfs_info *lfs = &cptvf->lfs;201 202 return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue,203 &lfs->lf[cpu_num]);204}205 206static int cpt_process_ccode(struct otx2_cptlfs_info *lfs,207 union otx2_cpt_res_s *cpt_status,208 struct otx2_cpt_inst_info *info,209 u32 *res_code)210{211 u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status);212 u8 ccode = lfs->ops->cpt_get_compcode(cpt_status);213 struct pci_dev *pdev = lfs->pdev;214 215 switch (ccode) {216 case OTX2_CPT_COMP_E_FAULT:217 dev_err(&pdev->dev,218 "Request failed with DMA fault\n");219 otx2_cpt_dump_sg_list(pdev, info->req);220 break;221 222 case OTX2_CPT_COMP_E_HWERR:223 dev_err(&pdev->dev,224 "Request failed with hardware error\n");225 otx2_cpt_dump_sg_list(pdev, info->req);226 break;227 228 case OTX2_CPT_COMP_E_INSTERR:229 dev_err(&pdev->dev,230 "Request failed with instruction error\n");231 otx2_cpt_dump_sg_list(pdev, info->req);232 break;233 234 case OTX2_CPT_COMP_E_NOTDONE:235 /* check for timeout */236 if (time_after_eq(jiffies, info->time_in +237 CPT_COMMAND_TIMEOUT * HZ))238 dev_warn(&pdev->dev,239 "Request timed out 0x%p", info->req);240 else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) {241 info->time_in = jiffies;242 info->extra_time++;243 }244 return 1;245 246 case OTX2_CPT_COMP_E_GOOD:247 case OTX2_CPT_COMP_E_WARN:248 /*249 * Check microcode completion code, it is only valid250 * when completion code is CPT_COMP_E::GOOD251 */252 if (uc_ccode != OTX2_CPT_UCC_SUCCESS) {253 /*254 * If requested hmac is truncated and ucode returns255 * s/g write length error then we report success256 * because ucode writes as many bytes of calculated257 * hmac as available in gather buffer and reports258 * s/g write length error if number of bytes in gather259 * buffer is less than full hmac size.260 */261 if (info->req->is_trunc_hmac &&262 uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) {263 *res_code = 0;264 break;265 }266 267 dev_err(&pdev->dev,268 "Request failed with software error code 0x%x\n",269 cpt_status->s.uc_compcode);270 otx2_cpt_dump_sg_list(pdev, info->req);271 break;272 }273 /* Request has been processed with success */274 *res_code = 0;275 break;276 277 default:278 dev_err(&pdev->dev,279 "Request returned invalid status %d\n", ccode);280 break;281 }282 return 0;283}284 285static inline void process_pending_queue(struct otx2_cptlfs_info *lfs,286 struct otx2_cpt_pending_queue *pqueue)287{288 struct otx2_cpt_pending_entry *resume_pentry = NULL;289 void (*callback)(int status, void *arg, void *req);290 struct otx2_cpt_pending_entry *pentry = NULL;291 union otx2_cpt_res_s *cpt_status = NULL;292 struct otx2_cpt_inst_info *info = NULL;293 struct otx2_cpt_req_info *req = NULL;294 struct crypto_async_request *areq;295 struct pci_dev *pdev = lfs->pdev;296 u32 res_code, resume_index;297 298 while (1) {299 spin_lock_bh(&pqueue->lock);300 pentry = &pqueue->head[pqueue->front];301 302 if (WARN_ON(!pentry)) {303 spin_unlock_bh(&pqueue->lock);304 break;305 }306 307 res_code = -EINVAL;308 if (unlikely(!pentry->busy)) {309 spin_unlock_bh(&pqueue->lock);310 break;311 }312 313 if (unlikely(!pentry->callback)) {314 dev_err(&pdev->dev, "Callback NULL\n");315 goto process_pentry;316 }317 318 info = pentry->info;319 if (unlikely(!info)) {320 dev_err(&pdev->dev, "Pending entry post arg NULL\n");321 goto process_pentry;322 }323 324 req = info->req;325 if (unlikely(!req)) {326 dev_err(&pdev->dev, "Request NULL\n");327 goto process_pentry;328 }329 330 cpt_status = pentry->completion_addr;331 if (unlikely(!cpt_status)) {332 dev_err(&pdev->dev, "Completion address NULL\n");333 goto process_pentry;334 }335 336 if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) {337 spin_unlock_bh(&pqueue->lock);338 return;339 }340 info->pdev = pdev;341 342process_pentry:343 /*344 * Check if we should inform sending side to resume345 * We do it CPT_IQ_RESUME_MARGIN elements in advance before346 * pending queue becomes empty347 */348 resume_index = modulo_inc(pqueue->front, pqueue->qlen,349 CPT_IQ_RESUME_MARGIN);350 resume_pentry = &pqueue->head[resume_index];351 if (resume_pentry &&352 resume_pentry->resume_sender) {353 resume_pentry->resume_sender = false;354 callback = resume_pentry->callback;355 areq = resume_pentry->areq;356 357 if (callback) {358 spin_unlock_bh(&pqueue->lock);359 360 /*361 * EINPROGRESS is an indication for sending362 * side that it can resume sending requests363 */364 callback(-EINPROGRESS, areq, info);365 spin_lock_bh(&pqueue->lock);366 }367 }368 369 callback = pentry->callback;370 areq = pentry->areq;371 free_pentry(pentry);372 373 pqueue->pending_count--;374 pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);375 spin_unlock_bh(&pqueue->lock);376 377 /*378 * Call callback after current pending entry has been379 * processed, we don't do it if the callback pointer is380 * invalid.381 */382 if (callback)383 callback(res_code, areq, info);384 }385}386 387void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe)388{389 process_pending_queue(wqe->lfs,390 &wqe->lfs->lf[wqe->lf_num].pqueue);391}392 393int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev)394{395 struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);396 397 return cptvf->lfs.kcrypto_eng_grp_num;398}399