663 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */3 4#include <linux/kernel.h>5#include <linux/nospec.h>6#include "cc_driver.h"7#include "cc_buffer_mgr.h"8#include "cc_request_mgr.h"9#include "cc_pm.h"10 11#define CC_MAX_POLL_ITER 1012/* The highest descriptor count in used */13#define CC_MAX_DESC_SEQ_LEN 2314 15struct cc_req_mgr_handle {16 /* Request manager resources */17 unsigned int hw_queue_size; /* HW capability */18 unsigned int min_free_hw_slots;19 unsigned int max_used_sw_slots;20 struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];21 u32 req_queue_head;22 u32 req_queue_tail;23 u32 axi_completed;24 u32 q_free_slots;25 /* This lock protects access to HW register26 * that must be single request at a time27 */28 spinlock_t hw_lock;29 struct cc_hw_desc compl_desc;30 u8 *dummy_comp_buff;31 dma_addr_t dummy_comp_buff_dma;32 33 /* backlog queue */34 struct list_head backlog;35 unsigned int bl_len;36 spinlock_t bl_lock; /* protect backlog queue */37 38#ifdef COMP_IN_WQ39 struct workqueue_struct *workq;40 struct delayed_work compwork;41#else42 struct tasklet_struct comptask;43#endif44};45 46struct cc_bl_item {47 struct cc_crypto_req creq;48 struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];49 unsigned int len;50 struct list_head list;51 bool notif;52};53 54static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = {55 { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT),56 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT),57 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT),58 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT),59 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT),60 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT),61 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT),62 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) },63 { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT),64 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT),65 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT),66 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT),67 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT),68 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT),69 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT),70 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) }71};72 73static void comp_handler(unsigned long devarg);74#ifdef COMP_IN_WQ75static void comp_work_handler(struct work_struct *work);76#endif77 78static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot)79{80 alg = array_index_nospec(alg, CC_CPP_NUM_ALGS);81 slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS);82 83 return cc_cpp_int_masks[alg][slot];84}85 86void cc_req_mgr_fini(struct cc_drvdata *drvdata)87{88 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;89 struct device *dev = drvdata_to_dev(drvdata);90 91 if (!req_mgr_h)92 return; /* Not allocated */93 94 if (req_mgr_h->dummy_comp_buff_dma) {95 dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,96 req_mgr_h->dummy_comp_buff_dma);97 }98 99 dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -100 req_mgr_h->min_free_hw_slots));101 dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);102 103#ifdef COMP_IN_WQ104 destroy_workqueue(req_mgr_h->workq);105#else106 /* Kill tasklet */107 tasklet_kill(&req_mgr_h->comptask);108#endif109 kfree_sensitive(req_mgr_h);110 drvdata->request_mgr_handle = NULL;111}112 113int cc_req_mgr_init(struct cc_drvdata *drvdata)114{115 struct cc_req_mgr_handle *req_mgr_h;116 struct device *dev = drvdata_to_dev(drvdata);117 int rc = 0;118 119 req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);120 if (!req_mgr_h) {121 rc = -ENOMEM;122 goto req_mgr_init_err;123 }124 125 drvdata->request_mgr_handle = req_mgr_h;126 127 spin_lock_init(&req_mgr_h->hw_lock);128 spin_lock_init(&req_mgr_h->bl_lock);129 INIT_LIST_HEAD(&req_mgr_h->backlog);130 131#ifdef COMP_IN_WQ132 dev_dbg(dev, "Initializing completion workqueue\n");133 req_mgr_h->workq = create_singlethread_workqueue("ccree");134 if (!req_mgr_h->workq) {135 dev_err(dev, "Failed creating work queue\n");136 rc = -ENOMEM;137 goto req_mgr_init_err;138 }139 INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);140#else141 dev_dbg(dev, "Initializing completion tasklet\n");142 tasklet_init(&req_mgr_h->comptask, comp_handler,143 (unsigned long)drvdata);144#endif145 req_mgr_h->hw_queue_size = cc_ioread(drvdata,146 CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));147 dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);148 if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {149 dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",150 req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);151 rc = -ENOMEM;152 goto req_mgr_init_err;153 }154 req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;155 req_mgr_h->max_used_sw_slots = 0;156 157 /* Allocate DMA word for "dummy" completion descriptor use */158 req_mgr_h->dummy_comp_buff =159 dma_alloc_coherent(dev, sizeof(u32),160 &req_mgr_h->dummy_comp_buff_dma,161 GFP_KERNEL);162 if (!req_mgr_h->dummy_comp_buff) {163 dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",164 sizeof(u32));165 rc = -ENOMEM;166 goto req_mgr_init_err;167 }168 169 /* Init. "dummy" completion descriptor */170 hw_desc_init(&req_mgr_h->compl_desc);171 set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));172 set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,173 sizeof(u32), NS_BIT, 1);174 set_flow_mode(&req_mgr_h->compl_desc, BYPASS);175 set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);176 177 return 0;178 179req_mgr_init_err:180 cc_req_mgr_fini(drvdata);181 return rc;182}183 184static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],185 unsigned int seq_len)186{187 int i, w;188 void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);189 struct device *dev = drvdata_to_dev(drvdata);190 191 /*192 * We do indeed write all 6 command words to the same193 * register. The HW supports this.194 */195 196 for (i = 0; i < seq_len; i++) {197 for (w = 0; w <= 5; w++)198 writel_relaxed(seq[i].word[w], reg);199 200 if (cc_dump_desc)201 dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",202 i, seq[i].word[0], seq[i].word[1],203 seq[i].word[2], seq[i].word[3],204 seq[i].word[4], seq[i].word[5]);205 }206}207 208/**209 * request_mgr_complete() - Completion will take place if and only if user210 * requested completion by cc_send_sync_request().211 *212 * @dev: Device pointer213 * @dx_compl_h: The completion event to signal214 * @dummy: unused error code215 */216static void request_mgr_complete(struct device *dev, void *dx_compl_h,217 int dummy)218{219 struct completion *this_compl = dx_compl_h;220 221 complete(this_compl);222}223 224static int cc_queues_status(struct cc_drvdata *drvdata,225 struct cc_req_mgr_handle *req_mgr_h,226 unsigned int total_seq_len)227{228 unsigned long poll_queue;229 struct device *dev = drvdata_to_dev(drvdata);230 231 /* SW queue is checked only once as it will not232 * be changed during the poll because the spinlock_bh233 * is held by the thread234 */235 if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==236 req_mgr_h->req_queue_tail) {237 dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",238 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);239 return -ENOSPC;240 }241 242 if (req_mgr_h->q_free_slots >= total_seq_len)243 return 0;244 245 /* Wait for space in HW queue. Poll constant num of iterations. */246 for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {247 req_mgr_h->q_free_slots =248 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));249 if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)250 req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;251 252 if (req_mgr_h->q_free_slots >= total_seq_len) {253 /* If there is enough place return */254 return 0;255 }256 257 dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",258 req_mgr_h->q_free_slots, total_seq_len);259 }260 /* No room in the HW queue try again later */261 dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",262 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,263 req_mgr_h->q_free_slots, total_seq_len);264 return -ENOSPC;265}266 267/**268 * cc_do_send_request() - Enqueue caller request to crypto hardware.269 * Need to be called with HW lock held and PM running270 *271 * @drvdata: Associated device driver context272 * @cc_req: The request to enqueue273 * @desc: The crypto sequence274 * @len: The crypto sequence length275 * @add_comp: If "true": add an artificial dout DMA to mark completion276 *277 */278static void cc_do_send_request(struct cc_drvdata *drvdata,279 struct cc_crypto_req *cc_req,280 struct cc_hw_desc *desc, unsigned int len,281 bool add_comp)282{283 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;284 unsigned int used_sw_slots;285 unsigned int total_seq_len = len; /*initial sequence length*/286 struct device *dev = drvdata_to_dev(drvdata);287 288 used_sw_slots = ((req_mgr_h->req_queue_head -289 req_mgr_h->req_queue_tail) &290 (MAX_REQUEST_QUEUE_SIZE - 1));291 if (used_sw_slots > req_mgr_h->max_used_sw_slots)292 req_mgr_h->max_used_sw_slots = used_sw_slots;293 294 /* Enqueue request - must be locked with HW lock*/295 req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;296 req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &297 (MAX_REQUEST_QUEUE_SIZE - 1);298 299 dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);300 301 /*302 * We are about to push command to the HW via the command registers303 * that may reference host memory. We need to issue a memory barrier304 * to make sure there are no outstanding memory writes305 */306 wmb();307 308 /* STAT_PHASE_4: Push sequence */309 310 enqueue_seq(drvdata, desc, len);311 312 if (add_comp) {313 enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);314 total_seq_len++;315 }316 317 if (req_mgr_h->q_free_slots < total_seq_len) {318 /* This situation should never occur. Maybe indicating problem319 * with resuming power. Set the free slot count to 0 and hope320 * for the best.321 */322 dev_err(dev, "HW free slot count mismatch.");323 req_mgr_h->q_free_slots = 0;324 } else {325 /* Update the free slots in HW queue */326 req_mgr_h->q_free_slots -= total_seq_len;327 }328}329 330static void cc_enqueue_backlog(struct cc_drvdata *drvdata,331 struct cc_bl_item *bli)332{333 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;334 struct device *dev = drvdata_to_dev(drvdata);335 336 spin_lock_bh(&mgr->bl_lock);337 list_add_tail(&bli->list, &mgr->backlog);338 ++mgr->bl_len;339 dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len);340 spin_unlock_bh(&mgr->bl_lock);341 tasklet_schedule(&mgr->comptask);342}343 344static void cc_proc_backlog(struct cc_drvdata *drvdata)345{346 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;347 struct cc_bl_item *bli;348 struct cc_crypto_req *creq;349 void *req;350 struct device *dev = drvdata_to_dev(drvdata);351 int rc;352 353 spin_lock(&mgr->bl_lock);354 355 while (mgr->bl_len) {356 bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);357 dev_dbg(dev, "---bl len: %d\n", mgr->bl_len);358 359 spin_unlock(&mgr->bl_lock);360 361 362 creq = &bli->creq;363 req = creq->user_arg;364 365 /*366 * Notify the request we're moving out of the backlog367 * but only if we haven't done so already.368 */369 if (!bli->notif) {370 creq->user_cb(dev, req, -EINPROGRESS);371 bli->notif = true;372 }373 374 spin_lock(&mgr->hw_lock);375 376 rc = cc_queues_status(drvdata, mgr, bli->len);377 if (rc) {378 /*379 * There is still no room in the FIFO for380 * this request. Bail out. We'll return here381 * on the next completion irq.382 */383 spin_unlock(&mgr->hw_lock);384 return;385 }386 387 cc_do_send_request(drvdata, &bli->creq, bli->desc, bli->len,388 false);389 spin_unlock(&mgr->hw_lock);390 391 /* Remove ourselves from the backlog list */392 spin_lock(&mgr->bl_lock);393 list_del(&bli->list);394 --mgr->bl_len;395 kfree(bli);396 }397 398 spin_unlock(&mgr->bl_lock);399}400 401int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,402 struct cc_hw_desc *desc, unsigned int len,403 struct crypto_async_request *req)404{405 int rc;406 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;407 struct device *dev = drvdata_to_dev(drvdata);408 bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;409 gfp_t flags = cc_gfp_flags(req);410 struct cc_bl_item *bli;411 412 rc = cc_pm_get(dev);413 if (rc) {414 dev_err(dev, "cc_pm_get returned %x\n", rc);415 return rc;416 }417 418 spin_lock_bh(&mgr->hw_lock);419 rc = cc_queues_status(drvdata, mgr, len);420 421#ifdef CC_DEBUG_FORCE_BACKLOG422 if (backlog_ok)423 rc = -ENOSPC;424#endif /* CC_DEBUG_FORCE_BACKLOG */425 426 if (rc == -ENOSPC && backlog_ok) {427 spin_unlock_bh(&mgr->hw_lock);428 429 bli = kmalloc(sizeof(*bli), flags);430 if (!bli) {431 cc_pm_put_suspend(dev);432 return -ENOMEM;433 }434 435 memcpy(&bli->creq, cc_req, sizeof(*cc_req));436 memcpy(&bli->desc, desc, len * sizeof(*desc));437 bli->len = len;438 bli->notif = false;439 cc_enqueue_backlog(drvdata, bli);440 return -EBUSY;441 }442 443 if (!rc) {444 cc_do_send_request(drvdata, cc_req, desc, len, false);445 rc = -EINPROGRESS;446 }447 448 spin_unlock_bh(&mgr->hw_lock);449 return rc;450}451 452int cc_send_sync_request(struct cc_drvdata *drvdata,453 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,454 unsigned int len)455{456 int rc;457 struct device *dev = drvdata_to_dev(drvdata);458 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;459 460 init_completion(&cc_req->seq_compl);461 cc_req->user_cb = request_mgr_complete;462 cc_req->user_arg = &cc_req->seq_compl;463 464 rc = cc_pm_get(dev);465 if (rc) {466 dev_err(dev, "cc_pm_get returned %x\n", rc);467 return rc;468 }469 470 while (true) {471 spin_lock_bh(&mgr->hw_lock);472 rc = cc_queues_status(drvdata, mgr, len + 1);473 474 if (!rc)475 break;476 477 spin_unlock_bh(&mgr->hw_lock);478 wait_for_completion_interruptible(&drvdata->hw_queue_avail);479 reinit_completion(&drvdata->hw_queue_avail);480 }481 482 cc_do_send_request(drvdata, cc_req, desc, len, true);483 spin_unlock_bh(&mgr->hw_lock);484 wait_for_completion(&cc_req->seq_compl);485 return 0;486}487 488/**489 * send_request_init() - Enqueue caller request to crypto hardware during init490 * process.491 * Assume this function is not called in the middle of a flow,492 * since we set QUEUE_LAST_IND flag in the last descriptor.493 *494 * @drvdata: Associated device driver context495 * @desc: The crypto sequence496 * @len: The crypto sequence length497 *498 * Return:499 * Returns "0" upon success500 */501int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,502 unsigned int len)503{504 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;505 unsigned int total_seq_len = len; /*initial sequence length*/506 int rc = 0;507 508 /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT.509 */510 rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);511 if (rc)512 return rc;513 514 set_queue_last_ind(drvdata, &desc[(len - 1)]);515 516 /*517 * We are about to push command to the HW via the command registers518 * that may reference host memory. We need to issue a memory barrier519 * to make sure there are no outstanding memory writes520 */521 wmb();522 enqueue_seq(drvdata, desc, len);523 524 /* Update the free slots in HW queue */525 req_mgr_h->q_free_slots =526 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));527 528 return 0;529}530 531void complete_request(struct cc_drvdata *drvdata)532{533 struct cc_req_mgr_handle *request_mgr_handle =534 drvdata->request_mgr_handle;535 536 complete(&drvdata->hw_queue_avail);537#ifdef COMP_IN_WQ538 queue_delayed_work(request_mgr_handle->workq,539 &request_mgr_handle->compwork, 0);540#else541 tasklet_schedule(&request_mgr_handle->comptask);542#endif543}544 545#ifdef COMP_IN_WQ546static void comp_work_handler(struct work_struct *work)547{548 struct cc_drvdata *drvdata =549 container_of(work, struct cc_drvdata, compwork.work);550 551 comp_handler((unsigned long)drvdata);552}553#endif554 555static void proc_completions(struct cc_drvdata *drvdata)556{557 struct cc_crypto_req *cc_req;558 struct device *dev = drvdata_to_dev(drvdata);559 struct cc_req_mgr_handle *request_mgr_handle =560 drvdata->request_mgr_handle;561 unsigned int *tail = &request_mgr_handle->req_queue_tail;562 unsigned int *head = &request_mgr_handle->req_queue_head;563 int rc;564 u32 mask;565 566 while (request_mgr_handle->axi_completed) {567 request_mgr_handle->axi_completed--;568 569 /* Dequeue request */570 if (*head == *tail) {571 /* We are supposed to handle a completion but our572 * queue is empty. This is not normal. Return and573 * hope for the best.574 */575 dev_err(dev, "Request queue is empty head == tail %u\n",576 *head);577 break;578 }579 580 cc_req = &request_mgr_handle->req_queue[*tail];581 582 if (cc_req->cpp.is_cpp) {583 584 dev_dbg(dev, "CPP request completion slot: %d alg:%d\n",585 cc_req->cpp.slot, cc_req->cpp.alg);586 mask = cc_cpp_int_mask(cc_req->cpp.alg,587 cc_req->cpp.slot);588 rc = (drvdata->irq & mask ? -EPERM : 0);589 dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask,590 drvdata->irq, rc);591 } else {592 dev_dbg(dev, "None CPP request completion\n");593 rc = 0;594 }595 596 if (cc_req->user_cb)597 cc_req->user_cb(dev, cc_req->user_arg, rc);598 *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);599 dev_dbg(dev, "Dequeue request tail=%u\n", *tail);600 dev_dbg(dev, "Request completed. axi_completed=%d\n",601 request_mgr_handle->axi_completed);602 cc_pm_put_suspend(dev);603 }604}605 606static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)607{608 return FIELD_GET(AXIM_MON_COMP_VALUE,609 cc_ioread(drvdata, drvdata->axim_mon_offset));610}611 612/* Deferred service handler, run as interrupt-fired tasklet */613static void comp_handler(unsigned long devarg)614{615 struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;616 struct cc_req_mgr_handle *request_mgr_handle =617 drvdata->request_mgr_handle;618 struct device *dev = drvdata_to_dev(drvdata);619 u32 irq;620 621 dev_dbg(dev, "Completion handler called!\n");622 irq = (drvdata->irq & drvdata->comp_mask);623 624 /* To avoid the interrupt from firing as we unmask it,625 * we clear it now626 */627 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);628 629 /* Avoid race with above clear: Test completion counter once more */630 631 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);632 633 dev_dbg(dev, "AXI completion after updated: %d\n",634 request_mgr_handle->axi_completed);635 636 while (request_mgr_handle->axi_completed) {637 do {638 drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR));639 irq = (drvdata->irq & drvdata->comp_mask);640 proc_completions(drvdata);641 642 /* At this point (after proc_completions()),643 * request_mgr_handle->axi_completed is 0.644 */645 request_mgr_handle->axi_completed +=646 cc_axi_comp_count(drvdata);647 } while (request_mgr_handle->axi_completed > 0);648 649 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);650 651 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);652 }653 654 /* after verifying that there is nothing to do,655 * unmask AXI completion interrupt656 */657 cc_iowrite(drvdata, CC_REG(HOST_IMR),658 cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask);659 660 cc_proc_backlog(drvdata);661 dev_dbg(dev, "Comp. handler done.\n");662}663