1412 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * IBM Accelerator Family 'GenWQE'4 *5 * (C) Copyright IBM Corp. 20136 *7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>9 * Author: Michael Jung <mijung@gmx.net>10 * Author: Michael Ruettger <michael@ibmra.de>11 */12 13/*14 * Device Driver Control Block (DDCB) queue support. Definition of15 * interrupt handlers for queue support as well as triggering the16 * health monitor code in case of problems. The current hardware uses17 * an MSI interrupt which is shared between error handling and18 * functional code.19 */20 21#include <linux/types.h>22#include <linux/sched.h>23#include <linux/wait.h>24#include <linux/pci.h>25#include <linux/string.h>26#include <linux/dma-mapping.h>27#include <linux/delay.h>28#include <linux/module.h>29#include <linux/interrupt.h>30#include <linux/crc-itu-t.h>31 32#include "card_base.h"33#include "card_ddcb.h"34 35/*36 * N: next DDCB, this is where the next DDCB will be put.37 * A: active DDCB, this is where the code will look for the next completion.38 * x: DDCB is enqueued, we are waiting for its completion.39 40 * Situation (1): Empty queue41 * +---+---+---+---+---+---+---+---+42 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |43 * | | | | | | | | |44 * +---+---+---+---+---+---+---+---+45 * A/N46 * enqueued_ddcbs = A - N = 2 - 2 = 047 *48 * Situation (2): Wrapped, N > A49 * +---+---+---+---+---+---+---+---+50 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |51 * | | | x | x | | | | |52 * +---+---+---+---+---+---+---+---+53 * A N54 * enqueued_ddcbs = N - A = 4 - 2 = 255 *56 * Situation (3): Queue wrapped, A > N57 * +---+---+---+---+---+---+---+---+58 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |59 * | x | x | | | x | x | x | x |60 * +---+---+---+---+---+---+---+---+61 * N A62 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 663 *64 * Situation (4a): Queue full N > A65 * +---+---+---+---+---+---+---+---+66 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |67 * | x | x | x | x | x | x | x | |68 * +---+---+---+---+---+---+---+---+69 * A N70 *71 * enqueued_ddcbs = N - A = 7 - 0 = 772 *73 * Situation (4a): Queue full A > N74 * +---+---+---+---+---+---+---+---+75 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |76 * | x | x | x | | x | x | x | x |77 * +---+---+---+---+---+---+---+---+78 * N A79 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 780 */81 82static int queue_empty(struct ddcb_queue *queue)83{84 return queue->ddcb_next == queue->ddcb_act;85}86 87static int queue_enqueued_ddcbs(struct ddcb_queue *queue)88{89 if (queue->ddcb_next >= queue->ddcb_act)90 return queue->ddcb_next - queue->ddcb_act;91 92 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);93}94 95static int queue_free_ddcbs(struct ddcb_queue *queue)96{97 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;98 99 if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */100 return 0;101 }102 return free_ddcbs;103}104 105/*106 * Use of the PRIV field in the DDCB for queue debugging:107 *108 * (1) Trying to get rid of a DDCB which saw a timeout:109 * pddcb->priv[6] = 0xcc; # cleared110 *111 * (2) Append a DDCB via NEXT bit:112 * pddcb->priv[7] = 0xaa; # appended113 *114 * (3) DDCB needed tapping:115 * pddcb->priv[7] = 0xbb; # tapped116 *117 * (4) DDCB marked as correctly finished:118 * pddcb->priv[6] = 0xff; # finished119 */120 121static inline void ddcb_mark_tapped(struct ddcb *pddcb)122{123 pddcb->priv[7] = 0xbb; /* tapped */124}125 126static inline void ddcb_mark_appended(struct ddcb *pddcb)127{128 pddcb->priv[7] = 0xaa; /* appended */129}130 131static inline void ddcb_mark_cleared(struct ddcb *pddcb)132{133 pddcb->priv[6] = 0xcc; /* cleared */134}135 136static inline void ddcb_mark_finished(struct ddcb *pddcb)137{138 pddcb->priv[6] = 0xff; /* finished */139}140 141static inline void ddcb_mark_unused(struct ddcb *pddcb)142{143 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */144}145 146/**147 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs148 * @buff: pointer to data buffer149 * @len: length of data for calculation150 * @init: initial crc (0xffff at start)151 *152 * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021)153 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff154 * should result in a crc16 of 0x89c3155 *156 * Return: crc16 checksum in big endian format !157 */158static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)159{160 return crc_itu_t(init, buff, len);161}162 163static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)164{165 int i;166 struct ddcb *pddcb;167 unsigned long flags;168 struct pci_dev *pci_dev = cd->pci_dev;169 170 spin_lock_irqsave(&cd->print_lock, flags);171 172 dev_info(&pci_dev->dev,173 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",174 cd->card_idx, queue->ddcb_act, queue->ddcb_next);175 176 pddcb = queue->ddcb_vaddr;177 for (i = 0; i < queue->ddcb_max; i++) {178 dev_err(&pci_dev->dev,179 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",180 i == queue->ddcb_act ? '>' : ' ',181 i,182 be16_to_cpu(pddcb->retc_16),183 be16_to_cpu(pddcb->seqnum_16),184 pddcb->hsi,185 pddcb->shi,186 be64_to_cpu(pddcb->priv_64),187 pddcb->cmd);188 pddcb++;189 }190 spin_unlock_irqrestore(&cd->print_lock, flags);191}192 193struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)194{195 struct ddcb_requ *req;196 197 req = kzalloc(sizeof(*req), GFP_KERNEL);198 if (!req)199 return NULL;200 201 return &req->cmd;202}203 204void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)205{206 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);207 208 kfree(req);209}210 211static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)212{213 return req->req_state;214}215 216static inline void ddcb_requ_set_state(struct ddcb_requ *req,217 enum genwqe_requ_state new_state)218{219 req->req_state = new_state;220}221 222static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)223{224 return req->cmd.ddata_addr != 0x0;225}226 227/**228 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB229 * @cd: pointer to genwqe device descriptor230 * @req: DDCB work request231 *232 * Status of ddcb_requ mirrors this hardware state, but is copied in233 * the ddcb_requ on interrupt/polling function. The lowlevel code234 * should check the hardware state directly, the higher level code235 * should check the copy.236 *237 * This function will also return true if the state of the queue is238 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the239 * shutdown case.240 */241static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)242{243 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||244 (cd->card_state != GENWQE_CARD_USED);245}246 247#define RET_DDCB_APPENDED 1248#define RET_DDCB_TAPPED 2249/**250 * enqueue_ddcb() - Enqueue a DDCB251 * @cd: pointer to genwqe device descriptor252 * @queue: queue this operation should be done on253 * @pddcb: pointer to ddcb structure254 * @ddcb_no: pointer to ddcb number being tapped255 *256 * Start execution of DDCB by tapping or append to queue via NEXT257 * bit. This is done by an atomic 'compare and swap' instruction and258 * checking SHI and HSI of the previous DDCB.259 *260 * This function must only be called with ddcb_lock held.261 *262 * Return: 1 if new DDCB is appended to previous263 * 2 if DDCB queue is tapped via register/simulation264 */265static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,266 struct ddcb *pddcb, int ddcb_no)267{268 unsigned int try;269 int prev_no;270 struct ddcb *prev_ddcb;271 __be32 old, new, icrc_hsi_shi;272 u64 num;273 274 /*275 * For performance checks a Dispatch Timestamp can be put into276 * DDCB It is supposed to use the SLU's free running counter,277 * but this requires PCIe cycles.278 */279 ddcb_mark_unused(pddcb);280 281 /* check previous DDCB if already fetched */282 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;283 prev_ddcb = &queue->ddcb_vaddr[prev_no];284 285 /*286 * It might have happened that the HSI.FETCHED bit is287 * set. Retry in this case. Therefore I expect maximum 2 times288 * trying.289 */290 ddcb_mark_appended(pddcb);291 for (try = 0; try < 2; try++) {292 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */293 294 /* try to append via NEXT bit if prev DDCB is not completed */295 if ((old & DDCB_COMPLETED_BE32) != 0x00000000)296 break;297 298 new = (old | DDCB_NEXT_BE32);299 300 wmb(); /* need to ensure write ordering */301 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);302 303 if (icrc_hsi_shi == old)304 return RET_DDCB_APPENDED; /* appended to queue */305 }306 307 /* Queue must be re-started by updating QUEUE_OFFSET */308 ddcb_mark_tapped(pddcb);309 num = (u64)ddcb_no << 8;310 311 wmb(); /* need to ensure write ordering */312 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */313 314 return RET_DDCB_TAPPED;315}316 317/**318 * copy_ddcb_results() - Copy output state from real DDCB to request319 * @req: pointer to requested DDCB parameters320 * @ddcb_no: pointer to ddcb number being tapped321 *322 * Copy DDCB ASV to request struct. There is no endian323 * conversion made, since data structure in ASV is still324 * unknown here.325 *326 * This is needed by:327 * - genwqe_purge_ddcb()328 * - genwqe_check_ddcb_queue()329 */330static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)331{332 struct ddcb_queue *queue = req->queue;333 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];334 335 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);336 337 /* copy status flags of the variant part */338 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16);339 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);340 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);341 342 req->cmd.attn = be16_to_cpu(pddcb->attn_16);343 req->cmd.progress = be32_to_cpu(pddcb->progress_32);344 req->cmd.retc = be16_to_cpu(pddcb->retc_16);345 346 if (ddcb_requ_collect_debug_data(req)) {347 int prev_no = (ddcb_no == 0) ?348 queue->ddcb_max - 1 : ddcb_no - 1;349 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];350 351 memcpy(&req->debug_data.ddcb_finished, pddcb,352 sizeof(req->debug_data.ddcb_finished));353 memcpy(&req->debug_data.ddcb_prev, prev_pddcb,354 sizeof(req->debug_data.ddcb_prev));355 }356}357 358/**359 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests.360 * @cd: pointer to genwqe device descriptor361 * @queue: queue to be checked362 *363 * Return: Number of DDCBs which were finished364 */365static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,366 struct ddcb_queue *queue)367{368 unsigned long flags;369 int ddcbs_finished = 0;370 struct pci_dev *pci_dev = cd->pci_dev;371 372 spin_lock_irqsave(&queue->ddcb_lock, flags);373 374 /* FIXME avoid soft locking CPU */375 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {376 377 struct ddcb *pddcb;378 struct ddcb_requ *req;379 u16 vcrc, vcrc_16, retc_16;380 381 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];382 383 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==384 0x00000000)385 goto go_home; /* not completed, continue waiting */386 387 wmb(); /* Add sync to decouple prev. read operations */388 389 /* Note: DDCB could be purged */390 req = queue->ddcb_req[queue->ddcb_act];391 if (req == NULL) {392 /* this occurs if DDCB is purged, not an error */393 /* Move active DDCB further; Nothing to do anymore. */394 goto pick_next_one;395 }396 397 /*398 * HSI=0x44 (fetched and completed), but RETC is399 * 0x101, or even worse 0x000.400 *401 * In case of seeing the queue in inconsistent state402 * we read the errcnts and the queue status to provide403 * a trigger for our PCIe analyzer stop capturing.404 */405 retc_16 = be16_to_cpu(pddcb->retc_16);406 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {407 u64 errcnts, status;408 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;409 410 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);411 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);412 413 dev_err(&pci_dev->dev,414 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",415 __func__, be16_to_cpu(pddcb->seqnum_16),416 pddcb->hsi, retc_16, errcnts, status,417 queue->ddcb_daddr + ddcb_offs);418 }419 420 copy_ddcb_results(req, queue->ddcb_act);421 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */422 423 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);424 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));425 426 ddcb_mark_finished(pddcb);427 428 /* calculate CRC_16 to see if VCRC is correct */429 vcrc = genwqe_crc16(pddcb->asv,430 VCRC_LENGTH(req->cmd.asv_length),431 0xffff);432 vcrc_16 = be16_to_cpu(pddcb->vcrc_16);433 if (vcrc != vcrc_16) {434 printk_ratelimited(KERN_ERR435 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",436 GENWQE_DEVNAME, dev_name(&pci_dev->dev),437 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),438 vcrc, vcrc_16);439 }440 441 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);442 queue->ddcbs_completed++;443 queue->ddcbs_in_flight--;444 445 /* wake up process waiting for this DDCB, and446 processes on the busy queue */447 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);448 wake_up_interruptible(&queue->busy_waitq);449 450pick_next_one:451 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;452 ddcbs_finished++;453 }454 455 go_home:456 spin_unlock_irqrestore(&queue->ddcb_lock, flags);457 return ddcbs_finished;458}459 460/**461 * __genwqe_wait_ddcb(): Waits until DDCB is completed462 * @cd: pointer to genwqe device descriptor463 * @req: pointer to requsted DDCB parameters464 *465 * The Service Layer will update the RETC in DDCB when processing is466 * pending or done.467 *468 * Return: > 0 remaining jiffies, DDCB completed469 * -ETIMEDOUT when timeout470 * -ERESTARTSYS when ^C471 * -EINVAL when unknown error condition472 *473 * When an error is returned the called needs to ensure that474 * purge_ddcb() is being called to get the &req removed from the475 * queue.476 */477int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)478{479 int rc;480 unsigned int ddcb_no;481 struct ddcb_queue *queue;482 struct pci_dev *pci_dev = cd->pci_dev;483 484 if (req == NULL)485 return -EINVAL;486 487 queue = req->queue;488 if (queue == NULL)489 return -EINVAL;490 491 ddcb_no = req->num;492 if (ddcb_no >= queue->ddcb_max)493 return -EINVAL;494 495 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],496 ddcb_requ_finished(cd, req),497 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);498 499 /*500 * We need to distinguish 3 cases here:501 * 1. rc == 0 timeout occurred502 * 2. rc == -ERESTARTSYS signal received503 * 3. rc > 0 remaining jiffies condition is true504 */505 if (rc == 0) {506 struct ddcb_queue *queue = req->queue;507 struct ddcb *pddcb;508 509 /*510 * Timeout may be caused by long task switching time.511 * When timeout happens, check if the request has512 * meanwhile completed.513 */514 genwqe_check_ddcb_queue(cd, req->queue);515 if (ddcb_requ_finished(cd, req))516 return rc;517 518 dev_err(&pci_dev->dev,519 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",520 __func__, req->num, rc, ddcb_requ_get_state(req),521 req);522 dev_err(&pci_dev->dev,523 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__,524 __genwqe_readq(cd, queue->IO_QUEUE_STATUS));525 526 pddcb = &queue->ddcb_vaddr[req->num];527 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));528 529 print_ddcb_info(cd, req->queue);530 return -ETIMEDOUT;531 532 } else if (rc == -ERESTARTSYS) {533 return rc;534 /*535 * EINTR: Stops the application536 * ERESTARTSYS: Restartable systemcall; called again537 */538 539 } else if (rc < 0) {540 dev_err(&pci_dev->dev,541 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",542 __func__, req->num, rc, ddcb_requ_get_state(req));543 return -EINVAL;544 }545 546 /* Severe error occured. Driver is forced to stop operation */547 if (cd->card_state != GENWQE_CARD_USED) {548 dev_err(&pci_dev->dev,549 "[%s] err: DDCB#%d forced to stop (rc=%d)\n",550 __func__, req->num, rc);551 return -EIO;552 }553 return rc;554}555 556/**557 * get_next_ddcb() - Get next available DDCB558 * @cd: pointer to genwqe device descriptor559 * @queue: DDCB queue560 * @num: internal DDCB number561 *562 * DDCB's content is completely cleared but presets for PRE and563 * SEQNUM. This function must only be called when ddcb_lock is held.564 *565 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.566 */567static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,568 struct ddcb_queue *queue,569 int *num)570{571 u64 *pu64;572 struct ddcb *pddcb;573 574 if (queue_free_ddcbs(queue) == 0) /* queue is full */575 return NULL;576 577 /* find new ddcb */578 pddcb = &queue->ddcb_vaddr[queue->ddcb_next];579 580 /* if it is not completed, we are not allowed to use it */581 /* barrier(); */582 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)583 return NULL;584 585 *num = queue->ddcb_next; /* internal DDCB number */586 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;587 588 /* clear important DDCB fields */589 pu64 = (u64 *)pddcb;590 pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */591 pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */592 593 /* destroy previous results in ASV */594 pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */595 pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */596 pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */597 pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */598 pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */599 600 pddcb->pre = DDCB_PRESET_PRE; /* 128 */601 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);602 return pddcb;603}604 605/**606 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue607 * @cd: genwqe device descriptor608 * @req: DDCB request609 *610 * This will fail when the request was already FETCHED. In this case611 * we need to wait until it is finished. Else the DDCB can be612 * reused. This function also ensures that the request data structure613 * is removed from ddcb_req[].614 *615 * Do not forget to call this function when genwqe_wait_ddcb() fails,616 * such that the request gets really removed from ddcb_req[].617 *618 * Return: 0 success619 */620int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)621{622 struct ddcb *pddcb = NULL;623 unsigned int t;624 unsigned long flags;625 struct ddcb_queue *queue = req->queue;626 struct pci_dev *pci_dev = cd->pci_dev;627 u64 queue_status;628 __be32 icrc_hsi_shi = 0x0000;629 __be32 old, new;630 631 /* unsigned long flags; */632 if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {633 dev_err(&pci_dev->dev,634 "[%s] err: software timeout is not set!\n", __func__);635 return -EFAULT;636 }637 638 pddcb = &queue->ddcb_vaddr[req->num];639 640 for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {641 642 spin_lock_irqsave(&queue->ddcb_lock, flags);643 644 /* Check if req was meanwhile finished */645 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)646 goto go_home;647 648 /* try to set PURGE bit if FETCHED/COMPLETED are not set */649 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */650 if ((old & DDCB_FETCHED_BE32) == 0x00000000) {651 652 new = (old | DDCB_PURGE_BE32);653 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,654 old, new);655 if (icrc_hsi_shi == old)656 goto finish_ddcb;657 }658 659 /* normal finish with HSI bit */660 barrier();661 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;662 if (icrc_hsi_shi & DDCB_COMPLETED_BE32)663 goto finish_ddcb;664 665 spin_unlock_irqrestore(&queue->ddcb_lock, flags);666 667 /*668 * Here the check_ddcb() function will most likely669 * discover this DDCB to be finished some point in670 * time. It will mark the req finished and free it up671 * in the list.672 */673 674 copy_ddcb_results(req, req->num); /* for the failing case */675 msleep(100); /* sleep for 1/10 second and try again */676 continue;677 678finish_ddcb:679 copy_ddcb_results(req, req->num);680 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);681 queue->ddcbs_in_flight--;682 queue->ddcb_req[req->num] = NULL; /* delete from array */683 ddcb_mark_cleared(pddcb);684 685 /* Move active DDCB further; Nothing to do here anymore. */686 687 /*688 * We need to ensure that there is at least one free689 * DDCB in the queue. To do that, we must update690 * ddcb_act only if the COMPLETED bit is set for the691 * DDCB we are working on else we treat that DDCB even692 * if we PURGED it as occupied (hardware is supposed693 * to set the COMPLETED bit yet!).694 */695 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;696 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&697 (queue->ddcb_act == req->num)) {698 queue->ddcb_act = ((queue->ddcb_act + 1) %699 queue->ddcb_max);700 }701go_home:702 spin_unlock_irqrestore(&queue->ddcb_lock, flags);703 return 0;704 }705 706 /*707 * If the card is dead and the queue is forced to stop, we708 * might see this in the queue status register.709 */710 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);711 712 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);713 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));714 715 dev_err(&pci_dev->dev,716 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",717 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,718 queue_status);719 720 print_ddcb_info(cd, req->queue);721 722 return -EFAULT;723}724 725int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)726{727 int len;728 struct pci_dev *pci_dev = cd->pci_dev;729 730 if (d == NULL) {731 dev_err(&pci_dev->dev,732 "[%s] err: invalid memory for debug data!\n",733 __func__);734 return -EFAULT;735 }736 737 len = sizeof(d->driver_version);738 snprintf(d->driver_version, len, "%s", DRV_VERSION);739 d->slu_unitcfg = cd->slu_unitcfg;740 d->app_unitcfg = cd->app_unitcfg;741 return 0;742}743 744/**745 * __genwqe_enqueue_ddcb() - Enqueue a DDCB746 * @cd: pointer to genwqe device descriptor747 * @req: pointer to DDCB execution request748 * @f_flags: file mode: blocking, non-blocking749 *750 * Return: 0 if enqueuing succeeded751 * -EIO if card is unusable/PCIe problems752 * -EBUSY if enqueuing failed753 */754int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,755 unsigned int f_flags)756{757 struct ddcb *pddcb;758 unsigned long flags;759 struct ddcb_queue *queue;760 struct pci_dev *pci_dev = cd->pci_dev;761 u16 icrc;762 763 retry:764 if (cd->card_state != GENWQE_CARD_USED) {765 printk_ratelimited(KERN_ERR766 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",767 GENWQE_DEVNAME, dev_name(&pci_dev->dev),768 __func__, req->num);769 return -EIO;770 }771 772 queue = req->queue = &cd->queue;773 774 /* FIXME circumvention to improve performance when no irq is775 * there.776 */777 if (GENWQE_POLLING_ENABLED)778 genwqe_check_ddcb_queue(cd, queue);779 780 /*781 * It must be ensured to process all DDCBs in successive782 * order. Use a lock here in order to prevent nested DDCB783 * enqueuing.784 */785 spin_lock_irqsave(&queue->ddcb_lock, flags);786 787 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */788 if (pddcb == NULL) {789 int rc;790 791 spin_unlock_irqrestore(&queue->ddcb_lock, flags);792 793 if (f_flags & O_NONBLOCK) {794 queue->return_on_busy++;795 return -EBUSY;796 }797 798 queue->wait_on_busy++;799 rc = wait_event_interruptible(queue->busy_waitq,800 queue_free_ddcbs(queue) != 0);801 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",802 __func__, rc);803 if (rc == -ERESTARTSYS)804 return rc; /* interrupted by a signal */805 806 goto retry;807 }808 809 if (queue->ddcb_req[req->num] != NULL) {810 spin_unlock_irqrestore(&queue->ddcb_lock, flags);811 812 dev_err(&pci_dev->dev,813 "[%s] picked DDCB %d with req=%p still in use!!\n",814 __func__, req->num, req);815 return -EFAULT;816 }817 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);818 queue->ddcb_req[req->num] = req;819 820 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);821 pddcb->cmd = req->cmd.cmd;822 pddcb->acfunc = req->cmd.acfunc; /* functional unit */823 824 /*825 * We know that we can get retc 0x104 with CRC error, do not826 * stop the queue in those cases for this command. XDIR = 1827 * does not work for old SLU versions.828 *829 * Last bitstream with the old XDIR behavior had SLU_ID830 * 0x34199.831 */832 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)833 pddcb->xdir = 0x1;834 else835 pddcb->xdir = 0x0;836 837 838 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |839 ((req->cmd.asv_length / 8)));840 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);841 842 /*843 * If copying the whole DDCB_ASIV_LENGTH is impacting844 * performance we need to change it to845 * req->cmd.asiv_length. But simulation benefits from some846 * non-architectured bits behind the architectured content.847 *848 * How much data is copied depends on the availability of the849 * ATS field, which was introduced late. If the ATS field is850 * supported ASIV is 8 bytes shorter than it used to be. Since851 * the ATS field is copied too, the code should do exactly852 * what it did before, but I wanted to make copying of the ATS853 * field very explicit.854 */855 if (genwqe_get_slu_id(cd) <= 0x2) {856 memcpy(&pddcb->__asiv[0], /* destination */857 &req->cmd.__asiv[0], /* source */858 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */859 } else {860 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);861 memcpy(&pddcb->n.asiv[0], /* destination */862 &req->cmd.asiv[0], /* source */863 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */864 }865 866 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */867 868 /*869 * Calculate CRC_16 for corresponding range PSP(7:4). Include870 * empty 4 bytes prior to the data.871 */872 icrc = genwqe_crc16((const u8 *)pddcb,873 ICRC_LENGTH(req->cmd.asiv_length), 0xffff);874 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);875 876 /* enable DDCB completion irq */877 if (!GENWQE_POLLING_ENABLED)878 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;879 880 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);881 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));882 883 if (ddcb_requ_collect_debug_data(req)) {884 /* use the kernel copy of debug data. copying back to885 user buffer happens later */886 887 genwqe_init_debug_data(cd, &req->debug_data);888 memcpy(&req->debug_data.ddcb_before, pddcb,889 sizeof(req->debug_data.ddcb_before));890 }891 892 enqueue_ddcb(cd, queue, pddcb, req->num);893 queue->ddcbs_in_flight++;894 895 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)896 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;897 898 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);899 spin_unlock_irqrestore(&queue->ddcb_lock, flags);900 wake_up_interruptible(&cd->queue_waitq);901 902 return 0;903}904 905/**906 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB907 * @cd: pointer to genwqe device descriptor908 * @cmd: user provided DDCB command909 * @f_flags: file mode: blocking, non-blocking910 */911int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,912 struct genwqe_ddcb_cmd *cmd,913 unsigned int f_flags)914{915 int rc = 0;916 struct pci_dev *pci_dev = cd->pci_dev;917 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);918 919 if (cmd->asiv_length > DDCB_ASIV_LENGTH) {920 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",921 __func__, cmd->asiv_length);922 return -EINVAL;923 }924 if (cmd->asv_length > DDCB_ASV_LENGTH) {925 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",926 __func__, cmd->asiv_length);927 return -EINVAL;928 }929 rc = __genwqe_enqueue_ddcb(cd, req, f_flags);930 if (rc != 0)931 return rc;932 933 rc = __genwqe_wait_ddcb(cd, req);934 if (rc < 0) /* error or signal interrupt */935 goto err_exit;936 937 if (ddcb_requ_collect_debug_data(req)) {938 if (copy_to_user((struct genwqe_debug_data __user *)939 (unsigned long)cmd->ddata_addr,940 &req->debug_data,941 sizeof(struct genwqe_debug_data)))942 return -EFAULT;943 }944 945 /*946 * Higher values than 0x102 indicate completion with faults,947 * lower values than 0x102 indicate processing faults. Note948 * that DDCB might have been purged. E.g. Cntl+C.949 */950 if (cmd->retc != DDCB_RETC_COMPLETE) {951 /* This might happen e.g. flash read, and needs to be952 handled by the upper layer code. */953 rc = -EBADMSG; /* not processed/error retc */954 }955 956 return rc;957 958 err_exit:959 __genwqe_purge_ddcb(cd, req);960 961 if (ddcb_requ_collect_debug_data(req)) {962 if (copy_to_user((struct genwqe_debug_data __user *)963 (unsigned long)cmd->ddata_addr,964 &req->debug_data,965 sizeof(struct genwqe_debug_data)))966 return -EFAULT;967 }968 return rc;969}970 971/**972 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished973 * @cd: pointer to genwqe device descriptor974 *975 * We use this as condition for our wait-queue code.976 */977static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)978{979 unsigned long flags;980 struct ddcb *pddcb;981 struct ddcb_queue *queue = &cd->queue;982 983 spin_lock_irqsave(&queue->ddcb_lock, flags);984 985 if (queue_empty(queue)) { /* empty queue */986 spin_unlock_irqrestore(&queue->ddcb_lock, flags);987 return 0;988 }989 990 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];991 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */992 spin_unlock_irqrestore(&queue->ddcb_lock, flags);993 return 1;994 }995 996 spin_unlock_irqrestore(&queue->ddcb_lock, flags);997 return 0;998}999 1000/**1001 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight1002 * @cd: pointer to genwqe device descriptor1003 *1004 * Keep track on the number of DDCBs which ware currently in the1005 * queue. This is needed for statistics as well as condition if we want1006 * to wait or better do polling in case of no interrupts available.1007 */1008int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)1009{1010 unsigned long flags;1011 int ddcbs_in_flight = 0;1012 struct ddcb_queue *queue = &cd->queue;1013 1014 spin_lock_irqsave(&queue->ddcb_lock, flags);1015 ddcbs_in_flight += queue->ddcbs_in_flight;1016 spin_unlock_irqrestore(&queue->ddcb_lock, flags);1017 1018 return ddcbs_in_flight;1019}1020 1021static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)1022{1023 int rc, i;1024 struct ddcb *pddcb;1025 u64 val64;1026 unsigned int queue_size;1027 struct pci_dev *pci_dev = cd->pci_dev;1028 1029 if (GENWQE_DDCB_MAX < 2)1030 return -EINVAL;1031 1032 queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);1033 1034 queue->ddcbs_in_flight = 0; /* statistics */1035 queue->ddcbs_max_in_flight = 0;1036 queue->ddcbs_completed = 0;1037 queue->return_on_busy = 0;1038 queue->wait_on_busy = 0;1039 1040 queue->ddcb_seq = 0x100; /* start sequence number */1041 queue->ddcb_max = GENWQE_DDCB_MAX;1042 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,1043 &queue->ddcb_daddr);1044 if (queue->ddcb_vaddr == NULL) {1045 dev_err(&pci_dev->dev,1046 "[%s] **err: could not allocate DDCB **\n", __func__);1047 return -ENOMEM;1048 }1049 queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),1050 GFP_KERNEL);1051 if (!queue->ddcb_req) {1052 rc = -ENOMEM;1053 goto free_ddcbs;1054 }1055 1056 queue->ddcb_waitqs = kcalloc(queue->ddcb_max,1057 sizeof(wait_queue_head_t),1058 GFP_KERNEL);1059 if (!queue->ddcb_waitqs) {1060 rc = -ENOMEM;1061 goto free_requs;1062 }1063 1064 for (i = 0; i < queue->ddcb_max; i++) {1065 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */1066 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;1067 pddcb->retc_16 = cpu_to_be16(0xfff);1068 1069 queue->ddcb_req[i] = NULL; /* requests */1070 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */1071 }1072 1073 queue->ddcb_act = 0;1074 queue->ddcb_next = 0; /* queue is empty */1075 1076 spin_lock_init(&queue->ddcb_lock);1077 init_waitqueue_head(&queue->busy_waitq);1078 1079 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */1080 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */1081 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);1082 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);1083 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64);1084 return 0;1085 1086 free_requs:1087 kfree(queue->ddcb_req);1088 queue->ddcb_req = NULL;1089 free_ddcbs:1090 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,1091 queue->ddcb_daddr);1092 queue->ddcb_vaddr = NULL;1093 queue->ddcb_daddr = 0ull;1094 return rc;1095 1096}1097 1098static int ddcb_queue_initialized(struct ddcb_queue *queue)1099{1100 return queue->ddcb_vaddr != NULL;1101}1102 1103static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)1104{1105 unsigned int queue_size;1106 1107 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);1108 1109 kfree(queue->ddcb_req);1110 queue->ddcb_req = NULL;1111 1112 if (queue->ddcb_vaddr) {1113 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,1114 queue->ddcb_daddr);1115 queue->ddcb_vaddr = NULL;1116 queue->ddcb_daddr = 0ull;1117 }1118}1119 1120static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)1121{1122 u64 gfir;1123 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;1124 struct pci_dev *pci_dev = cd->pci_dev;1125 1126 /*1127 * In case of fatal FIR error the queue is stopped, such that1128 * we can safely check it without risking anything.1129 */1130 cd->irqs_processed++;1131 wake_up_interruptible(&cd->queue_waitq);1132 1133 /*1134 * Checking for errors before kicking the queue might be1135 * safer, but slower for the good-case ... See above.1136 */1137 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);1138 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&1139 !pci_channel_offline(pci_dev)) {1140 1141 if (cd->use_platform_recovery) {1142 /*1143 * Since we use raw accessors, EEH errors won't be1144 * detected by the platform until we do a non-raw1145 * MMIO or config space read1146 */1147 readq(cd->mmio + IO_SLC_CFGREG_GFIR);1148 1149 /* Don't do anything if the PCI channel is frozen */1150 if (pci_channel_offline(pci_dev))1151 goto exit;1152 }1153 1154 wake_up_interruptible(&cd->health_waitq);1155 1156 /*1157 * By default GFIRs causes recovery actions. This1158 * count is just for debug when recovery is masked.1159 */1160 dev_err_ratelimited(&pci_dev->dev,1161 "[%s] GFIR=%016llx\n",1162 __func__, gfir);1163 }1164 1165 exit:1166 return IRQ_HANDLED;1167}1168 1169static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)1170{1171 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;1172 1173 cd->irqs_processed++;1174 wake_up_interruptible(&cd->queue_waitq);1175 1176 return IRQ_HANDLED;1177}1178 1179/**1180 * genwqe_card_thread() - Work thread for the DDCB queue1181 * @data: pointer to genwqe device descriptor1182 *1183 * The idea is to check if there are DDCBs in processing. If there are1184 * some finished DDCBs, we process them and wakeup the1185 * requestors. Otherwise we give other processes time using1186 * cond_resched().1187 */1188static int genwqe_card_thread(void *data)1189{1190 int should_stop = 0;1191 struct genwqe_dev *cd = (struct genwqe_dev *)data;1192 1193 while (!kthread_should_stop()) {1194 1195 genwqe_check_ddcb_queue(cd, &cd->queue);1196 1197 if (GENWQE_POLLING_ENABLED) {1198 wait_event_interruptible_timeout(1199 cd->queue_waitq,1200 genwqe_ddcbs_in_flight(cd) ||1201 (should_stop = kthread_should_stop()), 1);1202 } else {1203 wait_event_interruptible_timeout(1204 cd->queue_waitq,1205 genwqe_next_ddcb_ready(cd) ||1206 (should_stop = kthread_should_stop()), HZ);1207 }1208 if (should_stop)1209 break;1210 1211 /*1212 * Avoid soft lockups on heavy loads; we do not want1213 * to disable our interrupts.1214 */1215 cond_resched();1216 }1217 return 0;1218}1219 1220/**1221 * genwqe_setup_service_layer() - Setup DDCB queue1222 * @cd: pointer to genwqe device descriptor1223 *1224 * Allocate DDCBs. Configure Service Layer Controller (SLC).1225 *1226 * Return: 0 success1227 */1228int genwqe_setup_service_layer(struct genwqe_dev *cd)1229{1230 int rc;1231 struct ddcb_queue *queue;1232 struct pci_dev *pci_dev = cd->pci_dev;1233 1234 if (genwqe_is_privileged(cd)) {1235 rc = genwqe_card_reset(cd);1236 if (rc < 0) {1237 dev_err(&pci_dev->dev,1238 "[%s] err: reset failed.\n", __func__);1239 return rc;1240 }1241 genwqe_read_softreset(cd);1242 }1243 1244 queue = &cd->queue;1245 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG;1246 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS;1247 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;1248 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;1249 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET;1250 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP;1251 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME;1252 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;1253 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW;1254 1255 rc = setup_ddcb_queue(cd, queue);1256 if (rc != 0) {1257 rc = -ENODEV;1258 goto err_out;1259 }1260 1261 init_waitqueue_head(&cd->queue_waitq);1262 cd->card_thread = kthread_run(genwqe_card_thread, cd,1263 GENWQE_DEVNAME "%d_thread",1264 cd->card_idx);1265 if (IS_ERR(cd->card_thread)) {1266 rc = PTR_ERR(cd->card_thread);1267 cd->card_thread = NULL;1268 goto stop_free_queue;1269 }1270 1271 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);1272 if (rc)1273 goto stop_kthread;1274 1275 /*1276 * We must have all wait-queues initialized when we enable the1277 * interrupts. Otherwise we might crash if we get an early1278 * irq.1279 */1280 init_waitqueue_head(&cd->health_waitq);1281 1282 if (genwqe_is_privileged(cd)) {1283 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,1284 GENWQE_DEVNAME, cd);1285 } else {1286 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,1287 GENWQE_DEVNAME, cd);1288 }1289 if (rc < 0) {1290 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);1291 goto stop_irq_cap;1292 }1293 1294 cd->card_state = GENWQE_CARD_USED;1295 return 0;1296 1297 stop_irq_cap:1298 genwqe_reset_interrupt_capability(cd);1299 stop_kthread:1300 kthread_stop(cd->card_thread);1301 cd->card_thread = NULL;1302 stop_free_queue:1303 free_ddcb_queue(cd, queue);1304 err_out:1305 return rc;1306}1307 1308/**1309 * queue_wake_up_all() - Handles fatal error case1310 * @cd: pointer to genwqe device descriptor1311 *1312 * The PCI device got unusable and we have to stop all pending1313 * requests as fast as we can. The code after this must purge the1314 * DDCBs in question and ensure that all mappings are freed.1315 */1316static int queue_wake_up_all(struct genwqe_dev *cd)1317{1318 unsigned int i;1319 unsigned long flags;1320 struct ddcb_queue *queue = &cd->queue;1321 1322 spin_lock_irqsave(&queue->ddcb_lock, flags);1323 1324 for (i = 0; i < queue->ddcb_max; i++)1325 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);1326 1327 wake_up_interruptible(&queue->busy_waitq);1328 spin_unlock_irqrestore(&queue->ddcb_lock, flags);1329 1330 return 0;1331}1332 1333/**1334 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces1335 * @cd: pointer to genwqe device descriptor1336 *1337 * Relies on the pre-condition that there are no users of the card1338 * device anymore e.g. with open file-descriptors.1339 *1340 * This function must be robust enough to be called twice.1341 */1342int genwqe_finish_queue(struct genwqe_dev *cd)1343{1344 int i, rc = 0, in_flight;1345 int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;1346 struct pci_dev *pci_dev = cd->pci_dev;1347 struct ddcb_queue *queue = &cd->queue;1348 1349 if (!ddcb_queue_initialized(queue))1350 return 0;1351 1352 /* Do not wipe out the error state. */1353 if (cd->card_state == GENWQE_CARD_USED)1354 cd->card_state = GENWQE_CARD_UNUSED;1355 1356 /* Wake up all requests in the DDCB queue such that they1357 should be removed nicely. */1358 queue_wake_up_all(cd);1359 1360 /* We must wait to get rid of the DDCBs in flight */1361 for (i = 0; i < waitmax; i++) {1362 in_flight = genwqe_ddcbs_in_flight(cd);1363 1364 if (in_flight == 0)1365 break;1366 1367 dev_dbg(&pci_dev->dev,1368 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",1369 i, waitmax, in_flight);1370 1371 /*1372 * Severe severe error situation: The card itself has1373 * 16 DDCB queues, each queue has e.g. 32 entries,1374 * each DDBC has a hardware timeout of currently 2501375 * msec but the PFs have a hardware timeout of 8 sec1376 * ... so I take something large.1377 */1378 msleep(1000);1379 }1380 if (i == waitmax) {1381 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n",1382 __func__);1383 rc = -EIO;1384 }1385 return rc;1386}1387 1388/**1389 * genwqe_release_service_layer() - Shutdown DDCB queue1390 * @cd: genwqe device descriptor1391 *1392 * This function must be robust enough to be called twice.1393 */1394int genwqe_release_service_layer(struct genwqe_dev *cd)1395{1396 struct pci_dev *pci_dev = cd->pci_dev;1397 1398 if (!ddcb_queue_initialized(&cd->queue))1399 return 1;1400 1401 free_irq(pci_dev->irq, cd);1402 genwqe_reset_interrupt_capability(cd);1403 1404 if (cd->card_thread != NULL) {1405 kthread_stop(cd->card_thread);1406 cd->card_thread = NULL;1407 }1408 1409 free_ddcb_queue(cd, &cd->queue);1410 return 0;1411}1412