1392 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * IBM Accelerator Family 'GenWQE'4 *5 * (C) Copyright IBM Corp. 20136 *7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>9 * Author: Michael Jung <mijung@gmx.net>10 * Author: Michael Ruettger <michael@ibmra.de>11 */12 13/*14 * Character device representation of the GenWQE device. This allows15 * user-space applications to communicate with the card.16 */17 18#include <linux/kernel.h>19#include <linux/types.h>20#include <linux/module.h>21#include <linux/pci.h>22#include <linux/string.h>23#include <linux/fs.h>24#include <linux/sched/signal.h>25#include <linux/wait.h>26#include <linux/delay.h>27#include <linux/atomic.h>28 29#include "card_base.h"30#include "card_ddcb.h"31 32static int genwqe_open_files(struct genwqe_dev *cd)33{34 int rc;35 unsigned long flags;36 37 spin_lock_irqsave(&cd->file_lock, flags);38 rc = list_empty(&cd->file_list);39 spin_unlock_irqrestore(&cd->file_lock, flags);40 return !rc;41}42 43static void genwqe_add_file(struct genwqe_dev *cd, struct genwqe_file *cfile)44{45 unsigned long flags;46 47 cfile->opener = get_pid(task_tgid(current));48 spin_lock_irqsave(&cd->file_lock, flags);49 list_add(&cfile->list, &cd->file_list);50 spin_unlock_irqrestore(&cd->file_lock, flags);51}52 53static int genwqe_del_file(struct genwqe_dev *cd, struct genwqe_file *cfile)54{55 unsigned long flags;56 57 spin_lock_irqsave(&cd->file_lock, flags);58 list_del(&cfile->list);59 spin_unlock_irqrestore(&cd->file_lock, flags);60 put_pid(cfile->opener);61 62 return 0;63}64 65static void genwqe_add_pin(struct genwqe_file *cfile, struct dma_mapping *m)66{67 unsigned long flags;68 69 spin_lock_irqsave(&cfile->pin_lock, flags);70 list_add(&m->pin_list, &cfile->pin_list);71 spin_unlock_irqrestore(&cfile->pin_lock, flags);72}73 74static int genwqe_del_pin(struct genwqe_file *cfile, struct dma_mapping *m)75{76 unsigned long flags;77 78 spin_lock_irqsave(&cfile->pin_lock, flags);79 list_del(&m->pin_list);80 spin_unlock_irqrestore(&cfile->pin_lock, flags);81 82 return 0;83}84 85/**86 * genwqe_search_pin() - Search for the mapping for a userspace address87 * @cfile: Descriptor of opened file88 * @u_addr: User virtual address89 * @size: Size of buffer90 * @virt_addr: Virtual address to be updated91 *92 * Return: Pointer to the corresponding mapping NULL if not found93 */94static struct dma_mapping *genwqe_search_pin(struct genwqe_file *cfile,95 unsigned long u_addr,96 unsigned int size,97 void **virt_addr)98{99 unsigned long flags;100 struct dma_mapping *m;101 102 spin_lock_irqsave(&cfile->pin_lock, flags);103 104 list_for_each_entry(m, &cfile->pin_list, pin_list) {105 if ((((u64)m->u_vaddr) <= (u_addr)) &&106 (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {107 108 if (virt_addr)109 *virt_addr = m->k_vaddr +110 (u_addr - (u64)m->u_vaddr);111 112 spin_unlock_irqrestore(&cfile->pin_lock, flags);113 return m;114 }115 }116 spin_unlock_irqrestore(&cfile->pin_lock, flags);117 return NULL;118}119 120static void __genwqe_add_mapping(struct genwqe_file *cfile,121 struct dma_mapping *dma_map)122{123 unsigned long flags;124 125 spin_lock_irqsave(&cfile->map_lock, flags);126 list_add(&dma_map->card_list, &cfile->map_list);127 spin_unlock_irqrestore(&cfile->map_lock, flags);128}129 130static void __genwqe_del_mapping(struct genwqe_file *cfile,131 struct dma_mapping *dma_map)132{133 unsigned long flags;134 135 spin_lock_irqsave(&cfile->map_lock, flags);136 list_del(&dma_map->card_list);137 spin_unlock_irqrestore(&cfile->map_lock, flags);138}139 140 141/**142 * __genwqe_search_mapping() - Search for the mapping for a userspace address143 * @cfile: descriptor of opened file144 * @u_addr: user virtual address145 * @size: size of buffer146 * @dma_addr: DMA address to be updated147 * @virt_addr: Virtual address to be updated148 * Return: Pointer to the corresponding mapping NULL if not found149 */150static struct dma_mapping *__genwqe_search_mapping(struct genwqe_file *cfile,151 unsigned long u_addr,152 unsigned int size,153 dma_addr_t *dma_addr,154 void **virt_addr)155{156 unsigned long flags;157 struct dma_mapping *m;158 struct pci_dev *pci_dev = cfile->cd->pci_dev;159 160 spin_lock_irqsave(&cfile->map_lock, flags);161 list_for_each_entry(m, &cfile->map_list, card_list) {162 163 if ((((u64)m->u_vaddr) <= (u_addr)) &&164 (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {165 166 /* match found: current is as expected and167 addr is in range */168 if (dma_addr)169 *dma_addr = m->dma_addr +170 (u_addr - (u64)m->u_vaddr);171 172 if (virt_addr)173 *virt_addr = m->k_vaddr +174 (u_addr - (u64)m->u_vaddr);175 176 spin_unlock_irqrestore(&cfile->map_lock, flags);177 return m;178 }179 }180 spin_unlock_irqrestore(&cfile->map_lock, flags);181 182 dev_err(&pci_dev->dev,183 "[%s] Entry not found: u_addr=%lx, size=%x\n",184 __func__, u_addr, size);185 186 return NULL;187}188 189static void genwqe_remove_mappings(struct genwqe_file *cfile)190{191 int i = 0;192 struct list_head *node, *next;193 struct dma_mapping *dma_map;194 struct genwqe_dev *cd = cfile->cd;195 struct pci_dev *pci_dev = cfile->cd->pci_dev;196 197 list_for_each_safe(node, next, &cfile->map_list) {198 dma_map = list_entry(node, struct dma_mapping, card_list);199 200 list_del_init(&dma_map->card_list);201 202 /*203 * This is really a bug, because those things should204 * have been already tidied up.205 *206 * GENWQE_MAPPING_RAW should have been removed via mmunmap().207 * GENWQE_MAPPING_SGL_TEMP should be removed by tidy up code.208 */209 dev_err(&pci_dev->dev,210 "[%s] %d. cleanup mapping: u_vaddr=%p u_kaddr=%016lx dma_addr=%lx\n",211 __func__, i++, dma_map->u_vaddr,212 (unsigned long)dma_map->k_vaddr,213 (unsigned long)dma_map->dma_addr);214 215 if (dma_map->type == GENWQE_MAPPING_RAW) {216 /* we allocated this dynamically */217 __genwqe_free_consistent(cd, dma_map->size,218 dma_map->k_vaddr,219 dma_map->dma_addr);220 kfree(dma_map);221 } else if (dma_map->type == GENWQE_MAPPING_SGL_TEMP) {222 /* we use dma_map statically from the request */223 genwqe_user_vunmap(cd, dma_map);224 }225 }226}227 228static void genwqe_remove_pinnings(struct genwqe_file *cfile)229{230 struct list_head *node, *next;231 struct dma_mapping *dma_map;232 struct genwqe_dev *cd = cfile->cd;233 234 list_for_each_safe(node, next, &cfile->pin_list) {235 dma_map = list_entry(node, struct dma_mapping, pin_list);236 237 /*238 * This is not a bug, because a killed processed might239 * not call the unpin ioctl, which is supposed to free240 * the resources.241 *242 * Pinnings are dymically allocated and need to be243 * deleted.244 */245 list_del_init(&dma_map->pin_list);246 genwqe_user_vunmap(cd, dma_map);247 kfree(dma_map);248 }249}250 251/**252 * genwqe_kill_fasync() - Send signal to all processes with open GenWQE files253 * @cd: GenWQE device information254 * @sig: Signal to send out255 *256 * E.g. genwqe_send_signal(cd, SIGIO);257 */258static int genwqe_kill_fasync(struct genwqe_dev *cd, int sig)259{260 unsigned int files = 0;261 unsigned long flags;262 struct genwqe_file *cfile;263 264 spin_lock_irqsave(&cd->file_lock, flags);265 list_for_each_entry(cfile, &cd->file_list, list) {266 if (cfile->async_queue)267 kill_fasync(&cfile->async_queue, sig, POLL_HUP);268 files++;269 }270 spin_unlock_irqrestore(&cd->file_lock, flags);271 return files;272}273 274static int genwqe_terminate(struct genwqe_dev *cd)275{276 unsigned int files = 0;277 unsigned long flags;278 struct genwqe_file *cfile;279 280 spin_lock_irqsave(&cd->file_lock, flags);281 list_for_each_entry(cfile, &cd->file_list, list) {282 kill_pid(cfile->opener, SIGKILL, 1);283 files++;284 }285 spin_unlock_irqrestore(&cd->file_lock, flags);286 return files;287}288 289/**290 * genwqe_open() - file open291 * @inode: file system information292 * @filp: file handle293 *294 * This function is executed whenever an application calls295 * open("/dev/genwqe",..).296 *297 * Return: 0 if successful or <0 if errors298 */299static int genwqe_open(struct inode *inode, struct file *filp)300{301 struct genwqe_dev *cd;302 struct genwqe_file *cfile;303 304 cfile = kzalloc(sizeof(*cfile), GFP_KERNEL);305 if (cfile == NULL)306 return -ENOMEM;307 308 cd = container_of(inode->i_cdev, struct genwqe_dev, cdev_genwqe);309 cfile->cd = cd;310 cfile->filp = filp;311 cfile->client = NULL;312 313 spin_lock_init(&cfile->map_lock); /* list of raw memory allocations */314 INIT_LIST_HEAD(&cfile->map_list);315 316 spin_lock_init(&cfile->pin_lock); /* list of user pinned memory */317 INIT_LIST_HEAD(&cfile->pin_list);318 319 filp->private_data = cfile;320 321 genwqe_add_file(cd, cfile);322 return 0;323}324 325/**326 * genwqe_fasync() - Setup process to receive SIGIO.327 * @fd: file descriptor328 * @filp: file handle329 * @mode: file mode330 *331 * Sending a signal is working as following:332 *333 * if (cdev->async_queue)334 * kill_fasync(&cdev->async_queue, SIGIO, POLL_IN);335 *336 * Some devices also implement asynchronous notification to indicate337 * when the device can be written; in this case, of course,338 * kill_fasync must be called with a mode of POLL_OUT.339 */340static int genwqe_fasync(int fd, struct file *filp, int mode)341{342 struct genwqe_file *cdev = (struct genwqe_file *)filp->private_data;343 344 return fasync_helper(fd, filp, mode, &cdev->async_queue);345}346 347 348/**349 * genwqe_release() - file close350 * @inode: file system information351 * @filp: file handle352 *353 * This function is executed whenever an application calls 'close(fd_genwqe)'354 *355 * Return: always 0356 */357static int genwqe_release(struct inode *inode, struct file *filp)358{359 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;360 struct genwqe_dev *cd = cfile->cd;361 362 /* there must be no entries in these lists! */363 genwqe_remove_mappings(cfile);364 genwqe_remove_pinnings(cfile);365 366 /* remove this filp from the asynchronously notified filp's */367 genwqe_fasync(-1, filp, 0);368 369 /*370 * For this to work we must not release cd when this cfile is371 * not yet released, otherwise the list entry is invalid,372 * because the list itself gets reinstantiated!373 */374 genwqe_del_file(cd, cfile);375 kfree(cfile);376 return 0;377}378 379static void genwqe_vma_open(struct vm_area_struct *vma)380{381 /* nothing ... */382}383 384/**385 * genwqe_vma_close() - Called each time when vma is unmapped386 * @vma: VMA area to close387 *388 * Free memory which got allocated by GenWQE mmap().389 */390static void genwqe_vma_close(struct vm_area_struct *vma)391{392 unsigned long vsize = vma->vm_end - vma->vm_start;393 struct inode *inode = file_inode(vma->vm_file);394 struct dma_mapping *dma_map;395 struct genwqe_dev *cd = container_of(inode->i_cdev, struct genwqe_dev,396 cdev_genwqe);397 struct pci_dev *pci_dev = cd->pci_dev;398 dma_addr_t d_addr = 0;399 struct genwqe_file *cfile = vma->vm_private_data;400 401 dma_map = __genwqe_search_mapping(cfile, vma->vm_start, vsize,402 &d_addr, NULL);403 if (dma_map == NULL) {404 dev_err(&pci_dev->dev,405 " [%s] err: mapping not found: v=%lx, p=%lx s=%lx\n",406 __func__, vma->vm_start, vma->vm_pgoff << PAGE_SHIFT,407 vsize);408 return;409 }410 __genwqe_del_mapping(cfile, dma_map);411 __genwqe_free_consistent(cd, dma_map->size, dma_map->k_vaddr,412 dma_map->dma_addr);413 kfree(dma_map);414}415 416static const struct vm_operations_struct genwqe_vma_ops = {417 .open = genwqe_vma_open,418 .close = genwqe_vma_close,419};420 421/**422 * genwqe_mmap() - Provide contignous buffers to userspace423 * @filp: File pointer (unused)424 * @vma: VMA area to map425 *426 * We use mmap() to allocate contignous buffers used for DMA427 * transfers. After the buffer is allocated we remap it to user-space428 * and remember a reference to our dma_mapping data structure, where429 * we store the associated DMA address and allocated size.430 *431 * When we receive a DDCB execution request with the ATS bits set to432 * plain buffer, we lookup our dma_mapping list to find the433 * corresponding DMA address for the associated user-space address.434 */435static int genwqe_mmap(struct file *filp, struct vm_area_struct *vma)436{437 int rc;438 unsigned long pfn, vsize = vma->vm_end - vma->vm_start;439 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;440 struct genwqe_dev *cd = cfile->cd;441 struct dma_mapping *dma_map;442 443 if (vsize == 0)444 return -EINVAL;445 446 if (get_order(vsize) > MAX_PAGE_ORDER)447 return -ENOMEM;448 449 dma_map = kzalloc(sizeof(struct dma_mapping), GFP_KERNEL);450 if (dma_map == NULL)451 return -ENOMEM;452 453 genwqe_mapping_init(dma_map, GENWQE_MAPPING_RAW);454 dma_map->u_vaddr = (void *)vma->vm_start;455 dma_map->size = vsize;456 dma_map->nr_pages = DIV_ROUND_UP(vsize, PAGE_SIZE);457 dma_map->k_vaddr = __genwqe_alloc_consistent(cd, vsize,458 &dma_map->dma_addr);459 if (dma_map->k_vaddr == NULL) {460 rc = -ENOMEM;461 goto free_dma_map;462 }463 464 if (capable(CAP_SYS_ADMIN) && (vsize > sizeof(dma_addr_t)))465 *(dma_addr_t *)dma_map->k_vaddr = dma_map->dma_addr;466 467 pfn = virt_to_phys(dma_map->k_vaddr) >> PAGE_SHIFT;468 rc = remap_pfn_range(vma,469 vma->vm_start,470 pfn,471 vsize,472 vma->vm_page_prot);473 if (rc != 0) {474 rc = -EFAULT;475 goto free_dma_mem;476 }477 478 vma->vm_private_data = cfile;479 vma->vm_ops = &genwqe_vma_ops;480 __genwqe_add_mapping(cfile, dma_map);481 482 return 0;483 484 free_dma_mem:485 __genwqe_free_consistent(cd, dma_map->size,486 dma_map->k_vaddr,487 dma_map->dma_addr);488 free_dma_map:489 kfree(dma_map);490 return rc;491}492 493#define FLASH_BLOCK 0x40000 /* we use 256k blocks */494 495/**496 * do_flash_update() - Excute flash update (write image or CVPD)497 * @cfile: Descriptor of opened file498 * @load: details about image load499 *500 * Return: 0 if successful501 */502static int do_flash_update(struct genwqe_file *cfile,503 struct genwqe_bitstream *load)504{505 int rc = 0;506 int blocks_to_flash;507 dma_addr_t dma_addr;508 u64 flash = 0;509 size_t tocopy = 0;510 u8 __user *buf;511 u8 *xbuf;512 u32 crc;513 u8 cmdopts;514 struct genwqe_dev *cd = cfile->cd;515 struct file *filp = cfile->filp;516 struct pci_dev *pci_dev = cd->pci_dev;517 518 if ((load->size & 0x3) != 0)519 return -EINVAL;520 521 if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0)522 return -EINVAL;523 524 /* FIXME Bits have changed for new service layer! */525 switch ((char)load->partition) {526 case '0':527 cmdopts = 0x14;528 break; /* download/erase_first/part_0 */529 case '1':530 cmdopts = 0x1C;531 break; /* download/erase_first/part_1 */532 case 'v':533 cmdopts = 0x0C;534 break; /* download/erase_first/vpd */535 default:536 return -EINVAL;537 }538 539 buf = (u8 __user *)load->data_addr;540 xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);541 if (xbuf == NULL)542 return -ENOMEM;543 544 blocks_to_flash = load->size / FLASH_BLOCK;545 while (load->size) {546 struct genwqe_ddcb_cmd *req;547 548 /*549 * We must be 4 byte aligned. Buffer must be 0 appened550 * to have defined values when calculating CRC.551 */552 tocopy = min_t(size_t, load->size, FLASH_BLOCK);553 554 rc = copy_from_user(xbuf, buf, tocopy);555 if (rc) {556 rc = -EFAULT;557 goto free_buffer;558 }559 crc = genwqe_crc32(xbuf, tocopy, 0xffffffff);560 561 dev_dbg(&pci_dev->dev,562 "[%s] DMA: %lx CRC: %08x SZ: %ld %d\n",563 __func__, (unsigned long)dma_addr, crc, tocopy,564 blocks_to_flash);565 566 /* prepare DDCB for SLU process */567 req = ddcb_requ_alloc();568 if (req == NULL) {569 rc = -ENOMEM;570 goto free_buffer;571 }572 573 req->cmd = SLCMD_MOVE_FLASH;574 req->cmdopts = cmdopts;575 576 /* prepare invariant values */577 if (genwqe_get_slu_id(cd) <= 0x2) {578 *(__be64 *)&req->__asiv[0] = cpu_to_be64(dma_addr);579 *(__be64 *)&req->__asiv[8] = cpu_to_be64(tocopy);580 *(__be64 *)&req->__asiv[16] = cpu_to_be64(flash);581 *(__be32 *)&req->__asiv[24] = cpu_to_be32(0);582 req->__asiv[24] = load->uid;583 *(__be32 *)&req->__asiv[28] = cpu_to_be32(crc);584 585 /* for simulation only */586 *(__be64 *)&req->__asiv[88] = cpu_to_be64(load->slu_id);587 *(__be64 *)&req->__asiv[96] = cpu_to_be64(load->app_id);588 req->asiv_length = 32; /* bytes included in crc calc */589 } else { /* setup DDCB for ATS architecture */590 *(__be64 *)&req->asiv[0] = cpu_to_be64(dma_addr);591 *(__be32 *)&req->asiv[8] = cpu_to_be32(tocopy);592 *(__be32 *)&req->asiv[12] = cpu_to_be32(0); /* resvd */593 *(__be64 *)&req->asiv[16] = cpu_to_be64(flash);594 *(__be32 *)&req->asiv[24] = cpu_to_be32(load->uid<<24);595 *(__be32 *)&req->asiv[28] = cpu_to_be32(crc);596 597 /* for simulation only */598 *(__be64 *)&req->asiv[80] = cpu_to_be64(load->slu_id);599 *(__be64 *)&req->asiv[88] = cpu_to_be64(load->app_id);600 601 /* Rd only */602 req->ats = 0x4ULL << 44;603 req->asiv_length = 40; /* bytes included in crc calc */604 }605 req->asv_length = 8;606 607 /* For Genwqe5 we get back the calculated CRC */608 *(u64 *)&req->asv[0] = 0ULL; /* 0x80 */609 610 rc = __genwqe_execute_raw_ddcb(cd, req, filp->f_flags);611 612 load->retc = req->retc;613 load->attn = req->attn;614 load->progress = req->progress;615 616 if (rc < 0) {617 ddcb_requ_free(req);618 goto free_buffer;619 }620 621 if (req->retc != DDCB_RETC_COMPLETE) {622 rc = -EIO;623 ddcb_requ_free(req);624 goto free_buffer;625 }626 627 load->size -= tocopy;628 flash += tocopy;629 buf += tocopy;630 blocks_to_flash--;631 ddcb_requ_free(req);632 }633 634 free_buffer:635 __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);636 return rc;637}638 639static int do_flash_read(struct genwqe_file *cfile,640 struct genwqe_bitstream *load)641{642 int rc, blocks_to_flash;643 dma_addr_t dma_addr;644 u64 flash = 0;645 size_t tocopy = 0;646 u8 __user *buf;647 u8 *xbuf;648 u8 cmdopts;649 struct genwqe_dev *cd = cfile->cd;650 struct file *filp = cfile->filp;651 struct pci_dev *pci_dev = cd->pci_dev;652 struct genwqe_ddcb_cmd *cmd;653 654 if ((load->size & 0x3) != 0)655 return -EINVAL;656 657 if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0)658 return -EINVAL;659 660 /* FIXME Bits have changed for new service layer! */661 switch ((char)load->partition) {662 case '0':663 cmdopts = 0x12;664 break; /* upload/part_0 */665 case '1':666 cmdopts = 0x1A;667 break; /* upload/part_1 */668 case 'v':669 cmdopts = 0x0A;670 break; /* upload/vpd */671 default:672 return -EINVAL;673 }674 675 buf = (u8 __user *)load->data_addr;676 xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);677 if (xbuf == NULL)678 return -ENOMEM;679 680 blocks_to_flash = load->size / FLASH_BLOCK;681 while (load->size) {682 /*683 * We must be 4 byte aligned. Buffer must be 0 appened684 * to have defined values when calculating CRC.685 */686 tocopy = min_t(size_t, load->size, FLASH_BLOCK);687 688 dev_dbg(&pci_dev->dev,689 "[%s] DMA: %lx SZ: %ld %d\n",690 __func__, (unsigned long)dma_addr, tocopy,691 blocks_to_flash);692 693 /* prepare DDCB for SLU process */694 cmd = ddcb_requ_alloc();695 if (cmd == NULL) {696 rc = -ENOMEM;697 goto free_buffer;698 }699 cmd->cmd = SLCMD_MOVE_FLASH;700 cmd->cmdopts = cmdopts;701 702 /* prepare invariant values */703 if (genwqe_get_slu_id(cd) <= 0x2) {704 *(__be64 *)&cmd->__asiv[0] = cpu_to_be64(dma_addr);705 *(__be64 *)&cmd->__asiv[8] = cpu_to_be64(tocopy);706 *(__be64 *)&cmd->__asiv[16] = cpu_to_be64(flash);707 *(__be32 *)&cmd->__asiv[24] = cpu_to_be32(0);708 cmd->__asiv[24] = load->uid;709 *(__be32 *)&cmd->__asiv[28] = cpu_to_be32(0) /* CRC */;710 cmd->asiv_length = 32; /* bytes included in crc calc */711 } else { /* setup DDCB for ATS architecture */712 *(__be64 *)&cmd->asiv[0] = cpu_to_be64(dma_addr);713 *(__be32 *)&cmd->asiv[8] = cpu_to_be32(tocopy);714 *(__be32 *)&cmd->asiv[12] = cpu_to_be32(0); /* resvd */715 *(__be64 *)&cmd->asiv[16] = cpu_to_be64(flash);716 *(__be32 *)&cmd->asiv[24] = cpu_to_be32(load->uid<<24);717 *(__be32 *)&cmd->asiv[28] = cpu_to_be32(0); /* CRC */718 719 /* rd/wr */720 cmd->ats = 0x5ULL << 44;721 cmd->asiv_length = 40; /* bytes included in crc calc */722 }723 cmd->asv_length = 8;724 725 /* we only get back the calculated CRC */726 *(u64 *)&cmd->asv[0] = 0ULL; /* 0x80 */727 728 rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);729 730 load->retc = cmd->retc;731 load->attn = cmd->attn;732 load->progress = cmd->progress;733 734 if ((rc < 0) && (rc != -EBADMSG)) {735 ddcb_requ_free(cmd);736 goto free_buffer;737 }738 739 rc = copy_to_user(buf, xbuf, tocopy);740 if (rc) {741 rc = -EFAULT;742 ddcb_requ_free(cmd);743 goto free_buffer;744 }745 746 /* We know that we can get retc 0x104 with CRC err */747 if (((cmd->retc == DDCB_RETC_FAULT) &&748 (cmd->attn != 0x02)) || /* Normally ignore CRC error */749 ((cmd->retc == DDCB_RETC_COMPLETE) &&750 (cmd->attn != 0x00))) { /* Everything was fine */751 rc = -EIO;752 ddcb_requ_free(cmd);753 goto free_buffer;754 }755 756 load->size -= tocopy;757 flash += tocopy;758 buf += tocopy;759 blocks_to_flash--;760 ddcb_requ_free(cmd);761 }762 rc = 0;763 764 free_buffer:765 __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);766 return rc;767}768 769static int genwqe_pin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)770{771 int rc;772 struct genwqe_dev *cd = cfile->cd;773 struct pci_dev *pci_dev = cfile->cd->pci_dev;774 struct dma_mapping *dma_map;775 unsigned long map_addr;776 unsigned long map_size;777 778 if ((m->addr == 0x0) || (m->size == 0))779 return -EINVAL;780 if (m->size > ULONG_MAX - PAGE_SIZE - (m->addr & ~PAGE_MASK))781 return -EINVAL;782 783 map_addr = (m->addr & PAGE_MASK);784 map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);785 786 dma_map = kzalloc(sizeof(struct dma_mapping), GFP_KERNEL);787 if (dma_map == NULL)788 return -ENOMEM;789 790 genwqe_mapping_init(dma_map, GENWQE_MAPPING_SGL_PINNED);791 rc = genwqe_user_vmap(cd, dma_map, (void *)map_addr, map_size);792 if (rc != 0) {793 dev_err(&pci_dev->dev,794 "[%s] genwqe_user_vmap rc=%d\n", __func__, rc);795 kfree(dma_map);796 return rc;797 }798 799 genwqe_add_pin(cfile, dma_map);800 return 0;801}802 803static int genwqe_unpin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)804{805 struct genwqe_dev *cd = cfile->cd;806 struct dma_mapping *dma_map;807 unsigned long map_addr;808 unsigned long map_size;809 810 if (m->addr == 0x0)811 return -EINVAL;812 813 map_addr = (m->addr & PAGE_MASK);814 map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);815 816 dma_map = genwqe_search_pin(cfile, map_addr, map_size, NULL);817 if (dma_map == NULL)818 return -ENOENT;819 820 genwqe_del_pin(cfile, dma_map);821 genwqe_user_vunmap(cd, dma_map);822 kfree(dma_map);823 return 0;824}825 826/**827 * ddcb_cmd_cleanup() - Remove dynamically created fixup entries828 * @cfile: Descriptor of opened file829 * @req: DDCB work request830 *831 * Only if there are any. Pinnings are not removed.832 */833static int ddcb_cmd_cleanup(struct genwqe_file *cfile, struct ddcb_requ *req)834{835 unsigned int i;836 struct dma_mapping *dma_map;837 struct genwqe_dev *cd = cfile->cd;838 839 for (i = 0; i < DDCB_FIXUPS; i++) {840 dma_map = &req->dma_mappings[i];841 842 if (dma_mapping_used(dma_map)) {843 __genwqe_del_mapping(cfile, dma_map);844 genwqe_user_vunmap(cd, dma_map);845 }846 if (req->sgls[i].sgl != NULL)847 genwqe_free_sync_sgl(cd, &req->sgls[i]);848 }849 return 0;850}851 852/**853 * ddcb_cmd_fixups() - Establish DMA fixups/sglists for user memory references854 * @cfile: Descriptor of opened file855 * @req: DDCB work request856 *857 * Before the DDCB gets executed we need to handle the fixups. We858 * replace the user-space addresses with DMA addresses or do859 * additional setup work e.g. generating a scatter-gather list which860 * is used to describe the memory referred to in the fixup.861 */862static int ddcb_cmd_fixups(struct genwqe_file *cfile, struct ddcb_requ *req)863{864 int rc;865 unsigned int asiv_offs, i;866 struct genwqe_dev *cd = cfile->cd;867 struct genwqe_ddcb_cmd *cmd = &req->cmd;868 struct dma_mapping *m;869 870 for (i = 0, asiv_offs = 0x00; asiv_offs <= 0x58;871 i++, asiv_offs += 0x08) {872 873 u64 u_addr;874 dma_addr_t d_addr;875 u32 u_size = 0;876 u64 ats_flags;877 878 ats_flags = ATS_GET_FLAGS(cmd->ats, asiv_offs);879 880 switch (ats_flags) {881 882 case ATS_TYPE_DATA:883 break; /* nothing to do here */884 885 case ATS_TYPE_FLAT_RDWR:886 case ATS_TYPE_FLAT_RD: {887 u_addr = be64_to_cpu(*((__be64 *)&cmd->888 asiv[asiv_offs]));889 u_size = be32_to_cpu(*((__be32 *)&cmd->890 asiv[asiv_offs + 0x08]));891 892 /*893 * No data available. Ignore u_addr in this894 * case and set addr to 0. Hardware must not895 * fetch the buffer.896 */897 if (u_size == 0x0) {898 *((__be64 *)&cmd->asiv[asiv_offs]) =899 cpu_to_be64(0x0);900 break;901 }902 903 m = __genwqe_search_mapping(cfile, u_addr, u_size,904 &d_addr, NULL);905 if (m == NULL) {906 rc = -EFAULT;907 goto err_out;908 }909 910 *((__be64 *)&cmd->asiv[asiv_offs]) =911 cpu_to_be64(d_addr);912 break;913 }914 915 case ATS_TYPE_SGL_RDWR:916 case ATS_TYPE_SGL_RD: {917 int page_offs;918 919 u_addr = be64_to_cpu(*((__be64 *)920 &cmd->asiv[asiv_offs]));921 u_size = be32_to_cpu(*((__be32 *)922 &cmd->asiv[asiv_offs + 0x08]));923 924 /*925 * No data available. Ignore u_addr in this926 * case and set addr to 0. Hardware must not927 * fetch the empty sgl.928 */929 if (u_size == 0x0) {930 *((__be64 *)&cmd->asiv[asiv_offs]) =931 cpu_to_be64(0x0);932 break;933 }934 935 m = genwqe_search_pin(cfile, u_addr, u_size, NULL);936 if (m != NULL) {937 page_offs = (u_addr -938 (u64)m->u_vaddr)/PAGE_SIZE;939 } else {940 m = &req->dma_mappings[i];941 942 genwqe_mapping_init(m,943 GENWQE_MAPPING_SGL_TEMP);944 945 if (ats_flags == ATS_TYPE_SGL_RD)946 m->write = 0;947 948 rc = genwqe_user_vmap(cd, m, (void *)u_addr,949 u_size);950 if (rc != 0)951 goto err_out;952 953 __genwqe_add_mapping(cfile, m);954 page_offs = 0;955 }956 957 /* create genwqe style scatter gather list */958 rc = genwqe_alloc_sync_sgl(cd, &req->sgls[i],959 (void __user *)u_addr,960 u_size, m->write);961 if (rc != 0)962 goto err_out;963 964 genwqe_setup_sgl(cd, &req->sgls[i],965 &m->dma_list[page_offs]);966 967 *((__be64 *)&cmd->asiv[asiv_offs]) =968 cpu_to_be64(req->sgls[i].sgl_dma_addr);969 970 break;971 }972 default:973 rc = -EINVAL;974 goto err_out;975 }976 }977 return 0;978 979 err_out:980 ddcb_cmd_cleanup(cfile, req);981 return rc;982}983 984/**985 * genwqe_execute_ddcb() - Execute DDCB using userspace address fixups986 * @cfile: Descriptor of opened file987 * @cmd: Command identifier (passed from user)988 *989 * The code will build up the translation tables or lookup the990 * contignous memory allocation table to find the right translations991 * and DMA addresses.992 */993static int genwqe_execute_ddcb(struct genwqe_file *cfile,994 struct genwqe_ddcb_cmd *cmd)995{996 int rc;997 struct genwqe_dev *cd = cfile->cd;998 struct file *filp = cfile->filp;999 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);1000 1001 rc = ddcb_cmd_fixups(cfile, req);1002 if (rc != 0)1003 return rc;1004 1005 rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);1006 ddcb_cmd_cleanup(cfile, req);1007 return rc;1008}1009 1010static int do_execute_ddcb(struct genwqe_file *cfile,1011 unsigned long arg, int raw)1012{1013 int rc;1014 struct genwqe_ddcb_cmd *cmd;1015 struct genwqe_dev *cd = cfile->cd;1016 struct file *filp = cfile->filp;1017 1018 cmd = ddcb_requ_alloc();1019 if (cmd == NULL)1020 return -ENOMEM;1021 1022 if (copy_from_user(cmd, (void __user *)arg, sizeof(*cmd))) {1023 ddcb_requ_free(cmd);1024 return -EFAULT;1025 }1026 1027 if (!raw)1028 rc = genwqe_execute_ddcb(cfile, cmd);1029 else1030 rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);1031 1032 /* Copy back only the modifed fields. Do not copy ASIV1033 back since the copy got modified by the driver. */1034 if (copy_to_user((void __user *)arg, cmd,1035 sizeof(*cmd) - DDCB_ASIV_LENGTH)) {1036 ddcb_requ_free(cmd);1037 return -EFAULT;1038 }1039 1040 ddcb_requ_free(cmd);1041 return rc;1042}1043 1044/**1045 * genwqe_ioctl() - IO control1046 * @filp: file handle1047 * @cmd: command identifier (passed from user)1048 * @arg: argument (passed from user)1049 *1050 * Return: 0 success1051 */1052static long genwqe_ioctl(struct file *filp, unsigned int cmd,1053 unsigned long arg)1054{1055 int rc = 0;1056 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;1057 struct genwqe_dev *cd = cfile->cd;1058 struct pci_dev *pci_dev = cd->pci_dev;1059 struct genwqe_reg_io __user *io;1060 u64 val;1061 u32 reg_offs;1062 1063 /* Return -EIO if card hit EEH */1064 if (pci_channel_offline(pci_dev))1065 return -EIO;1066 1067 if (_IOC_TYPE(cmd) != GENWQE_IOC_CODE)1068 return -EINVAL;1069 1070 switch (cmd) {1071 1072 case GENWQE_GET_CARD_STATE:1073 put_user(cd->card_state, (enum genwqe_card_state __user *)arg);1074 return 0;1075 1076 /* Register access */1077 case GENWQE_READ_REG64: {1078 io = (struct genwqe_reg_io __user *)arg;1079 1080 if (get_user(reg_offs, &io->num))1081 return -EFAULT;1082 1083 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))1084 return -EINVAL;1085 1086 val = __genwqe_readq(cd, reg_offs);1087 put_user(val, &io->val64);1088 return 0;1089 }1090 1091 case GENWQE_WRITE_REG64: {1092 io = (struct genwqe_reg_io __user *)arg;1093 1094 if (!capable(CAP_SYS_ADMIN))1095 return -EPERM;1096 1097 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)1098 return -EPERM;1099 1100 if (get_user(reg_offs, &io->num))1101 return -EFAULT;1102 1103 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))1104 return -EINVAL;1105 1106 if (get_user(val, &io->val64))1107 return -EFAULT;1108 1109 __genwqe_writeq(cd, reg_offs, val);1110 return 0;1111 }1112 1113 case GENWQE_READ_REG32: {1114 io = (struct genwqe_reg_io __user *)arg;1115 1116 if (get_user(reg_offs, &io->num))1117 return -EFAULT;1118 1119 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))1120 return -EINVAL;1121 1122 val = __genwqe_readl(cd, reg_offs);1123 put_user(val, &io->val64);1124 return 0;1125 }1126 1127 case GENWQE_WRITE_REG32: {1128 io = (struct genwqe_reg_io __user *)arg;1129 1130 if (!capable(CAP_SYS_ADMIN))1131 return -EPERM;1132 1133 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)1134 return -EPERM;1135 1136 if (get_user(reg_offs, &io->num))1137 return -EFAULT;1138 1139 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))1140 return -EINVAL;1141 1142 if (get_user(val, &io->val64))1143 return -EFAULT;1144 1145 __genwqe_writel(cd, reg_offs, val);1146 return 0;1147 }1148 1149 /* Flash update/reading */1150 case GENWQE_SLU_UPDATE: {1151 struct genwqe_bitstream load;1152 1153 if (!genwqe_is_privileged(cd))1154 return -EPERM;1155 1156 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)1157 return -EPERM;1158 1159 if (copy_from_user(&load, (void __user *)arg,1160 sizeof(load)))1161 return -EFAULT;1162 1163 rc = do_flash_update(cfile, &load);1164 1165 if (copy_to_user((void __user *)arg, &load, sizeof(load)))1166 return -EFAULT;1167 1168 return rc;1169 }1170 1171 case GENWQE_SLU_READ: {1172 struct genwqe_bitstream load;1173 1174 if (!genwqe_is_privileged(cd))1175 return -EPERM;1176 1177 if (genwqe_flash_readback_fails(cd))1178 return -ENOSPC; /* known to fail for old versions */1179 1180 if (copy_from_user(&load, (void __user *)arg, sizeof(load)))1181 return -EFAULT;1182 1183 rc = do_flash_read(cfile, &load);1184 1185 if (copy_to_user((void __user *)arg, &load, sizeof(load)))1186 return -EFAULT;1187 1188 return rc;1189 }1190 1191 /* memory pinning and unpinning */1192 case GENWQE_PIN_MEM: {1193 struct genwqe_mem m;1194 1195 if (copy_from_user(&m, (void __user *)arg, sizeof(m)))1196 return -EFAULT;1197 1198 return genwqe_pin_mem(cfile, &m);1199 }1200 1201 case GENWQE_UNPIN_MEM: {1202 struct genwqe_mem m;1203 1204 if (copy_from_user(&m, (void __user *)arg, sizeof(m)))1205 return -EFAULT;1206 1207 return genwqe_unpin_mem(cfile, &m);1208 }1209 1210 /* launch an DDCB and wait for completion */1211 case GENWQE_EXECUTE_DDCB:1212 return do_execute_ddcb(cfile, arg, 0);1213 1214 case GENWQE_EXECUTE_RAW_DDCB: {1215 1216 if (!capable(CAP_SYS_ADMIN))1217 return -EPERM;1218 1219 return do_execute_ddcb(cfile, arg, 1);1220 }1221 1222 default:1223 return -EINVAL;1224 }1225 1226 return rc;1227}1228 1229static const struct file_operations genwqe_fops = {1230 .owner = THIS_MODULE,1231 .open = genwqe_open,1232 .fasync = genwqe_fasync,1233 .mmap = genwqe_mmap,1234 .unlocked_ioctl = genwqe_ioctl,1235 .compat_ioctl = compat_ptr_ioctl,1236 .release = genwqe_release,1237};1238 1239static int genwqe_device_initialized(struct genwqe_dev *cd)1240{1241 return cd->dev != NULL;1242}1243 1244/**1245 * genwqe_device_create() - Create and configure genwqe char device1246 * @cd: genwqe device descriptor1247 *1248 * This function must be called before we create any more genwqe1249 * character devices, because it is allocating the major and minor1250 * number which are supposed to be used by the client drivers.1251 */1252int genwqe_device_create(struct genwqe_dev *cd)1253{1254 int rc;1255 struct pci_dev *pci_dev = cd->pci_dev;1256 1257 /*1258 * Here starts the individual setup per client. It must1259 * initialize its own cdev data structure with its own fops.1260 * The appropriate devnum needs to be created. The ranges must1261 * not overlap.1262 */1263 rc = alloc_chrdev_region(&cd->devnum_genwqe, 0,1264 GENWQE_MAX_MINOR, GENWQE_DEVNAME);1265 if (rc < 0) {1266 dev_err(&pci_dev->dev, "err: alloc_chrdev_region failed\n");1267 goto err_dev;1268 }1269 1270 cdev_init(&cd->cdev_genwqe, &genwqe_fops);1271 cd->cdev_genwqe.owner = THIS_MODULE;1272 1273 rc = cdev_add(&cd->cdev_genwqe, cd->devnum_genwqe, 1);1274 if (rc < 0) {1275 dev_err(&pci_dev->dev, "err: cdev_add failed\n");1276 goto err_add;1277 }1278 1279 /*1280 * Finally the device in /dev/... must be created. The rule is1281 * to use card%d_clientname for each created device.1282 */1283 cd->dev = device_create_with_groups(cd->class_genwqe,1284 &cd->pci_dev->dev,1285 cd->devnum_genwqe, cd,1286 genwqe_attribute_groups,1287 GENWQE_DEVNAME "%u_card",1288 cd->card_idx);1289 if (IS_ERR(cd->dev)) {1290 rc = PTR_ERR(cd->dev);1291 goto err_cdev;1292 }1293 1294 genwqe_init_debugfs(cd);1295 1296 return 0;1297 1298 err_cdev:1299 cdev_del(&cd->cdev_genwqe);1300 err_add:1301 unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);1302 err_dev:1303 cd->dev = NULL;1304 return rc;1305}1306 1307static int genwqe_inform_and_stop_processes(struct genwqe_dev *cd)1308{1309 int rc;1310 unsigned int i;1311 struct pci_dev *pci_dev = cd->pci_dev;1312 1313 if (!genwqe_open_files(cd))1314 return 0;1315 1316 dev_warn(&pci_dev->dev, "[%s] send SIGIO and wait ...\n", __func__);1317 1318 rc = genwqe_kill_fasync(cd, SIGIO);1319 if (rc > 0) {1320 /* give kill_timeout seconds to close file descriptors ... */1321 for (i = 0; (i < GENWQE_KILL_TIMEOUT) &&1322 genwqe_open_files(cd); i++) {1323 dev_info(&pci_dev->dev, " %d sec ...", i);1324 1325 cond_resched();1326 msleep(1000);1327 }1328 1329 /* if no open files we can safely continue, else ... */1330 if (!genwqe_open_files(cd))1331 return 0;1332 1333 dev_warn(&pci_dev->dev,1334 "[%s] send SIGKILL and wait ...\n", __func__);1335 1336 rc = genwqe_terminate(cd);1337 if (rc) {1338 /* Give kill_timout more seconds to end processes */1339 for (i = 0; (i < GENWQE_KILL_TIMEOUT) &&1340 genwqe_open_files(cd); i++) {1341 dev_warn(&pci_dev->dev, " %d sec ...", i);1342 1343 cond_resched();1344 msleep(1000);1345 }1346 }1347 }1348 return 0;1349}1350 1351/**1352 * genwqe_device_remove() - Remove genwqe's char device1353 * @cd: GenWQE device information1354 *1355 * This function must be called after the client devices are removed1356 * because it will free the major/minor number range for the genwqe1357 * drivers.1358 *1359 * This function must be robust enough to be called twice.1360 */1361int genwqe_device_remove(struct genwqe_dev *cd)1362{1363 int rc;1364 struct pci_dev *pci_dev = cd->pci_dev;1365 1366 if (!genwqe_device_initialized(cd))1367 return 1;1368 1369 genwqe_inform_and_stop_processes(cd);1370 1371 /*1372 * We currently do wait until all filedescriptors are1373 * closed. This leads to a problem when we abort the1374 * application which will decrease this reference from1375 * 1/unused to 0/illegal and not from 2/used 1/empty.1376 */1377 rc = kref_read(&cd->cdev_genwqe.kobj.kref);1378 if (rc != 1) {1379 dev_err(&pci_dev->dev,1380 "[%s] err: cdev_genwqe...refcount=%d\n", __func__, rc);1381 panic("Fatal err: cannot free resources with pending references!");1382 }1383 1384 genqwe_exit_debugfs(cd);1385 device_destroy(cd->class_genwqe, cd->devnum_genwqe);1386 cdev_del(&cd->cdev_genwqe);1387 unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);1388 cd->dev = NULL;1389 1390 return 0;1391}1392