2095 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */3 4#include <linux/init.h>5#include <linux/kernel.h>6#include <linux/module.h>7#include <linux/pci.h>8#include <linux/device.h>9#include <linux/iommu.h>10#include <uapi/linux/idxd.h>11#include <linux/highmem.h>12#include <linux/sched/smt.h>13#include <crypto/internal/acompress.h>14 15#include "idxd.h"16#include "iaa_crypto.h"17#include "iaa_crypto_stats.h"18 19#ifdef pr_fmt20#undef pr_fmt21#endif22 23#define pr_fmt(fmt) "idxd: " IDXD_SUBDRIVER_NAME ": " fmt24 25#define IAA_ALG_PRIORITY 30026 27/* number of iaa instances probed */28static unsigned int nr_iaa;29static unsigned int nr_cpus;30static unsigned int nr_nodes;31static unsigned int nr_cpus_per_node;32 33/* Number of physical cpus sharing each iaa instance */34static unsigned int cpus_per_iaa;35 36static struct crypto_comp *deflate_generic_tfm;37 38/* Per-cpu lookup table for balanced wqs */39static struct wq_table_entry __percpu *wq_table;40 41static struct idxd_wq *wq_table_next_wq(int cpu)42{43 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);44 45 if (++entry->cur_wq >= entry->n_wqs)46 entry->cur_wq = 0;47 48 if (!entry->wqs[entry->cur_wq])49 return NULL;50 51 pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__,52 entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id,53 entry->wqs[entry->cur_wq]->id, cpu);54 55 return entry->wqs[entry->cur_wq];56}57 58static void wq_table_add(int cpu, struct idxd_wq *wq)59{60 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);61 62 if (WARN_ON(entry->n_wqs == entry->max_wqs))63 return;64 65 entry->wqs[entry->n_wqs++] = wq;66 67 pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__,68 entry->wqs[entry->n_wqs - 1]->idxd->id,69 entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu);70}71 72static void wq_table_free_entry(int cpu)73{74 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);75 76 kfree(entry->wqs);77 memset(entry, 0, sizeof(*entry));78}79 80static void wq_table_clear_entry(int cpu)81{82 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);83 84 entry->n_wqs = 0;85 entry->cur_wq = 0;86 memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *));87}88 89LIST_HEAD(iaa_devices);90DEFINE_MUTEX(iaa_devices_lock);91 92/* If enabled, IAA hw crypto algos are registered, unavailable otherwise */93static bool iaa_crypto_enabled;94static bool iaa_crypto_registered;95 96/* Verify results of IAA compress or not */97static bool iaa_verify_compress = true;98 99static ssize_t verify_compress_show(struct device_driver *driver, char *buf)100{101 return sprintf(buf, "%d\n", iaa_verify_compress);102}103 104static ssize_t verify_compress_store(struct device_driver *driver,105 const char *buf, size_t count)106{107 int ret = -EBUSY;108 109 mutex_lock(&iaa_devices_lock);110 111 if (iaa_crypto_enabled)112 goto out;113 114 ret = kstrtobool(buf, &iaa_verify_compress);115 if (ret)116 goto out;117 118 ret = count;119out:120 mutex_unlock(&iaa_devices_lock);121 122 return ret;123}124static DRIVER_ATTR_RW(verify_compress);125 126/*127 * The iaa crypto driver supports three 'sync' methods determining how128 * compressions and decompressions are performed:129 *130 * - sync: the compression or decompression completes before131 * returning. This is the mode used by the async crypto132 * interface when the sync mode is set to 'sync' and by133 * the sync crypto interface regardless of setting.134 *135 * - async: the compression or decompression is submitted and returns136 * immediately. Completion interrupts are not used so137 * the caller is responsible for polling the descriptor138 * for completion. This mode is applicable to only the139 * async crypto interface and is ignored for anything140 * else.141 *142 * - async_irq: the compression or decompression is submitted and143 * returns immediately. Completion interrupts are144 * enabled so the caller can wait for the completion and145 * yield to other threads. When the compression or146 * decompression completes, the completion is signaled147 * and the caller awakened. This mode is applicable to148 * only the async crypto interface and is ignored for149 * anything else.150 *151 * These modes can be set using the iaa_crypto sync_mode driver152 * attribute.153 */154 155/* Use async mode */156static bool async_mode;157/* Use interrupts */158static bool use_irq;159 160/**161 * set_iaa_sync_mode - Set IAA sync mode162 * @name: The name of the sync mode163 *164 * Make the IAA sync mode named @name the current sync mode used by165 * compression/decompression.166 */167 168static int set_iaa_sync_mode(const char *name)169{170 int ret = 0;171 172 if (sysfs_streq(name, "sync")) {173 async_mode = false;174 use_irq = false;175 } else if (sysfs_streq(name, "async")) {176 async_mode = true;177 use_irq = false;178 } else if (sysfs_streq(name, "async_irq")) {179 async_mode = true;180 use_irq = true;181 } else {182 ret = -EINVAL;183 }184 185 return ret;186}187 188static ssize_t sync_mode_show(struct device_driver *driver, char *buf)189{190 int ret = 0;191 192 if (!async_mode && !use_irq)193 ret = sprintf(buf, "%s\n", "sync");194 else if (async_mode && !use_irq)195 ret = sprintf(buf, "%s\n", "async");196 else if (async_mode && use_irq)197 ret = sprintf(buf, "%s\n", "async_irq");198 199 return ret;200}201 202static ssize_t sync_mode_store(struct device_driver *driver,203 const char *buf, size_t count)204{205 int ret = -EBUSY;206 207 mutex_lock(&iaa_devices_lock);208 209 if (iaa_crypto_enabled)210 goto out;211 212 ret = set_iaa_sync_mode(buf);213 if (ret == 0)214 ret = count;215out:216 mutex_unlock(&iaa_devices_lock);217 218 return ret;219}220static DRIVER_ATTR_RW(sync_mode);221 222static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX];223 224static int find_empty_iaa_compression_mode(void)225{226 int i = -EINVAL;227 228 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {229 if (iaa_compression_modes[i])230 continue;231 break;232 }233 234 return i;235}236 237static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx)238{239 struct iaa_compression_mode *mode;240 int i;241 242 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {243 mode = iaa_compression_modes[i];244 if (!mode)245 continue;246 247 if (!strcmp(mode->name, name)) {248 *idx = i;249 return iaa_compression_modes[i];250 }251 }252 253 return NULL;254}255 256static void free_iaa_compression_mode(struct iaa_compression_mode *mode)257{258 kfree(mode->name);259 kfree(mode->ll_table);260 kfree(mode->d_table);261 262 kfree(mode);263}264 265/*266 * IAA Compression modes are defined by an ll_table and a d_table.267 * These tables are typically generated and captured using statistics268 * collected from running actual compress/decompress workloads.269 *270 * A module or other kernel code can add and remove compression modes271 * with a given name using the exported @add_iaa_compression_mode()272 * and @remove_iaa_compression_mode functions.273 *274 * When a new compression mode is added, the tables are saved in a275 * global compression mode list. When IAA devices are added, a276 * per-IAA device dma mapping is created for each IAA device, for each277 * compression mode. These are the tables used to do the actual278 * compression/deccompression and are unmapped if/when the devices are279 * removed. Currently, compression modes must be added before any280 * device is added, and removed after all devices have been removed.281 */282 283/**284 * remove_iaa_compression_mode - Remove an IAA compression mode285 * @name: The name the compression mode will be known as286 *287 * Remove the IAA compression mode named @name.288 */289void remove_iaa_compression_mode(const char *name)290{291 struct iaa_compression_mode *mode;292 int idx;293 294 mutex_lock(&iaa_devices_lock);295 296 if (!list_empty(&iaa_devices))297 goto out;298 299 mode = find_iaa_compression_mode(name, &idx);300 if (mode) {301 free_iaa_compression_mode(mode);302 iaa_compression_modes[idx] = NULL;303 }304out:305 mutex_unlock(&iaa_devices_lock);306}307EXPORT_SYMBOL_GPL(remove_iaa_compression_mode);308 309/**310 * add_iaa_compression_mode - Add an IAA compression mode311 * @name: The name the compression mode will be known as312 * @ll_table: The ll table313 * @ll_table_size: The ll table size in bytes314 * @d_table: The d table315 * @d_table_size: The d table size in bytes316 * @init: Optional callback function to init the compression mode data317 * @free: Optional callback function to free the compression mode data318 *319 * Add a new IAA compression mode named @name.320 *321 * Returns 0 if successful, errcode otherwise.322 */323int add_iaa_compression_mode(const char *name,324 const u32 *ll_table,325 int ll_table_size,326 const u32 *d_table,327 int d_table_size,328 iaa_dev_comp_init_fn_t init,329 iaa_dev_comp_free_fn_t free)330{331 struct iaa_compression_mode *mode;332 int idx, ret = -ENOMEM;333 334 mutex_lock(&iaa_devices_lock);335 336 if (!list_empty(&iaa_devices)) {337 ret = -EBUSY;338 goto out;339 }340 341 mode = kzalloc(sizeof(*mode), GFP_KERNEL);342 if (!mode)343 goto out;344 345 mode->name = kstrdup(name, GFP_KERNEL);346 if (!mode->name)347 goto free;348 349 if (ll_table) {350 mode->ll_table = kmemdup(ll_table, ll_table_size, GFP_KERNEL);351 if (!mode->ll_table)352 goto free;353 mode->ll_table_size = ll_table_size;354 }355 356 if (d_table) {357 mode->d_table = kmemdup(d_table, d_table_size, GFP_KERNEL);358 if (!mode->d_table)359 goto free;360 mode->d_table_size = d_table_size;361 }362 363 mode->init = init;364 mode->free = free;365 366 idx = find_empty_iaa_compression_mode();367 if (idx < 0)368 goto free;369 370 pr_debug("IAA compression mode %s added at idx %d\n",371 mode->name, idx);372 373 iaa_compression_modes[idx] = mode;374 375 ret = 0;376out:377 mutex_unlock(&iaa_devices_lock);378 379 return ret;380free:381 free_iaa_compression_mode(mode);382 goto out;383}384EXPORT_SYMBOL_GPL(add_iaa_compression_mode);385 386static struct iaa_device_compression_mode *387get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx)388{389 return iaa_device->compression_modes[idx];390}391 392static void free_device_compression_mode(struct iaa_device *iaa_device,393 struct iaa_device_compression_mode *device_mode)394{395 size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;396 struct device *dev = &iaa_device->idxd->pdev->dev;397 398 kfree(device_mode->name);399 400 if (device_mode->aecs_comp_table)401 dma_free_coherent(dev, size, device_mode->aecs_comp_table,402 device_mode->aecs_comp_table_dma_addr);403 kfree(device_mode);404}405 406#define IDXD_OP_FLAG_AECS_RW_TGLS 0x400000407#define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC)408#define IAX_AECS_COMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)409#define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)410#define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \411 IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \412 IDXD_OP_FLAG_AECS_RW_TGLS)413 414static int check_completion(struct device *dev,415 struct iax_completion_record *comp,416 bool compress,417 bool only_once);418 419static int init_device_compression_mode(struct iaa_device *iaa_device,420 struct iaa_compression_mode *mode,421 int idx, struct idxd_wq *wq)422{423 size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;424 struct device *dev = &iaa_device->idxd->pdev->dev;425 struct iaa_device_compression_mode *device_mode;426 int ret = -ENOMEM;427 428 device_mode = kzalloc(sizeof(*device_mode), GFP_KERNEL);429 if (!device_mode)430 return -ENOMEM;431 432 device_mode->name = kstrdup(mode->name, GFP_KERNEL);433 if (!device_mode->name)434 goto free;435 436 device_mode->aecs_comp_table = dma_alloc_coherent(dev, size,437 &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL);438 if (!device_mode->aecs_comp_table)439 goto free;440 441 /* Add Huffman table to aecs */442 memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table));443 memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size);444 memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size);445 446 if (mode->init) {447 ret = mode->init(device_mode);448 if (ret)449 goto free;450 }451 452 /* mode index should match iaa_compression_modes idx */453 iaa_device->compression_modes[idx] = device_mode;454 455 pr_debug("IAA %s compression mode initialized for iaa device %d\n",456 mode->name, iaa_device->idxd->id);457 458 ret = 0;459out:460 return ret;461free:462 pr_debug("IAA %s compression mode initialization failed for iaa device %d\n",463 mode->name, iaa_device->idxd->id);464 465 free_device_compression_mode(iaa_device, device_mode);466 goto out;467}468 469static int init_device_compression_modes(struct iaa_device *iaa_device,470 struct idxd_wq *wq)471{472 struct iaa_compression_mode *mode;473 int i, ret = 0;474 475 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {476 mode = iaa_compression_modes[i];477 if (!mode)478 continue;479 480 ret = init_device_compression_mode(iaa_device, mode, i, wq);481 if (ret)482 break;483 }484 485 return ret;486}487 488static void remove_device_compression_modes(struct iaa_device *iaa_device)489{490 struct iaa_device_compression_mode *device_mode;491 int i;492 493 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {494 device_mode = iaa_device->compression_modes[i];495 if (!device_mode)496 continue;497 498 if (iaa_compression_modes[i]->free)499 iaa_compression_modes[i]->free(device_mode);500 free_device_compression_mode(iaa_device, device_mode);501 iaa_device->compression_modes[i] = NULL;502 }503}504 505static struct iaa_device *iaa_device_alloc(void)506{507 struct iaa_device *iaa_device;508 509 iaa_device = kzalloc(sizeof(*iaa_device), GFP_KERNEL);510 if (!iaa_device)511 return NULL;512 513 INIT_LIST_HEAD(&iaa_device->wqs);514 515 return iaa_device;516}517 518static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)519{520 struct iaa_wq *iaa_wq;521 522 list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {523 if (iaa_wq->wq == wq)524 return true;525 }526 527 return false;528}529 530static struct iaa_device *add_iaa_device(struct idxd_device *idxd)531{532 struct iaa_device *iaa_device;533 534 iaa_device = iaa_device_alloc();535 if (!iaa_device)536 return NULL;537 538 iaa_device->idxd = idxd;539 540 list_add_tail(&iaa_device->list, &iaa_devices);541 542 nr_iaa++;543 544 return iaa_device;545}546 547static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq)548{549 int ret = 0;550 551 ret = init_device_compression_modes(iaa_device, iaa_wq->wq);552 if (ret)553 return ret;554 555 return ret;556}557 558static void del_iaa_device(struct iaa_device *iaa_device)559{560 list_del(&iaa_device->list);561 562 nr_iaa--;563}564 565static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq,566 struct iaa_wq **new_wq)567{568 struct idxd_device *idxd = iaa_device->idxd;569 struct pci_dev *pdev = idxd->pdev;570 struct device *dev = &pdev->dev;571 struct iaa_wq *iaa_wq;572 573 iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL);574 if (!iaa_wq)575 return -ENOMEM;576 577 iaa_wq->wq = wq;578 iaa_wq->iaa_device = iaa_device;579 idxd_wq_set_private(wq, iaa_wq);580 581 list_add_tail(&iaa_wq->list, &iaa_device->wqs);582 583 iaa_device->n_wq++;584 585 if (new_wq)586 *new_wq = iaa_wq;587 588 dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n",589 wq->id, iaa_device->idxd->id, iaa_device->n_wq);590 591 return 0;592}593 594static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)595{596 struct idxd_device *idxd = iaa_device->idxd;597 struct pci_dev *pdev = idxd->pdev;598 struct device *dev = &pdev->dev;599 struct iaa_wq *iaa_wq;600 601 list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {602 if (iaa_wq->wq == wq) {603 list_del(&iaa_wq->list);604 iaa_device->n_wq--;605 606 dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n",607 wq->id, iaa_device->idxd->id,608 iaa_device->n_wq, nr_iaa);609 610 if (iaa_device->n_wq == 0)611 del_iaa_device(iaa_device);612 break;613 }614 }615}616 617static void clear_wq_table(void)618{619 int cpu;620 621 for (cpu = 0; cpu < nr_cpus; cpu++)622 wq_table_clear_entry(cpu);623 624 pr_debug("cleared wq table\n");625}626 627static void free_iaa_device(struct iaa_device *iaa_device)628{629 if (!iaa_device)630 return;631 632 remove_device_compression_modes(iaa_device);633 kfree(iaa_device);634}635 636static void __free_iaa_wq(struct iaa_wq *iaa_wq)637{638 struct iaa_device *iaa_device;639 640 if (!iaa_wq)641 return;642 643 iaa_device = iaa_wq->iaa_device;644 if (iaa_device->n_wq == 0)645 free_iaa_device(iaa_wq->iaa_device);646}647 648static void free_iaa_wq(struct iaa_wq *iaa_wq)649{650 struct idxd_wq *wq;651 652 __free_iaa_wq(iaa_wq);653 654 wq = iaa_wq->wq;655 656 kfree(iaa_wq);657 idxd_wq_set_private(wq, NULL);658}659 660static int iaa_wq_get(struct idxd_wq *wq)661{662 struct idxd_device *idxd = wq->idxd;663 struct iaa_wq *iaa_wq;664 int ret = 0;665 666 spin_lock(&idxd->dev_lock);667 iaa_wq = idxd_wq_get_private(wq);668 if (iaa_wq && !iaa_wq->remove) {669 iaa_wq->ref++;670 idxd_wq_get(wq);671 } else {672 ret = -ENODEV;673 }674 spin_unlock(&idxd->dev_lock);675 676 return ret;677}678 679static int iaa_wq_put(struct idxd_wq *wq)680{681 struct idxd_device *idxd = wq->idxd;682 struct iaa_wq *iaa_wq;683 bool free = false;684 int ret = 0;685 686 spin_lock(&idxd->dev_lock);687 iaa_wq = idxd_wq_get_private(wq);688 if (iaa_wq) {689 iaa_wq->ref--;690 if (iaa_wq->ref == 0 && iaa_wq->remove) {691 idxd_wq_set_private(wq, NULL);692 free = true;693 }694 idxd_wq_put(wq);695 } else {696 ret = -ENODEV;697 }698 spin_unlock(&idxd->dev_lock);699 if (free) {700 __free_iaa_wq(iaa_wq);701 kfree(iaa_wq);702 }703 704 return ret;705}706 707static void free_wq_table(void)708{709 int cpu;710 711 for (cpu = 0; cpu < nr_cpus; cpu++)712 wq_table_free_entry(cpu);713 714 free_percpu(wq_table);715 716 pr_debug("freed wq table\n");717}718 719static int alloc_wq_table(int max_wqs)720{721 struct wq_table_entry *entry;722 int cpu;723 724 wq_table = alloc_percpu(struct wq_table_entry);725 if (!wq_table)726 return -ENOMEM;727 728 for (cpu = 0; cpu < nr_cpus; cpu++) {729 entry = per_cpu_ptr(wq_table, cpu);730 entry->wqs = kcalloc(max_wqs, sizeof(struct wq *), GFP_KERNEL);731 if (!entry->wqs) {732 free_wq_table();733 return -ENOMEM;734 }735 736 entry->max_wqs = max_wqs;737 }738 739 pr_debug("initialized wq table\n");740 741 return 0;742}743 744static int save_iaa_wq(struct idxd_wq *wq)745{746 struct iaa_device *iaa_device, *found = NULL;747 struct idxd_device *idxd;748 struct pci_dev *pdev;749 struct device *dev;750 int ret = 0;751 752 list_for_each_entry(iaa_device, &iaa_devices, list) {753 if (iaa_device->idxd == wq->idxd) {754 idxd = iaa_device->idxd;755 pdev = idxd->pdev;756 dev = &pdev->dev;757 /*758 * Check to see that we don't already have this wq.759 * Shouldn't happen but we don't control probing.760 */761 if (iaa_has_wq(iaa_device, wq)) {762 dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n",763 iaa_device);764 goto out;765 }766 767 found = iaa_device;768 769 ret = add_iaa_wq(iaa_device, wq, NULL);770 if (ret)771 goto out;772 773 break;774 }775 }776 777 if (!found) {778 struct iaa_device *new_device;779 struct iaa_wq *new_wq;780 781 new_device = add_iaa_device(wq->idxd);782 if (!new_device) {783 ret = -ENOMEM;784 goto out;785 }786 787 ret = add_iaa_wq(new_device, wq, &new_wq);788 if (ret) {789 del_iaa_device(new_device);790 free_iaa_device(new_device);791 goto out;792 }793 794 ret = init_iaa_device(new_device, new_wq);795 if (ret) {796 del_iaa_wq(new_device, new_wq->wq);797 del_iaa_device(new_device);798 free_iaa_wq(new_wq);799 goto out;800 }801 }802 803 if (WARN_ON(nr_iaa == 0))804 return -EINVAL;805 806 cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;807 if (!cpus_per_iaa)808 cpus_per_iaa = 1;809out:810 return 0;811}812 813static void remove_iaa_wq(struct idxd_wq *wq)814{815 struct iaa_device *iaa_device;816 817 list_for_each_entry(iaa_device, &iaa_devices, list) {818 if (iaa_has_wq(iaa_device, wq)) {819 del_iaa_wq(iaa_device, wq);820 break;821 }822 }823 824 if (nr_iaa) {825 cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;826 if (!cpus_per_iaa)827 cpus_per_iaa = 1;828 } else829 cpus_per_iaa = 1;830}831 832static int wq_table_add_wqs(int iaa, int cpu)833{834 struct iaa_device *iaa_device, *found_device = NULL;835 int ret = 0, cur_iaa = 0, n_wqs_added = 0;836 struct idxd_device *idxd;837 struct iaa_wq *iaa_wq;838 struct pci_dev *pdev;839 struct device *dev;840 841 list_for_each_entry(iaa_device, &iaa_devices, list) {842 idxd = iaa_device->idxd;843 pdev = idxd->pdev;844 dev = &pdev->dev;845 846 if (cur_iaa != iaa) {847 cur_iaa++;848 continue;849 }850 851 found_device = iaa_device;852 dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n",853 found_device->idxd->id, cur_iaa);854 break;855 }856 857 if (!found_device) {858 found_device = list_first_entry_or_null(&iaa_devices,859 struct iaa_device, list);860 if (!found_device) {861 pr_debug("couldn't find any iaa devices with wqs!\n");862 ret = -EINVAL;863 goto out;864 }865 cur_iaa = 0;866 867 idxd = found_device->idxd;868 pdev = idxd->pdev;869 dev = &pdev->dev;870 dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n",871 found_device->idxd->id, cur_iaa);872 }873 874 list_for_each_entry(iaa_wq, &found_device->wqs, list) {875 wq_table_add(cpu, iaa_wq->wq);876 pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n",877 cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id);878 n_wqs_added++;879 }880 881 if (!n_wqs_added) {882 pr_debug("couldn't find any iaa wqs!\n");883 ret = -EINVAL;884 goto out;885 }886out:887 return ret;888}889 890/*891 * Rebalance the wq table so that given a cpu, it's easy to find the892 * closest IAA instance. The idea is to try to choose the most893 * appropriate IAA instance for a caller and spread available894 * workqueues around to clients.895 */896static void rebalance_wq_table(void)897{898 const struct cpumask *node_cpus;899 int node, cpu, iaa = -1;900 901 if (nr_iaa == 0)902 return;903 904 pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n",905 nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa);906 907 clear_wq_table();908 909 if (nr_iaa == 1) {910 for (cpu = 0; cpu < nr_cpus; cpu++) {911 if (WARN_ON(wq_table_add_wqs(0, cpu))) {912 pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu);913 return;914 }915 }916 917 return;918 }919 920 for_each_node_with_cpus(node) {921 node_cpus = cpumask_of_node(node);922 923 for (cpu = 0; cpu < cpumask_weight(node_cpus); cpu++) {924 int node_cpu = cpumask_nth(cpu, node_cpus);925 926 if (WARN_ON(node_cpu >= nr_cpu_ids)) {927 pr_debug("node_cpu %d doesn't exist!\n", node_cpu);928 return;929 }930 931 if ((cpu % cpus_per_iaa) == 0)932 iaa++;933 934 if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) {935 pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu);936 return;937 }938 }939 }940}941 942static inline int check_completion(struct device *dev,943 struct iax_completion_record *comp,944 bool compress,945 bool only_once)946{947 char *op_str = compress ? "compress" : "decompress";948 int ret = 0;949 950 while (!comp->status) {951 if (only_once)952 return -EAGAIN;953 cpu_relax();954 }955 956 if (comp->status != IAX_COMP_SUCCESS) {957 if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) {958 ret = -ETIMEDOUT;959 dev_dbg(dev, "%s timed out, size=0x%x\n",960 op_str, comp->output_size);961 update_completion_timeout_errs();962 goto out;963 }964 965 if (comp->status == IAA_ANALYTICS_ERROR &&966 comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) {967 ret = -E2BIG;968 dev_dbg(dev, "compressed > uncompressed size,"969 " not compressing, size=0x%x\n",970 comp->output_size);971 update_completion_comp_buf_overflow_errs();972 goto out;973 }974 975 if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) {976 ret = -EOVERFLOW;977 goto out;978 }979 980 ret = -EINVAL;981 dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n",982 op_str, comp->status, comp->error_code, comp->output_size);983 print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, comp, 64, 0);984 update_completion_einval_errs();985 986 goto out;987 }988out:989 return ret;990}991 992static int deflate_generic_decompress(struct acomp_req *req)993{994 void *src, *dst;995 int ret;996 997 src = kmap_local_page(sg_page(req->src)) + req->src->offset;998 dst = kmap_local_page(sg_page(req->dst)) + req->dst->offset;999 1000 ret = crypto_comp_decompress(deflate_generic_tfm,1001 src, req->slen, dst, &req->dlen);1002 1003 kunmap_local(src);1004 kunmap_local(dst);1005 1006 update_total_sw_decomp_calls();1007 1008 return ret;1009}1010 1011static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,1012 struct acomp_req *req,1013 dma_addr_t *src_addr, dma_addr_t *dst_addr);1014 1015static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,1016 struct idxd_wq *wq,1017 dma_addr_t src_addr, unsigned int slen,1018 dma_addr_t dst_addr, unsigned int *dlen,1019 u32 compression_crc);1020 1021static void iaa_desc_complete(struct idxd_desc *idxd_desc,1022 enum idxd_complete_type comp_type,1023 bool free_desc, void *__ctx,1024 u32 *status)1025{1026 struct iaa_device_compression_mode *active_compression_mode;1027 struct iaa_compression_ctx *compression_ctx;1028 struct crypto_ctx *ctx = __ctx;1029 struct iaa_device *iaa_device;1030 struct idxd_device *idxd;1031 struct iaa_wq *iaa_wq;1032 struct pci_dev *pdev;1033 struct device *dev;1034 int ret, err = 0;1035 1036 compression_ctx = crypto_tfm_ctx(ctx->tfm);1037 1038 iaa_wq = idxd_wq_get_private(idxd_desc->wq);1039 iaa_device = iaa_wq->iaa_device;1040 idxd = iaa_device->idxd;1041 pdev = idxd->pdev;1042 dev = &pdev->dev;1043 1044 active_compression_mode = get_iaa_device_compression_mode(iaa_device,1045 compression_ctx->mode);1046 dev_dbg(dev, "%s: compression mode %s,"1047 " ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__,1048 active_compression_mode->name,1049 ctx->src_addr, ctx->dst_addr);1050 1051 ret = check_completion(dev, idxd_desc->iax_completion,1052 ctx->compress, false);1053 if (ret) {1054 dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);1055 if (!ctx->compress &&1056 idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {1057 pr_warn("%s: falling back to deflate-generic decompress, "1058 "analytics error code %x\n", __func__,1059 idxd_desc->iax_completion->error_code);1060 ret = deflate_generic_decompress(ctx->req);1061 if (ret) {1062 dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",1063 __func__, ret);1064 err = -EIO;1065 goto err;1066 }1067 } else {1068 err = -EIO;1069 goto err;1070 }1071 } else {1072 ctx->req->dlen = idxd_desc->iax_completion->output_size;1073 }1074 1075 /* Update stats */1076 if (ctx->compress) {1077 update_total_comp_bytes_out(ctx->req->dlen);1078 update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen);1079 } else {1080 update_total_decomp_bytes_in(ctx->req->slen);1081 update_wq_decomp_bytes(iaa_wq->wq, ctx->req->slen);1082 }1083 1084 if (ctx->compress && compression_ctx->verify_compress) {1085 dma_addr_t src_addr, dst_addr;1086 u32 compression_crc;1087 1088 compression_crc = idxd_desc->iax_completion->crc;1089 1090 ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);1091 if (ret) {1092 dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);1093 err = -EIO;1094 goto out;1095 }1096 1097 ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr,1098 ctx->req->slen, dst_addr, &ctx->req->dlen,1099 compression_crc);1100 if (ret) {1101 dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret);1102 err = -EIO;1103 }1104 1105 dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE);1106 dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE);1107 1108 goto out;1109 }1110err:1111 dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);1112 dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);1113out:1114 if (ret != 0)1115 dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);1116 1117 if (ctx->req->base.complete)1118 acomp_request_complete(ctx->req, err);1119 1120 if (free_desc)1121 idxd_free_desc(idxd_desc->wq, idxd_desc);1122 iaa_wq_put(idxd_desc->wq);1123}1124 1125static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,1126 struct idxd_wq *wq,1127 dma_addr_t src_addr, unsigned int slen,1128 dma_addr_t dst_addr, unsigned int *dlen,1129 u32 *compression_crc,1130 bool disable_async)1131{1132 struct iaa_device_compression_mode *active_compression_mode;1133 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);1134 struct iaa_device *iaa_device;1135 struct idxd_desc *idxd_desc;1136 struct iax_hw_desc *desc;1137 struct idxd_device *idxd;1138 struct iaa_wq *iaa_wq;1139 struct pci_dev *pdev;1140 struct device *dev;1141 int ret = 0;1142 1143 iaa_wq = idxd_wq_get_private(wq);1144 iaa_device = iaa_wq->iaa_device;1145 idxd = iaa_device->idxd;1146 pdev = idxd->pdev;1147 dev = &pdev->dev;1148 1149 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);1150 1151 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);1152 if (IS_ERR(idxd_desc)) {1153 dev_dbg(dev, "idxd descriptor allocation failed\n");1154 dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc));1155 return PTR_ERR(idxd_desc);1156 }1157 desc = idxd_desc->iax_hw;1158 1159 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR |1160 IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC;1161 desc->opcode = IAX_OPCODE_COMPRESS;1162 desc->compr_flags = IAA_COMP_FLAGS;1163 desc->priv = 0;1164 1165 desc->src1_addr = (u64)src_addr;1166 desc->src1_size = slen;1167 desc->dst_addr = (u64)dst_addr;1168 desc->max_dst_size = *dlen;1169 desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr;1170 desc->src2_size = sizeof(struct aecs_comp_table_record);1171 desc->completion_addr = idxd_desc->compl_dma;1172 1173 if (ctx->use_irq && !disable_async) {1174 desc->flags |= IDXD_OP_FLAG_RCI;1175 1176 idxd_desc->crypto.req = req;1177 idxd_desc->crypto.tfm = tfm;1178 idxd_desc->crypto.src_addr = src_addr;1179 idxd_desc->crypto.dst_addr = dst_addr;1180 idxd_desc->crypto.compress = true;1181 1182 dev_dbg(dev, "%s use_async_irq: compression mode %s,"1183 " src_addr %llx, dst_addr %llx\n", __func__,1184 active_compression_mode->name,1185 src_addr, dst_addr);1186 } else if (ctx->async_mode && !disable_async)1187 req->base.data = idxd_desc;1188 1189 dev_dbg(dev, "%s: compression mode %s,"1190 " desc->src1_addr %llx, desc->src1_size %d,"1191 " desc->dst_addr %llx, desc->max_dst_size %d,"1192 " desc->src2_addr %llx, desc->src2_size %d\n", __func__,1193 active_compression_mode->name,1194 desc->src1_addr, desc->src1_size, desc->dst_addr,1195 desc->max_dst_size, desc->src2_addr, desc->src2_size);1196 1197 ret = idxd_submit_desc(wq, idxd_desc);1198 if (ret) {1199 dev_dbg(dev, "submit_desc failed ret=%d\n", ret);1200 goto err;1201 }1202 1203 /* Update stats */1204 update_total_comp_calls();1205 update_wq_comp_calls(wq);1206 1207 if (ctx->async_mode && !disable_async) {1208 ret = -EINPROGRESS;1209 dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);1210 goto out;1211 }1212 1213 ret = check_completion(dev, idxd_desc->iax_completion, true, false);1214 if (ret) {1215 dev_dbg(dev, "check_completion failed ret=%d\n", ret);1216 goto err;1217 }1218 1219 *dlen = idxd_desc->iax_completion->output_size;1220 1221 /* Update stats */1222 update_total_comp_bytes_out(*dlen);1223 update_wq_comp_bytes(wq, *dlen);1224 1225 *compression_crc = idxd_desc->iax_completion->crc;1226 1227 if (!ctx->async_mode || disable_async)1228 idxd_free_desc(wq, idxd_desc);1229out:1230 return ret;1231err:1232 idxd_free_desc(wq, idxd_desc);1233 dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);1234 1235 goto out;1236}1237 1238static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,1239 struct acomp_req *req,1240 dma_addr_t *src_addr, dma_addr_t *dst_addr)1241{1242 int ret = 0;1243 int nr_sgs;1244 1245 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);1246 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);1247 1248 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);1249 if (nr_sgs <= 0 || nr_sgs > 1) {1250 dev_dbg(dev, "verify: couldn't map src sg for iaa device %d,"1251 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,1252 iaa_wq->wq->id, ret);1253 ret = -EIO;1254 goto out;1255 }1256 *src_addr = sg_dma_address(req->src);1257 dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"1258 " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs,1259 req->src, req->slen, sg_dma_len(req->src));1260 1261 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);1262 if (nr_sgs <= 0 || nr_sgs > 1) {1263 dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d,"1264 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,1265 iaa_wq->wq->id, ret);1266 ret = -EIO;1267 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);1268 goto out;1269 }1270 *dst_addr = sg_dma_address(req->dst);1271 dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"1272 " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs,1273 req->dst, req->dlen, sg_dma_len(req->dst));1274out:1275 return ret;1276}1277 1278static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,1279 struct idxd_wq *wq,1280 dma_addr_t src_addr, unsigned int slen,1281 dma_addr_t dst_addr, unsigned int *dlen,1282 u32 compression_crc)1283{1284 struct iaa_device_compression_mode *active_compression_mode;1285 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);1286 struct iaa_device *iaa_device;1287 struct idxd_desc *idxd_desc;1288 struct iax_hw_desc *desc;1289 struct idxd_device *idxd;1290 struct iaa_wq *iaa_wq;1291 struct pci_dev *pdev;1292 struct device *dev;1293 int ret = 0;1294 1295 iaa_wq = idxd_wq_get_private(wq);1296 iaa_device = iaa_wq->iaa_device;1297 idxd = iaa_device->idxd;1298 pdev = idxd->pdev;1299 dev = &pdev->dev;1300 1301 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);1302 1303 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);1304 if (IS_ERR(idxd_desc)) {1305 dev_dbg(dev, "idxd descriptor allocation failed\n");1306 dev_dbg(dev, "iaa compress failed: ret=%ld\n",1307 PTR_ERR(idxd_desc));1308 return PTR_ERR(idxd_desc);1309 }1310 desc = idxd_desc->iax_hw;1311 1312 /* Verify (optional) - decompress and check crc, suppress dest write */1313 1314 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;1315 desc->opcode = IAX_OPCODE_DECOMPRESS;1316 desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT;1317 desc->priv = 0;1318 1319 desc->src1_addr = (u64)dst_addr;1320 desc->src1_size = *dlen;1321 desc->dst_addr = (u64)src_addr;1322 desc->max_dst_size = slen;1323 desc->completion_addr = idxd_desc->compl_dma;1324 1325 dev_dbg(dev, "(verify) compression mode %s,"1326 " desc->src1_addr %llx, desc->src1_size %d,"1327 " desc->dst_addr %llx, desc->max_dst_size %d,"1328 " desc->src2_addr %llx, desc->src2_size %d\n",1329 active_compression_mode->name,1330 desc->src1_addr, desc->src1_size, desc->dst_addr,1331 desc->max_dst_size, desc->src2_addr, desc->src2_size);1332 1333 ret = idxd_submit_desc(wq, idxd_desc);1334 if (ret) {1335 dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret);1336 goto err;1337 }1338 1339 ret = check_completion(dev, idxd_desc->iax_completion, false, false);1340 if (ret) {1341 dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret);1342 goto err;1343 }1344 1345 if (compression_crc != idxd_desc->iax_completion->crc) {1346 ret = -EINVAL;1347 dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:"1348 " comp=0x%x, decomp=0x%x\n", compression_crc,1349 idxd_desc->iax_completion->crc);1350 print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET,1351 8, 1, idxd_desc->iax_completion, 64, 0);1352 goto err;1353 }1354 1355 idxd_free_desc(wq, idxd_desc);1356out:1357 return ret;1358err:1359 idxd_free_desc(wq, idxd_desc);1360 dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);1361 1362 goto out;1363}1364 1365static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,1366 struct idxd_wq *wq,1367 dma_addr_t src_addr, unsigned int slen,1368 dma_addr_t dst_addr, unsigned int *dlen,1369 bool disable_async)1370{1371 struct iaa_device_compression_mode *active_compression_mode;1372 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);1373 struct iaa_device *iaa_device;1374 struct idxd_desc *idxd_desc;1375 struct iax_hw_desc *desc;1376 struct idxd_device *idxd;1377 struct iaa_wq *iaa_wq;1378 struct pci_dev *pdev;1379 struct device *dev;1380 int ret = 0;1381 1382 iaa_wq = idxd_wq_get_private(wq);1383 iaa_device = iaa_wq->iaa_device;1384 idxd = iaa_device->idxd;1385 pdev = idxd->pdev;1386 dev = &pdev->dev;1387 1388 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);1389 1390 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);1391 if (IS_ERR(idxd_desc)) {1392 dev_dbg(dev, "idxd descriptor allocation failed\n");1393 dev_dbg(dev, "iaa decompress failed: ret=%ld\n",1394 PTR_ERR(idxd_desc));1395 return PTR_ERR(idxd_desc);1396 }1397 desc = idxd_desc->iax_hw;1398 1399 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;1400 desc->opcode = IAX_OPCODE_DECOMPRESS;1401 desc->max_dst_size = PAGE_SIZE;1402 desc->decompr_flags = IAA_DECOMP_FLAGS;1403 desc->priv = 0;1404 1405 desc->src1_addr = (u64)src_addr;1406 desc->dst_addr = (u64)dst_addr;1407 desc->max_dst_size = *dlen;1408 desc->src1_size = slen;1409 desc->completion_addr = idxd_desc->compl_dma;1410 1411 if (ctx->use_irq && !disable_async) {1412 desc->flags |= IDXD_OP_FLAG_RCI;1413 1414 idxd_desc->crypto.req = req;1415 idxd_desc->crypto.tfm = tfm;1416 idxd_desc->crypto.src_addr = src_addr;1417 idxd_desc->crypto.dst_addr = dst_addr;1418 idxd_desc->crypto.compress = false;1419 1420 dev_dbg(dev, "%s: use_async_irq compression mode %s,"1421 " src_addr %llx, dst_addr %llx\n", __func__,1422 active_compression_mode->name,1423 src_addr, dst_addr);1424 } else if (ctx->async_mode && !disable_async)1425 req->base.data = idxd_desc;1426 1427 dev_dbg(dev, "%s: decompression mode %s,"1428 " desc->src1_addr %llx, desc->src1_size %d,"1429 " desc->dst_addr %llx, desc->max_dst_size %d,"1430 " desc->src2_addr %llx, desc->src2_size %d\n", __func__,1431 active_compression_mode->name,1432 desc->src1_addr, desc->src1_size, desc->dst_addr,1433 desc->max_dst_size, desc->src2_addr, desc->src2_size);1434 1435 ret = idxd_submit_desc(wq, idxd_desc);1436 if (ret) {1437 dev_dbg(dev, "submit_desc failed ret=%d\n", ret);1438 goto err;1439 }1440 1441 /* Update stats */1442 update_total_decomp_calls();1443 update_wq_decomp_calls(wq);1444 1445 if (ctx->async_mode && !disable_async) {1446 ret = -EINPROGRESS;1447 dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);1448 goto out;1449 }1450 1451 ret = check_completion(dev, idxd_desc->iax_completion, false, false);1452 if (ret) {1453 dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);1454 if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {1455 pr_warn("%s: falling back to deflate-generic decompress, "1456 "analytics error code %x\n", __func__,1457 idxd_desc->iax_completion->error_code);1458 ret = deflate_generic_decompress(req);1459 if (ret) {1460 dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",1461 __func__, ret);1462 goto err;1463 }1464 } else {1465 goto err;1466 }1467 } else {1468 req->dlen = idxd_desc->iax_completion->output_size;1469 }1470 1471 *dlen = req->dlen;1472 1473 if (!ctx->async_mode || disable_async)1474 idxd_free_desc(wq, idxd_desc);1475 1476 /* Update stats */1477 update_total_decomp_bytes_in(slen);1478 update_wq_decomp_bytes(wq, slen);1479out:1480 return ret;1481err:1482 idxd_free_desc(wq, idxd_desc);1483 dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret);1484 1485 goto out;1486}1487 1488static int iaa_comp_acompress(struct acomp_req *req)1489{1490 struct iaa_compression_ctx *compression_ctx;1491 struct crypto_tfm *tfm = req->base.tfm;1492 dma_addr_t src_addr, dst_addr;1493 bool disable_async = false;1494 int nr_sgs, cpu, ret = 0;1495 struct iaa_wq *iaa_wq;1496 u32 compression_crc;1497 struct idxd_wq *wq;1498 struct device *dev;1499 int order = -1;1500 1501 compression_ctx = crypto_tfm_ctx(tfm);1502 1503 if (!iaa_crypto_enabled) {1504 pr_debug("iaa_crypto disabled, not compressing\n");1505 return -ENODEV;1506 }1507 1508 if (!req->src || !req->slen) {1509 pr_debug("invalid src, not compressing\n");1510 return -EINVAL;1511 }1512 1513 cpu = get_cpu();1514 wq = wq_table_next_wq(cpu);1515 put_cpu();1516 if (!wq) {1517 pr_debug("no wq configured for cpu=%d\n", cpu);1518 return -ENODEV;1519 }1520 1521 ret = iaa_wq_get(wq);1522 if (ret) {1523 pr_debug("no wq available for cpu=%d\n", cpu);1524 return -ENODEV;1525 }1526 1527 iaa_wq = idxd_wq_get_private(wq);1528 1529 if (!req->dst) {1530 gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;1531 1532 /* incompressible data will always be < 2 * slen */1533 req->dlen = 2 * req->slen;1534 order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE);1535 req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL);1536 if (!req->dst) {1537 ret = -ENOMEM;1538 order = -1;1539 goto out;1540 }1541 disable_async = true;1542 }1543 1544 dev = &wq->idxd->pdev->dev;1545 1546 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);1547 if (nr_sgs <= 0 || nr_sgs > 1) {1548 dev_dbg(dev, "couldn't map src sg for iaa device %d,"1549 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,1550 iaa_wq->wq->id, ret);1551 ret = -EIO;1552 goto out;1553 }1554 src_addr = sg_dma_address(req->src);1555 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"1556 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,1557 req->src, req->slen, sg_dma_len(req->src));1558 1559 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);1560 if (nr_sgs <= 0 || nr_sgs > 1) {1561 dev_dbg(dev, "couldn't map dst sg for iaa device %d,"1562 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,1563 iaa_wq->wq->id, ret);1564 ret = -EIO;1565 goto err_map_dst;1566 }1567 dst_addr = sg_dma_address(req->dst);1568 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"1569 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,1570 req->dst, req->dlen, sg_dma_len(req->dst));1571 1572 ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr,1573 &req->dlen, &compression_crc, disable_async);1574 if (ret == -EINPROGRESS)1575 return ret;1576 1577 if (!ret && compression_ctx->verify_compress) {1578 ret = iaa_remap_for_verify(dev, iaa_wq, req, &src_addr, &dst_addr);1579 if (ret) {1580 dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);1581 goto out;1582 }1583 1584 ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen,1585 dst_addr, &req->dlen, compression_crc);1586 if (ret)1587 dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret);1588 1589 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);1590 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);1591 1592 goto out;1593 }1594 1595 if (ret)1596 dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);1597 1598 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);1599err_map_dst:1600 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);1601out:1602 iaa_wq_put(wq);1603 1604 if (order >= 0)1605 sgl_free_order(req->dst, order);1606 1607 return ret;1608}1609 1610static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req)1611{1612 gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?1613 GFP_KERNEL : GFP_ATOMIC;1614 struct crypto_tfm *tfm = req->base.tfm;1615 dma_addr_t src_addr, dst_addr;1616 int nr_sgs, cpu, ret = 0;1617 struct iaa_wq *iaa_wq;1618 struct device *dev;1619 struct idxd_wq *wq;1620 int order = -1;1621 1622 cpu = get_cpu();1623 wq = wq_table_next_wq(cpu);1624 put_cpu();1625 if (!wq) {1626 pr_debug("no wq configured for cpu=%d\n", cpu);1627 return -ENODEV;1628 }1629 1630 ret = iaa_wq_get(wq);1631 if (ret) {1632 pr_debug("no wq available for cpu=%d\n", cpu);1633 return -ENODEV;1634 }1635 1636 iaa_wq = idxd_wq_get_private(wq);1637 1638 dev = &wq->idxd->pdev->dev;1639 1640 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);1641 if (nr_sgs <= 0 || nr_sgs > 1) {1642 dev_dbg(dev, "couldn't map src sg for iaa device %d,"1643 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,1644 iaa_wq->wq->id, ret);1645 ret = -EIO;1646 goto out;1647 }1648 src_addr = sg_dma_address(req->src);1649 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"1650 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,1651 req->src, req->slen, sg_dma_len(req->src));1652 1653 req->dlen = 4 * req->slen; /* start with ~avg comp rato */1654alloc_dest:1655 order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE);1656 req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL);1657 if (!req->dst) {1658 ret = -ENOMEM;1659 order = -1;1660 goto out;1661 }1662 1663 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);1664 if (nr_sgs <= 0 || nr_sgs > 1) {1665 dev_dbg(dev, "couldn't map dst sg for iaa device %d,"1666 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,1667 iaa_wq->wq->id, ret);1668 ret = -EIO;1669 goto err_map_dst;1670 }1671 1672 dst_addr = sg_dma_address(req->dst);1673 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"1674 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,1675 req->dst, req->dlen, sg_dma_len(req->dst));1676 ret = iaa_decompress(tfm, req, wq, src_addr, req->slen,1677 dst_addr, &req->dlen, true);1678 if (ret == -EOVERFLOW) {1679 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);1680 req->dlen *= 2;1681 if (req->dlen > CRYPTO_ACOMP_DST_MAX)1682 goto err_map_dst;1683 goto alloc_dest;1684 }1685 1686 if (ret != 0)1687 dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);1688 1689 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);1690err_map_dst:1691 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);1692out:1693 iaa_wq_put(wq);1694 1695 if (order >= 0)1696 sgl_free_order(req->dst, order);1697 1698 return ret;1699}1700 1701static int iaa_comp_adecompress(struct acomp_req *req)1702{1703 struct crypto_tfm *tfm = req->base.tfm;1704 dma_addr_t src_addr, dst_addr;1705 int nr_sgs, cpu, ret = 0;1706 struct iaa_wq *iaa_wq;1707 struct device *dev;1708 struct idxd_wq *wq;1709 1710 if (!iaa_crypto_enabled) {1711 pr_debug("iaa_crypto disabled, not decompressing\n");1712 return -ENODEV;1713 }1714 1715 if (!req->src || !req->slen) {1716 pr_debug("invalid src, not decompressing\n");1717 return -EINVAL;1718 }1719 1720 if (!req->dst)1721 return iaa_comp_adecompress_alloc_dest(req);1722 1723 cpu = get_cpu();1724 wq = wq_table_next_wq(cpu);1725 put_cpu();1726 if (!wq) {1727 pr_debug("no wq configured for cpu=%d\n", cpu);1728 return -ENODEV;1729 }1730 1731 ret = iaa_wq_get(wq);1732 if (ret) {1733 pr_debug("no wq available for cpu=%d\n", cpu);1734 return -ENODEV;1735 }1736 1737 iaa_wq = idxd_wq_get_private(wq);1738 1739 dev = &wq->idxd->pdev->dev;1740 1741 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);1742 if (nr_sgs <= 0 || nr_sgs > 1) {1743 dev_dbg(dev, "couldn't map src sg for iaa device %d,"1744 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,1745 iaa_wq->wq->id, ret);1746 ret = -EIO;1747 goto out;1748 }1749 src_addr = sg_dma_address(req->src);1750 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"1751 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,1752 req->src, req->slen, sg_dma_len(req->src));1753 1754 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);1755 if (nr_sgs <= 0 || nr_sgs > 1) {1756 dev_dbg(dev, "couldn't map dst sg for iaa device %d,"1757 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,1758 iaa_wq->wq->id, ret);1759 ret = -EIO;1760 goto err_map_dst;1761 }1762 dst_addr = sg_dma_address(req->dst);1763 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"1764 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,1765 req->dst, req->dlen, sg_dma_len(req->dst));1766 1767 ret = iaa_decompress(tfm, req, wq, src_addr, req->slen,1768 dst_addr, &req->dlen, false);1769 if (ret == -EINPROGRESS)1770 return ret;1771 1772 if (ret != 0)1773 dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);1774 1775 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);1776err_map_dst:1777 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);1778out:1779 iaa_wq_put(wq);1780 1781 return ret;1782}1783 1784static void compression_ctx_init(struct iaa_compression_ctx *ctx)1785{1786 ctx->verify_compress = iaa_verify_compress;1787 ctx->async_mode = async_mode;1788 ctx->use_irq = use_irq;1789}1790 1791static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm)1792{1793 struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);1794 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);1795 1796 compression_ctx_init(ctx);1797 1798 ctx->mode = IAA_MODE_FIXED;1799 1800 return 0;1801}1802 1803static void dst_free(struct scatterlist *sgl)1804{1805 /*1806 * Called for req->dst = NULL cases but we free elsewhere1807 * using sgl_free_order().1808 */1809}1810 1811static struct acomp_alg iaa_acomp_fixed_deflate = {1812 .init = iaa_comp_init_fixed,1813 .compress = iaa_comp_acompress,1814 .decompress = iaa_comp_adecompress,1815 .dst_free = dst_free,1816 .base = {1817 .cra_name = "deflate",1818 .cra_driver_name = "deflate-iaa",1819 .cra_flags = CRYPTO_ALG_ASYNC,1820 .cra_ctxsize = sizeof(struct iaa_compression_ctx),1821 .cra_module = THIS_MODULE,1822 .cra_priority = IAA_ALG_PRIORITY,1823 }1824};1825 1826static int iaa_register_compression_device(void)1827{1828 int ret;1829 1830 ret = crypto_register_acomp(&iaa_acomp_fixed_deflate);1831 if (ret) {1832 pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret);1833 goto out;1834 }1835 1836 iaa_crypto_registered = true;1837out:1838 return ret;1839}1840 1841static int iaa_unregister_compression_device(void)1842{1843 if (iaa_crypto_registered)1844 crypto_unregister_acomp(&iaa_acomp_fixed_deflate);1845 1846 return 0;1847}1848 1849static int iaa_crypto_probe(struct idxd_dev *idxd_dev)1850{1851 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);1852 struct idxd_device *idxd = wq->idxd;1853 struct idxd_driver_data *data = idxd->data;1854 struct device *dev = &idxd_dev->conf_dev;1855 bool first_wq = false;1856 int ret = 0;1857 1858 if (idxd->state != IDXD_DEV_ENABLED)1859 return -ENXIO;1860 1861 if (data->type != IDXD_TYPE_IAX)1862 return -ENODEV;1863 1864 mutex_lock(&wq->wq_lock);1865 1866 if (idxd_wq_get_private(wq)) {1867 mutex_unlock(&wq->wq_lock);1868 return -EBUSY;1869 }1870 1871 if (!idxd_wq_driver_name_match(wq, dev)) {1872 dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n",1873 idxd->id, wq->id, wq->driver_name, dev->driver->name);1874 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;1875 ret = -ENODEV;1876 goto err;1877 }1878 1879 wq->type = IDXD_WQT_KERNEL;1880 1881 ret = idxd_drv_enable_wq(wq);1882 if (ret < 0) {1883 dev_dbg(dev, "enable wq %d.%d failed: %d\n",1884 idxd->id, wq->id, ret);1885 ret = -ENXIO;1886 goto err;1887 }1888 1889 mutex_lock(&iaa_devices_lock);1890 1891 if (list_empty(&iaa_devices)) {1892 ret = alloc_wq_table(wq->idxd->max_wqs);1893 if (ret)1894 goto err_alloc;1895 first_wq = true;1896 }1897 1898 ret = save_iaa_wq(wq);1899 if (ret)1900 goto err_save;1901 1902 rebalance_wq_table();1903 1904 if (first_wq) {1905 iaa_crypto_enabled = true;1906 ret = iaa_register_compression_device();1907 if (ret != 0) {1908 iaa_crypto_enabled = false;1909 dev_dbg(dev, "IAA compression device registration failed\n");1910 goto err_register;1911 }1912 try_module_get(THIS_MODULE);1913 1914 pr_info("iaa_crypto now ENABLED\n");1915 }1916 1917 mutex_unlock(&iaa_devices_lock);1918out:1919 mutex_unlock(&wq->wq_lock);1920 1921 return ret;1922 1923err_register:1924 remove_iaa_wq(wq);1925 free_iaa_wq(idxd_wq_get_private(wq));1926err_save:1927 if (first_wq)1928 free_wq_table();1929err_alloc:1930 mutex_unlock(&iaa_devices_lock);1931 idxd_drv_disable_wq(wq);1932err:1933 wq->type = IDXD_WQT_NONE;1934 1935 goto out;1936}1937 1938static void iaa_crypto_remove(struct idxd_dev *idxd_dev)1939{1940 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);1941 struct idxd_device *idxd = wq->idxd;1942 struct iaa_wq *iaa_wq;1943 bool free = false;1944 1945 idxd_wq_quiesce(wq);1946 1947 mutex_lock(&wq->wq_lock);1948 mutex_lock(&iaa_devices_lock);1949 1950 remove_iaa_wq(wq);1951 1952 spin_lock(&idxd->dev_lock);1953 iaa_wq = idxd_wq_get_private(wq);1954 if (!iaa_wq) {1955 spin_unlock(&idxd->dev_lock);1956 pr_err("%s: no iaa_wq available to remove\n", __func__);1957 goto out;1958 }1959 1960 if (iaa_wq->ref) {1961 iaa_wq->remove = true;1962 } else {1963 wq = iaa_wq->wq;1964 idxd_wq_set_private(wq, NULL);1965 free = true;1966 }1967 spin_unlock(&idxd->dev_lock);1968 if (free) {1969 __free_iaa_wq(iaa_wq);1970 kfree(iaa_wq);1971 }1972 1973 idxd_drv_disable_wq(wq);1974 rebalance_wq_table();1975 1976 if (nr_iaa == 0) {1977 iaa_crypto_enabled = false;1978 free_wq_table();1979 module_put(THIS_MODULE);1980 1981 pr_info("iaa_crypto now DISABLED\n");1982 }1983out:1984 mutex_unlock(&iaa_devices_lock);1985 mutex_unlock(&wq->wq_lock);1986}1987 1988static enum idxd_dev_type dev_types[] = {1989 IDXD_DEV_WQ,1990 IDXD_DEV_NONE,1991};1992 1993static struct idxd_device_driver iaa_crypto_driver = {1994 .probe = iaa_crypto_probe,1995 .remove = iaa_crypto_remove,1996 .name = IDXD_SUBDRIVER_NAME,1997 .type = dev_types,1998 .desc_complete = iaa_desc_complete,1999};2000 2001static int __init iaa_crypto_init_module(void)2002{2003 int ret = 0;2004 int node;2005 2006 nr_cpus = num_possible_cpus();2007 for_each_node_with_cpus(node)2008 nr_nodes++;2009 if (!nr_nodes) {2010 pr_err("IAA couldn't find any nodes with cpus\n");2011 return -ENODEV;2012 }2013 nr_cpus_per_node = nr_cpus / nr_nodes;2014 2015 if (crypto_has_comp("deflate-generic", 0, 0))2016 deflate_generic_tfm = crypto_alloc_comp("deflate-generic", 0, 0);2017 2018 if (IS_ERR_OR_NULL(deflate_generic_tfm)) {2019 pr_err("IAA could not alloc %s tfm: errcode = %ld\n",2020 "deflate-generic", PTR_ERR(deflate_generic_tfm));2021 return -ENOMEM;2022 }2023 2024 ret = iaa_aecs_init_fixed();2025 if (ret < 0) {2026 pr_debug("IAA fixed compression mode init failed\n");2027 goto err_aecs_init;2028 }2029 2030 ret = idxd_driver_register(&iaa_crypto_driver);2031 if (ret) {2032 pr_debug("IAA wq sub-driver registration failed\n");2033 goto err_driver_reg;2034 }2035 2036 ret = driver_create_file(&iaa_crypto_driver.drv,2037 &driver_attr_verify_compress);2038 if (ret) {2039 pr_debug("IAA verify_compress attr creation failed\n");2040 goto err_verify_attr_create;2041 }2042 2043 ret = driver_create_file(&iaa_crypto_driver.drv,2044 &driver_attr_sync_mode);2045 if (ret) {2046 pr_debug("IAA sync mode attr creation failed\n");2047 goto err_sync_attr_create;2048 }2049 2050 if (iaa_crypto_debugfs_init())2051 pr_warn("debugfs init failed, stats not available\n");2052 2053 pr_debug("initialized\n");2054out:2055 return ret;2056 2057err_sync_attr_create:2058 driver_remove_file(&iaa_crypto_driver.drv,2059 &driver_attr_verify_compress);2060err_verify_attr_create:2061 idxd_driver_unregister(&iaa_crypto_driver);2062err_driver_reg:2063 iaa_aecs_cleanup_fixed();2064err_aecs_init:2065 crypto_free_comp(deflate_generic_tfm);2066 2067 goto out;2068}2069 2070static void __exit iaa_crypto_cleanup_module(void)2071{2072 if (iaa_unregister_compression_device())2073 pr_debug("IAA compression device unregister failed\n");2074 2075 iaa_crypto_debugfs_cleanup();2076 driver_remove_file(&iaa_crypto_driver.drv,2077 &driver_attr_sync_mode);2078 driver_remove_file(&iaa_crypto_driver.drv,2079 &driver_attr_verify_compress);2080 idxd_driver_unregister(&iaa_crypto_driver);2081 iaa_aecs_cleanup_fixed();2082 crypto_free_comp(deflate_generic_tfm);2083 2084 pr_debug("cleaned up\n");2085}2086 2087MODULE_IMPORT_NS(IDXD);2088MODULE_LICENSE("GPL");2089MODULE_ALIAS_IDXD_DEVICE(0);2090MODULE_AUTHOR("Intel Corporation");2091MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver");2092 2093module_init(iaa_crypto_init_module);2094module_exit(iaa_crypto_cleanup_module);2095