1274 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Driver for IBM Power 842 compression accelerator4 *5 * Copyright (C) IBM Corporation, 20126 *7 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>8 * Seth Jennings <sjenning@linux.vnet.ibm.com>9 */10 11#include <asm/vio.h>12#include <asm/hvcall.h>13#include <asm/vas.h>14 15#include "nx-842.h"16#include "nx_csbcpb.h" /* struct nx_csbcpb */17 18MODULE_LICENSE("GPL");19MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");20MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");21MODULE_ALIAS_CRYPTO("842");22MODULE_ALIAS_CRYPTO("842-nx");23 24/*25 * Coprocessor type specific capabilities from the hypervisor.26 */27struct hv_nx_cop_caps {28 __be64 descriptor;29 __be64 req_max_processed_len; /* Max bytes in one GZIP request */30 __be64 min_compress_len; /* Min compression size in bytes */31 __be64 min_decompress_len; /* Min decompression size in bytes */32} __packed __aligned(0x1000);33 34/*35 * Coprocessor type specific capabilities.36 */37struct nx_cop_caps {38 u64 descriptor;39 u64 req_max_processed_len; /* Max bytes in one GZIP request */40 u64 min_compress_len; /* Min compression in bytes */41 u64 min_decompress_len; /* Min decompression in bytes */42};43 44static u64 caps_feat;45static struct nx_cop_caps nx_cop_caps;46 47static struct nx842_constraints nx842_pseries_constraints = {48 .alignment = DDE_BUFFER_ALIGN,49 .multiple = DDE_BUFFER_LAST_MULT,50 .minimum = DDE_BUFFER_LAST_MULT,51 .maximum = PAGE_SIZE, /* dynamic, max_sync_size */52};53 54static int check_constraints(unsigned long buf, unsigned int *len, bool in)55{56 if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {57 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",58 in ? "input" : "output", buf,59 nx842_pseries_constraints.alignment);60 return -EINVAL;61 }62 if (*len % nx842_pseries_constraints.multiple) {63 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",64 in ? "input" : "output", *len,65 nx842_pseries_constraints.multiple);66 if (in)67 return -EINVAL;68 *len = round_down(*len, nx842_pseries_constraints.multiple);69 }70 if (*len < nx842_pseries_constraints.minimum) {71 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",72 in ? "input" : "output", *len,73 nx842_pseries_constraints.minimum);74 return -EINVAL;75 }76 if (*len > nx842_pseries_constraints.maximum) {77 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",78 in ? "input" : "output", *len,79 nx842_pseries_constraints.maximum);80 if (in)81 return -EINVAL;82 *len = nx842_pseries_constraints.maximum;83 }84 return 0;85}86 87/* I assume we need to align the CSB? */88#define WORKMEM_ALIGN (256)89 90struct nx842_workmem {91 /* scatterlist */92 char slin[4096];93 char slout[4096];94 /* coprocessor status/parameter block */95 struct nx_csbcpb csbcpb;96 97 char padding[WORKMEM_ALIGN];98} __aligned(WORKMEM_ALIGN);99 100/* Macros for fields within nx_csbcpb */101/* Check the valid bit within the csbcpb valid field */102#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))103 104/* CE macros operate on the completion_extension field bits in the csbcpb.105 * CE0 0=full completion, 1=partial completion106 * CE1 0=CE0 indicates completion, 1=termination (output may be modified)107 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */108#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))109#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))110#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))111 112/* The NX unit accepts data only on 4K page boundaries */113#define NX842_HW_PAGE_SIZE (4096)114#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))115 116struct ibm_nx842_counters {117 atomic64_t comp_complete;118 atomic64_t comp_failed;119 atomic64_t decomp_complete;120 atomic64_t decomp_failed;121 atomic64_t swdecomp;122 atomic64_t comp_times[32];123 atomic64_t decomp_times[32];124};125 126struct nx842_devdata {127 struct vio_dev *vdev;128 struct device *dev;129 struct ibm_nx842_counters *counters;130 unsigned int max_sg_len;131 unsigned int max_sync_size;132 unsigned int max_sync_sg;133};134 135static struct nx842_devdata __rcu *devdata;136static DEFINE_SPINLOCK(devdata_mutex);137 138#define NX842_COUNTER_INC(_x) \139static inline void nx842_inc_##_x( \140 const struct nx842_devdata *dev) { \141 if (dev) \142 atomic64_inc(&dev->counters->_x); \143}144NX842_COUNTER_INC(comp_complete);145NX842_COUNTER_INC(comp_failed);146NX842_COUNTER_INC(decomp_complete);147NX842_COUNTER_INC(decomp_failed);148NX842_COUNTER_INC(swdecomp);149 150#define NX842_HIST_SLOTS 16151 152static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)153{154 int bucket = fls(time);155 156 if (bucket)157 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);158 159 atomic64_inc(×[bucket]);160}161 162/* NX unit operation flags */163#define NX842_OP_COMPRESS 0x0164#define NX842_OP_CRC 0x1165#define NX842_OP_DECOMPRESS 0x2166#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)167#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)168#define NX842_OP_ASYNC (1<<23)169#define NX842_OP_NOTIFY (1<<22)170#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)171 172static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)173{174 /* No use of DMA mappings within the driver. */175 return 0;176}177 178struct nx842_slentry {179 __be64 ptr; /* Real address (use __pa()) */180 __be64 len;181};182 183/* pHyp scatterlist entry */184struct nx842_scatterlist {185 int entry_nr; /* number of slentries */186 struct nx842_slentry *entries; /* ptr to array of slentries */187};188 189/* Does not include sizeof(entry_nr) in the size */190static inline unsigned long nx842_get_scatterlist_size(191 struct nx842_scatterlist *sl)192{193 return sl->entry_nr * sizeof(struct nx842_slentry);194}195 196static int nx842_build_scatterlist(unsigned long buf, int len,197 struct nx842_scatterlist *sl)198{199 unsigned long entrylen;200 struct nx842_slentry *entry;201 202 sl->entry_nr = 0;203 204 entry = sl->entries;205 while (len) {206 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));207 entrylen = min_t(int, len,208 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));209 entry->len = cpu_to_be64(entrylen);210 211 len -= entrylen;212 buf += entrylen;213 214 sl->entry_nr++;215 entry++;216 }217 218 return 0;219}220 221static int nx842_validate_result(struct device *dev,222 struct cop_status_block *csb)223{224 /* The csb must be valid after returning from vio_h_cop_sync */225 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {226 dev_err(dev, "%s: cspcbp not valid upon completion.\n",227 __func__);228 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",229 csb->valid,230 csb->crb_seq_number,231 csb->completion_code,232 csb->completion_extension);233 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",234 be32_to_cpu(csb->processed_byte_count),235 (unsigned long)be64_to_cpu(csb->address));236 return -EIO;237 }238 239 /* Check return values from the hardware in the CSB */240 switch (csb->completion_code) {241 case 0: /* Completed without error */242 break;243 case 64: /* Compression ok, but output larger than input */244 dev_dbg(dev, "%s: output size larger than input size\n",245 __func__);246 break;247 case 13: /* Output buffer too small */248 dev_dbg(dev, "%s: Out of space in output buffer\n",249 __func__);250 return -ENOSPC;251 case 65: /* Calculated CRC doesn't match the passed value */252 dev_dbg(dev, "%s: CRC mismatch for decompression\n",253 __func__);254 return -EINVAL;255 case 66: /* Input data contains an illegal template field */256 case 67: /* Template indicates data past the end of the input stream */257 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",258 __func__, csb->completion_code);259 return -EINVAL;260 default:261 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",262 __func__, csb->completion_code);263 return -EIO;264 }265 266 /* Hardware sanity check */267 if (!NX842_CSBCPB_CE2(csb->completion_extension)) {268 dev_err(dev, "%s: No error returned by hardware, but "269 "data returned is unusable, contact support.\n"270 "(Additional info: csbcbp->processed bytes "271 "does not specify processed bytes for the "272 "target buffer.)\n", __func__);273 return -EIO;274 }275 276 return 0;277}278 279/**280 * nx842_pseries_compress - Compress data using the 842 algorithm281 *282 * Compression provide by the NX842 coprocessor on IBM Power systems.283 * The input buffer is compressed and the result is stored in the284 * provided output buffer.285 *286 * Upon return from this function @outlen contains the length of the287 * compressed data. If there is an error then @outlen will be 0 and an288 * error will be specified by the return code from this function.289 *290 * @in: Pointer to input buffer291 * @inlen: Length of input buffer292 * @out: Pointer to output buffer293 * @outlen: Length of output buffer294 * @wmem: ptr to buffer for working memory, size determined by295 * nx842_pseries_driver.workmem_size296 *297 * Returns:298 * 0 Success, output of length @outlen stored in the buffer at @out299 * -ENOMEM Unable to allocate internal buffers300 * -ENOSPC Output buffer is to small301 * -EIO Internal error302 * -ENODEV Hardware unavailable303 */304static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,305 unsigned char *out, unsigned int *outlen,306 void *wmem)307{308 struct nx842_devdata *local_devdata;309 struct device *dev = NULL;310 struct nx842_workmem *workmem;311 struct nx842_scatterlist slin, slout;312 struct nx_csbcpb *csbcpb;313 int ret = 0;314 unsigned long inbuf, outbuf;315 struct vio_pfo_op op = {316 .done = NULL,317 .handle = 0,318 .timeout = 0,319 };320 unsigned long start = get_tb();321 322 inbuf = (unsigned long)in;323 if (check_constraints(inbuf, &inlen, true))324 return -EINVAL;325 326 outbuf = (unsigned long)out;327 if (check_constraints(outbuf, outlen, false))328 return -EINVAL;329 330 rcu_read_lock();331 local_devdata = rcu_dereference(devdata);332 if (!local_devdata || !local_devdata->dev) {333 rcu_read_unlock();334 return -ENODEV;335 }336 dev = local_devdata->dev;337 338 /* Init scatterlist */339 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);340 slin.entries = (struct nx842_slentry *)workmem->slin;341 slout.entries = (struct nx842_slentry *)workmem->slout;342 343 /* Init operation */344 op.flags = NX842_OP_COMPRESS_CRC;345 csbcpb = &workmem->csbcpb;346 memset(csbcpb, 0, sizeof(*csbcpb));347 op.csbcpb = nx842_get_pa(csbcpb);348 349 if ((inbuf & NX842_HW_PAGE_MASK) ==350 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {351 /* Create direct DDE */352 op.in = nx842_get_pa((void *)inbuf);353 op.inlen = inlen;354 } else {355 /* Create indirect DDE (scatterlist) */356 nx842_build_scatterlist(inbuf, inlen, &slin);357 op.in = nx842_get_pa(slin.entries);358 op.inlen = -nx842_get_scatterlist_size(&slin);359 }360 361 if ((outbuf & NX842_HW_PAGE_MASK) ==362 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {363 /* Create direct DDE */364 op.out = nx842_get_pa((void *)outbuf);365 op.outlen = *outlen;366 } else {367 /* Create indirect DDE (scatterlist) */368 nx842_build_scatterlist(outbuf, *outlen, &slout);369 op.out = nx842_get_pa(slout.entries);370 op.outlen = -nx842_get_scatterlist_size(&slout);371 }372 373 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",374 __func__, (unsigned long)op.in, (long)op.inlen,375 (unsigned long)op.out, (long)op.outlen);376 377 /* Send request to pHyp */378 ret = vio_h_cop_sync(local_devdata->vdev, &op);379 380 /* Check for pHyp error */381 if (ret) {382 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",383 __func__, ret, op.hcall_err);384 ret = -EIO;385 goto unlock;386 }387 388 /* Check for hardware error */389 ret = nx842_validate_result(dev, &csbcpb->csb);390 if (ret)391 goto unlock;392 393 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);394 dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);395 396unlock:397 if (ret)398 nx842_inc_comp_failed(local_devdata);399 else {400 nx842_inc_comp_complete(local_devdata);401 ibm_nx842_incr_hist(local_devdata->counters->comp_times,402 (get_tb() - start) / tb_ticks_per_usec);403 }404 rcu_read_unlock();405 return ret;406}407 408/**409 * nx842_pseries_decompress - Decompress data using the 842 algorithm410 *411 * Decompression provide by the NX842 coprocessor on IBM Power systems.412 * The input buffer is decompressed and the result is stored in the413 * provided output buffer. The size allocated to the output buffer is414 * provided by the caller of this function in @outlen. Upon return from415 * this function @outlen contains the length of the decompressed data.416 * If there is an error then @outlen will be 0 and an error will be417 * specified by the return code from this function.418 *419 * @in: Pointer to input buffer420 * @inlen: Length of input buffer421 * @out: Pointer to output buffer422 * @outlen: Length of output buffer423 * @wmem: ptr to buffer for working memory, size determined by424 * nx842_pseries_driver.workmem_size425 *426 * Returns:427 * 0 Success, output of length @outlen stored in the buffer at @out428 * -ENODEV Hardware decompression device is unavailable429 * -ENOMEM Unable to allocate internal buffers430 * -ENOSPC Output buffer is to small431 * -EINVAL Bad input data encountered when attempting decompress432 * -EIO Internal error433 */434static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,435 unsigned char *out, unsigned int *outlen,436 void *wmem)437{438 struct nx842_devdata *local_devdata;439 struct device *dev = NULL;440 struct nx842_workmem *workmem;441 struct nx842_scatterlist slin, slout;442 struct nx_csbcpb *csbcpb;443 int ret = 0;444 unsigned long inbuf, outbuf;445 struct vio_pfo_op op = {446 .done = NULL,447 .handle = 0,448 .timeout = 0,449 };450 unsigned long start = get_tb();451 452 /* Ensure page alignment and size */453 inbuf = (unsigned long)in;454 if (check_constraints(inbuf, &inlen, true))455 return -EINVAL;456 457 outbuf = (unsigned long)out;458 if (check_constraints(outbuf, outlen, false))459 return -EINVAL;460 461 rcu_read_lock();462 local_devdata = rcu_dereference(devdata);463 if (!local_devdata || !local_devdata->dev) {464 rcu_read_unlock();465 return -ENODEV;466 }467 dev = local_devdata->dev;468 469 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);470 471 /* Init scatterlist */472 slin.entries = (struct nx842_slentry *)workmem->slin;473 slout.entries = (struct nx842_slentry *)workmem->slout;474 475 /* Init operation */476 op.flags = NX842_OP_DECOMPRESS_CRC;477 csbcpb = &workmem->csbcpb;478 memset(csbcpb, 0, sizeof(*csbcpb));479 op.csbcpb = nx842_get_pa(csbcpb);480 481 if ((inbuf & NX842_HW_PAGE_MASK) ==482 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {483 /* Create direct DDE */484 op.in = nx842_get_pa((void *)inbuf);485 op.inlen = inlen;486 } else {487 /* Create indirect DDE (scatterlist) */488 nx842_build_scatterlist(inbuf, inlen, &slin);489 op.in = nx842_get_pa(slin.entries);490 op.inlen = -nx842_get_scatterlist_size(&slin);491 }492 493 if ((outbuf & NX842_HW_PAGE_MASK) ==494 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {495 /* Create direct DDE */496 op.out = nx842_get_pa((void *)outbuf);497 op.outlen = *outlen;498 } else {499 /* Create indirect DDE (scatterlist) */500 nx842_build_scatterlist(outbuf, *outlen, &slout);501 op.out = nx842_get_pa(slout.entries);502 op.outlen = -nx842_get_scatterlist_size(&slout);503 }504 505 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",506 __func__, (unsigned long)op.in, (long)op.inlen,507 (unsigned long)op.out, (long)op.outlen);508 509 /* Send request to pHyp */510 ret = vio_h_cop_sync(local_devdata->vdev, &op);511 512 /* Check for pHyp error */513 if (ret) {514 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",515 __func__, ret, op.hcall_err);516 goto unlock;517 }518 519 /* Check for hardware error */520 ret = nx842_validate_result(dev, &csbcpb->csb);521 if (ret)522 goto unlock;523 524 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);525 526unlock:527 if (ret)528 /* decompress fail */529 nx842_inc_decomp_failed(local_devdata);530 else {531 nx842_inc_decomp_complete(local_devdata);532 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,533 (get_tb() - start) / tb_ticks_per_usec);534 }535 536 rcu_read_unlock();537 return ret;538}539 540/**541 * nx842_OF_set_defaults -- Set default (disabled) values for devdata542 *543 * @devdata: struct nx842_devdata to update544 *545 * Returns:546 * 0 on success547 * -ENOENT if @devdata ptr is NULL548 */549static int nx842_OF_set_defaults(struct nx842_devdata *devdata)550{551 if (devdata) {552 devdata->max_sync_size = 0;553 devdata->max_sync_sg = 0;554 devdata->max_sg_len = 0;555 return 0;556 } else557 return -ENOENT;558}559 560/**561 * nx842_OF_upd_status -- Check the device info from OF status prop562 *563 * The status property indicates if the accelerator is enabled. If the564 * device is in the OF tree it indicates that the hardware is present.565 * The status field indicates if the device is enabled when the status566 * is 'okay'. Otherwise the device driver will be disabled.567 *568 * @devdata: struct nx842_devdata to use for dev_info569 * @prop: struct property point containing the maxsyncop for the update570 *571 * Returns:572 * 0 - Device is available573 * -ENODEV - Device is not available574 */575static int nx842_OF_upd_status(struct nx842_devdata *devdata,576 struct property *prop)577{578 const char *status = (const char *)prop->value;579 580 if (!strncmp(status, "okay", (size_t)prop->length))581 return 0;582 if (!strncmp(status, "disabled", (size_t)prop->length))583 return -ENODEV;584 dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);585 586 return -EINVAL;587}588 589/**590 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop591 *592 * Definition of the 'ibm,max-sg-len' OF property:593 * This field indicates the maximum byte length of a scatter list594 * for the platform facility. It is a single cell encoded as with encode-int.595 *596 * Example:597 * # od -x ibm,max-sg-len598 * 0000000 0000 0ff0599 *600 * In this example, the maximum byte length of a scatter list is601 * 0x0ff0 (4,080).602 *603 * @devdata: struct nx842_devdata to update604 * @prop: struct property point containing the maxsyncop for the update605 *606 * Returns:607 * 0 on success608 * -EINVAL on failure609 */610static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,611 struct property *prop) {612 int ret = 0;613 const unsigned int maxsglen = of_read_number(prop->value, 1);614 615 if (prop->length != sizeof(maxsglen)) {616 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);617 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,618 prop->length, sizeof(maxsglen));619 ret = -EINVAL;620 } else {621 devdata->max_sg_len = min_t(unsigned int,622 maxsglen, NX842_HW_PAGE_SIZE);623 }624 625 return ret;626}627 628/**629 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop630 *631 * Definition of the 'ibm,max-sync-cop' OF property:632 * Two series of cells. The first series of cells represents the maximums633 * that can be synchronously compressed. The second series of cells634 * represents the maximums that can be synchronously decompressed.635 * 1. The first cell in each series contains the count of the number of636 * data length, scatter list elements pairs that follow – each being637 * of the form638 * a. One cell data byte length639 * b. One cell total number of scatter list elements640 *641 * Example:642 * # od -x ibm,max-sync-cop643 * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001644 * 0000020 0000 1000 0000 01fe645 *646 * In this example, compression supports 0x1000 (4,096) data byte length647 * and 0x1fe (510) total scatter list elements. Decompression supports648 * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list649 * elements.650 *651 * @devdata: struct nx842_devdata to update652 * @prop: struct property point containing the maxsyncop for the update653 *654 * Returns:655 * 0 on success656 * -EINVAL on failure657 */658static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,659 struct property *prop) {660 int ret = 0;661 unsigned int comp_data_limit, decomp_data_limit;662 unsigned int comp_sg_limit, decomp_sg_limit;663 const struct maxsynccop_t {664 __be32 comp_elements;665 __be32 comp_data_limit;666 __be32 comp_sg_limit;667 __be32 decomp_elements;668 __be32 decomp_data_limit;669 __be32 decomp_sg_limit;670 } *maxsynccop;671 672 if (prop->length != sizeof(*maxsynccop)) {673 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);674 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,675 sizeof(*maxsynccop));676 ret = -EINVAL;677 goto out;678 }679 680 maxsynccop = (const struct maxsynccop_t *)prop->value;681 comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);682 comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);683 decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);684 decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);685 686 /* Use one limit rather than separate limits for compression and687 * decompression. Set a maximum for this so as not to exceed the688 * size that the header can support and round the value down to689 * the hardware page size (4K) */690 devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);691 692 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,693 65536);694 695 if (devdata->max_sync_size < 4096) {696 dev_err(devdata->dev, "%s: hardware max data size (%u) is "697 "less than the driver minimum, unable to use "698 "the hardware device\n",699 __func__, devdata->max_sync_size);700 ret = -EINVAL;701 goto out;702 }703 704 nx842_pseries_constraints.maximum = devdata->max_sync_size;705 706 devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);707 if (devdata->max_sync_sg < 1) {708 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "709 "less than the driver minimum, unable to use "710 "the hardware device\n",711 __func__, devdata->max_sync_sg);712 ret = -EINVAL;713 goto out;714 }715 716out:717 return ret;718}719 720/**721 * nx842_OF_upd -- Handle OF properties updates for the device.722 *723 * Set all properties from the OF tree. Optionally, a new property724 * can be provided by the @new_prop pointer to overwrite an existing value.725 * The device will remain disabled until all values are valid, this function726 * will return an error for updates unless all values are valid.727 *728 * @new_prop: If not NULL, this property is being updated. If NULL, update729 * all properties from the current values in the OF tree.730 *731 * Returns:732 * 0 - Success733 * -ENOMEM - Could not allocate memory for new devdata structure734 * -EINVAL - property value not found, new_prop is not a recognized735 * property for the device or property value is not valid.736 * -ENODEV - Device is not available737 */738static int nx842_OF_upd(struct property *new_prop)739{740 struct nx842_devdata *old_devdata = NULL;741 struct nx842_devdata *new_devdata = NULL;742 struct device_node *of_node = NULL;743 struct property *status = NULL;744 struct property *maxsglen = NULL;745 struct property *maxsyncop = NULL;746 int ret = 0;747 unsigned long flags;748 749 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);750 if (!new_devdata)751 return -ENOMEM;752 753 spin_lock_irqsave(&devdata_mutex, flags);754 old_devdata = rcu_dereference_check(devdata,755 lockdep_is_held(&devdata_mutex));756 if (old_devdata)757 of_node = old_devdata->dev->of_node;758 759 if (!old_devdata || !of_node) {760 pr_err("%s: device is not available\n", __func__);761 spin_unlock_irqrestore(&devdata_mutex, flags);762 kfree(new_devdata);763 return -ENODEV;764 }765 766 memcpy(new_devdata, old_devdata, sizeof(*old_devdata));767 new_devdata->counters = old_devdata->counters;768 769 /* Set ptrs for existing properties */770 status = of_find_property(of_node, "status", NULL);771 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);772 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);773 if (!status || !maxsglen || !maxsyncop) {774 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);775 ret = -EINVAL;776 goto error_out;777 }778 779 /*780 * If this is a property update, there are only certain properties that781 * we care about. Bail if it isn't in the below list782 */783 if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||784 strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||785 strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))786 goto out;787 788 /* Perform property updates */789 ret = nx842_OF_upd_status(new_devdata, status);790 if (ret)791 goto error_out;792 793 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);794 if (ret)795 goto error_out;796 797 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);798 if (ret)799 goto error_out;800 801out:802 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",803 __func__, new_devdata->max_sync_size,804 old_devdata->max_sync_size);805 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",806 __func__, new_devdata->max_sync_sg,807 old_devdata->max_sync_sg);808 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",809 __func__, new_devdata->max_sg_len,810 old_devdata->max_sg_len);811 812 rcu_assign_pointer(devdata, new_devdata);813 spin_unlock_irqrestore(&devdata_mutex, flags);814 synchronize_rcu();815 dev_set_drvdata(new_devdata->dev, new_devdata);816 kfree(old_devdata);817 return 0;818 819error_out:820 if (new_devdata) {821 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);822 nx842_OF_set_defaults(new_devdata);823 rcu_assign_pointer(devdata, new_devdata);824 spin_unlock_irqrestore(&devdata_mutex, flags);825 synchronize_rcu();826 dev_set_drvdata(new_devdata->dev, new_devdata);827 kfree(old_devdata);828 } else {829 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);830 spin_unlock_irqrestore(&devdata_mutex, flags);831 }832 833 if (!ret)834 ret = -EINVAL;835 return ret;836}837 838/**839 * nx842_OF_notifier - Process updates to OF properties for the device840 *841 * @np: notifier block842 * @action: notifier action843 * @data: struct of_reconfig_data pointer844 *845 * Returns:846 * NOTIFY_OK on success847 * NOTIFY_BAD encoded with error number on failure, use848 * notifier_to_errno() to decode this value849 */850static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,851 void *data)852{853 struct of_reconfig_data *upd = data;854 struct nx842_devdata *local_devdata;855 struct device_node *node = NULL;856 857 rcu_read_lock();858 local_devdata = rcu_dereference(devdata);859 if (local_devdata)860 node = local_devdata->dev->of_node;861 862 if (local_devdata &&863 action == OF_RECONFIG_UPDATE_PROPERTY &&864 !strcmp(upd->dn->name, node->name)) {865 rcu_read_unlock();866 nx842_OF_upd(upd->prop);867 } else868 rcu_read_unlock();869 870 return NOTIFY_OK;871}872 873static struct notifier_block nx842_of_nb = {874 .notifier_call = nx842_OF_notifier,875};876 877#define nx842_counter_read(_name) \878static ssize_t nx842_##_name##_show(struct device *dev, \879 struct device_attribute *attr, \880 char *buf) { \881 struct nx842_devdata *local_devdata; \882 int p = 0; \883 rcu_read_lock(); \884 local_devdata = rcu_dereference(devdata); \885 if (local_devdata) \886 p = snprintf(buf, PAGE_SIZE, "%lld\n", \887 atomic64_read(&local_devdata->counters->_name)); \888 rcu_read_unlock(); \889 return p; \890}891 892#define NX842DEV_COUNTER_ATTR_RO(_name) \893 nx842_counter_read(_name); \894 static struct device_attribute dev_attr_##_name = __ATTR(_name, \895 0444, \896 nx842_##_name##_show,\897 NULL);898 899NX842DEV_COUNTER_ATTR_RO(comp_complete);900NX842DEV_COUNTER_ATTR_RO(comp_failed);901NX842DEV_COUNTER_ATTR_RO(decomp_complete);902NX842DEV_COUNTER_ATTR_RO(decomp_failed);903NX842DEV_COUNTER_ATTR_RO(swdecomp);904 905static ssize_t nx842_timehist_show(struct device *,906 struct device_attribute *, char *);907 908static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,909 nx842_timehist_show, NULL);910static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,911 0444, nx842_timehist_show, NULL);912 913static ssize_t nx842_timehist_show(struct device *dev,914 struct device_attribute *attr, char *buf) {915 char *p = buf;916 struct nx842_devdata *local_devdata;917 atomic64_t *times;918 int bytes_remain = PAGE_SIZE;919 int bytes;920 int i;921 922 rcu_read_lock();923 local_devdata = rcu_dereference(devdata);924 if (!local_devdata) {925 rcu_read_unlock();926 return 0;927 }928 929 if (attr == &dev_attr_comp_times)930 times = local_devdata->counters->comp_times;931 else if (attr == &dev_attr_decomp_times)932 times = local_devdata->counters->decomp_times;933 else {934 rcu_read_unlock();935 return 0;936 }937 938 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {939 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",940 i ? (2<<(i-1)) : 0, (2<<i)-1,941 atomic64_read(×[i]));942 bytes_remain -= bytes;943 p += bytes;944 }945 /* The last bucket holds everything over946 * 2<<(NX842_HIST_SLOTS - 2) us */947 bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",948 2<<(NX842_HIST_SLOTS - 2),949 atomic64_read(×[(NX842_HIST_SLOTS - 1)]));950 p += bytes;951 952 rcu_read_unlock();953 return p - buf;954}955 956static struct attribute *nx842_sysfs_entries[] = {957 &dev_attr_comp_complete.attr,958 &dev_attr_comp_failed.attr,959 &dev_attr_decomp_complete.attr,960 &dev_attr_decomp_failed.attr,961 &dev_attr_swdecomp.attr,962 &dev_attr_comp_times.attr,963 &dev_attr_decomp_times.attr,964 NULL,965};966 967static const struct attribute_group nx842_attribute_group = {968 .name = NULL, /* put in device directory */969 .attrs = nx842_sysfs_entries,970};971 972#define nxcop_caps_read(_name) \973static ssize_t nxcop_##_name##_show(struct device *dev, \974 struct device_attribute *attr, char *buf) \975{ \976 return sprintf(buf, "%lld\n", nx_cop_caps._name); \977}978 979#define NXCT_ATTR_RO(_name) \980 nxcop_caps_read(_name); \981 static struct device_attribute dev_attr_##_name = __ATTR(_name, \982 0444, \983 nxcop_##_name##_show, \984 NULL);985 986NXCT_ATTR_RO(req_max_processed_len);987NXCT_ATTR_RO(min_compress_len);988NXCT_ATTR_RO(min_decompress_len);989 990static struct attribute *nxcop_caps_sysfs_entries[] = {991 &dev_attr_req_max_processed_len.attr,992 &dev_attr_min_compress_len.attr,993 &dev_attr_min_decompress_len.attr,994 NULL,995};996 997static const struct attribute_group nxcop_caps_attr_group = {998 .name = "nx_gzip_caps",999 .attrs = nxcop_caps_sysfs_entries,1000};1001 1002static struct nx842_driver nx842_pseries_driver = {1003 .name = KBUILD_MODNAME,1004 .owner = THIS_MODULE,1005 .workmem_size = sizeof(struct nx842_workmem),1006 .constraints = &nx842_pseries_constraints,1007 .compress = nx842_pseries_compress,1008 .decompress = nx842_pseries_decompress,1009};1010 1011static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)1012{1013 return nx842_crypto_init(tfm, &nx842_pseries_driver);1014}1015 1016static struct crypto_alg nx842_pseries_alg = {1017 .cra_name = "842",1018 .cra_driver_name = "842-nx",1019 .cra_priority = 300,1020 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,1021 .cra_ctxsize = sizeof(struct nx842_crypto_ctx),1022 .cra_module = THIS_MODULE,1023 .cra_init = nx842_pseries_crypto_init,1024 .cra_exit = nx842_crypto_exit,1025 .cra_u = { .compress = {1026 .coa_compress = nx842_crypto_compress,1027 .coa_decompress = nx842_crypto_decompress } }1028};1029 1030static int nx842_probe(struct vio_dev *viodev,1031 const struct vio_device_id *id)1032{1033 struct nx842_devdata *old_devdata, *new_devdata = NULL;1034 unsigned long flags;1035 int ret = 0;1036 1037 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);1038 if (!new_devdata)1039 return -ENOMEM;1040 1041 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),1042 GFP_NOFS);1043 if (!new_devdata->counters) {1044 kfree(new_devdata);1045 return -ENOMEM;1046 }1047 1048 spin_lock_irqsave(&devdata_mutex, flags);1049 old_devdata = rcu_dereference_check(devdata,1050 lockdep_is_held(&devdata_mutex));1051 1052 if (old_devdata && old_devdata->vdev != NULL) {1053 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);1054 ret = -1;1055 goto error_unlock;1056 }1057 1058 dev_set_drvdata(&viodev->dev, NULL);1059 1060 new_devdata->vdev = viodev;1061 new_devdata->dev = &viodev->dev;1062 nx842_OF_set_defaults(new_devdata);1063 1064 rcu_assign_pointer(devdata, new_devdata);1065 spin_unlock_irqrestore(&devdata_mutex, flags);1066 synchronize_rcu();1067 kfree(old_devdata);1068 1069 of_reconfig_notifier_register(&nx842_of_nb);1070 1071 ret = nx842_OF_upd(NULL);1072 if (ret)1073 goto error;1074 1075 ret = crypto_register_alg(&nx842_pseries_alg);1076 if (ret) {1077 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);1078 goto error;1079 }1080 1081 rcu_read_lock();1082 dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));1083 rcu_read_unlock();1084 1085 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {1086 dev_err(&viodev->dev, "could not create sysfs device attributes\n");1087 ret = -1;1088 goto error;1089 }1090 1091 if (caps_feat) {1092 if (sysfs_create_group(&viodev->dev.kobj,1093 &nxcop_caps_attr_group)) {1094 dev_err(&viodev->dev,1095 "Could not create sysfs NX capability entries\n");1096 ret = -1;1097 goto error;1098 }1099 }1100 1101 return 0;1102 1103error_unlock:1104 spin_unlock_irqrestore(&devdata_mutex, flags);1105 if (new_devdata)1106 kfree(new_devdata->counters);1107 kfree(new_devdata);1108error:1109 return ret;1110}1111 1112static void nx842_remove(struct vio_dev *viodev)1113{1114 struct nx842_devdata *old_devdata;1115 unsigned long flags;1116 1117 pr_info("Removing IBM Power 842 compression device\n");1118 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);1119 1120 if (caps_feat)1121 sysfs_remove_group(&viodev->dev.kobj, &nxcop_caps_attr_group);1122 1123 crypto_unregister_alg(&nx842_pseries_alg);1124 1125 spin_lock_irqsave(&devdata_mutex, flags);1126 old_devdata = rcu_dereference_check(devdata,1127 lockdep_is_held(&devdata_mutex));1128 of_reconfig_notifier_unregister(&nx842_of_nb);1129 RCU_INIT_POINTER(devdata, NULL);1130 spin_unlock_irqrestore(&devdata_mutex, flags);1131 synchronize_rcu();1132 dev_set_drvdata(&viodev->dev, NULL);1133 if (old_devdata)1134 kfree(old_devdata->counters);1135 kfree(old_devdata);1136}1137 1138/*1139 * Get NX capabilities from the hypervisor.1140 * Only NXGZIP capabilities are provided by the hypersvisor right1141 * now and these values are available to user space with sysfs.1142 */1143static void __init nxcop_get_capabilities(void)1144{1145 struct hv_vas_all_caps *hv_caps;1146 struct hv_nx_cop_caps *hv_nxc;1147 int rc;1148 1149 hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL);1150 if (!hv_caps)1151 return;1152 /*1153 * Get NX overall capabilities with feature type=01154 */1155 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0,1156 (u64)virt_to_phys(hv_caps));1157 if (rc)1158 goto out;1159 1160 caps_feat = be64_to_cpu(hv_caps->feat_type);1161 /*1162 * NX-GZIP feature available1163 */1164 if (caps_feat & VAS_NX_GZIP_FEAT_BIT) {1165 hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL);1166 if (!hv_nxc)1167 goto out;1168 /*1169 * Get capabilities for NX-GZIP feature1170 */1171 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES,1172 VAS_NX_GZIP_FEAT,1173 (u64)virt_to_phys(hv_nxc));1174 } else {1175 pr_err("NX-GZIP feature is not available\n");1176 rc = -EINVAL;1177 }1178 1179 if (!rc) {1180 nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor);1181 nx_cop_caps.req_max_processed_len =1182 be64_to_cpu(hv_nxc->req_max_processed_len);1183 nx_cop_caps.min_compress_len =1184 be64_to_cpu(hv_nxc->min_compress_len);1185 nx_cop_caps.min_decompress_len =1186 be64_to_cpu(hv_nxc->min_decompress_len);1187 } else {1188 caps_feat = 0;1189 }1190 1191 kfree(hv_nxc);1192out:1193 kfree(hv_caps);1194}1195 1196static const struct vio_device_id nx842_vio_driver_ids[] = {1197 {"ibm,compression-v1", "ibm,compression"},1198 {"", ""},1199};1200MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);1201 1202static struct vio_driver nx842_vio_driver = {1203 .name = KBUILD_MODNAME,1204 .probe = nx842_probe,1205 .remove = nx842_remove,1206 .get_desired_dma = nx842_get_desired_dma,1207 .id_table = nx842_vio_driver_ids,1208};1209 1210static int __init nx842_pseries_init(void)1211{1212 struct nx842_devdata *new_devdata;1213 struct device_node *np;1214 int ret;1215 1216 np = of_find_compatible_node(NULL, NULL, "ibm,compression");1217 if (!np)1218 return -ENODEV;1219 of_node_put(np);1220 1221 RCU_INIT_POINTER(devdata, NULL);1222 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);1223 if (!new_devdata)1224 return -ENOMEM;1225 1226 RCU_INIT_POINTER(devdata, new_devdata);1227 /*1228 * Get NX capabilities from the hypervisor.1229 */1230 nxcop_get_capabilities();1231 1232 ret = vio_register_driver(&nx842_vio_driver);1233 if (ret) {1234 pr_err("Could not register VIO driver %d\n", ret);1235 1236 kfree(new_devdata);1237 return ret;1238 }1239 1240 ret = vas_register_api_pseries(THIS_MODULE, VAS_COP_TYPE_GZIP,1241 "nx-gzip");1242 1243 if (ret)1244 pr_err("NX-GZIP is not supported. Returned=%d\n", ret);1245 1246 return 0;1247}1248 1249module_init(nx842_pseries_init);1250 1251static void __exit nx842_pseries_exit(void)1252{1253 struct nx842_devdata *old_devdata;1254 unsigned long flags;1255 1256 vas_unregister_api_pseries();1257 1258 crypto_unregister_alg(&nx842_pseries_alg);1259 1260 spin_lock_irqsave(&devdata_mutex, flags);1261 old_devdata = rcu_dereference_check(devdata,1262 lockdep_is_held(&devdata_mutex));1263 RCU_INIT_POINTER(devdata, NULL);1264 spin_unlock_irqrestore(&devdata_mutex, flags);1265 synchronize_rcu();1266 if (old_devdata && old_devdata->dev)1267 dev_set_drvdata(old_devdata->dev, NULL);1268 kfree(old_devdata);1269 vio_unregister_driver(&nx842_vio_driver);1270}1271 1272module_exit(nx842_pseries_exit);1273 1274