1136 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Driver for IBM PowerNV compression accelerator4 *5 * Copyright (C) 2015 Dan Streetman, IBM Corp6 */7 8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt9 10#include "nx-842.h"11 12#include <linux/timer.h>13 14#include <asm/prom.h>15#include <asm/icswx.h>16#include <asm/vas.h>17#include <asm/reg.h>18#include <asm/opal-api.h>19#include <asm/opal.h>20 21MODULE_LICENSE("GPL");22MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");23MODULE_DESCRIPTION("H/W Compression driver for IBM PowerNV processors");24MODULE_ALIAS_CRYPTO("842");25MODULE_ALIAS_CRYPTO("842-nx");26 27#define WORKMEM_ALIGN (CRB_ALIGN)28#define CSB_WAIT_MAX (5000) /* ms */29#define VAS_RETRIES (10)30 31struct nx842_workmem {32 /* Below fields must be properly aligned */33 struct coprocessor_request_block crb; /* CRB_ALIGN align */34 struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */35 struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */36 /* Above fields must be properly aligned */37 38 ktime_t start;39 40 char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */41} __packed __aligned(WORKMEM_ALIGN);42 43struct nx_coproc {44 unsigned int chip_id;45 unsigned int ct; /* Can be 842 or GZIP high/normal*/46 unsigned int ci; /* Coprocessor instance, used with icswx */47 struct {48 struct vas_window *rxwin;49 int id;50 } vas;51 struct list_head list;52};53 54/*55 * Send the request to NX engine on the chip for the corresponding CPU56 * where the process is executing. Use with VAS function.57 */58static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);59 60/* no cpu hotplug on powernv, so this list never changes after init */61static LIST_HEAD(nx_coprocs);62static unsigned int nx842_ct; /* used in icswx function */63 64/*65 * Using same values as in skiboot or coprocessor type representing66 * in NX workbook.67 */68#define NX_CT_GZIP (2) /* on P9 and later */69#define NX_CT_842 (3)70 71static int (*nx842_powernv_exec)(const unsigned char *in,72 unsigned int inlen, unsigned char *out,73 unsigned int *outlenp, void *workmem, int fc);74 75/*76 * setup_indirect_dde - Setup an indirect DDE77 *78 * The DDE is setup with the DDE count, byte count, and address of79 * first direct DDE in the list.80 */81static void setup_indirect_dde(struct data_descriptor_entry *dde,82 struct data_descriptor_entry *ddl,83 unsigned int dde_count, unsigned int byte_count)84{85 dde->flags = 0;86 dde->count = dde_count;87 dde->index = 0;88 dde->length = cpu_to_be32(byte_count);89 dde->address = cpu_to_be64(nx842_get_pa(ddl));90}91 92/*93 * setup_direct_dde - Setup single DDE from buffer94 *95 * The DDE is setup with the buffer and length. The buffer must be properly96 * aligned. The used length is returned.97 * Returns:98 * N Successfully set up DDE with N bytes99 */100static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,101 unsigned long pa, unsigned int len)102{103 unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));104 105 dde->flags = 0;106 dde->count = 0;107 dde->index = 0;108 dde->length = cpu_to_be32(l);109 dde->address = cpu_to_be64(pa);110 111 return l;112}113 114/*115 * setup_ddl - Setup DDL from buffer116 *117 * Returns:118 * 0 Successfully set up DDL119 */120static int setup_ddl(struct data_descriptor_entry *dde,121 struct data_descriptor_entry *ddl,122 unsigned char *buf, unsigned int len,123 bool in)124{125 unsigned long pa = nx842_get_pa(buf);126 int i, ret, total_len = len;127 128 if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {129 pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",130 in ? "input" : "output", pa, DDE_BUFFER_ALIGN);131 return -EINVAL;132 }133 134 /* only need to check last mult; since buffer must be135 * DDE_BUFFER_ALIGN aligned, and that is a multiple of136 * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers137 * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.138 */139 if (len % DDE_BUFFER_LAST_MULT) {140 pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",141 in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);142 if (in)143 return -EINVAL;144 len = round_down(len, DDE_BUFFER_LAST_MULT);145 }146 147 /* use a single direct DDE */148 if (len <= LEN_ON_PAGE(pa)) {149 ret = setup_direct_dde(dde, pa, len);150 WARN_ON(ret < len);151 return 0;152 }153 154 /* use the DDL */155 for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {156 ret = setup_direct_dde(&ddl[i], pa, len);157 buf += ret;158 len -= ret;159 pa = nx842_get_pa(buf);160 }161 162 if (len > 0) {163 pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",164 total_len, in ? "input" : "output", len);165 if (in)166 return -EMSGSIZE;167 total_len -= len;168 }169 setup_indirect_dde(dde, ddl, i, total_len);170 171 return 0;172}173 174#define CSB_ERR(csb, msg, ...) \175 pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \176 ##__VA_ARGS__, (csb)->flags, \177 (csb)->cs, (csb)->cc, (csb)->ce, \178 be32_to_cpu((csb)->count))179 180#define CSB_ERR_ADDR(csb, msg, ...) \181 CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \182 (unsigned long)be64_to_cpu((csb)->address))183 184static int wait_for_csb(struct nx842_workmem *wmem,185 struct coprocessor_status_block *csb)186{187 ktime_t start = wmem->start, now = ktime_get();188 ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);189 190 while (!(READ_ONCE(csb->flags) & CSB_V)) {191 cpu_relax();192 now = ktime_get();193 if (ktime_after(now, timeout))194 break;195 }196 197 /* hw has updated csb and output buffer */198 barrier();199 200 /* check CSB flags */201 if (!(csb->flags & CSB_V)) {202 CSB_ERR(csb, "CSB still not valid after %ld us, giving up",203 (long)ktime_us_delta(now, start));204 return -ETIMEDOUT;205 }206 if (csb->flags & CSB_F) {207 CSB_ERR(csb, "Invalid CSB format");208 return -EPROTO;209 }210 if (csb->flags & CSB_CH) {211 CSB_ERR(csb, "Invalid CSB chaining state");212 return -EPROTO;213 }214 215 /* verify CSB completion sequence is 0 */216 if (csb->cs) {217 CSB_ERR(csb, "Invalid CSB completion sequence");218 return -EPROTO;219 }220 221 /* check CSB Completion Code */222 switch (csb->cc) {223 /* no error */224 case CSB_CC_SUCCESS:225 break;226 case CSB_CC_TPBC_GT_SPBC:227 /* not an error, but the compressed data is228 * larger than the uncompressed data :(229 */230 break;231 232 /* input data errors */233 case CSB_CC_OPERAND_OVERLAP:234 /* input and output buffers overlap */235 CSB_ERR(csb, "Operand Overlap error");236 return -EINVAL;237 case CSB_CC_INVALID_OPERAND:238 CSB_ERR(csb, "Invalid operand");239 return -EINVAL;240 case CSB_CC_NOSPC:241 /* output buffer too small */242 return -ENOSPC;243 case CSB_CC_ABORT:244 CSB_ERR(csb, "Function aborted");245 return -EINTR;246 case CSB_CC_CRC_MISMATCH:247 CSB_ERR(csb, "CRC mismatch");248 return -EINVAL;249 case CSB_CC_TEMPL_INVALID:250 CSB_ERR(csb, "Compressed data template invalid");251 return -EINVAL;252 case CSB_CC_TEMPL_OVERFLOW:253 CSB_ERR(csb, "Compressed data template shows data past end");254 return -EINVAL;255 case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */256 /*257 * DDE byte count exceeds the limit specified in Maximum258 * byte count register.259 */260 CSB_ERR(csb, "DDE byte count exceeds the limit");261 return -EINVAL;262 263 /* these should not happen */264 case CSB_CC_INVALID_ALIGN:265 /* setup_ddl should have detected this */266 CSB_ERR_ADDR(csb, "Invalid alignment");267 return -EINVAL;268 case CSB_CC_DATA_LENGTH:269 /* setup_ddl should have detected this */270 CSB_ERR(csb, "Invalid data length");271 return -EINVAL;272 case CSB_CC_WR_TRANSLATION:273 case CSB_CC_TRANSLATION:274 case CSB_CC_TRANSLATION_DUP1:275 case CSB_CC_TRANSLATION_DUP2:276 case CSB_CC_TRANSLATION_DUP3:277 case CSB_CC_TRANSLATION_DUP4:278 case CSB_CC_TRANSLATION_DUP5:279 case CSB_CC_TRANSLATION_DUP6:280 /* should not happen, we use physical addrs */281 CSB_ERR_ADDR(csb, "Translation error");282 return -EPROTO;283 case CSB_CC_WR_PROTECTION:284 case CSB_CC_PROTECTION:285 case CSB_CC_PROTECTION_DUP1:286 case CSB_CC_PROTECTION_DUP2:287 case CSB_CC_PROTECTION_DUP3:288 case CSB_CC_PROTECTION_DUP4:289 case CSB_CC_PROTECTION_DUP5:290 case CSB_CC_PROTECTION_DUP6:291 /* should not happen, we use physical addrs */292 CSB_ERR_ADDR(csb, "Protection error");293 return -EPROTO;294 case CSB_CC_PRIVILEGE:295 /* shouldn't happen, we're in HYP mode */296 CSB_ERR(csb, "Insufficient Privilege error");297 return -EPROTO;298 case CSB_CC_EXCESSIVE_DDE:299 /* shouldn't happen, setup_ddl doesn't use many dde's */300 CSB_ERR(csb, "Too many DDEs in DDL");301 return -EINVAL;302 case CSB_CC_TRANSPORT:303 case CSB_CC_INVALID_CRB: /* P9 or later */304 /* shouldn't happen, we setup CRB correctly */305 CSB_ERR(csb, "Invalid CRB");306 return -EINVAL;307 case CSB_CC_INVALID_DDE: /* P9 or later */308 /*309 * shouldn't happen, setup_direct/indirect_dde creates310 * DDE right311 */312 CSB_ERR(csb, "Invalid DDE");313 return -EINVAL;314 case CSB_CC_SEGMENTED_DDL:315 /* shouldn't happen, setup_ddl creates DDL right */316 CSB_ERR(csb, "Segmented DDL error");317 return -EINVAL;318 case CSB_CC_DDE_OVERFLOW:319 /* shouldn't happen, setup_ddl creates DDL right */320 CSB_ERR(csb, "DDE overflow error");321 return -EINVAL;322 case CSB_CC_SESSION:323 /* should not happen with ICSWX */324 CSB_ERR(csb, "Session violation error");325 return -EPROTO;326 case CSB_CC_CHAIN:327 /* should not happen, we don't use chained CRBs */328 CSB_ERR(csb, "Chained CRB error");329 return -EPROTO;330 case CSB_CC_SEQUENCE:331 /* should not happen, we don't use chained CRBs */332 CSB_ERR(csb, "CRB sequence number error");333 return -EPROTO;334 case CSB_CC_UNKNOWN_CODE:335 CSB_ERR(csb, "Unknown subfunction code");336 return -EPROTO;337 338 /* hardware errors */339 case CSB_CC_RD_EXTERNAL:340 case CSB_CC_RD_EXTERNAL_DUP1:341 case CSB_CC_RD_EXTERNAL_DUP2:342 case CSB_CC_RD_EXTERNAL_DUP3:343 CSB_ERR_ADDR(csb, "Read error outside coprocessor");344 return -EPROTO;345 case CSB_CC_WR_EXTERNAL:346 CSB_ERR_ADDR(csb, "Write error outside coprocessor");347 return -EPROTO;348 case CSB_CC_INTERNAL:349 CSB_ERR(csb, "Internal error in coprocessor");350 return -EPROTO;351 case CSB_CC_PROVISION:352 CSB_ERR(csb, "Storage provision error");353 return -EPROTO;354 case CSB_CC_HW:355 CSB_ERR(csb, "Correctable hardware error");356 return -EPROTO;357 case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */358 CSB_ERR(csb, "Job did not finish within allowed time");359 return -EPROTO;360 361 default:362 CSB_ERR(csb, "Invalid CC %d", csb->cc);363 return -EPROTO;364 }365 366 /* check Completion Extension state */367 if (csb->ce & CSB_CE_TERMINATION) {368 CSB_ERR(csb, "CSB request was terminated");369 return -EPROTO;370 }371 if (csb->ce & CSB_CE_INCOMPLETE) {372 CSB_ERR(csb, "CSB request not complete");373 return -EPROTO;374 }375 if (!(csb->ce & CSB_CE_TPBC)) {376 CSB_ERR(csb, "TPBC not provided, unknown target length");377 return -EPROTO;378 }379 380 /* successful completion */381 pr_debug_ratelimited("Processed %u bytes in %lu us\n",382 be32_to_cpu(csb->count),383 (unsigned long)ktime_us_delta(now, start));384 385 return 0;386}387 388static int nx842_config_crb(const unsigned char *in, unsigned int inlen,389 unsigned char *out, unsigned int outlen,390 struct nx842_workmem *wmem)391{392 struct coprocessor_request_block *crb;393 struct coprocessor_status_block *csb;394 u64 csb_addr;395 int ret;396 397 crb = &wmem->crb;398 csb = &crb->csb;399 400 /* Clear any previous values */401 memset(crb, 0, sizeof(*crb));402 403 /* set up DDLs */404 ret = setup_ddl(&crb->source, wmem->ddl_in,405 (unsigned char *)in, inlen, true);406 if (ret)407 return ret;408 409 ret = setup_ddl(&crb->target, wmem->ddl_out,410 out, outlen, false);411 if (ret)412 return ret;413 414 /* set up CRB's CSB addr */415 csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;416 csb_addr |= CRB_CSB_AT; /* Addrs are phys */417 crb->csb_addr = cpu_to_be64(csb_addr);418 419 return 0;420}421 422/**423 * nx842_exec_icswx - compress/decompress data using the 842 algorithm424 *425 * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.426 * This compresses or decompresses the provided input buffer into the provided427 * output buffer.428 *429 * Upon return from this function @outlen contains the length of the430 * output data. If there is an error then @outlen will be 0 and an431 * error will be specified by the return code from this function.432 *433 * The @workmem buffer should only be used by one function call at a time.434 *435 * @in: input buffer pointer436 * @inlen: input buffer size437 * @out: output buffer pointer438 * @outlenp: output buffer size pointer439 * @workmem: working memory buffer pointer, size determined by440 * nx842_powernv_driver.workmem_size441 * @fc: function code, see CCW Function Codes in nx-842.h442 *443 * Returns:444 * 0 Success, output of length @outlenp stored in the buffer at @out445 * -ENODEV Hardware unavailable446 * -ENOSPC Output buffer is to small447 * -EMSGSIZE Input buffer too large448 * -EINVAL buffer constraints do not fix nx842_constraints449 * -EPROTO hardware error during operation450 * -ETIMEDOUT hardware did not complete operation in reasonable time451 * -EINTR operation was aborted452 */453static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen,454 unsigned char *out, unsigned int *outlenp,455 void *workmem, int fc)456{457 struct coprocessor_request_block *crb;458 struct coprocessor_status_block *csb;459 struct nx842_workmem *wmem;460 int ret;461 u32 ccw;462 unsigned int outlen = *outlenp;463 464 wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);465 466 *outlenp = 0;467 468 /* shoudn't happen, we don't load without a coproc */469 if (!nx842_ct) {470 pr_err_ratelimited("coprocessor CT is 0");471 return -ENODEV;472 }473 474 ret = nx842_config_crb(in, inlen, out, outlen, wmem);475 if (ret)476 return ret;477 478 crb = &wmem->crb;479 csb = &crb->csb;480 481 /* set up CCW */482 ccw = 0;483 ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);484 ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */485 ccw = SET_FIELD(CCW_FC_842, ccw, fc);486 487 wmem->start = ktime_get();488 489 /* do ICSWX */490 ret = icswx(cpu_to_be32(ccw), crb);491 492 pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,493 (unsigned int)ccw,494 (unsigned int)be32_to_cpu(crb->ccw));495 496 /*497 * NX842 coprocessor sets 3rd bit in CR register with XER[S0].498 * XER[S0] is the integer summary overflow bit which is nothing499 * to do NX. Since this bit can be set with other return values,500 * mask this bit.501 */502 ret &= ~ICSWX_XERS0;503 504 switch (ret) {505 case ICSWX_INITIATED:506 ret = wait_for_csb(wmem, csb);507 break;508 case ICSWX_BUSY:509 pr_debug_ratelimited("842 Coprocessor busy\n");510 ret = -EBUSY;511 break;512 case ICSWX_REJECTED:513 pr_err_ratelimited("ICSWX rejected\n");514 ret = -EPROTO;515 break;516 }517 518 if (!ret)519 *outlenp = be32_to_cpu(csb->count);520 521 return ret;522}523 524/**525 * nx842_exec_vas - compress/decompress data using the 842 algorithm526 *527 * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.528 * This compresses or decompresses the provided input buffer into the provided529 * output buffer.530 *531 * Upon return from this function @outlen contains the length of the532 * output data. If there is an error then @outlen will be 0 and an533 * error will be specified by the return code from this function.534 *535 * The @workmem buffer should only be used by one function call at a time.536 *537 * @in: input buffer pointer538 * @inlen: input buffer size539 * @out: output buffer pointer540 * @outlenp: output buffer size pointer541 * @workmem: working memory buffer pointer, size determined by542 * nx842_powernv_driver.workmem_size543 * @fc: function code, see CCW Function Codes in nx-842.h544 *545 * Returns:546 * 0 Success, output of length @outlenp stored in the buffer547 * at @out548 * -ENODEV Hardware unavailable549 * -ENOSPC Output buffer is to small550 * -EMSGSIZE Input buffer too large551 * -EINVAL buffer constraints do not fix nx842_constraints552 * -EPROTO hardware error during operation553 * -ETIMEDOUT hardware did not complete operation in reasonable time554 * -EINTR operation was aborted555 */556static int nx842_exec_vas(const unsigned char *in, unsigned int inlen,557 unsigned char *out, unsigned int *outlenp,558 void *workmem, int fc)559{560 struct coprocessor_request_block *crb;561 struct coprocessor_status_block *csb;562 struct nx842_workmem *wmem;563 struct vas_window *txwin;564 int ret, i = 0;565 u32 ccw;566 unsigned int outlen = *outlenp;567 568 wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);569 570 *outlenp = 0;571 572 crb = &wmem->crb;573 csb = &crb->csb;574 575 ret = nx842_config_crb(in, inlen, out, outlen, wmem);576 if (ret)577 return ret;578 579 ccw = 0;580 ccw = SET_FIELD(CCW_FC_842, ccw, fc);581 crb->ccw = cpu_to_be32(ccw);582 583 do {584 wmem->start = ktime_get();585 preempt_disable();586 txwin = this_cpu_read(cpu_txwin);587 588 /*589 * VAS copy CRB into L2 cache. Refer <asm/vas.h>.590 * @crb and @offset.591 */592 vas_copy_crb(crb, 0);593 594 /*595 * VAS paste previously copied CRB to NX.596 * @txwin, @offset and @last (must be true).597 */598 ret = vas_paste_crb(txwin, 0, 1);599 preempt_enable();600 /*601 * Retry copy/paste function for VAS failures.602 */603 } while (ret && (i++ < VAS_RETRIES));604 605 if (ret) {606 pr_err_ratelimited("VAS copy/paste failed\n");607 return ret;608 }609 610 ret = wait_for_csb(wmem, csb);611 if (!ret)612 *outlenp = be32_to_cpu(csb->count);613 614 return ret;615}616 617/**618 * nx842_powernv_compress - Compress data using the 842 algorithm619 *620 * Compression provided by the NX842 coprocessor on IBM PowerNV systems.621 * The input buffer is compressed and the result is stored in the622 * provided output buffer.623 *624 * Upon return from this function @outlen contains the length of the625 * compressed data. If there is an error then @outlen will be 0 and an626 * error will be specified by the return code from this function.627 *628 * @in: input buffer pointer629 * @inlen: input buffer size630 * @out: output buffer pointer631 * @outlenp: output buffer size pointer632 * @wmem: working memory buffer pointer, size determined by633 * nx842_powernv_driver.workmem_size634 *635 * Returns: see @nx842_powernv_exec()636 */637static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,638 unsigned char *out, unsigned int *outlenp,639 void *wmem)640{641 return nx842_powernv_exec(in, inlen, out, outlenp,642 wmem, CCW_FC_842_COMP_CRC);643}644 645/**646 * nx842_powernv_decompress - Decompress data using the 842 algorithm647 *648 * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.649 * The input buffer is decompressed and the result is stored in the650 * provided output buffer.651 *652 * Upon return from this function @outlen contains the length of the653 * decompressed data. If there is an error then @outlen will be 0 and an654 * error will be specified by the return code from this function.655 *656 * @in: input buffer pointer657 * @inlen: input buffer size658 * @out: output buffer pointer659 * @outlenp: output buffer size pointer660 * @wmem: working memory buffer pointer, size determined by661 * nx842_powernv_driver.workmem_size662 *663 * Returns: see @nx842_powernv_exec()664 */665static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,666 unsigned char *out, unsigned int *outlenp,667 void *wmem)668{669 return nx842_powernv_exec(in, inlen, out, outlenp,670 wmem, CCW_FC_842_DECOMP_CRC);671}672 673static inline void nx_add_coprocs_list(struct nx_coproc *coproc,674 int chipid)675{676 coproc->chip_id = chipid;677 INIT_LIST_HEAD(&coproc->list);678 list_add(&coproc->list, &nx_coprocs);679}680 681static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc)682{683 struct vas_window *txwin = NULL;684 struct vas_tx_win_attr txattr;685 686 /*687 * Kernel requests will be high priority. So open send688 * windows only for high priority RxFIFO entries.689 */690 vas_init_tx_win_attr(&txattr, coproc->ct);691 txattr.lpid = 0; /* lpid is 0 for kernel requests */692 693 /*694 * Open a VAS send window which is used to send request to NX.695 */696 txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr);697 if (IS_ERR(txwin))698 pr_err("ibm,nx-842: Can not open TX window: %ld\n",699 PTR_ERR(txwin));700 701 return txwin;702}703 704/*705 * Identify chip ID for each CPU, open send wndow for the corresponding NX706 * engine and save txwin in percpu cpu_txwin.707 * cpu_txwin is used in copy/paste operation for each compression /708 * decompression request.709 */710static int nx_open_percpu_txwins(void)711{712 struct nx_coproc *coproc, *n;713 unsigned int i, chip_id;714 715 for_each_possible_cpu(i) {716 struct vas_window *txwin = NULL;717 718 chip_id = cpu_to_chip_id(i);719 720 list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {721 /*722 * Kernel requests use only high priority FIFOs. So723 * open send windows for these FIFOs.724 * GZIP is not supported in kernel right now.725 */726 727 if (coproc->ct != VAS_COP_TYPE_842_HIPRI)728 continue;729 730 if (coproc->chip_id == chip_id) {731 txwin = nx_alloc_txwin(coproc);732 if (IS_ERR(txwin))733 return PTR_ERR(txwin);734 735 per_cpu(cpu_txwin, i) = txwin;736 break;737 }738 }739 740 if (!per_cpu(cpu_txwin, i)) {741 /* shouldn't happen, Each chip will have NX engine */742 pr_err("NX engine is not available for CPU %d\n", i);743 return -EINVAL;744 }745 }746 747 return 0;748}749 750static int __init nx_set_ct(struct nx_coproc *coproc, const char *priority,751 int high, int normal)752{753 if (!strcmp(priority, "High"))754 coproc->ct = high;755 else if (!strcmp(priority, "Normal"))756 coproc->ct = normal;757 else {758 pr_err("Invalid RxFIFO priority value\n");759 return -EINVAL;760 }761 762 return 0;763}764 765static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,766 int vasid, int type, int *ct)767{768 struct vas_window *rxwin = NULL;769 struct vas_rx_win_attr rxattr;770 u32 lpid, pid, tid, fifo_size;771 struct nx_coproc *coproc;772 u64 rx_fifo;773 const char *priority;774 int ret;775 776 ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo);777 if (ret) {778 pr_err("Missing rx-fifo-address property\n");779 return ret;780 }781 782 ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size);783 if (ret) {784 pr_err("Missing rx-fifo-size property\n");785 return ret;786 }787 788 ret = of_property_read_u32(dn, "lpid", &lpid);789 if (ret) {790 pr_err("Missing lpid property\n");791 return ret;792 }793 794 ret = of_property_read_u32(dn, "pid", &pid);795 if (ret) {796 pr_err("Missing pid property\n");797 return ret;798 }799 800 ret = of_property_read_u32(dn, "tid", &tid);801 if (ret) {802 pr_err("Missing tid property\n");803 return ret;804 }805 806 ret = of_property_read_string(dn, "priority", &priority);807 if (ret) {808 pr_err("Missing priority property\n");809 return ret;810 }811 812 coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);813 if (!coproc)814 return -ENOMEM;815 816 if (type == NX_CT_842)817 ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI,818 VAS_COP_TYPE_842);819 else if (type == NX_CT_GZIP)820 ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_GZIP_HIPRI,821 VAS_COP_TYPE_GZIP);822 823 if (ret)824 goto err_out;825 826 vas_init_rx_win_attr(&rxattr, coproc->ct);827 rxattr.rx_fifo = rx_fifo;828 rxattr.rx_fifo_size = fifo_size;829 rxattr.lnotify_lpid = lpid;830 rxattr.lnotify_pid = pid;831 rxattr.lnotify_tid = tid;832 /*833 * Maximum RX window credits can not be more than #CRBs in834 * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns.835 */836 rxattr.wcreds_max = fifo_size / CRB_SIZE;837 838 /*839 * Open a VAS receice window which is used to configure RxFIFO840 * for NX.841 */842 rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr);843 if (IS_ERR(rxwin)) {844 ret = PTR_ERR(rxwin);845 pr_err("setting RxFIFO with VAS failed: %d\n",846 ret);847 goto err_out;848 }849 850 coproc->vas.rxwin = rxwin;851 coproc->vas.id = vasid;852 nx_add_coprocs_list(coproc, chip_id);853 854 /*855 * (lpid, pid, tid) combination has to be unique for each856 * coprocessor instance in the system. So to make it857 * unique, skiboot uses coprocessor type such as 842 or858 * GZIP for pid and provides this value to kernel in pid859 * device-tree property.860 */861 *ct = pid;862 863 return 0;864 865err_out:866 kfree(coproc);867 return ret;868}869 870static int __init nx_coproc_init(int chip_id, int ct_842, int ct_gzip)871{872 int ret = 0;873 874 if (opal_check_token(OPAL_NX_COPROC_INIT)) {875 ret = opal_nx_coproc_init(chip_id, ct_842);876 877 if (!ret)878 ret = opal_nx_coproc_init(chip_id, ct_gzip);879 880 if (ret) {881 ret = opal_error_code(ret);882 pr_err("Failed to initialize NX for chip(%d): %d\n",883 chip_id, ret);884 }885 } else886 pr_warn("Firmware doesn't support NX initialization\n");887 888 return ret;889}890 891static int __init find_nx_device_tree(struct device_node *dn, int chip_id,892 int vasid, int type, char *devname,893 int *ct)894{895 int ret = 0;896 897 if (of_device_is_compatible(dn, devname)) {898 ret = vas_cfg_coproc_info(dn, chip_id, vasid, type, ct);899 if (ret)900 of_node_put(dn);901 }902 903 return ret;904}905 906static int __init nx_powernv_probe_vas(struct device_node *pn)907{908 int chip_id, vasid, ret = 0;909 int ct_842 = 0, ct_gzip = 0;910 struct device_node *dn;911 912 chip_id = of_get_ibm_chip_id(pn);913 if (chip_id < 0) {914 pr_err("ibm,chip-id missing\n");915 return -EINVAL;916 }917 918 vasid = chip_to_vas_id(chip_id);919 if (vasid < 0) {920 pr_err("Unable to map chip_id %d to vasid\n", chip_id);921 return -EINVAL;922 }923 924 for_each_child_of_node(pn, dn) {925 ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842,926 "ibm,p9-nx-842", &ct_842);927 928 if (!ret)929 ret = find_nx_device_tree(dn, chip_id, vasid,930 NX_CT_GZIP, "ibm,p9-nx-gzip", &ct_gzip);931 932 if (ret) {933 of_node_put(dn);934 return ret;935 }936 }937 938 if (!ct_842 || !ct_gzip) {939 pr_err("NX FIFO nodes are missing\n");940 return -EINVAL;941 }942 943 /*944 * Initialize NX instance for both high and normal priority FIFOs.945 */946 ret = nx_coproc_init(chip_id, ct_842, ct_gzip);947 948 return ret;949}950 951static int __init nx842_powernv_probe(struct device_node *dn)952{953 struct nx_coproc *coproc;954 unsigned int ct, ci;955 int chip_id;956 957 chip_id = of_get_ibm_chip_id(dn);958 if (chip_id < 0) {959 pr_err("ibm,chip-id missing\n");960 return -EINVAL;961 }962 963 if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) {964 pr_err("ibm,842-coprocessor-type missing\n");965 return -EINVAL;966 }967 968 if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) {969 pr_err("ibm,842-coprocessor-instance missing\n");970 return -EINVAL;971 }972 973 coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);974 if (!coproc)975 return -ENOMEM;976 977 coproc->ct = ct;978 coproc->ci = ci;979 nx_add_coprocs_list(coproc, chip_id);980 981 pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);982 983 if (!nx842_ct)984 nx842_ct = ct;985 else if (nx842_ct != ct)986 pr_err("NX842 chip %d, CT %d != first found CT %d\n",987 chip_id, ct, nx842_ct);988 989 return 0;990}991 992static void nx_delete_coprocs(void)993{994 struct nx_coproc *coproc, *n;995 struct vas_window *txwin;996 int i;997 998 /*999 * close percpu txwins that are opened for the corresponding coproc.1000 */1001 for_each_possible_cpu(i) {1002 txwin = per_cpu(cpu_txwin, i);1003 if (txwin)1004 vas_win_close(txwin);1005 1006 per_cpu(cpu_txwin, i) = NULL;1007 }1008 1009 list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {1010 if (coproc->vas.rxwin)1011 vas_win_close(coproc->vas.rxwin);1012 1013 list_del(&coproc->list);1014 kfree(coproc);1015 }1016}1017 1018static struct nx842_constraints nx842_powernv_constraints = {1019 .alignment = DDE_BUFFER_ALIGN,1020 .multiple = DDE_BUFFER_LAST_MULT,1021 .minimum = DDE_BUFFER_LAST_MULT,1022 .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE,1023};1024 1025static struct nx842_driver nx842_powernv_driver = {1026 .name = KBUILD_MODNAME,1027 .owner = THIS_MODULE,1028 .workmem_size = sizeof(struct nx842_workmem),1029 .constraints = &nx842_powernv_constraints,1030 .compress = nx842_powernv_compress,1031 .decompress = nx842_powernv_decompress,1032};1033 1034static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)1035{1036 return nx842_crypto_init(tfm, &nx842_powernv_driver);1037}1038 1039static struct crypto_alg nx842_powernv_alg = {1040 .cra_name = "842",1041 .cra_driver_name = "842-nx",1042 .cra_priority = 300,1043 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,1044 .cra_ctxsize = sizeof(struct nx842_crypto_ctx),1045 .cra_module = THIS_MODULE,1046 .cra_init = nx842_powernv_crypto_init,1047 .cra_exit = nx842_crypto_exit,1048 .cra_u = { .compress = {1049 .coa_compress = nx842_crypto_compress,1050 .coa_decompress = nx842_crypto_decompress } }1051};1052 1053static __init int nx_compress_powernv_init(void)1054{1055 struct device_node *dn;1056 int ret;1057 1058 /* verify workmem size/align restrictions */1059 BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);1060 BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);1061 BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);1062 /* verify buffer size/align restrictions */1063 BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);1064 BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);1065 BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);1066 1067 for_each_compatible_node(dn, NULL, "ibm,power9-nx") {1068 ret = nx_powernv_probe_vas(dn);1069 if (ret) {1070 nx_delete_coprocs();1071 of_node_put(dn);1072 return ret;1073 }1074 }1075 1076 if (list_empty(&nx_coprocs)) {1077 for_each_compatible_node(dn, NULL, "ibm,power-nx")1078 nx842_powernv_probe(dn);1079 1080 if (!nx842_ct)1081 return -ENODEV;1082 1083 nx842_powernv_exec = nx842_exec_icswx;1084 } else {1085 /*1086 * Register VAS user space API for NX GZIP so1087 * that user space can use GZIP engine.1088 * Using high FIFO priority for kernel requests and1089 * normal FIFO priority is assigned for userspace.1090 * 842 compression is supported only in kernel.1091 */1092 ret = vas_register_api_powernv(THIS_MODULE, VAS_COP_TYPE_GZIP,1093 "nx-gzip");1094 1095 /*1096 * GZIP is not supported in kernel right now.1097 * So open tx windows only for 842.1098 */1099 if (!ret)1100 ret = nx_open_percpu_txwins();1101 1102 if (ret) {1103 nx_delete_coprocs();1104 return ret;1105 }1106 1107 nx842_powernv_exec = nx842_exec_vas;1108 }1109 1110 ret = crypto_register_alg(&nx842_powernv_alg);1111 if (ret) {1112 nx_delete_coprocs();1113 return ret;1114 }1115 1116 return 0;1117}1118module_init(nx_compress_powernv_init);1119 1120static void __exit nx_compress_powernv_exit(void)1121{1122 /*1123 * GZIP engine is supported only in power9 or later and nx842_ct1124 * is used on power8 (icswx).1125 * VAS API for NX GZIP is registered during init for user space1126 * use. So delete this API use for GZIP engine.1127 */1128 if (!nx842_ct)1129 vas_unregister_api_powernv();1130 1131 crypto_unregister_alg(&nx842_powernv_alg);1132 1133 nx_delete_coprocs();1134}1135module_exit(nx_compress_powernv_exit);1136