651 lines · c
1/* SPDX-License-Identifier: GPL-2.0-or-later */2/*3 * Hardware interface of the NX-GZIP compression accelerator4 *5 * Copyright (C) IBM Corporation, 20206 *7 * Author: Bulent Abali <abali@us.ibm.com>8 *9 */10 11#ifndef _NXU_H12#define _NXU_H13 14#include <stdint.h>15#include <endian.h>16#include "nx.h"17 18/* deflate */19#define LLSZ 28620#define DSZ 3021 22/* nx */23#define DHTSZ 1824#define DHT_MAXSZ 28825#define MAX_DDE_COUNT 25626 27/* util */28#ifdef NXDBG29#define NXPRT(X) X30#else31#define NXPRT(X)32#endif33 34#ifdef NXTIMER35#include <sys/platform/ppc.h>36#define NX_CLK(X) X37#define nx_get_time() __ppc_get_timebase()38#define nx_get_freq() __ppc_get_timebase_freq()39#else40#define NX_CLK(X)41#define nx_get_time() (-1)42#define nx_get_freq() (-1)43#endif44 45#define NX_MAX_FAULTS 50046 47/*48 * Definitions of acronyms used here. See49 * P9 NX Gzip Accelerator User's Manual for details:50 * https://github.com/libnxz/power-gzip/blob/develop/doc/power_nx_gzip_um.pdf51 *52 * adler/crc: 32 bit checksums appended to stream tail53 * ce: completion extension54 * cpb: coprocessor parameter block (metadata)55 * crb: coprocessor request block (command)56 * csb: coprocessor status block (status)57 * dht: dynamic huffman table58 * dde: data descriptor element (address, length)59 * ddl: list of ddes60 * dh/fh: dynamic and fixed huffman types61 * fc: coprocessor function code62 * histlen: history/dictionary length63 * history: sliding window of up to 32KB of data64 * lzcount: Deflate LZ symbol counts65 * rembytecnt: remaining byte count66 * sfbt: source final block type; last block's type during decomp67 * spbc: source processed byte count68 * subc: source unprocessed bit count69 * tebc: target ending bit count; valid bits in the last byte70 * tpbc: target processed byte count71 * vas: virtual accelerator switch; the user mode interface72 */73 74union nx_qw_t {75 uint32_t word[4];76 uint64_t dword[2];77} __aligned(16);78 79/*80 * Note: NX registers with fewer than 32 bits are declared by81 * convention as uint32_t variables in unions. If *_offset and *_mask82 * are defined for a variable, then use get_ put_ macros to83 * conveniently access the register fields for endian conversions.84 */85 86struct nx_dde_t {87 /* Data Descriptor Element, Section 6.4 */88 union {89 uint32_t dde_count;90 /* When dde_count == 0 ddead is a pointer to a data buffer;91 * ddebc is the buffer length bytes.92 * When dde_count > 0 dde is an indirect dde; ddead is a93 * pointer to a contiguous list of direct ddes; ddebc is the94 * total length of all data pointed to by the list of direct95 * ddes. Note that only one level of indirection is permitted.96 * See Section 6.4 of the user manual for additional details.97 */98 };99 uint32_t ddebc; /* dde byte count */100 uint64_t ddead; /* dde address */101} __aligned(16);102 103struct nx_csb_t {104 /* Coprocessor Status Block, Section 6.6 */105 union {106 uint32_t csb_v;107 /* Valid bit. v must be set to 0 by the program108 * before submitting the coprocessor command.109 * Software can poll for the v bit110 */111 112 uint32_t csb_f;113 /* 16B CSB size. Written to 0 by DMA when it writes the CPB */114 115 uint32_t csb_cs;116 /* cs completion sequence; unused */117 118 uint32_t csb_cc;119 /* cc completion code; cc != 0 exception occurred */120 121 uint32_t csb_ce;122 /* ce completion extension */123 124 };125 uint32_t tpbc;126 /* target processed byte count TPBC */127 128 uint64_t fsaddr;129 /* Section 6.12.1 CSB NonZero error summary. FSA Failing storage130 * address. Address where error occurred. When available, written131 * to A field of CSB132 */133} __aligned(16);134 135struct nx_ccb_t {136 /* Coprocessor Completion Block, Section 6.7 */137 138 uint32_t reserved[3];139 union {140 /* When crb.c==0 (no ccb defined) it is reserved;141 * When crb.c==1 (ccb defined) it is cm142 */143 144 uint32_t ccb_cm;145 /* Signal interrupt of crb.c==1 and cm==1 */146 147 uint32_t word;148 /* generic access to the 32bit word */149 };150} __aligned(16);151 152struct vas_stamped_crb_t {153 /*154 * CRB operand of the paste coprocessor instruction is stamped155 * in quadword 4 with the information shown here as its written156 * in to the receive FIFO of the coprocessor157 */158 159 union {160 uint32_t vas_buf_num;161 /* Verification only vas buffer number which correlates to162 * the low order bits of the atag in the paste command163 */164 165 uint32_t send_wc_id;166 /* Pointer to Send Window Context that provides for NX address167 * translation information, such as MSR and LPCR bits, job168 * completion interrupt RA, PSWID, and job utilization counter.169 */170 171 };172 union {173 uint32_t recv_wc_id;174 /* Pointer to Receive Window Context. NX uses this to return175 * credits to a Receive FIFO as entries are dequeued.176 */177 178 };179 uint32_t reserved2;180 union {181 uint32_t vas_invalid;182 /* Invalid bit. If this bit is 1 the CRB is discarded by183 * NX upon fetching from the receive FIFO. If this bit is 0184 * the CRB is processed normally. The bit is stamped to 0185 * by VAS and may be written to 1 by hypervisor while186 * the CRB is in the receive FIFO (in memory).187 */188 189 };190};191 192struct nx_stamped_fault_crb_t {193 /*194 * A CRB that has a translation fault is stamped by NX in quadword 4195 * and pasted to the Fault Send Window in VAS.196 */197 uint64_t fsa;198 union {199 uint32_t nxsf_t;200 uint32_t nxsf_fs;201 };202 uint32_t pswid;203};204 205union stamped_crb_t {206 struct vas_stamped_crb_t vas;207 struct nx_stamped_fault_crb_t nx;208};209 210struct nx_gzip_cpb_t {211 /*212 * Coprocessor Parameter Block In/Out are used to pass metadata213 * to/from accelerator. Tables 6.5 and 6.6 of the user manual.214 */215 216 /* CPBInput */217 218 struct {219 union {220 union nx_qw_t qw0;221 struct {222 uint32_t in_adler; /* bits 0:31 */223 uint32_t in_crc; /* bits 32:63 */224 union {225 uint32_t in_histlen; /* bits 64:75 */226 uint32_t in_subc; /* bits 93:95 */227 };228 union {229 /* bits 108:111 */230 uint32_t in_sfbt;231 /* bits 112:127 */232 uint32_t in_rembytecnt;233 /* bits 116:127 */234 uint32_t in_dhtlen;235 };236 };237 };238 union {239 union nx_qw_t in_dht[DHTSZ]; /* qw[1:18] */240 char in_dht_char[DHT_MAXSZ]; /* byte access */241 };242 union nx_qw_t reserved[5]; /* qw[19:23] */243 };244 245 /* CPBOutput */246 247 volatile struct {248 union {249 union nx_qw_t qw24;250 struct {251 uint32_t out_adler; /* bits 0:31 qw[24] */252 uint32_t out_crc; /* bits 32:63 qw[24] */253 union {254 /* bits 77:79 qw[24] */255 uint32_t out_tebc;256 /* bits 80:95 qw[24] */257 uint32_t out_subc;258 };259 union {260 /* bits 108:111 qw[24] */261 uint32_t out_sfbt;262 /* bits 112:127 qw[24] */263 uint32_t out_rembytecnt;264 /* bits 116:127 qw[24] */265 uint32_t out_dhtlen;266 };267 };268 };269 union {270 union nx_qw_t qw25[79]; /* qw[25:103] */271 /* qw[25] compress no lzcounts or wrap */272 uint32_t out_spbc_comp_wrap;273 uint32_t out_spbc_wrap; /* qw[25] wrap */274 /* qw[25] compress no lzcounts */275 uint32_t out_spbc_comp;276 /* 286 LL and 30 D symbol counts */277 uint32_t out_lzcount[LLSZ+DSZ];278 struct {279 union nx_qw_t out_dht[DHTSZ]; /* qw[25:42] */280 /* qw[43] decompress */281 uint32_t out_spbc_decomp;282 };283 };284 /* qw[104] compress with lzcounts */285 uint32_t out_spbc_comp_with_count;286 };287} __aligned(128);288 289struct nx_gzip_crb_t {290 union { /* byte[0:3] */291 uint32_t gzip_fc; /* bits[24-31] */292 };293 uint32_t reserved1; /* byte[4:7] */294 union {295 uint64_t csb_address; /* byte[8:15] */296 struct {297 uint32_t reserved2;298 union {299 uint32_t crb_c;300 /* c==0 no ccb defined */301 302 uint32_t crb_at;303 /* at==0 address type is ignored;304 * all addrs effective assumed.305 */306 307 };308 };309 };310 struct nx_dde_t source_dde; /* byte[16:31] */311 struct nx_dde_t target_dde; /* byte[32:47] */312 volatile struct nx_ccb_t ccb; /* byte[48:63] */313 volatile union {314 /* byte[64:239] shift csb by 128 bytes out of the crb; csb was315 * in crb earlier; JReilly says csb written with partial inject316 */317 union nx_qw_t reserved64[11];318 union stamped_crb_t stamp; /* byte[64:79] */319 };320 volatile struct nx_csb_t csb;321} __aligned(128);322 323struct nx_gzip_crb_cpb_t {324 struct nx_gzip_crb_t crb;325 struct nx_gzip_cpb_t cpb;326} __aligned(2048);327 328 329/*330 * NX hardware convention has the msb bit on the left numbered 0.331 * The defines below has *_offset defined as the right most bit332 * position of a field. x of size_mask(x) is the field width in bits.333 */334 335#define size_mask(x) ((1U<<(x))-1)336 337/*338 * Offsets and Widths within the containing 32 bits of the various NX339 * gzip hardware registers. Use the getnn/putnn macros to access340 * these regs341 */342 343#define dde_count_mask size_mask(8)344#define dde_count_offset 23345 346/* CSB */347 348#define csb_v_mask size_mask(1)349#define csb_v_offset 0350#define csb_f_mask size_mask(1)351#define csb_f_offset 6352#define csb_cs_mask size_mask(8)353#define csb_cs_offset 15354#define csb_cc_mask size_mask(8)355#define csb_cc_offset 23356#define csb_ce_mask size_mask(8)357#define csb_ce_offset 31358 359/* CCB */360 361#define ccb_cm_mask size_mask(3)362#define ccb_cm_offset 31363 364/* VAS stamped CRB fields */365 366#define vas_buf_num_mask size_mask(6)367#define vas_buf_num_offset 5368#define send_wc_id_mask size_mask(16)369#define send_wc_id_offset 31370#define recv_wc_id_mask size_mask(16)371#define recv_wc_id_offset 31372#define vas_invalid_mask size_mask(1)373#define vas_invalid_offset 31374 375/* NX stamped fault CRB fields */376 377#define nxsf_t_mask size_mask(1)378#define nxsf_t_offset 23379#define nxsf_fs_mask size_mask(8)380#define nxsf_fs_offset 31381 382/* CPB input */383 384#define in_histlen_mask size_mask(12)385#define in_histlen_offset 11386#define in_dhtlen_mask size_mask(12)387#define in_dhtlen_offset 31388#define in_subc_mask size_mask(3)389#define in_subc_offset 31390#define in_sfbt_mask size_mask(4)391#define in_sfbt_offset 15392#define in_rembytecnt_mask size_mask(16)393#define in_rembytecnt_offset 31394 395/* CPB output */396 397#define out_tebc_mask size_mask(3)398#define out_tebc_offset 15399#define out_subc_mask size_mask(16)400#define out_subc_offset 31401#define out_sfbt_mask size_mask(4)402#define out_sfbt_offset 15403#define out_rembytecnt_mask size_mask(16)404#define out_rembytecnt_offset 31405#define out_dhtlen_mask size_mask(12)406#define out_dhtlen_offset 31407 408/* CRB */409 410#define gzip_fc_mask size_mask(8)411#define gzip_fc_offset 31412#define crb_c_mask size_mask(1)413#define crb_c_offset 28414#define crb_at_mask size_mask(1)415#define crb_at_offset 30416#define csb_address_mask ~(15UL) /* mask off bottom 4b */417 418/*419 * Access macros for the registers. Do not access registers directly420 * because of the endian conversion. P9 processor may run either as421 * Little or Big endian. However the NX coprocessor regs are always422 * big endian.423 * Use the 32 and 64b macros to access respective424 * register sizes.425 * Use nn forms for the register fields shorter than 32 bits.426 */427 428#define getnn(ST, REG) ((be32toh(ST.REG) >> (31-REG##_offset)) \429 & REG##_mask)430#define getpnn(ST, REG) ((be32toh((ST)->REG) >> (31-REG##_offset)) \431 & REG##_mask)432#define get32(ST, REG) (be32toh(ST.REG))433#define getp32(ST, REG) (be32toh((ST)->REG))434#define get64(ST, REG) (be64toh(ST.REG))435#define getp64(ST, REG) (be64toh((ST)->REG))436 437#define unget32(ST, REG) (get32(ST, REG) & ~((REG##_mask) \438 << (31-REG##_offset)))439/* get 32bits less the REG field */440 441#define ungetp32(ST, REG) (getp32(ST, REG) & ~((REG##_mask) \442 << (31-REG##_offset)))443/* get 32bits less the REG field */444 445#define clear_regs(ST) memset((void *)(&(ST)), 0, sizeof(ST))446#define clear_dde(ST) do { ST.dde_count = ST.ddebc = 0; ST.ddead = 0; \447 } while (0)448#define clearp_dde(ST) do { (ST)->dde_count = (ST)->ddebc = 0; \449 (ST)->ddead = 0; \450 } while (0)451#define clear_struct(ST) memset((void *)(&(ST)), 0, sizeof(ST))452#define putnn(ST, REG, X) (ST.REG = htobe32(unget32(ST, REG) | (((X) \453 & REG##_mask) << (31-REG##_offset))))454#define putpnn(ST, REG, X) ((ST)->REG = htobe32(ungetp32(ST, REG) \455 | (((X) & REG##_mask) << (31-REG##_offset))))456 457#define put32(ST, REG, X) (ST.REG = htobe32(X))458#define putp32(ST, REG, X) ((ST)->REG = htobe32(X))459#define put64(ST, REG, X) (ST.REG = htobe64(X))460#define putp64(ST, REG, X) ((ST)->REG = htobe64(X))461 462/*463 * Completion extension ce(0) ce(1) ce(2). Bits ce(3-7)464 * unused. Section 6.6 Figure 6.7.465 */466 467#define get_csb_ce(ST) ((uint32_t)getnn(ST, csb_ce))468#define get_csb_ce_ms3b(ST) (get_csb_ce(ST) >> 5)469#define put_csb_ce_ms3b(ST, X) putnn(ST, csb_ce, ((uint32_t)(X) << 5))470 471#define CSB_CE_PARTIAL 0x4472#define CSB_CE_TERMINATE 0x2473#define CSB_CE_TPBC_VALID 0x1474 475#define csb_ce_termination(X) (!!((X) & CSB_CE_TERMINATE))476/* termination, output buffers may be modified, SPBC/TPBC invalid Fig.6-7 */477 478#define csb_ce_check_completion(X) (!csb_ce_termination(X))479/* if not terminated then check full or partial completion */480 481#define csb_ce_partial_completion(X) (!!((X) & CSB_CE_PARTIAL))482#define csb_ce_full_completion(X) (!csb_ce_partial_completion(X))483#define csb_ce_tpbc_valid(X) (!!((X) & CSB_CE_TPBC_VALID))484/* TPBC indicates successfully stored data count */485 486#define csb_ce_default_err(X) csb_ce_termination(X)487/* most error CEs have CE(0)=0 and CE(1)=1 */488 489#define csb_ce_cc3_partial(X) csb_ce_partial_completion(X)490/* some CC=3 are partially completed, Table 6-8 */491 492#define csb_ce_cc64(X) ((X)&(CSB_CE_PARTIAL \493 | CSB_CE_TERMINATE) == 0)494/* Compression: when TPBC>SPBC then CC=64 Table 6-8; target didn't495 * compress smaller than source.496 */497 498/* Decompress SFBT combinations Tables 5-3, 6-4, 6-6 */499 500#define SFBT_BFINAL 0x1501#define SFBT_LIT 0x4502#define SFBT_FHT 0x5503#define SFBT_DHT 0x6504#define SFBT_HDR 0x7505 506/*507 * NX gzip function codes. Table 6.2.508 * Bits 0:4 are the FC. Bit 5 is used by the DMA controller to509 * select one of the two Byte Count Limits.510 */511 512#define GZIP_FC_LIMIT_MASK 0x01513#define GZIP_FC_COMPRESS_FHT 0x00514#define GZIP_FC_COMPRESS_DHT 0x02515#define GZIP_FC_COMPRESS_FHT_COUNT 0x04516#define GZIP_FC_COMPRESS_DHT_COUNT 0x06517#define GZIP_FC_COMPRESS_RESUME_FHT 0x08518#define GZIP_FC_COMPRESS_RESUME_DHT 0x0a519#define GZIP_FC_COMPRESS_RESUME_FHT_COUNT 0x0c520#define GZIP_FC_COMPRESS_RESUME_DHT_COUNT 0x0e521#define GZIP_FC_DECOMPRESS 0x10522#define GZIP_FC_DECOMPRESS_SINGLE_BLK_N_SUSPEND 0x12523#define GZIP_FC_DECOMPRESS_RESUME 0x14524#define GZIP_FC_DECOMPRESS_RESUME_SINGLE_BLK_N_SUSPEND 0x16525#define GZIP_FC_WRAP 0x1e526 527#define fc_is_compress(fc) (((fc) & 0x10) == 0)528#define fc_has_count(fc) (fc_is_compress(fc) && (((fc) & 0x4) != 0))529 530/* CSB.CC Error codes */531 532#define ERR_NX_OK 0533#define ERR_NX_ALIGNMENT 1534#define ERR_NX_OPOVERLAP 2535#define ERR_NX_DATA_LENGTH 3536#define ERR_NX_TRANSLATION 5537#define ERR_NX_PROTECTION 6538#define ERR_NX_EXTERNAL_UE7 7539#define ERR_NX_INVALID_OP 8540#define ERR_NX_PRIVILEGE 9541#define ERR_NX_INTERNAL_UE 10542#define ERR_NX_EXTERN_UE_WR 12543#define ERR_NX_TARGET_SPACE 13544#define ERR_NX_EXCESSIVE_DDE 14545#define ERR_NX_TRANSL_WR 15546#define ERR_NX_PROTECT_WR 16547#define ERR_NX_SUBFUNCTION 17548#define ERR_NX_FUNC_ABORT 18549#define ERR_NX_BYTE_MAX 19550#define ERR_NX_CORRUPT_CRB 20551#define ERR_NX_INVALID_CRB 21552#define ERR_NX_INVALID_DDE 30553#define ERR_NX_SEGMENTED_DDL 31554#define ERR_NX_DDE_OVERFLOW 33555#define ERR_NX_TPBC_GT_SPBC 64556#define ERR_NX_MISSING_CODE 66557#define ERR_NX_INVALID_DIST 67558#define ERR_NX_INVALID_DHT 68559#define ERR_NX_EXTERNAL_UE90 90560#define ERR_NX_WDOG_TIMER 224561#define ERR_NX_AT_FAULT 250562#define ERR_NX_INTR_SERVER 252563#define ERR_NX_UE253 253564#define ERR_NX_NO_HW 254565#define ERR_NX_HUNG_OP 255566#define ERR_NX_END 256567 568/* initial values for non-resume operations */569#define INIT_CRC 0 /* crc32(0L, Z_NULL, 0) */570#define INIT_ADLER 1 /* adler32(0L, Z_NULL, 0) adler is initialized to 1 */571 572/* prototypes */573int nxu_submit_job(struct nx_gzip_crb_cpb_t *c, void *handle);574 575extern void nxu_sigsegv_handler(int sig, siginfo_t *info, void *ctx);576extern int nxu_touch_pages(void *buf, long buf_len, long page_len, int wr);577 578/* caller supplies a print buffer 4*sizeof(crb) */579 580char *nx_crb_str(struct nx_gzip_crb_t *crb, char *prbuf);581char *nx_cpb_str(struct nx_gzip_cpb_t *cpb, char *prbuf);582char *nx_prt_hex(void *cp, int sz, char *prbuf);583char *nx_lzcount_str(struct nx_gzip_cpb_t *cpb, char *prbuf);584char *nx_strerror(int e);585 586#ifdef NX_SIM587#include <stdio.h>588int nx_sim_init(void *ctx);589int nx_sim_end(void *ctx);590int nxu_run_sim_job(struct nx_gzip_crb_cpb_t *c, void *ctx);591#endif /* NX_SIM */592 593/* Deflate stream manipulation */594 595#define set_final_bit(x) (x |= (unsigned char)1)596#define clr_final_bit(x) (x &= ~(unsigned char)1)597 598#define append_empty_fh_blk(p, b) do { *(p) = (2 | (1&(b))); *((p)+1) = 0; \599 } while (0)600/* append 10 bits 0000001b 00...... ;601 * assumes appending starts on a byte boundary; b is the final bit.602 */603 604 605#ifdef NX_842606 607/* 842 Engine */608 609struct nx_eft_crb_t {610 union { /* byte[0:3] */611 uint32_t eft_fc; /* bits[29-31] */612 };613 uint32_t reserved1; /* byte[4:7] */614 union {615 uint64_t csb_address; /* byte[8:15] */616 struct {617 uint32_t reserved2;618 union {619 uint32_t crb_c;620 /* c==0 no ccb defined */621 622 uint32_t crb_at;623 /* at==0 address type is ignored;624 * all addrs effective assumed.625 */626 627 };628 };629 };630 struct nx_dde_t source_dde; /* byte[16:31] */631 struct nx_dde_t target_dde; /* byte[32:47] */632 struct nx_ccb_t ccb; /* byte[48:63] */633 union {634 union nx_qw_t reserved64[3]; /* byte[64:96] */635 };636 struct nx_csb_t csb;637} __aligned(128);638 639/* 842 CRB */640 641#define EFT_FC_MASK size_mask(3)642#define EFT_FC_OFFSET 31643#define EFT_FC_COMPRESS 0x0644#define EFT_FC_COMPRESS_WITH_CRC 0x1645#define EFT_FC_DECOMPRESS 0x2646#define EFT_FC_DECOMPRESS_WITH_CRC 0x3647#define EFT_FC_BLK_DATA_MOVE 0x4648#endif /* NX_842 */649 650#endif /* _NXU_H */651