317 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2 3/*4 * Copyright 2020 IBM Corp.5 *6 * Author: Bulent Abali <abali@us.ibm.com>7 *8 */9#include <stdio.h>10#include <stdlib.h>11#include <string.h>12#include <unistd.h>13#include <stdint.h>14#include <sys/types.h>15#include <sys/stat.h>16#include <sys/time.h>17#include <sys/fcntl.h>18#include <sys/mman.h>19#include <endian.h>20#include <bits/endian.h>21#include <sys/ioctl.h>22#include <assert.h>23#include <errno.h>24#include <signal.h>25#include "vas-api.h"26#include "nx.h"27#include "copy-paste.h"28#include "nxu.h"29#include "nx_dbg.h"30#include <sys/platform/ppc.h>31 32#define barrier()33#define hwsync() ({ asm volatile("sync" ::: "memory"); })34 35#ifndef NX_NO_CPU_PRI36#define cpu_pri_default() ({ asm volatile ("or 2, 2, 2"); })37#define cpu_pri_low() ({ asm volatile ("or 31, 31, 31"); })38#else39#define cpu_pri_default()40#define cpu_pri_low()41#endif42 43void *nx_fault_storage_address;44 45struct nx_handle {46 int fd;47 int function;48 void *paste_addr;49};50 51static int open_device_nodes(char *devname, int pri, struct nx_handle *handle)52{53 int rc, fd;54 void *addr;55 struct vas_tx_win_open_attr txattr;56 57 fd = open(devname, O_RDWR);58 if (fd < 0) {59 fprintf(stderr, " open device name %s\n", devname);60 return -errno;61 }62 63 memset(&txattr, 0, sizeof(txattr));64 txattr.version = 1;65 txattr.vas_id = pri;66 rc = ioctl(fd, VAS_TX_WIN_OPEN, (unsigned long)&txattr);67 if (rc < 0) {68 fprintf(stderr, "ioctl() n %d, error %d\n", rc, errno);69 rc = -errno;70 goto out;71 }72 73 addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0ULL);74 if (addr == MAP_FAILED) {75 fprintf(stderr, "mmap() failed, errno %d\n", errno);76 rc = -errno;77 goto out;78 }79 handle->fd = fd;80 handle->paste_addr = (void *)((char *)addr + 0x400);81 82 rc = 0;83out:84 close(fd);85 return rc;86}87 88void *nx_function_begin(int function, int pri)89{90 int rc;91 char *devname = "/dev/crypto/nx-gzip";92 struct nx_handle *nxhandle;93 94 if (function != NX_FUNC_COMP_GZIP) {95 errno = EINVAL;96 fprintf(stderr, " NX_FUNC_COMP_GZIP not found\n");97 return NULL;98 }99 100 101 nxhandle = malloc(sizeof(*nxhandle));102 if (!nxhandle) {103 errno = ENOMEM;104 fprintf(stderr, " No memory\n");105 return NULL;106 }107 108 nxhandle->function = function;109 rc = open_device_nodes(devname, pri, nxhandle);110 if (rc < 0) {111 errno = -rc;112 fprintf(stderr, " open_device_nodes failed\n");113 return NULL;114 }115 116 return nxhandle;117}118 119int nx_function_end(void *handle)120{121 int rc = 0;122 struct nx_handle *nxhandle = handle;123 124 rc = munmap(nxhandle->paste_addr - 0x400, 4096);125 if (rc < 0) {126 fprintf(stderr, "munmap() failed, errno %d\n", errno);127 return rc;128 }129 close(nxhandle->fd);130 free(nxhandle);131 132 return rc;133}134 135static int nx_wait_for_csb(struct nx_gzip_crb_cpb_t *cmdp)136{137 long poll = 0;138 uint64_t t;139 140 /* Save power and let other threads use the h/w. top may show141 * 100% but only because OS doesn't know we slowed the this142 * h/w thread while polling. We're letting other threads have143 * higher throughput on the core.144 */145 cpu_pri_low();146 147#define CSB_MAX_POLL 200000000UL148#define USLEEP_TH 300000UL149 150 t = __ppc_get_timebase();151 152 while (getnn(cmdp->crb.csb, csb_v) == 0) {153 ++poll;154 hwsync();155 156 cpu_pri_low();157 158 /* usleep(0) takes around 29000 ticks ~60 us.159 * 300000 is spinning for about 600 us then160 * start sleeping.161 */162 if ((__ppc_get_timebase() - t) > USLEEP_TH) {163 cpu_pri_default();164 usleep(1);165 }166 167 if (poll > CSB_MAX_POLL)168 break;169 170 /* Fault address from signal handler */171 if (nx_fault_storage_address) {172 cpu_pri_default();173 return -EAGAIN;174 }175 176 }177 178 cpu_pri_default();179 180 /* hw has updated csb and output buffer */181 hwsync();182 183 /* Check CSB flags. */184 if (getnn(cmdp->crb.csb, csb_v) == 0) {185 fprintf(stderr, "CSB still not valid after %d polls.\n",186 (int) poll);187 prt_err("CSB still not valid after %d polls, giving up.\n",188 (int) poll);189 return -ETIMEDOUT;190 }191 192 return 0;193}194 195static int nxu_run_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)196{197 int i, ret, retries;198 struct nx_handle *nxhandle = handle;199 200 assert(handle != NULL);201 i = 0;202 retries = 5000;203 while (i++ < retries) {204 hwsync();205 vas_copy(&cmdp->crb, 0);206 ret = vas_paste(nxhandle->paste_addr, 0);207 hwsync();208 209 NXPRT(fprintf(stderr, "Paste attempt %d/%d returns 0x%x\n",210 i, retries, ret));211 212 if ((ret == 2) || (ret == 3)) {213 214 ret = nx_wait_for_csb(cmdp);215 if (!ret) {216 goto out;217 } else if (ret == -EAGAIN) {218 long x;219 220 prt_err("Touching address %p, 0x%lx\n",221 nx_fault_storage_address,222 *(long *) nx_fault_storage_address);223 x = *(long *) nx_fault_storage_address;224 *(long *) nx_fault_storage_address = x;225 nx_fault_storage_address = 0;226 continue;227 } else {228 prt_err("wait_for_csb() returns %d\n", ret);229 break;230 }231 } else {232 if (i < 10) {233 /* spin for few ticks */234#define SPIN_TH 500UL235 uint64_t fail_spin;236 237 fail_spin = __ppc_get_timebase();238 while ((__ppc_get_timebase() - fail_spin) <239 SPIN_TH)240 ;241 } else {242 /* sleep */243 unsigned int pr = 0;244 245 if (pr++ % 100 == 0) {246 prt_err("Paste attempt %d/", i);247 prt_err("%d, failed pid= %d\n", retries,248 getpid());249 }250 usleep(1);251 }252 continue;253 }254 }255 256out:257 cpu_pri_default();258 259 return ret;260}261 262int nxu_submit_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)263{264 int cc;265 266 cc = nxu_run_job(cmdp, handle);267 268 if (!cc)269 cc = getnn(cmdp->crb.csb, csb_cc); /* CC Table 6-8 */270 271 return cc;272}273 274 275void nxu_sigsegv_handler(int sig, siginfo_t *info, void *ctx)276{277 fprintf(stderr, "%d: Got signal %d si_code %d, si_addr %p\n", getpid(),278 sig, info->si_code, info->si_addr);279 280 nx_fault_storage_address = info->si_addr;281}282 283/*284 * Fault in pages prior to NX job submission. wr=1 may be required to285 * touch writeable pages. System zero pages do not fault-in the page as286 * intended. Typically set wr=1 for NX target pages and set wr=0 for NX287 * source pages.288 */289int nxu_touch_pages(void *buf, long buf_len, long page_len, int wr)290{291 char *begin = buf;292 char *end = (char *) buf + buf_len - 1;293 volatile char t;294 295 assert(buf_len >= 0 && !!buf);296 297 NXPRT(fprintf(stderr, "touch %p %p len 0x%lx wr=%d\n", buf,298 (buf + buf_len), buf_len, wr));299 300 if (buf_len <= 0 || buf == NULL)301 return -1;302 303 do {304 t = *begin;305 if (wr)306 *begin = t;307 begin = begin + page_len;308 } while (begin < end);309 310 /* When buf_sz is small or buf tail is in another page */311 t = *end;312 if (wr)313 *end = t;314 315 return 0;316}317