990 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.4 */5 6/*7 * Oracle Data Analytics Accelerator (DAX)8 *9 * DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M810 * (DAX2) processor chips, and has direct access to the CPU's L311 * caches as well as physical memory. It can perform several12 * operations on data streams with various input and output formats.13 * The driver provides a transport mechanism only and has limited14 * knowledge of the various opcodes and data formats. A user space15 * library provides high level services and translates these into low16 * level commands which are then passed into the driver and17 * subsequently the hypervisor and the coprocessor. The library is18 * the recommended way for applications to use the coprocessor, and19 * the driver interface is not intended for general use.20 *21 * See Documentation/arch/sparc/oradax/oracle-dax.rst for more details.22 */23 24#include <linux/uaccess.h>25#include <linux/module.h>26#include <linux/delay.h>27#include <linux/cdev.h>28#include <linux/slab.h>29#include <linux/mm.h>30 31#include <asm/hypervisor.h>32#include <asm/mdesc.h>33#include <asm/oradax.h>34 35MODULE_LICENSE("GPL");36MODULE_DESCRIPTION("Driver for Oracle Data Analytics Accelerator");37 38#define DAX_DBG_FLG_BASIC 0x0139#define DAX_DBG_FLG_STAT 0x0240#define DAX_DBG_FLG_INFO 0x0441#define DAX_DBG_FLG_ALL 0xff42 43#define dax_err(fmt, ...) pr_err("%s: " fmt "\n", __func__, ##__VA_ARGS__)44#define dax_info(fmt, ...) pr_info("%s: " fmt "\n", __func__, ##__VA_ARGS__)45 46#define dax_dbg(fmt, ...) do { \47 if (dax_debug & DAX_DBG_FLG_BASIC)\48 dax_info(fmt, ##__VA_ARGS__); \49 } while (0)50#define dax_stat_dbg(fmt, ...) do { \51 if (dax_debug & DAX_DBG_FLG_STAT) \52 dax_info(fmt, ##__VA_ARGS__); \53 } while (0)54#define dax_info_dbg(fmt, ...) do { \55 if (dax_debug & DAX_DBG_FLG_INFO) \56 dax_info(fmt, ##__VA_ARGS__); \57 } while (0)58 59#define DAX1_MINOR 160#define DAX1_MAJOR 161#define DAX2_MINOR 062#define DAX2_MAJOR 263 64#define DAX1_STR "ORCL,sun4v-dax"65#define DAX2_STR "ORCL,sun4v-dax2"66 67#define DAX_CA_ELEMS (DAX_MMAP_LEN / sizeof(struct dax_cca))68 69#define DAX_CCB_USEC 10070#define DAX_CCB_RETRIES 1000071 72/* stream types */73enum {74 OUT,75 PRI,76 SEC,77 TBL,78 NUM_STREAM_TYPES79};80 81/* completion status */82#define CCA_STAT_NOT_COMPLETED 083#define CCA_STAT_COMPLETED 184#define CCA_STAT_FAILED 285#define CCA_STAT_KILLED 386#define CCA_STAT_NOT_RUN 487#define CCA_STAT_PIPE_OUT 588#define CCA_STAT_PIPE_SRC 689#define CCA_STAT_PIPE_DST 790 91/* completion err */92#define CCA_ERR_SUCCESS 0x0 /* no error */93#define CCA_ERR_OVERFLOW 0x1 /* buffer overflow */94#define CCA_ERR_DECODE 0x2 /* CCB decode error */95#define CCA_ERR_PAGE_OVERFLOW 0x3 /* page overflow */96#define CCA_ERR_KILLED 0x7 /* command was killed */97#define CCA_ERR_TIMEOUT 0x8 /* Timeout */98#define CCA_ERR_ADI 0x9 /* ADI error */99#define CCA_ERR_DATA_FMT 0xA /* data format error */100#define CCA_ERR_OTHER_NO_RETRY 0xE /* Other error, do not retry */101#define CCA_ERR_OTHER_RETRY 0xF /* Other error, retry */102#define CCA_ERR_PARTIAL_SYMBOL 0x80 /* QP partial symbol warning */103 104/* CCB address types */105#define DAX_ADDR_TYPE_NONE 0106#define DAX_ADDR_TYPE_VA_ALT 1 /* secondary context */107#define DAX_ADDR_TYPE_RA 2 /* real address */108#define DAX_ADDR_TYPE_VA 3 /* virtual address */109 110/* dax_header_t opcode */111#define DAX_OP_SYNC_NOP 0x0112#define DAX_OP_EXTRACT 0x1113#define DAX_OP_SCAN_VALUE 0x2114#define DAX_OP_SCAN_RANGE 0x3115#define DAX_OP_TRANSLATE 0x4116#define DAX_OP_SELECT 0x5117#define DAX_OP_INVERT 0x10 /* OR with translate, scan opcodes */118 119struct dax_header {120 u32 ccb_version:4; /* 31:28 CCB Version */121 /* 27:24 Sync Flags */122 u32 pipe:1; /* Pipeline */123 u32 longccb:1; /* Longccb. Set for scan with lu2, lu3, lu4. */124 u32 cond:1; /* Conditional */125 u32 serial:1; /* Serial */126 u32 opcode:8; /* 23:16 Opcode */127 /* 15:0 Address Type. */128 u32 reserved:3; /* 15:13 reserved */129 u32 table_addr_type:2; /* 12:11 Huffman Table Address Type */130 u32 out_addr_type:3; /* 10:8 Destination Address Type */131 u32 sec_addr_type:3; /* 7:5 Secondary Source Address Type */132 u32 pri_addr_type:3; /* 4:2 Primary Source Address Type */133 u32 cca_addr_type:2; /* 1:0 Completion Address Type */134};135 136struct dax_control {137 u32 pri_fmt:4; /* 31:28 Primary Input Format */138 u32 pri_elem_size:5; /* 27:23 Primary Input Element Size(less1) */139 u32 pri_offset:3; /* 22:20 Primary Input Starting Offset */140 u32 sec_encoding:1; /* 19 Secondary Input Encoding */141 /* (must be 0 for Select) */142 u32 sec_offset:3; /* 18:16 Secondary Input Starting Offset */143 u32 sec_elem_size:2; /* 15:14 Secondary Input Element Size */144 /* (must be 0 for Select) */145 u32 out_fmt:2; /* 13:12 Output Format */146 u32 out_elem_size:2; /* 11:10 Output Element Size */147 u32 misc:10; /* 9:0 Opcode specific info */148};149 150struct dax_data_access {151 u64 flow_ctrl:2; /* 63:62 Flow Control Type */152 u64 pipe_target:2; /* 61:60 Pipeline Target */153 u64 out_buf_size:20; /* 59:40 Output Buffer Size */154 /* (cachelines less 1) */155 u64 unused1:8; /* 39:32 Reserved, Set to 0 */156 u64 out_alloc:5; /* 31:27 Output Allocation */157 u64 unused2:1; /* 26 Reserved */158 u64 pri_len_fmt:2; /* 25:24 Input Length Format */159 u64 pri_len:24; /* 23:0 Input Element/Byte/Bit Count */160 /* (less 1) */161};162 163struct dax_ccb {164 struct dax_header hdr; /* CCB Header */165 struct dax_control ctrl;/* Control Word */166 void *ca; /* Completion Address */167 void *pri; /* Primary Input Address */168 struct dax_data_access dac; /* Data Access Control */169 void *sec; /* Secondary Input Address */170 u64 dword5; /* depends on opcode */171 void *out; /* Output Address */172 void *tbl; /* Table Address or bitmap */173};174 175struct dax_cca {176 u8 status; /* user may mwait on this address */177 u8 err; /* user visible error notification */178 u8 rsvd[2]; /* reserved */179 u32 n_remaining; /* for QP partial symbol warning */180 u32 output_sz; /* output in bytes */181 u32 rsvd2; /* reserved */182 u64 run_cycles; /* run time in OCND2 cycles */183 u64 run_stats; /* nothing reported in version 1.0 */184 u32 n_processed; /* number input elements */185 u32 rsvd3[5]; /* reserved */186 u64 retval; /* command return value */187 u64 rsvd4[8]; /* reserved */188};189 190/* per thread CCB context */191struct dax_ctx {192 struct dax_ccb *ccb_buf;193 u64 ccb_buf_ra; /* cached RA of ccb_buf */194 struct dax_cca *ca_buf;195 u64 ca_buf_ra; /* cached RA of ca_buf */196 struct page *pages[DAX_CA_ELEMS][NUM_STREAM_TYPES];197 /* array of locked pages */198 struct task_struct *owner; /* thread that owns ctx */199 struct task_struct *client; /* requesting thread */200 union ccb_result result;201 u32 ccb_count;202 u32 fail_count;203};204 205/* driver public entry points */206static int dax_open(struct inode *inode, struct file *file);207static ssize_t dax_read(struct file *filp, char __user *buf,208 size_t count, loff_t *ppos);209static ssize_t dax_write(struct file *filp, const char __user *buf,210 size_t count, loff_t *ppos);211static int dax_devmap(struct file *f, struct vm_area_struct *vma);212static int dax_close(struct inode *i, struct file *f);213 214static const struct file_operations dax_fops = {215 .owner = THIS_MODULE,216 .open = dax_open,217 .read = dax_read,218 .write = dax_write,219 .mmap = dax_devmap,220 .release = dax_close,221};222 223static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf,224 size_t count, loff_t *ppos);225static int dax_ccb_info(u64 ca, struct ccb_info_result *info);226static int dax_ccb_kill(u64 ca, u16 *kill_res);227 228static struct cdev c_dev;229static dev_t first;230static const struct class cl = {231 .name = DAX_NAME,232};233 234static int max_ccb_version;235static int dax_debug;236module_param(dax_debug, int, 0644);237MODULE_PARM_DESC(dax_debug, "Debug flags");238 239static int __init dax_attach(void)240{241 unsigned long dummy, hv_rv, major, minor, minor_requested, max_ccbs;242 struct mdesc_handle *hp = mdesc_grab();243 char *prop, *dax_name;244 bool found = false;245 int len, ret = 0;246 u64 pn;247 248 if (hp == NULL) {249 dax_err("Unable to grab mdesc");250 return -ENODEV;251 }252 253 mdesc_for_each_node_by_name(hp, pn, "virtual-device") {254 prop = (char *)mdesc_get_property(hp, pn, "name", &len);255 if (prop == NULL)256 continue;257 if (strncmp(prop, "dax", strlen("dax")))258 continue;259 dax_dbg("Found node 0x%llx = %s", pn, prop);260 261 prop = (char *)mdesc_get_property(hp, pn, "compatible", &len);262 if (prop == NULL)263 continue;264 dax_dbg("Found node 0x%llx = %s", pn, prop);265 found = true;266 break;267 }268 269 if (!found) {270 dax_err("No DAX device found");271 ret = -ENODEV;272 goto done;273 }274 275 if (strncmp(prop, DAX2_STR, strlen(DAX2_STR)) == 0) {276 dax_name = DAX_NAME "2";277 major = DAX2_MAJOR;278 minor_requested = DAX2_MINOR;279 max_ccb_version = 1;280 dax_dbg("MD indicates DAX2 coprocessor");281 } else if (strncmp(prop, DAX1_STR, strlen(DAX1_STR)) == 0) {282 dax_name = DAX_NAME "1";283 major = DAX1_MAJOR;284 minor_requested = DAX1_MINOR;285 max_ccb_version = 0;286 dax_dbg("MD indicates DAX1 coprocessor");287 } else {288 dax_err("Unknown dax type: %s", prop);289 ret = -ENODEV;290 goto done;291 }292 293 minor = minor_requested;294 dax_dbg("Registering DAX HV api with major %ld minor %ld", major,295 minor);296 if (sun4v_hvapi_register(HV_GRP_DAX, major, &minor)) {297 dax_err("hvapi_register failed");298 ret = -ENODEV;299 goto done;300 } else {301 dax_dbg("Max minor supported by HV = %ld (major %ld)", minor,302 major);303 minor = min(minor, minor_requested);304 dax_dbg("registered DAX major %ld minor %ld", major, minor);305 }306 307 /* submit a zero length ccb array to query coprocessor queue size */308 hv_rv = sun4v_ccb_submit(0, 0, HV_CCB_QUERY_CMD, 0, &max_ccbs, &dummy);309 if (hv_rv != 0) {310 dax_err("get_hwqueue_size failed with status=%ld and max_ccbs=%ld",311 hv_rv, max_ccbs);312 ret = -ENODEV;313 goto done;314 }315 316 if (max_ccbs != DAX_MAX_CCBS) {317 dax_err("HV reports unsupported max_ccbs=%ld", max_ccbs);318 ret = -ENODEV;319 goto done;320 }321 322 if (alloc_chrdev_region(&first, 0, 1, DAX_NAME) < 0) {323 dax_err("alloc_chrdev_region failed");324 ret = -ENXIO;325 goto done;326 }327 328 ret = class_register(&cl);329 if (ret)330 goto class_error;331 332 if (device_create(&cl, NULL, first, NULL, dax_name) == NULL) {333 dax_err("device_create failed");334 ret = -ENXIO;335 goto device_error;336 }337 338 cdev_init(&c_dev, &dax_fops);339 if (cdev_add(&c_dev, first, 1) == -1) {340 dax_err("cdev_add failed");341 ret = -ENXIO;342 goto cdev_error;343 }344 345 pr_info("Attached DAX module\n");346 goto done;347 348cdev_error:349 device_destroy(&cl, first);350device_error:351 class_unregister(&cl);352class_error:353 unregister_chrdev_region(first, 1);354done:355 mdesc_release(hp);356 return ret;357}358module_init(dax_attach);359 360static void __exit dax_detach(void)361{362 pr_info("Cleaning up DAX module\n");363 cdev_del(&c_dev);364 device_destroy(&cl, first);365 class_unregister(&cl);366 unregister_chrdev_region(first, 1);367}368module_exit(dax_detach);369 370/* map completion area */371static int dax_devmap(struct file *f, struct vm_area_struct *vma)372{373 struct dax_ctx *ctx = (struct dax_ctx *)f->private_data;374 size_t len = vma->vm_end - vma->vm_start;375 376 dax_dbg("len=0x%lx, flags=0x%lx", len, vma->vm_flags);377 378 if (ctx->owner != current) {379 dax_dbg("devmap called from wrong thread");380 return -EINVAL;381 }382 383 if (len != DAX_MMAP_LEN) {384 dax_dbg("len(%lu) != DAX_MMAP_LEN(%d)", len, DAX_MMAP_LEN);385 return -EINVAL;386 }387 388 /* completion area is mapped read-only for user */389 if (vma->vm_flags & VM_WRITE)390 return -EPERM;391 vm_flags_clear(vma, VM_MAYWRITE);392 393 if (remap_pfn_range(vma, vma->vm_start, ctx->ca_buf_ra >> PAGE_SHIFT,394 len, vma->vm_page_prot))395 return -EAGAIN;396 397 dax_dbg("mmapped completion area at uva 0x%lx", vma->vm_start);398 return 0;399}400 401/* Unlock user pages. Called during dequeue or device close */402static void dax_unlock_pages(struct dax_ctx *ctx, int ccb_index, int nelem)403{404 int i, j;405 406 for (i = ccb_index; i < ccb_index + nelem; i++) {407 for (j = 0; j < NUM_STREAM_TYPES; j++) {408 struct page *p = ctx->pages[i][j];409 410 if (p) {411 dax_dbg("freeing page %p", p);412 unpin_user_pages_dirty_lock(&p, 1, j == OUT);413 ctx->pages[i][j] = NULL;414 }415 }416 }417}418 419static int dax_lock_page(void *va, struct page **p)420{421 int ret;422 423 dax_dbg("uva %p", va);424 425 ret = pin_user_pages_fast((unsigned long)va, 1, FOLL_WRITE, p);426 if (ret == 1) {427 dax_dbg("locked page %p, for VA %p", *p, va);428 return 0;429 }430 431 dax_dbg("pin_user_pages failed, va=%p, ret=%d", va, ret);432 return -1;433}434 435static int dax_lock_pages(struct dax_ctx *ctx, int idx,436 int nelem, u64 *err_va)437{438 int i;439 440 for (i = 0; i < nelem; i++) {441 struct dax_ccb *ccbp = &ctx->ccb_buf[i];442 443 /*444 * For each address in the CCB whose type is virtual,445 * lock the page and change the type to virtual alternate446 * context. On error, return the offending address in447 * err_va.448 */449 if (ccbp->hdr.out_addr_type == DAX_ADDR_TYPE_VA) {450 dax_dbg("output");451 if (dax_lock_page(ccbp->out,452 &ctx->pages[i + idx][OUT]) != 0) {453 *err_va = (u64)ccbp->out;454 goto error;455 }456 ccbp->hdr.out_addr_type = DAX_ADDR_TYPE_VA_ALT;457 }458 459 if (ccbp->hdr.pri_addr_type == DAX_ADDR_TYPE_VA) {460 dax_dbg("input");461 if (dax_lock_page(ccbp->pri,462 &ctx->pages[i + idx][PRI]) != 0) {463 *err_va = (u64)ccbp->pri;464 goto error;465 }466 ccbp->hdr.pri_addr_type = DAX_ADDR_TYPE_VA_ALT;467 }468 469 if (ccbp->hdr.sec_addr_type == DAX_ADDR_TYPE_VA) {470 dax_dbg("sec input");471 if (dax_lock_page(ccbp->sec,472 &ctx->pages[i + idx][SEC]) != 0) {473 *err_va = (u64)ccbp->sec;474 goto error;475 }476 ccbp->hdr.sec_addr_type = DAX_ADDR_TYPE_VA_ALT;477 }478 479 if (ccbp->hdr.table_addr_type == DAX_ADDR_TYPE_VA) {480 dax_dbg("tbl");481 if (dax_lock_page(ccbp->tbl,482 &ctx->pages[i + idx][TBL]) != 0) {483 *err_va = (u64)ccbp->tbl;484 goto error;485 }486 ccbp->hdr.table_addr_type = DAX_ADDR_TYPE_VA_ALT;487 }488 489 /* skip over 2nd 64 bytes of long CCB */490 if (ccbp->hdr.longccb)491 i++;492 }493 return DAX_SUBMIT_OK;494 495error:496 dax_unlock_pages(ctx, idx, nelem);497 return DAX_SUBMIT_ERR_NOACCESS;498}499 500static void dax_ccb_wait(struct dax_ctx *ctx, int idx)501{502 int ret, nretries;503 u16 kill_res;504 505 dax_dbg("idx=%d", idx);506 507 for (nretries = 0; nretries < DAX_CCB_RETRIES; nretries++) {508 if (ctx->ca_buf[idx].status == CCA_STAT_NOT_COMPLETED)509 udelay(DAX_CCB_USEC);510 else511 return;512 }513 dax_dbg("ctx (%p): CCB[%d] timed out, wait usec=%d, retries=%d. Killing ccb",514 (void *)ctx, idx, DAX_CCB_USEC, DAX_CCB_RETRIES);515 516 ret = dax_ccb_kill(ctx->ca_buf_ra + idx * sizeof(struct dax_cca),517 &kill_res);518 dax_dbg("Kill CCB[%d] %s", idx, ret ? "failed" : "succeeded");519}520 521static int dax_close(struct inode *ino, struct file *f)522{523 struct dax_ctx *ctx = (struct dax_ctx *)f->private_data;524 int i;525 526 f->private_data = NULL;527 528 for (i = 0; i < DAX_CA_ELEMS; i++) {529 if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) {530 dax_dbg("CCB[%d] not completed", i);531 dax_ccb_wait(ctx, i);532 }533 dax_unlock_pages(ctx, i, 1);534 }535 536 kfree(ctx->ccb_buf);537 kfree(ctx->ca_buf);538 dax_stat_dbg("CCBs: %d good, %d bad", ctx->ccb_count, ctx->fail_count);539 kfree(ctx);540 541 return 0;542}543 544static ssize_t dax_read(struct file *f, char __user *buf,545 size_t count, loff_t *ppos)546{547 struct dax_ctx *ctx = f->private_data;548 549 if (ctx->client != current)550 return -EUSERS;551 552 ctx->client = NULL;553 554 if (count != sizeof(union ccb_result))555 return -EINVAL;556 if (copy_to_user(buf, &ctx->result, sizeof(union ccb_result)))557 return -EFAULT;558 return count;559}560 561static ssize_t dax_write(struct file *f, const char __user *buf,562 size_t count, loff_t *ppos)563{564 struct dax_ctx *ctx = f->private_data;565 struct dax_command hdr;566 unsigned long ca;567 int i, idx, ret;568 569 if (ctx->client != NULL)570 return -EINVAL;571 572 if (count == 0 || count > DAX_MAX_CCBS * sizeof(struct dax_ccb))573 return -EINVAL;574 575 if (count % sizeof(struct dax_ccb) == 0)576 return dax_ccb_exec(ctx, buf, count, ppos); /* CCB EXEC */577 578 if (count != sizeof(struct dax_command))579 return -EINVAL;580 581 /* immediate command */582 if (ctx->owner != current)583 return -EUSERS;584 585 if (copy_from_user(&hdr, buf, sizeof(hdr)))586 return -EFAULT;587 588 ca = ctx->ca_buf_ra + hdr.ca_offset;589 590 switch (hdr.command) {591 case CCB_KILL:592 if (hdr.ca_offset >= DAX_MMAP_LEN) {593 dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)",594 hdr.ca_offset, DAX_MMAP_LEN);595 return -EINVAL;596 }597 598 ret = dax_ccb_kill(ca, &ctx->result.kill.action);599 if (ret != 0) {600 dax_dbg("dax_ccb_kill failed (ret=%d)", ret);601 return ret;602 }603 604 dax_info_dbg("killed (ca_offset %d)", hdr.ca_offset);605 idx = hdr.ca_offset / sizeof(struct dax_cca);606 ctx->ca_buf[idx].status = CCA_STAT_KILLED;607 ctx->ca_buf[idx].err = CCA_ERR_KILLED;608 ctx->client = current;609 return count;610 611 case CCB_INFO:612 if (hdr.ca_offset >= DAX_MMAP_LEN) {613 dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)",614 hdr.ca_offset, DAX_MMAP_LEN);615 return -EINVAL;616 }617 618 ret = dax_ccb_info(ca, &ctx->result.info);619 if (ret != 0) {620 dax_dbg("dax_ccb_info failed (ret=%d)", ret);621 return ret;622 }623 624 dax_info_dbg("info succeeded on ca_offset %d", hdr.ca_offset);625 ctx->client = current;626 return count;627 628 case CCB_DEQUEUE:629 for (i = 0; i < DAX_CA_ELEMS; i++) {630 if (ctx->ca_buf[i].status !=631 CCA_STAT_NOT_COMPLETED)632 dax_unlock_pages(ctx, i, 1);633 }634 return count;635 636 default:637 return -EINVAL;638 }639}640 641static int dax_open(struct inode *inode, struct file *f)642{643 struct dax_ctx *ctx = NULL;644 int i;645 646 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);647 if (ctx == NULL)648 goto done;649 650 ctx->ccb_buf = kcalloc(DAX_MAX_CCBS, sizeof(struct dax_ccb),651 GFP_KERNEL);652 if (ctx->ccb_buf == NULL)653 goto done;654 655 ctx->ccb_buf_ra = virt_to_phys(ctx->ccb_buf);656 dax_dbg("ctx->ccb_buf=0x%p, ccb_buf_ra=0x%llx",657 (void *)ctx->ccb_buf, ctx->ccb_buf_ra);658 659 /* allocate CCB completion area buffer */660 ctx->ca_buf = kzalloc(DAX_MMAP_LEN, GFP_KERNEL);661 if (ctx->ca_buf == NULL)662 goto alloc_error;663 for (i = 0; i < DAX_CA_ELEMS; i++)664 ctx->ca_buf[i].status = CCA_STAT_COMPLETED;665 666 ctx->ca_buf_ra = virt_to_phys(ctx->ca_buf);667 dax_dbg("ctx=0x%p, ctx->ca_buf=0x%p, ca_buf_ra=0x%llx",668 (void *)ctx, (void *)ctx->ca_buf, ctx->ca_buf_ra);669 670 ctx->owner = current;671 f->private_data = ctx;672 return 0;673 674alloc_error:675 kfree(ctx->ccb_buf);676done:677 kfree(ctx);678 return -ENOMEM;679}680 681static char *dax_hv_errno(unsigned long hv_ret, int *ret)682{683 switch (hv_ret) {684 case HV_EBADALIGN:685 *ret = -EFAULT;686 return "HV_EBADALIGN";687 case HV_ENORADDR:688 *ret = -EFAULT;689 return "HV_ENORADDR";690 case HV_EINVAL:691 *ret = -EINVAL;692 return "HV_EINVAL";693 case HV_EWOULDBLOCK:694 *ret = -EAGAIN;695 return "HV_EWOULDBLOCK";696 case HV_ENOACCESS:697 *ret = -EPERM;698 return "HV_ENOACCESS";699 default:700 break;701 }702 703 *ret = -EIO;704 return "UNKNOWN";705}706 707static int dax_ccb_kill(u64 ca, u16 *kill_res)708{709 unsigned long hv_ret;710 int count, ret = 0;711 char *err_str;712 713 for (count = 0; count < DAX_CCB_RETRIES; count++) {714 dax_dbg("attempting kill on ca_ra 0x%llx", ca);715 hv_ret = sun4v_ccb_kill(ca, kill_res);716 717 if (hv_ret == HV_EOK) {718 dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca,719 *kill_res);720 } else {721 err_str = dax_hv_errno(hv_ret, &ret);722 dax_dbg("%s (ca_ra 0x%llx)", err_str, ca);723 }724 725 if (ret != -EAGAIN)726 return ret;727 dax_info_dbg("ccb_kill count = %d", count);728 udelay(DAX_CCB_USEC);729 }730 731 return -EAGAIN;732}733 734static int dax_ccb_info(u64 ca, struct ccb_info_result *info)735{736 unsigned long hv_ret;737 char *err_str;738 int ret = 0;739 740 dax_dbg("attempting info on ca_ra 0x%llx", ca);741 hv_ret = sun4v_ccb_info(ca, info);742 743 if (hv_ret == HV_EOK) {744 dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca, info->state);745 if (info->state == DAX_CCB_ENQUEUED) {746 dax_info_dbg("dax_unit %d, queue_num %d, queue_pos %d",747 info->inst_num, info->q_num, info->q_pos);748 }749 } else {750 err_str = dax_hv_errno(hv_ret, &ret);751 dax_dbg("%s (ca_ra 0x%llx)", err_str, ca);752 }753 754 return ret;755}756 757static void dax_prt_ccbs(struct dax_ccb *ccb, int nelem)758{759 int i, j;760 u64 *ccbp;761 762 dax_dbg("ccb buffer:");763 for (i = 0; i < nelem; i++) {764 ccbp = (u64 *)&ccb[i];765 dax_dbg(" %sccb[%d]", ccb[i].hdr.longccb ? "long " : "", i);766 for (j = 0; j < 8; j++)767 dax_dbg("\tccb[%d].dwords[%d]=0x%llx",768 i, j, *(ccbp + j));769 }770}771 772/*773 * Validates user CCB content. Also sets completion address and address types774 * for all addresses contained in CCB.775 */776static int dax_preprocess_usr_ccbs(struct dax_ctx *ctx, int idx, int nelem)777{778 int i;779 780 /*781 * The user is not allowed to specify real address types in782 * the CCB header. This must be enforced by the kernel before783 * submitting the CCBs to HV. The only allowed values for all784 * address fields are VA or IMM785 */786 for (i = 0; i < nelem; i++) {787 struct dax_ccb *ccbp = &ctx->ccb_buf[i];788 unsigned long ca_offset;789 790 if (ccbp->hdr.ccb_version > max_ccb_version)791 return DAX_SUBMIT_ERR_CCB_INVAL;792 793 switch (ccbp->hdr.opcode) {794 case DAX_OP_SYNC_NOP:795 case DAX_OP_EXTRACT:796 case DAX_OP_SCAN_VALUE:797 case DAX_OP_SCAN_RANGE:798 case DAX_OP_TRANSLATE:799 case DAX_OP_SCAN_VALUE | DAX_OP_INVERT:800 case DAX_OP_SCAN_RANGE | DAX_OP_INVERT:801 case DAX_OP_TRANSLATE | DAX_OP_INVERT:802 case DAX_OP_SELECT:803 break;804 default:805 return DAX_SUBMIT_ERR_CCB_INVAL;806 }807 808 if (ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_VA &&809 ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_NONE) {810 dax_dbg("invalid out_addr_type in user CCB[%d]", i);811 return DAX_SUBMIT_ERR_CCB_INVAL;812 }813 814 if (ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_VA &&815 ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_NONE) {816 dax_dbg("invalid pri_addr_type in user CCB[%d]", i);817 return DAX_SUBMIT_ERR_CCB_INVAL;818 }819 820 if (ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_VA &&821 ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_NONE) {822 dax_dbg("invalid sec_addr_type in user CCB[%d]", i);823 return DAX_SUBMIT_ERR_CCB_INVAL;824 }825 826 if (ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_VA &&827 ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_NONE) {828 dax_dbg("invalid table_addr_type in user CCB[%d]", i);829 return DAX_SUBMIT_ERR_CCB_INVAL;830 }831 832 /* set completion (real) address and address type */833 ccbp->hdr.cca_addr_type = DAX_ADDR_TYPE_RA;834 ca_offset = (idx + i) * sizeof(struct dax_cca);835 ccbp->ca = (void *)ctx->ca_buf_ra + ca_offset;836 memset(&ctx->ca_buf[idx + i], 0, sizeof(struct dax_cca));837 838 dax_dbg("ccb[%d]=%p, ca_offset=0x%lx, compl RA=0x%llx",839 i, ccbp, ca_offset, ctx->ca_buf_ra + ca_offset);840 841 /* skip over 2nd 64 bytes of long CCB */842 if (ccbp->hdr.longccb)843 i++;844 }845 846 return DAX_SUBMIT_OK;847}848 849static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf,850 size_t count, loff_t *ppos)851{852 unsigned long accepted_len, hv_rv;853 int i, idx, nccbs, naccepted;854 855 ctx->client = current;856 idx = *ppos;857 nccbs = count / sizeof(struct dax_ccb);858 859 if (ctx->owner != current) {860 dax_dbg("wrong thread");861 ctx->result.exec.status = DAX_SUBMIT_ERR_THR_INIT;862 return 0;863 }864 dax_dbg("args: ccb_buf_len=%ld, idx=%d", count, idx);865 866 /* for given index and length, verify ca_buf range exists */867 if (idx < 0 || idx > (DAX_CA_ELEMS - nccbs)) {868 ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL;869 return 0;870 }871 872 /*873 * Copy CCBs into kernel buffer to prevent modification by the874 * user in between validation and submission.875 */876 if (copy_from_user(ctx->ccb_buf, buf, count)) {877 dax_dbg("copyin of user CCB buffer failed");878 ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS;879 return 0;880 }881 882 /* check to see if ca_buf[idx] .. ca_buf[idx + nccbs] are available */883 for (i = idx; i < idx + nccbs; i++) {884 if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) {885 dax_dbg("CA range not available, dequeue needed");886 ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL;887 return 0;888 }889 }890 dax_unlock_pages(ctx, idx, nccbs);891 892 ctx->result.exec.status = dax_preprocess_usr_ccbs(ctx, idx, nccbs);893 if (ctx->result.exec.status != DAX_SUBMIT_OK)894 return 0;895 896 ctx->result.exec.status = dax_lock_pages(ctx, idx, nccbs,897 &ctx->result.exec.status_data);898 if (ctx->result.exec.status != DAX_SUBMIT_OK)899 return 0;900 901 if (dax_debug & DAX_DBG_FLG_BASIC)902 dax_prt_ccbs(ctx->ccb_buf, nccbs);903 904 hv_rv = sun4v_ccb_submit(ctx->ccb_buf_ra, count,905 HV_CCB_QUERY_CMD | HV_CCB_VA_SECONDARY, 0,906 &accepted_len, &ctx->result.exec.status_data);907 908 switch (hv_rv) {909 case HV_EOK:910 /*911 * Hcall succeeded with no errors but the accepted912 * length may be less than the requested length. The913 * only way the driver can resubmit the remainder is914 * to wait for completion of the submitted CCBs since915 * there is no way to guarantee the ordering semantics916 * required by the client applications. Therefore we917 * let the user library deal with resubmissions.918 */919 ctx->result.exec.status = DAX_SUBMIT_OK;920 break;921 case HV_EWOULDBLOCK:922 /*923 * This is a transient HV API error. The user library924 * can retry.925 */926 dax_dbg("hcall returned HV_EWOULDBLOCK");927 ctx->result.exec.status = DAX_SUBMIT_ERR_WOULDBLOCK;928 break;929 case HV_ENOMAP:930 /*931 * HV was unable to translate a VA. The VA it could932 * not translate is returned in the status_data param.933 */934 dax_dbg("hcall returned HV_ENOMAP");935 ctx->result.exec.status = DAX_SUBMIT_ERR_NOMAP;936 break;937 case HV_EINVAL:938 /*939 * This is the result of an invalid user CCB as HV is940 * validating some of the user CCB fields. Pass this941 * error back to the user. There is no supporting info942 * to isolate the invalid field.943 */944 dax_dbg("hcall returned HV_EINVAL");945 ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_INVAL;946 break;947 case HV_ENOACCESS:948 /*949 * HV found a VA that did not have the appropriate950 * permissions (such as the w bit). The VA in question951 * is returned in status_data param.952 */953 dax_dbg("hcall returned HV_ENOACCESS");954 ctx->result.exec.status = DAX_SUBMIT_ERR_NOACCESS;955 break;956 case HV_EUNAVAILABLE:957 /*958 * The requested CCB operation could not be performed959 * at this time. Return the specific unavailable code960 * in the status_data field.961 */962 dax_dbg("hcall returned HV_EUNAVAILABLE");963 ctx->result.exec.status = DAX_SUBMIT_ERR_UNAVAIL;964 break;965 default:966 ctx->result.exec.status = DAX_SUBMIT_ERR_INTERNAL;967 dax_dbg("unknown hcall return value (%ld)", hv_rv);968 break;969 }970 971 /* unlock pages associated with the unaccepted CCBs */972 naccepted = accepted_len / sizeof(struct dax_ccb);973 dax_unlock_pages(ctx, idx + naccepted, nccbs - naccepted);974 975 /* mark unaccepted CCBs as not completed */976 for (i = idx + naccepted; i < idx + nccbs; i++)977 ctx->ca_buf[i].status = CCA_STAT_COMPLETED;978 979 ctx->ccb_count += naccepted;980 ctx->fail_count += nccbs - naccepted;981 982 dax_dbg("hcall rv=%ld, accepted_len=%ld, status_data=0x%llx, ret status=%d",983 hv_rv, accepted_len, ctx->result.exec.status_data,984 ctx->result.exec.status);985 986 if (count == accepted_len)987 ctx->client = NULL; /* no read needed to complete protocol */988 return accepted_len;989}990