1209 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright 2015 IBM Corp.4 */5 6#include <linux/spinlock.h>7#include <linux/uaccess.h>8#include <linux/delay.h>9#include <linux/irqdomain.h>10#include <linux/platform_device.h>11 12#include "cxl.h"13#include "hcalls.h"14#include "trace.h"15 16#define CXL_ERROR_DETECTED_EVENT 117#define CXL_SLOT_RESET_EVENT 218#define CXL_RESUME_EVENT 319 20static void pci_error_handlers(struct cxl_afu *afu,21 int bus_error_event,22 pci_channel_state_t state)23{24 struct pci_dev *afu_dev;25 struct pci_driver *afu_drv;26 const struct pci_error_handlers *err_handler;27 28 if (afu->phb == NULL)29 return;30 31 list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {32 afu_drv = to_pci_driver(afu_dev->dev.driver);33 if (!afu_drv)34 continue;35 36 err_handler = afu_drv->err_handler;37 switch (bus_error_event) {38 case CXL_ERROR_DETECTED_EVENT:39 afu_dev->error_state = state;40 41 if (err_handler &&42 err_handler->error_detected)43 err_handler->error_detected(afu_dev, state);44 break;45 case CXL_SLOT_RESET_EVENT:46 afu_dev->error_state = state;47 48 if (err_handler &&49 err_handler->slot_reset)50 err_handler->slot_reset(afu_dev);51 break;52 case CXL_RESUME_EVENT:53 if (err_handler &&54 err_handler->resume)55 err_handler->resume(afu_dev);56 break;57 }58 }59}60 61static irqreturn_t guest_handle_psl_slice_error(struct cxl_context *ctx, u64 dsisr,62 u64 errstat)63{64 pr_devel("in %s\n", __func__);65 dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%.16llx\n", errstat);66 67 return cxl_ops->ack_irq(ctx, 0, errstat);68}69 70static ssize_t guest_collect_vpd(struct cxl *adapter, struct cxl_afu *afu,71 void *buf, size_t len)72{73 unsigned int entries, mod;74 unsigned long **vpd_buf = NULL;75 struct sg_list *le;76 int rc = 0, i, tocopy;77 u64 out = 0;78 79 if (buf == NULL)80 return -EINVAL;81 82 /* number of entries in the list */83 entries = len / SG_BUFFER_SIZE;84 mod = len % SG_BUFFER_SIZE;85 if (mod)86 entries++;87 88 if (entries > SG_MAX_ENTRIES) {89 entries = SG_MAX_ENTRIES;90 len = SG_MAX_ENTRIES * SG_BUFFER_SIZE;91 mod = 0;92 }93 94 vpd_buf = kcalloc(entries, sizeof(unsigned long *), GFP_KERNEL);95 if (!vpd_buf)96 return -ENOMEM;97 98 le = (struct sg_list *)get_zeroed_page(GFP_KERNEL);99 if (!le) {100 rc = -ENOMEM;101 goto err1;102 }103 104 for (i = 0; i < entries; i++) {105 vpd_buf[i] = (unsigned long *)get_zeroed_page(GFP_KERNEL);106 if (!vpd_buf[i]) {107 rc = -ENOMEM;108 goto err2;109 }110 le[i].phys_addr = cpu_to_be64(virt_to_phys(vpd_buf[i]));111 le[i].len = cpu_to_be64(SG_BUFFER_SIZE);112 if ((i == (entries - 1)) && mod)113 le[i].len = cpu_to_be64(mod);114 }115 116 if (adapter)117 rc = cxl_h_collect_vpd_adapter(adapter->guest->handle,118 virt_to_phys(le), entries, &out);119 else120 rc = cxl_h_collect_vpd(afu->guest->handle, 0,121 virt_to_phys(le), entries, &out);122 pr_devel("length of available (entries: %i), vpd: %#llx\n",123 entries, out);124 125 if (!rc) {126 /*127 * hcall returns in 'out' the size of available VPDs.128 * It fills the buffer with as much data as possible.129 */130 if (out < len)131 len = out;132 rc = len;133 if (out) {134 for (i = 0; i < entries; i++) {135 if (len < SG_BUFFER_SIZE)136 tocopy = len;137 else138 tocopy = SG_BUFFER_SIZE;139 memcpy(buf, vpd_buf[i], tocopy);140 buf += tocopy;141 len -= tocopy;142 }143 }144 }145err2:146 for (i = 0; i < entries; i++) {147 if (vpd_buf[i])148 free_page((unsigned long) vpd_buf[i]);149 }150 free_page((unsigned long) le);151err1:152 kfree(vpd_buf);153 return rc;154}155 156static int guest_get_irq_info(struct cxl_context *ctx, struct cxl_irq_info *info)157{158 return cxl_h_collect_int_info(ctx->afu->guest->handle, ctx->process_token, info);159}160 161static irqreturn_t guest_psl_irq(int irq, void *data)162{163 struct cxl_context *ctx = data;164 struct cxl_irq_info irq_info;165 int rc;166 167 pr_devel("%d: received PSL interrupt %i\n", ctx->pe, irq);168 rc = guest_get_irq_info(ctx, &irq_info);169 if (rc) {170 WARN(1, "Unable to get IRQ info: %i\n", rc);171 return IRQ_HANDLED;172 }173 174 rc = cxl_irq_psl8(irq, ctx, &irq_info);175 return rc;176}177 178static int afu_read_error_state(struct cxl_afu *afu, int *state_out)179{180 u64 state;181 int rc = 0;182 183 if (!afu)184 return -EIO;185 186 rc = cxl_h_read_error_state(afu->guest->handle, &state);187 if (!rc) {188 WARN_ON(state != H_STATE_NORMAL &&189 state != H_STATE_DISABLE &&190 state != H_STATE_TEMP_UNAVAILABLE &&191 state != H_STATE_PERM_UNAVAILABLE);192 *state_out = state & 0xffffffff;193 }194 return rc;195}196 197static irqreturn_t guest_slice_irq_err(int irq, void *data)198{199 struct cxl_afu *afu = data;200 int rc;201 u64 serr, afu_error, dsisr;202 203 rc = cxl_h_get_fn_error_interrupt(afu->guest->handle, &serr);204 if (rc) {205 dev_crit(&afu->dev, "Couldn't read PSL_SERR_An: %d\n", rc);206 return IRQ_HANDLED;207 }208 afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An);209 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);210 cxl_afu_decode_psl_serr(afu, serr);211 dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error);212 dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr);213 214 rc = cxl_h_ack_fn_error_interrupt(afu->guest->handle, serr);215 if (rc)216 dev_crit(&afu->dev, "Couldn't ack slice error interrupt: %d\n",217 rc);218 219 return IRQ_HANDLED;220}221 222 223static int irq_alloc_range(struct cxl *adapter, int len, int *irq)224{225 int i, n;226 struct irq_avail *cur;227 228 for (i = 0; i < adapter->guest->irq_nranges; i++) {229 cur = &adapter->guest->irq_avail[i];230 n = bitmap_find_next_zero_area(cur->bitmap, cur->range,231 0, len, 0);232 if (n < cur->range) {233 bitmap_set(cur->bitmap, n, len);234 *irq = cur->offset + n;235 pr_devel("guest: allocate IRQs %#x->%#x\n",236 *irq, *irq + len - 1);237 238 return 0;239 }240 }241 return -ENOSPC;242}243 244static int irq_free_range(struct cxl *adapter, int irq, int len)245{246 int i, n;247 struct irq_avail *cur;248 249 if (len == 0)250 return -ENOENT;251 252 for (i = 0; i < adapter->guest->irq_nranges; i++) {253 cur = &adapter->guest->irq_avail[i];254 if (irq >= cur->offset &&255 (irq + len) <= (cur->offset + cur->range)) {256 n = irq - cur->offset;257 bitmap_clear(cur->bitmap, n, len);258 pr_devel("guest: release IRQs %#x->%#x\n",259 irq, irq + len - 1);260 return 0;261 }262 }263 return -ENOENT;264}265 266static int guest_reset(struct cxl *adapter)267{268 struct cxl_afu *afu = NULL;269 int i, rc;270 271 pr_devel("Adapter reset request\n");272 spin_lock(&adapter->afu_list_lock);273 for (i = 0; i < adapter->slices; i++) {274 if ((afu = adapter->afu[i])) {275 pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,276 pci_channel_io_frozen);277 cxl_context_detach_all(afu);278 }279 }280 281 rc = cxl_h_reset_adapter(adapter->guest->handle);282 for (i = 0; i < adapter->slices; i++) {283 if (!rc && (afu = adapter->afu[i])) {284 pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,285 pci_channel_io_normal);286 pci_error_handlers(afu, CXL_RESUME_EVENT, 0);287 }288 }289 spin_unlock(&adapter->afu_list_lock);290 return rc;291}292 293static int guest_alloc_one_irq(struct cxl *adapter)294{295 int irq;296 297 spin_lock(&adapter->guest->irq_alloc_lock);298 if (irq_alloc_range(adapter, 1, &irq))299 irq = -ENOSPC;300 spin_unlock(&adapter->guest->irq_alloc_lock);301 return irq;302}303 304static void guest_release_one_irq(struct cxl *adapter, int irq)305{306 spin_lock(&adapter->guest->irq_alloc_lock);307 irq_free_range(adapter, irq, 1);308 spin_unlock(&adapter->guest->irq_alloc_lock);309}310 311static int guest_alloc_irq_ranges(struct cxl_irq_ranges *irqs,312 struct cxl *adapter, unsigned int num)313{314 int i, try, irq;315 316 memset(irqs, 0, sizeof(struct cxl_irq_ranges));317 318 spin_lock(&adapter->guest->irq_alloc_lock);319 for (i = 0; i < CXL_IRQ_RANGES && num; i++) {320 try = num;321 while (try) {322 if (irq_alloc_range(adapter, try, &irq) == 0)323 break;324 try /= 2;325 }326 if (!try)327 goto error;328 irqs->offset[i] = irq;329 irqs->range[i] = try;330 num -= try;331 }332 if (num)333 goto error;334 spin_unlock(&adapter->guest->irq_alloc_lock);335 return 0;336 337error:338 for (i = 0; i < CXL_IRQ_RANGES; i++)339 irq_free_range(adapter, irqs->offset[i], irqs->range[i]);340 spin_unlock(&adapter->guest->irq_alloc_lock);341 return -ENOSPC;342}343 344static void guest_release_irq_ranges(struct cxl_irq_ranges *irqs,345 struct cxl *adapter)346{347 int i;348 349 spin_lock(&adapter->guest->irq_alloc_lock);350 for (i = 0; i < CXL_IRQ_RANGES; i++)351 irq_free_range(adapter, irqs->offset[i], irqs->range[i]);352 spin_unlock(&adapter->guest->irq_alloc_lock);353}354 355static int guest_register_serr_irq(struct cxl_afu *afu)356{357 afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",358 dev_name(&afu->dev));359 if (!afu->err_irq_name)360 return -ENOMEM;361 362 if (!(afu->serr_virq = cxl_map_irq(afu->adapter, afu->serr_hwirq,363 guest_slice_irq_err, afu, afu->err_irq_name))) {364 kfree(afu->err_irq_name);365 afu->err_irq_name = NULL;366 return -ENOMEM;367 }368 369 return 0;370}371 372static void guest_release_serr_irq(struct cxl_afu *afu)373{374 cxl_unmap_irq(afu->serr_virq, afu);375 cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);376 kfree(afu->err_irq_name);377}378 379static int guest_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)380{381 return cxl_h_control_faults(ctx->afu->guest->handle, ctx->process_token,382 tfc >> 32, (psl_reset_mask != 0));383}384 385static void disable_afu_irqs(struct cxl_context *ctx)386{387 irq_hw_number_t hwirq;388 unsigned int virq;389 int r, i;390 391 pr_devel("Disabling AFU(%d) interrupts\n", ctx->afu->slice);392 for (r = 0; r < CXL_IRQ_RANGES; r++) {393 hwirq = ctx->irqs.offset[r];394 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {395 virq = irq_find_mapping(NULL, hwirq);396 disable_irq(virq);397 }398 }399}400 401static void enable_afu_irqs(struct cxl_context *ctx)402{403 irq_hw_number_t hwirq;404 unsigned int virq;405 int r, i;406 407 pr_devel("Enabling AFU(%d) interrupts\n", ctx->afu->slice);408 for (r = 0; r < CXL_IRQ_RANGES; r++) {409 hwirq = ctx->irqs.offset[r];410 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {411 virq = irq_find_mapping(NULL, hwirq);412 enable_irq(virq);413 }414 }415}416 417static int _guest_afu_cr_readXX(int sz, struct cxl_afu *afu, int cr_idx,418 u64 offset, u64 *val)419{420 unsigned long cr;421 char c;422 int rc = 0;423 424 if (afu->crs_len < sz)425 return -ENOENT;426 427 if (unlikely(offset >= afu->crs_len))428 return -ERANGE;429 430 cr = get_zeroed_page(GFP_KERNEL);431 if (!cr)432 return -ENOMEM;433 434 rc = cxl_h_get_config(afu->guest->handle, cr_idx, offset,435 virt_to_phys((void *)cr), sz);436 if (rc)437 goto err;438 439 switch (sz) {440 case 1:441 c = *((char *) cr);442 *val = c;443 break;444 case 2:445 *val = in_le16((u16 *)cr);446 break;447 case 4:448 *val = in_le32((unsigned *)cr);449 break;450 case 8:451 *val = in_le64((u64 *)cr);452 break;453 default:454 WARN_ON(1);455 }456err:457 free_page(cr);458 return rc;459}460 461static int guest_afu_cr_read32(struct cxl_afu *afu, int cr_idx, u64 offset,462 u32 *out)463{464 int rc;465 u64 val;466 467 rc = _guest_afu_cr_readXX(4, afu, cr_idx, offset, &val);468 if (!rc)469 *out = (u32) val;470 return rc;471}472 473static int guest_afu_cr_read16(struct cxl_afu *afu, int cr_idx, u64 offset,474 u16 *out)475{476 int rc;477 u64 val;478 479 rc = _guest_afu_cr_readXX(2, afu, cr_idx, offset, &val);480 if (!rc)481 *out = (u16) val;482 return rc;483}484 485static int guest_afu_cr_read8(struct cxl_afu *afu, int cr_idx, u64 offset,486 u8 *out)487{488 int rc;489 u64 val;490 491 rc = _guest_afu_cr_readXX(1, afu, cr_idx, offset, &val);492 if (!rc)493 *out = (u8) val;494 return rc;495}496 497static int guest_afu_cr_read64(struct cxl_afu *afu, int cr_idx, u64 offset,498 u64 *out)499{500 return _guest_afu_cr_readXX(8, afu, cr_idx, offset, out);501}502 503static int guest_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)504{505 /* config record is not writable from guest */506 return -EPERM;507}508 509static int guest_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)510{511 /* config record is not writable from guest */512 return -EPERM;513}514 515static int guest_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)516{517 /* config record is not writable from guest */518 return -EPERM;519}520 521static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)522{523 struct cxl_process_element_hcall *elem;524 struct cxl *adapter = ctx->afu->adapter;525 const struct cred *cred;526 u32 pid, idx;527 int rc, r, i;528 u64 mmio_addr, mmio_size;529 __be64 flags = 0;530 531 /* Must be 8 byte aligned and cannot cross a 4096 byte boundary */532 if (!(elem = (struct cxl_process_element_hcall *)533 get_zeroed_page(GFP_KERNEL)))534 return -ENOMEM;535 536 elem->version = cpu_to_be64(CXL_PROCESS_ELEMENT_VERSION);537 if (ctx->kernel) {538 pid = 0;539 flags |= CXL_PE_TRANSLATION_ENABLED;540 flags |= CXL_PE_PRIVILEGED_PROCESS;541 if (mfmsr() & MSR_SF)542 flags |= CXL_PE_64_BIT;543 } else {544 pid = current->pid;545 flags |= CXL_PE_PROBLEM_STATE;546 flags |= CXL_PE_TRANSLATION_ENABLED;547 if (!test_tsk_thread_flag(current, TIF_32BIT))548 flags |= CXL_PE_64_BIT;549 cred = get_current_cred();550 if (uid_eq(cred->euid, GLOBAL_ROOT_UID))551 flags |= CXL_PE_PRIVILEGED_PROCESS;552 put_cred(cred);553 }554 elem->flags = cpu_to_be64(flags);555 elem->common.tid = cpu_to_be32(0); /* Unused */556 elem->common.pid = cpu_to_be32(pid);557 elem->common.csrp = cpu_to_be64(0); /* disable */558 elem->common.u.psl8.aurp0 = cpu_to_be64(0); /* disable */559 elem->common.u.psl8.aurp1 = cpu_to_be64(0); /* disable */560 561 cxl_prefault(ctx, wed);562 563 elem->common.u.psl8.sstp0 = cpu_to_be64(ctx->sstp0);564 elem->common.u.psl8.sstp1 = cpu_to_be64(ctx->sstp1);565 566 /*567 * Ensure we have at least one interrupt allocated to take faults for568 * kernel contexts that may not have allocated any AFU IRQs at all:569 */570 if (ctx->irqs.range[0] == 0) {571 rc = afu_register_irqs(ctx, 0);572 if (rc)573 goto out_free;574 }575 576 for (r = 0; r < CXL_IRQ_RANGES; r++) {577 for (i = 0; i < ctx->irqs.range[r]; i++) {578 if (r == 0 && i == 0) {579 elem->pslVirtualIsn = cpu_to_be32(ctx->irqs.offset[0]);580 } else {581 idx = ctx->irqs.offset[r] + i - adapter->guest->irq_base_offset;582 elem->applicationVirtualIsnBitmap[idx / 8] |= 0x80 >> (idx % 8);583 }584 }585 }586 elem->common.amr = cpu_to_be64(amr);587 elem->common.wed = cpu_to_be64(wed);588 589 disable_afu_irqs(ctx);590 591 rc = cxl_h_attach_process(ctx->afu->guest->handle, elem,592 &ctx->process_token, &mmio_addr, &mmio_size);593 if (rc == H_SUCCESS) {594 if (ctx->master || !ctx->afu->pp_psa) {595 ctx->psn_phys = ctx->afu->psn_phys;596 ctx->psn_size = ctx->afu->adapter->ps_size;597 } else {598 ctx->psn_phys = mmio_addr;599 ctx->psn_size = mmio_size;600 }601 if (ctx->afu->pp_psa && mmio_size &&602 ctx->afu->pp_size == 0) {603 /*604 * There's no property in the device tree to read the605 * pp_size. We only find out at the 1st attach.606 * Compared to bare-metal, it is too late and we607 * should really lock here. However, on powerVM,608 * pp_size is really only used to display in /sys.609 * Being discussed with pHyp for their next release.610 */611 ctx->afu->pp_size = mmio_size;612 }613 /* from PAPR: process element is bytes 4-7 of process token */614 ctx->external_pe = ctx->process_token & 0xFFFFFFFF;615 pr_devel("CXL pe=%i is known as %i for pHyp, mmio_size=%#llx",616 ctx->pe, ctx->external_pe, ctx->psn_size);617 ctx->pe_inserted = true;618 enable_afu_irqs(ctx);619 }620 621out_free:622 free_page((u64)elem);623 return rc;624}625 626static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)627{628 pr_devel("in %s\n", __func__);629 630 ctx->kernel = kernel;631 if (ctx->afu->current_mode == CXL_MODE_DIRECTED)632 return attach_afu_directed(ctx, wed, amr);633 634 /* dedicated mode not supported on FW840 */635 636 return -EINVAL;637}638 639static int detach_afu_directed(struct cxl_context *ctx)640{641 if (!ctx->pe_inserted)642 return 0;643 if (cxl_h_detach_process(ctx->afu->guest->handle, ctx->process_token))644 return -1;645 return 0;646}647 648static int guest_detach_process(struct cxl_context *ctx)649{650 pr_devel("in %s\n", __func__);651 trace_cxl_detach(ctx);652 653 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))654 return -EIO;655 656 if (ctx->afu->current_mode == CXL_MODE_DIRECTED)657 return detach_afu_directed(ctx);658 659 return -EINVAL;660}661 662static void guest_release_afu(struct device *dev)663{664 struct cxl_afu *afu = to_cxl_afu(dev);665 666 pr_devel("%s\n", __func__);667 668 idr_destroy(&afu->contexts_idr);669 670 kfree(afu->guest);671 kfree(afu);672}673 674ssize_t cxl_guest_read_afu_vpd(struct cxl_afu *afu, void *buf, size_t len)675{676 return guest_collect_vpd(NULL, afu, buf, len);677}678 679#define ERR_BUFF_MAX_COPY_SIZE PAGE_SIZE680static ssize_t guest_afu_read_err_buffer(struct cxl_afu *afu, char *buf,681 loff_t off, size_t count)682{683 void *tbuf = NULL;684 int rc = 0;685 686 tbuf = (void *) get_zeroed_page(GFP_KERNEL);687 if (!tbuf)688 return -ENOMEM;689 690 rc = cxl_h_get_afu_err(afu->guest->handle,691 off & 0x7,692 virt_to_phys(tbuf),693 count);694 if (rc)695 goto err;696 697 if (count > ERR_BUFF_MAX_COPY_SIZE)698 count = ERR_BUFF_MAX_COPY_SIZE - (off & 0x7);699 memcpy(buf, tbuf, count);700err:701 free_page((u64)tbuf);702 703 return rc;704}705 706static int guest_afu_check_and_enable(struct cxl_afu *afu)707{708 return 0;709}710 711static bool guest_support_attributes(const char *attr_name,712 enum cxl_attrs type)713{714 switch (type) {715 case CXL_ADAPTER_ATTRS:716 if ((strcmp(attr_name, "base_image") == 0) ||717 (strcmp(attr_name, "load_image_on_perst") == 0) ||718 (strcmp(attr_name, "perst_reloads_same_image") == 0) ||719 (strcmp(attr_name, "image_loaded") == 0))720 return false;721 break;722 case CXL_AFU_MASTER_ATTRS:723 if ((strcmp(attr_name, "pp_mmio_off") == 0))724 return false;725 break;726 case CXL_AFU_ATTRS:727 break;728 default:729 break;730 }731 732 return true;733}734 735static int activate_afu_directed(struct cxl_afu *afu)736{737 int rc;738 739 dev_info(&afu->dev, "Activating AFU(%d) directed mode\n", afu->slice);740 741 afu->current_mode = CXL_MODE_DIRECTED;742 743 afu->num_procs = afu->max_procs_virtualised;744 745 if ((rc = cxl_chardev_m_afu_add(afu)))746 return rc;747 748 if ((rc = cxl_sysfs_afu_m_add(afu)))749 goto err;750 751 if ((rc = cxl_chardev_s_afu_add(afu)))752 goto err1;753 754 return 0;755err1:756 cxl_sysfs_afu_m_remove(afu);757err:758 cxl_chardev_afu_remove(afu);759 return rc;760}761 762static int guest_afu_activate_mode(struct cxl_afu *afu, int mode)763{764 if (!mode)765 return 0;766 if (!(mode & afu->modes_supported))767 return -EINVAL;768 769 if (mode == CXL_MODE_DIRECTED)770 return activate_afu_directed(afu);771 772 if (mode == CXL_MODE_DEDICATED)773 dev_err(&afu->dev, "Dedicated mode not supported\n");774 775 return -EINVAL;776}777 778static int deactivate_afu_directed(struct cxl_afu *afu)779{780 dev_info(&afu->dev, "Deactivating AFU(%d) directed mode\n", afu->slice);781 782 afu->current_mode = 0;783 afu->num_procs = 0;784 785 cxl_sysfs_afu_m_remove(afu);786 cxl_chardev_afu_remove(afu);787 788 cxl_ops->afu_reset(afu);789 790 return 0;791}792 793static int guest_afu_deactivate_mode(struct cxl_afu *afu, int mode)794{795 if (!mode)796 return 0;797 if (!(mode & afu->modes_supported))798 return -EINVAL;799 800 if (mode == CXL_MODE_DIRECTED)801 return deactivate_afu_directed(afu);802 return 0;803}804 805static int guest_afu_reset(struct cxl_afu *afu)806{807 pr_devel("AFU(%d) reset request\n", afu->slice);808 return cxl_h_reset_afu(afu->guest->handle);809}810 811static int guest_map_slice_regs(struct cxl_afu *afu)812{813 if (!(afu->p2n_mmio = ioremap(afu->guest->p2n_phys, afu->guest->p2n_size))) {814 dev_err(&afu->dev, "Error mapping AFU(%d) MMIO regions\n",815 afu->slice);816 return -ENOMEM;817 }818 return 0;819}820 821static void guest_unmap_slice_regs(struct cxl_afu *afu)822{823 if (afu->p2n_mmio)824 iounmap(afu->p2n_mmio);825}826 827static int afu_update_state(struct cxl_afu *afu)828{829 int rc, cur_state;830 831 rc = afu_read_error_state(afu, &cur_state);832 if (rc)833 return rc;834 835 if (afu->guest->previous_state == cur_state)836 return 0;837 838 pr_devel("AFU(%d) update state to %#x\n", afu->slice, cur_state);839 840 switch (cur_state) {841 case H_STATE_NORMAL:842 afu->guest->previous_state = cur_state;843 break;844 845 case H_STATE_DISABLE:846 pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,847 pci_channel_io_frozen);848 849 cxl_context_detach_all(afu);850 if ((rc = cxl_ops->afu_reset(afu)))851 pr_devel("reset hcall failed %d\n", rc);852 853 rc = afu_read_error_state(afu, &cur_state);854 if (!rc && cur_state == H_STATE_NORMAL) {855 pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,856 pci_channel_io_normal);857 pci_error_handlers(afu, CXL_RESUME_EVENT, 0);858 }859 afu->guest->previous_state = 0;860 break;861 862 case H_STATE_TEMP_UNAVAILABLE:863 afu->guest->previous_state = cur_state;864 break;865 866 case H_STATE_PERM_UNAVAILABLE:867 dev_err(&afu->dev, "AFU is in permanent error state\n");868 pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,869 pci_channel_io_perm_failure);870 afu->guest->previous_state = cur_state;871 break;872 873 default:874 pr_err("Unexpected AFU(%d) error state: %#x\n",875 afu->slice, cur_state);876 return -EINVAL;877 }878 879 return rc;880}881 882static void afu_handle_errstate(struct work_struct *work)883{884 struct cxl_afu_guest *afu_guest =885 container_of(to_delayed_work(work), struct cxl_afu_guest, work_err);886 887 if (!afu_update_state(afu_guest->parent) &&888 afu_guest->previous_state == H_STATE_PERM_UNAVAILABLE)889 return;890 891 if (afu_guest->handle_err)892 schedule_delayed_work(&afu_guest->work_err,893 msecs_to_jiffies(3000));894}895 896static bool guest_link_ok(struct cxl *cxl, struct cxl_afu *afu)897{898 int state;899 900 if (afu && (!afu_read_error_state(afu, &state))) {901 if (state == H_STATE_NORMAL)902 return true;903 }904 905 return false;906}907 908static int afu_properties_look_ok(struct cxl_afu *afu)909{910 if (afu->pp_irqs < 0) {911 dev_err(&afu->dev, "Unexpected per-process minimum interrupt value\n");912 return -EINVAL;913 }914 915 if (afu->max_procs_virtualised < 1) {916 dev_err(&afu->dev, "Unexpected max number of processes virtualised value\n");917 return -EINVAL;918 }919 920 return 0;921}922 923int cxl_guest_init_afu(struct cxl *adapter, int slice, struct device_node *afu_np)924{925 struct cxl_afu *afu;926 bool free = true;927 int rc;928 929 pr_devel("in %s - AFU(%d)\n", __func__, slice);930 if (!(afu = cxl_alloc_afu(adapter, slice)))931 return -ENOMEM;932 933 if (!(afu->guest = kzalloc(sizeof(struct cxl_afu_guest), GFP_KERNEL))) {934 kfree(afu);935 return -ENOMEM;936 }937 938 if ((rc = dev_set_name(&afu->dev, "afu%i.%i",939 adapter->adapter_num,940 slice)))941 goto err1;942 943 adapter->slices++;944 945 if ((rc = cxl_of_read_afu_handle(afu, afu_np)))946 goto err1;947 948 if ((rc = cxl_ops->afu_reset(afu)))949 goto err1;950 951 if ((rc = cxl_of_read_afu_properties(afu, afu_np)))952 goto err1;953 954 if ((rc = afu_properties_look_ok(afu)))955 goto err1;956 957 if ((rc = guest_map_slice_regs(afu)))958 goto err1;959 960 if ((rc = guest_register_serr_irq(afu)))961 goto err2;962 963 /*964 * After we call this function we must not free the afu directly, even965 * if it returns an error!966 */967 if ((rc = cxl_register_afu(afu)))968 goto err_put_dev;969 970 if ((rc = cxl_sysfs_afu_add(afu)))971 goto err_del_dev;972 973 /*974 * pHyp doesn't expose the programming models supported by the975 * AFU. pHyp currently only supports directed mode. If it adds976 * dedicated mode later, this version of cxl has no way to977 * detect it. So we'll initialize the driver, but the first978 * attach will fail.979 * Being discussed with pHyp to do better (likely new property)980 */981 if (afu->max_procs_virtualised == 1)982 afu->modes_supported = CXL_MODE_DEDICATED;983 else984 afu->modes_supported = CXL_MODE_DIRECTED;985 986 if ((rc = cxl_afu_select_best_mode(afu)))987 goto err_remove_sysfs;988 989 adapter->afu[afu->slice] = afu;990 991 afu->enabled = true;992 993 /*994 * wake up the cpu periodically to check the state995 * of the AFU using "afu" stored in the guest structure.996 */997 afu->guest->parent = afu;998 afu->guest->handle_err = true;999 INIT_DELAYED_WORK(&afu->guest->work_err, afu_handle_errstate);1000 schedule_delayed_work(&afu->guest->work_err, msecs_to_jiffies(1000));1001 1002 if ((rc = cxl_pci_vphb_add(afu)))1003 dev_info(&afu->dev, "Can't register vPHB\n");1004 1005 return 0;1006 1007err_remove_sysfs:1008 cxl_sysfs_afu_remove(afu);1009err_del_dev:1010 device_del(&afu->dev);1011err_put_dev:1012 put_device(&afu->dev);1013 free = false;1014 guest_release_serr_irq(afu);1015err2:1016 guest_unmap_slice_regs(afu);1017err1:1018 if (free) {1019 kfree(afu->guest);1020 kfree(afu);1021 }1022 return rc;1023}1024 1025void cxl_guest_remove_afu(struct cxl_afu *afu)1026{1027 if (!afu)1028 return;1029 1030 /* flush and stop pending job */1031 afu->guest->handle_err = false;1032 flush_delayed_work(&afu->guest->work_err);1033 1034 cxl_pci_vphb_remove(afu);1035 cxl_sysfs_afu_remove(afu);1036 1037 spin_lock(&afu->adapter->afu_list_lock);1038 afu->adapter->afu[afu->slice] = NULL;1039 spin_unlock(&afu->adapter->afu_list_lock);1040 1041 cxl_context_detach_all(afu);1042 cxl_ops->afu_deactivate_mode(afu, afu->current_mode);1043 guest_release_serr_irq(afu);1044 guest_unmap_slice_regs(afu);1045 1046 device_unregister(&afu->dev);1047}1048 1049static void free_adapter(struct cxl *adapter)1050{1051 struct irq_avail *cur;1052 int i;1053 1054 if (adapter->guest) {1055 if (adapter->guest->irq_avail) {1056 for (i = 0; i < adapter->guest->irq_nranges; i++) {1057 cur = &adapter->guest->irq_avail[i];1058 bitmap_free(cur->bitmap);1059 }1060 kfree(adapter->guest->irq_avail);1061 }1062 kfree(adapter->guest->status);1063 kfree(adapter->guest);1064 }1065 cxl_remove_adapter_nr(adapter);1066 kfree(adapter);1067}1068 1069static int properties_look_ok(struct cxl *adapter)1070{1071 /* The absence of this property means that the operational1072 * status is unknown or okay1073 */1074 if (strlen(adapter->guest->status) &&1075 strcmp(adapter->guest->status, "okay")) {1076 pr_err("ABORTING:Bad operational status of the device\n");1077 return -EINVAL;1078 }1079 1080 return 0;1081}1082 1083ssize_t cxl_guest_read_adapter_vpd(struct cxl *adapter, void *buf, size_t len)1084{1085 return guest_collect_vpd(adapter, NULL, buf, len);1086}1087 1088void cxl_guest_remove_adapter(struct cxl *adapter)1089{1090 pr_devel("in %s\n", __func__);1091 1092 cxl_sysfs_adapter_remove(adapter);1093 1094 cxl_guest_remove_chardev(adapter);1095 device_unregister(&adapter->dev);1096}1097 1098static void release_adapter(struct device *dev)1099{1100 free_adapter(to_cxl_adapter(dev));1101}1102 1103struct cxl *cxl_guest_init_adapter(struct device_node *np, struct platform_device *pdev)1104{1105 struct cxl *adapter;1106 bool free = true;1107 int rc;1108 1109 if (!(adapter = cxl_alloc_adapter()))1110 return ERR_PTR(-ENOMEM);1111 1112 if (!(adapter->guest = kzalloc(sizeof(struct cxl_guest), GFP_KERNEL))) {1113 free_adapter(adapter);1114 return ERR_PTR(-ENOMEM);1115 }1116 1117 adapter->slices = 0;1118 adapter->guest->pdev = pdev;1119 adapter->dev.parent = &pdev->dev;1120 adapter->dev.release = release_adapter;1121 dev_set_drvdata(&pdev->dev, adapter);1122 1123 /*1124 * Hypervisor controls PSL timebase initialization (p1 register).1125 * On FW840, PSL is initialized.1126 */1127 adapter->psl_timebase_synced = true;1128 1129 if ((rc = cxl_of_read_adapter_handle(adapter, np)))1130 goto err1;1131 1132 if ((rc = cxl_of_read_adapter_properties(adapter, np)))1133 goto err1;1134 1135 if ((rc = properties_look_ok(adapter)))1136 goto err1;1137 1138 if ((rc = cxl_guest_add_chardev(adapter)))1139 goto err1;1140 1141 /*1142 * After we call this function we must not free the adapter directly,1143 * even if it returns an error!1144 */1145 if ((rc = cxl_register_adapter(adapter)))1146 goto err_put_dev;1147 1148 if ((rc = cxl_sysfs_adapter_add(adapter)))1149 goto err_del_dev;1150 1151 /* release the context lock as the adapter is configured */1152 cxl_adapter_context_unlock(adapter);1153 1154 return adapter;1155 1156err_del_dev:1157 device_del(&adapter->dev);1158err_put_dev:1159 put_device(&adapter->dev);1160 free = false;1161 cxl_guest_remove_chardev(adapter);1162err1:1163 if (free)1164 free_adapter(adapter);1165 return ERR_PTR(rc);1166}1167 1168void cxl_guest_reload_module(struct cxl *adapter)1169{1170 struct platform_device *pdev;1171 1172 pdev = adapter->guest->pdev;1173 cxl_guest_remove_adapter(adapter);1174 1175 cxl_of_probe(pdev);1176}1177 1178const struct cxl_backend_ops cxl_guest_ops = {1179 .module = THIS_MODULE,1180 .adapter_reset = guest_reset,1181 .alloc_one_irq = guest_alloc_one_irq,1182 .release_one_irq = guest_release_one_irq,1183 .alloc_irq_ranges = guest_alloc_irq_ranges,1184 .release_irq_ranges = guest_release_irq_ranges,1185 .setup_irq = NULL,1186 .handle_psl_slice_error = guest_handle_psl_slice_error,1187 .psl_interrupt = guest_psl_irq,1188 .ack_irq = guest_ack_irq,1189 .attach_process = guest_attach_process,1190 .detach_process = guest_detach_process,1191 .update_ivtes = NULL,1192 .support_attributes = guest_support_attributes,1193 .link_ok = guest_link_ok,1194 .release_afu = guest_release_afu,1195 .afu_read_err_buffer = guest_afu_read_err_buffer,1196 .afu_check_and_enable = guest_afu_check_and_enable,1197 .afu_activate_mode = guest_afu_activate_mode,1198 .afu_deactivate_mode = guest_afu_deactivate_mode,1199 .afu_reset = guest_afu_reset,1200 .afu_cr_read8 = guest_afu_cr_read8,1201 .afu_cr_read16 = guest_afu_cr_read16,1202 .afu_cr_read32 = guest_afu_cr_read32,1203 .afu_cr_read64 = guest_afu_cr_read64,1204 .afu_cr_write8 = guest_afu_cr_write8,1205 .afu_cr_write16 = guest_afu_cr_write16,1206 .afu_cr_write32 = guest_afu_cr_write32,1207 .read_adapter_vpd = cxl_guest_read_adapter_vpd,1208};1209