1128 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2020 Intel Corporation. */3 4#include <linux/io-64-nonatomic-lo-hi.h>5#include <linux/firmware.h>6#include <linux/device.h>7#include <linux/slab.h>8#include <linux/idr.h>9#include <linux/pci.h>10#include <cxlmem.h>11#include "trace.h"12#include "core.h"13 14static DECLARE_RWSEM(cxl_memdev_rwsem);15 16/*17 * An entire PCI topology full of devices should be enough for any18 * config19 */20#define CXL_MEM_MAX_DEVS 6553621 22static int cxl_mem_major;23static DEFINE_IDA(cxl_memdev_ida);24 25static void cxl_memdev_release(struct device *dev)26{27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);28 29 ida_free(&cxl_memdev_ida, cxlmd->id);30 kfree(cxlmd);31}32 33static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,34 kgid_t *gid)35{36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));37}38 39static ssize_t firmware_version_show(struct device *dev,40 struct device_attribute *attr, char *buf)41{42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);43 struct cxl_dev_state *cxlds = cxlmd->cxlds;44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);45 46 if (!mds)47 return sysfs_emit(buf, "\n");48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version);49}50static DEVICE_ATTR_RO(firmware_version);51 52static ssize_t payload_max_show(struct device *dev,53 struct device_attribute *attr, char *buf)54{55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);56 struct cxl_dev_state *cxlds = cxlmd->cxlds;57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);58 59 if (!mds)60 return sysfs_emit(buf, "\n");61 return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);62}63static DEVICE_ATTR_RO(payload_max);64 65static ssize_t label_storage_size_show(struct device *dev,66 struct device_attribute *attr, char *buf)67{68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);69 struct cxl_dev_state *cxlds = cxlmd->cxlds;70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);71 72 if (!mds)73 return sysfs_emit(buf, "\n");74 return sysfs_emit(buf, "%zu\n", mds->lsa_size);75}76static DEVICE_ATTR_RO(label_storage_size);77 78static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,79 char *buf)80{81 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);82 struct cxl_dev_state *cxlds = cxlmd->cxlds;83 unsigned long long len = resource_size(&cxlds->ram_res);84 85 return sysfs_emit(buf, "%#llx\n", len);86}87 88static struct device_attribute dev_attr_ram_size =89 __ATTR(size, 0444, ram_size_show, NULL);90 91static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,92 char *buf)93{94 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);95 struct cxl_dev_state *cxlds = cxlmd->cxlds;96 unsigned long long len = resource_size(&cxlds->pmem_res);97 98 return sysfs_emit(buf, "%#llx\n", len);99}100 101static struct device_attribute dev_attr_pmem_size =102 __ATTR(size, 0444, pmem_size_show, NULL);103 104static ssize_t serial_show(struct device *dev, struct device_attribute *attr,105 char *buf)106{107 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);108 struct cxl_dev_state *cxlds = cxlmd->cxlds;109 110 return sysfs_emit(buf, "%#llx\n", cxlds->serial);111}112static DEVICE_ATTR_RO(serial);113 114static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,115 char *buf)116{117 return sysfs_emit(buf, "%d\n", dev_to_node(dev));118}119static DEVICE_ATTR_RO(numa_node);120 121static ssize_t security_state_show(struct device *dev,122 struct device_attribute *attr,123 char *buf)124{125 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);126 struct cxl_dev_state *cxlds = cxlmd->cxlds;127 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;128 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);129 unsigned long state = mds->security.state;130 int rc = 0;131 132 /* sync with latest submission state */133 mutex_lock(&cxl_mbox->mbox_mutex);134 if (mds->security.sanitize_active)135 rc = sysfs_emit(buf, "sanitize\n");136 mutex_unlock(&cxl_mbox->mbox_mutex);137 if (rc)138 return rc;139 140 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))141 return sysfs_emit(buf, "disabled\n");142 if (state & CXL_PMEM_SEC_STATE_FROZEN ||143 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||144 state & CXL_PMEM_SEC_STATE_USER_PLIMIT)145 return sysfs_emit(buf, "frozen\n");146 if (state & CXL_PMEM_SEC_STATE_LOCKED)147 return sysfs_emit(buf, "locked\n");148 else149 return sysfs_emit(buf, "unlocked\n");150}151static struct device_attribute dev_attr_security_state =152 __ATTR(state, 0444, security_state_show, NULL);153 154static ssize_t security_sanitize_store(struct device *dev,155 struct device_attribute *attr,156 const char *buf, size_t len)157{158 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);159 bool sanitize;160 ssize_t rc;161 162 if (kstrtobool(buf, &sanitize) || !sanitize)163 return -EINVAL;164 165 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE);166 if (rc)167 return rc;168 169 return len;170}171static struct device_attribute dev_attr_security_sanitize =172 __ATTR(sanitize, 0200, NULL, security_sanitize_store);173 174static ssize_t security_erase_store(struct device *dev,175 struct device_attribute *attr,176 const char *buf, size_t len)177{178 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);179 ssize_t rc;180 bool erase;181 182 if (kstrtobool(buf, &erase) || !erase)183 return -EINVAL;184 185 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE);186 if (rc)187 return rc;188 189 return len;190}191static struct device_attribute dev_attr_security_erase =192 __ATTR(erase, 0200, NULL, security_erase_store);193 194static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)195{196 struct cxl_dev_state *cxlds = cxlmd->cxlds;197 u64 offset, length;198 int rc = 0;199 200 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */201 if (resource_size(&cxlds->pmem_res)) {202 offset = cxlds->pmem_res.start;203 length = resource_size(&cxlds->pmem_res);204 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);205 if (rc)206 return rc;207 }208 if (resource_size(&cxlds->ram_res)) {209 offset = cxlds->ram_res.start;210 length = resource_size(&cxlds->ram_res);211 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);212 /*213 * Invalid Physical Address is not an error for214 * volatile addresses. Device support is optional.215 */216 if (rc == -EFAULT)217 rc = 0;218 }219 return rc;220}221 222int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)223{224 struct cxl_port *port;225 int rc;226 227 port = cxlmd->endpoint;228 if (!port || !is_cxl_endpoint(port))229 return -EINVAL;230 231 rc = down_read_interruptible(&cxl_region_rwsem);232 if (rc)233 return rc;234 235 rc = down_read_interruptible(&cxl_dpa_rwsem);236 if (rc) {237 up_read(&cxl_region_rwsem);238 return rc;239 }240 241 if (cxl_num_decoders_committed(port) == 0) {242 /* No regions mapped to this memdev */243 rc = cxl_get_poison_by_memdev(cxlmd);244 } else {245 /* Regions mapped, collect poison by endpoint */246 rc = cxl_get_poison_by_endpoint(port);247 }248 up_read(&cxl_dpa_rwsem);249 up_read(&cxl_region_rwsem);250 251 return rc;252}253EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL);254 255static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)256{257 struct cxl_dev_state *cxlds = cxlmd->cxlds;258 259 if (!IS_ENABLED(CONFIG_DEBUG_FS))260 return 0;261 262 if (!resource_size(&cxlds->dpa_res)) {263 dev_dbg(cxlds->dev, "device has no dpa resource\n");264 return -EINVAL;265 }266 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) {267 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",268 dpa, &cxlds->dpa_res);269 return -EINVAL;270 }271 if (!IS_ALIGNED(dpa, 64)) {272 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);273 return -EINVAL;274 }275 276 return 0;277}278 279int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)280{281 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;282 struct cxl_mbox_inject_poison inject;283 struct cxl_poison_record record;284 struct cxl_mbox_cmd mbox_cmd;285 struct cxl_region *cxlr;286 int rc;287 288 if (!IS_ENABLED(CONFIG_DEBUG_FS))289 return 0;290 291 rc = down_read_interruptible(&cxl_region_rwsem);292 if (rc)293 return rc;294 295 rc = down_read_interruptible(&cxl_dpa_rwsem);296 if (rc) {297 up_read(&cxl_region_rwsem);298 return rc;299 }300 301 rc = cxl_validate_poison_dpa(cxlmd, dpa);302 if (rc)303 goto out;304 305 inject.address = cpu_to_le64(dpa);306 mbox_cmd = (struct cxl_mbox_cmd) {307 .opcode = CXL_MBOX_OP_INJECT_POISON,308 .size_in = sizeof(inject),309 .payload_in = &inject,310 };311 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);312 if (rc)313 goto out;314 315 cxlr = cxl_dpa_to_region(cxlmd, dpa);316 if (cxlr)317 dev_warn_once(cxl_mbox->host,318 "poison inject dpa:%#llx region: %s\n", dpa,319 dev_name(&cxlr->dev));320 321 record = (struct cxl_poison_record) {322 .address = cpu_to_le64(dpa),323 .length = cpu_to_le32(1),324 };325 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);326out:327 up_read(&cxl_dpa_rwsem);328 up_read(&cxl_region_rwsem);329 330 return rc;331}332EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);333 334int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)335{336 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;337 struct cxl_mbox_clear_poison clear;338 struct cxl_poison_record record;339 struct cxl_mbox_cmd mbox_cmd;340 struct cxl_region *cxlr;341 int rc;342 343 if (!IS_ENABLED(CONFIG_DEBUG_FS))344 return 0;345 346 rc = down_read_interruptible(&cxl_region_rwsem);347 if (rc)348 return rc;349 350 rc = down_read_interruptible(&cxl_dpa_rwsem);351 if (rc) {352 up_read(&cxl_region_rwsem);353 return rc;354 }355 356 rc = cxl_validate_poison_dpa(cxlmd, dpa);357 if (rc)358 goto out;359 360 /*361 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command362 * is defined to accept 64 bytes of write-data, along with the363 * address to clear. This driver uses zeroes as write-data.364 */365 clear = (struct cxl_mbox_clear_poison) {366 .address = cpu_to_le64(dpa)367 };368 369 mbox_cmd = (struct cxl_mbox_cmd) {370 .opcode = CXL_MBOX_OP_CLEAR_POISON,371 .size_in = sizeof(clear),372 .payload_in = &clear,373 };374 375 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);376 if (rc)377 goto out;378 379 cxlr = cxl_dpa_to_region(cxlmd, dpa);380 if (cxlr)381 dev_warn_once(cxl_mbox->host,382 "poison clear dpa:%#llx region: %s\n", dpa,383 dev_name(&cxlr->dev));384 385 record = (struct cxl_poison_record) {386 .address = cpu_to_le64(dpa),387 .length = cpu_to_le32(1),388 };389 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);390out:391 up_read(&cxl_dpa_rwsem);392 up_read(&cxl_region_rwsem);393 394 return rc;395}396EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL);397 398static struct attribute *cxl_memdev_attributes[] = {399 &dev_attr_serial.attr,400 &dev_attr_firmware_version.attr,401 &dev_attr_payload_max.attr,402 &dev_attr_label_storage_size.attr,403 &dev_attr_numa_node.attr,404 NULL,405};406 407static ssize_t pmem_qos_class_show(struct device *dev,408 struct device_attribute *attr, char *buf)409{410 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);411 struct cxl_dev_state *cxlds = cxlmd->cxlds;412 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);413 414 return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class);415}416 417static struct device_attribute dev_attr_pmem_qos_class =418 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL);419 420static struct attribute *cxl_memdev_pmem_attributes[] = {421 &dev_attr_pmem_size.attr,422 &dev_attr_pmem_qos_class.attr,423 NULL,424};425 426static ssize_t ram_qos_class_show(struct device *dev,427 struct device_attribute *attr, char *buf)428{429 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);430 struct cxl_dev_state *cxlds = cxlmd->cxlds;431 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);432 433 return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class);434}435 436static struct device_attribute dev_attr_ram_qos_class =437 __ATTR(qos_class, 0444, ram_qos_class_show, NULL);438 439static struct attribute *cxl_memdev_ram_attributes[] = {440 &dev_attr_ram_size.attr,441 &dev_attr_ram_qos_class.attr,442 NULL,443};444 445static struct attribute *cxl_memdev_security_attributes[] = {446 &dev_attr_security_state.attr,447 &dev_attr_security_sanitize.attr,448 &dev_attr_security_erase.attr,449 NULL,450};451 452static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,453 int n)454{455 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)456 return 0;457 return a->mode;458}459 460static struct attribute_group cxl_memdev_attribute_group = {461 .attrs = cxl_memdev_attributes,462 .is_visible = cxl_memdev_visible,463};464 465static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)466{467 struct device *dev = kobj_to_dev(kobj);468 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);469 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);470 471 if (a == &dev_attr_ram_qos_class.attr)472 if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID)473 return 0;474 475 return a->mode;476}477 478static struct attribute_group cxl_memdev_ram_attribute_group = {479 .name = "ram",480 .attrs = cxl_memdev_ram_attributes,481 .is_visible = cxl_ram_visible,482};483 484static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n)485{486 struct device *dev = kobj_to_dev(kobj);487 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);488 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);489 490 if (a == &dev_attr_pmem_qos_class.attr)491 if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID)492 return 0;493 494 return a->mode;495}496 497static struct attribute_group cxl_memdev_pmem_attribute_group = {498 .name = "pmem",499 .attrs = cxl_memdev_pmem_attributes,500 .is_visible = cxl_pmem_visible,501};502 503static umode_t cxl_memdev_security_visible(struct kobject *kobj,504 struct attribute *a, int n)505{506 struct device *dev = kobj_to_dev(kobj);507 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);508 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);509 510 if (a == &dev_attr_security_sanitize.attr &&511 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))512 return 0;513 514 if (a == &dev_attr_security_erase.attr &&515 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))516 return 0;517 518 return a->mode;519}520 521static struct attribute_group cxl_memdev_security_attribute_group = {522 .name = "security",523 .attrs = cxl_memdev_security_attributes,524 .is_visible = cxl_memdev_security_visible,525};526 527static const struct attribute_group *cxl_memdev_attribute_groups[] = {528 &cxl_memdev_attribute_group,529 &cxl_memdev_ram_attribute_group,530 &cxl_memdev_pmem_attribute_group,531 &cxl_memdev_security_attribute_group,532 NULL,533};534 535void cxl_memdev_update_perf(struct cxl_memdev *cxlmd)536{537 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group);538 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group);539}540EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, CXL);541 542static const struct device_type cxl_memdev_type = {543 .name = "cxl_memdev",544 .release = cxl_memdev_release,545 .devnode = cxl_memdev_devnode,546 .groups = cxl_memdev_attribute_groups,547};548 549bool is_cxl_memdev(const struct device *dev)550{551 return dev->type == &cxl_memdev_type;552}553EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);554 555/**556 * set_exclusive_cxl_commands() - atomically disable user cxl commands557 * @mds: The device state to operate on558 * @cmds: bitmap of commands to mark exclusive559 *560 * Grab the cxl_memdev_rwsem in write mode to flush in-flight561 * invocations of the ioctl path and then disable future execution of562 * commands with the command ids set in @cmds.563 */564void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,565 unsigned long *cmds)566{567 down_write(&cxl_memdev_rwsem);568 bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds,569 CXL_MEM_COMMAND_ID_MAX);570 up_write(&cxl_memdev_rwsem);571}572EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);573 574/**575 * clear_exclusive_cxl_commands() - atomically enable user cxl commands576 * @mds: The device state to modify577 * @cmds: bitmap of commands to mark available for userspace578 */579void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,580 unsigned long *cmds)581{582 down_write(&cxl_memdev_rwsem);583 bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds,584 CXL_MEM_COMMAND_ID_MAX);585 up_write(&cxl_memdev_rwsem);586}587EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);588 589static void cxl_memdev_shutdown(struct device *dev)590{591 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);592 593 down_write(&cxl_memdev_rwsem);594 cxlmd->cxlds = NULL;595 up_write(&cxl_memdev_rwsem);596}597 598static void cxl_memdev_unregister(void *_cxlmd)599{600 struct cxl_memdev *cxlmd = _cxlmd;601 struct device *dev = &cxlmd->dev;602 603 cdev_device_del(&cxlmd->cdev, dev);604 cxl_memdev_shutdown(dev);605 put_device(dev);606}607 608static void detach_memdev(struct work_struct *work)609{610 struct cxl_memdev *cxlmd;611 612 cxlmd = container_of(work, typeof(*cxlmd), detach_work);613 device_release_driver(&cxlmd->dev);614 put_device(&cxlmd->dev);615}616 617static struct lock_class_key cxl_memdev_key;618 619static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,620 const struct file_operations *fops)621{622 struct cxl_memdev *cxlmd;623 struct device *dev;624 struct cdev *cdev;625 int rc;626 627 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);628 if (!cxlmd)629 return ERR_PTR(-ENOMEM);630 631 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);632 if (rc < 0)633 goto err;634 cxlmd->id = rc;635 cxlmd->depth = -1;636 637 dev = &cxlmd->dev;638 device_initialize(dev);639 lockdep_set_class(&dev->mutex, &cxl_memdev_key);640 dev->parent = cxlds->dev;641 dev->bus = &cxl_bus_type;642 dev->devt = MKDEV(cxl_mem_major, cxlmd->id);643 dev->type = &cxl_memdev_type;644 device_set_pm_not_required(dev);645 INIT_WORK(&cxlmd->detach_work, detach_memdev);646 647 cdev = &cxlmd->cdev;648 cdev_init(cdev, fops);649 return cxlmd;650 651err:652 kfree(cxlmd);653 return ERR_PTR(rc);654}655 656static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,657 unsigned long arg)658{659 switch (cmd) {660 case CXL_MEM_QUERY_COMMANDS:661 return cxl_query_cmd(cxlmd, (void __user *)arg);662 case CXL_MEM_SEND_COMMAND:663 return cxl_send_cmd(cxlmd, (void __user *)arg);664 default:665 return -ENOTTY;666 }667}668 669static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,670 unsigned long arg)671{672 struct cxl_memdev *cxlmd = file->private_data;673 struct cxl_dev_state *cxlds;674 int rc = -ENXIO;675 676 down_read(&cxl_memdev_rwsem);677 cxlds = cxlmd->cxlds;678 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)679 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);680 up_read(&cxl_memdev_rwsem);681 682 return rc;683}684 685static int cxl_memdev_open(struct inode *inode, struct file *file)686{687 struct cxl_memdev *cxlmd =688 container_of(inode->i_cdev, typeof(*cxlmd), cdev);689 690 get_device(&cxlmd->dev);691 file->private_data = cxlmd;692 693 return 0;694}695 696static int cxl_memdev_release_file(struct inode *inode, struct file *file)697{698 struct cxl_memdev *cxlmd =699 container_of(inode->i_cdev, typeof(*cxlmd), cdev);700 701 put_device(&cxlmd->dev);702 703 return 0;704}705 706/**707 * cxl_mem_get_fw_info - Get Firmware info708 * @mds: The device data for the operation709 *710 * Retrieve firmware info for the device specified.711 *712 * Return: 0 if no error: or the result of the mailbox command.713 *714 * See CXL-3.0 8.2.9.3.1 Get FW Info715 */716static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)717{718 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;719 struct cxl_mbox_get_fw_info info;720 struct cxl_mbox_cmd mbox_cmd;721 int rc;722 723 mbox_cmd = (struct cxl_mbox_cmd) {724 .opcode = CXL_MBOX_OP_GET_FW_INFO,725 .size_out = sizeof(info),726 .payload_out = &info,727 };728 729 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);730 if (rc < 0)731 return rc;732 733 mds->fw.num_slots = info.num_slots;734 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,735 info.slot_info);736 737 return 0;738}739 740/**741 * cxl_mem_activate_fw - Activate Firmware742 * @mds: The device data for the operation743 * @slot: slot number to activate744 *745 * Activate firmware in a given slot for the device specified.746 *747 * Return: 0 if no error: or the result of the mailbox command.748 *749 * See CXL-3.0 8.2.9.3.3 Activate FW750 */751static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)752{753 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;754 struct cxl_mbox_activate_fw activate;755 struct cxl_mbox_cmd mbox_cmd;756 757 if (slot == 0 || slot > mds->fw.num_slots)758 return -EINVAL;759 760 mbox_cmd = (struct cxl_mbox_cmd) {761 .opcode = CXL_MBOX_OP_ACTIVATE_FW,762 .size_in = sizeof(activate),763 .payload_in = &activate,764 };765 766 /* Only offline activation supported for now */767 activate.action = CXL_FW_ACTIVATE_OFFLINE;768 activate.slot = slot;769 770 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);771}772 773/**774 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer775 * @mds: The device data for the operation776 *777 * Abort an in-progress firmware transfer for the device specified.778 *779 * Return: 0 if no error: or the result of the mailbox command.780 *781 * See CXL-3.0 8.2.9.3.2 Transfer FW782 */783static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)784{785 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;786 struct cxl_mbox_transfer_fw *transfer;787 struct cxl_mbox_cmd mbox_cmd;788 int rc;789 790 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);791 if (!transfer)792 return -ENOMEM;793 794 /* Set a 1s poll interval and a total wait time of 30s */795 mbox_cmd = (struct cxl_mbox_cmd) {796 .opcode = CXL_MBOX_OP_TRANSFER_FW,797 .size_in = sizeof(*transfer),798 .payload_in = transfer,799 .poll_interval_ms = 1000,800 .poll_count = 30,801 };802 803 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;804 805 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);806 kfree(transfer);807 return rc;808}809 810static void cxl_fw_cleanup(struct fw_upload *fwl)811{812 struct cxl_memdev_state *mds = fwl->dd_handle;813 814 mds->fw.next_slot = 0;815}816 817static int cxl_fw_do_cancel(struct fw_upload *fwl)818{819 struct cxl_memdev_state *mds = fwl->dd_handle;820 struct cxl_dev_state *cxlds = &mds->cxlds;821 struct cxl_memdev *cxlmd = cxlds->cxlmd;822 int rc;823 824 rc = cxl_mem_abort_fw_xfer(mds);825 if (rc < 0)826 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);827 828 return FW_UPLOAD_ERR_CANCELED;829}830 831static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,832 u32 size)833{834 struct cxl_memdev_state *mds = fwl->dd_handle;835 struct cxl_mbox_transfer_fw *transfer;836 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;837 838 if (!size)839 return FW_UPLOAD_ERR_INVALID_SIZE;840 841 mds->fw.oneshot = struct_size(transfer, data, size) <842 cxl_mbox->payload_size;843 844 if (cxl_mem_get_fw_info(mds))845 return FW_UPLOAD_ERR_HW_ERROR;846 847 /*848 * So far no state has been changed, hence no other cleanup is849 * necessary. Simply return the cancelled status.850 */851 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))852 return FW_UPLOAD_ERR_CANCELED;853 854 return FW_UPLOAD_ERR_NONE;855}856 857static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,858 u32 offset, u32 size, u32 *written)859{860 struct cxl_memdev_state *mds = fwl->dd_handle;861 struct cxl_dev_state *cxlds = &mds->cxlds;862 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;863 struct cxl_memdev *cxlmd = cxlds->cxlmd;864 struct cxl_mbox_transfer_fw *transfer;865 struct cxl_mbox_cmd mbox_cmd;866 u32 cur_size, remaining;867 size_t size_in;868 int rc;869 870 *written = 0;871 872 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */873 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {874 dev_err(&cxlmd->dev,875 "misaligned offset for FW transfer slice (%u)\n",876 offset);877 return FW_UPLOAD_ERR_RW_ERROR;878 }879 880 /*881 * Pick transfer size based on mds->payload_size @size must bw 128-byte882 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and883 * sizeof(*transfer) is 128. These constraints imply that @cur_size884 * will always be 128b aligned.885 */886 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));887 888 remaining = size - cur_size;889 size_in = struct_size(transfer, data, cur_size);890 891 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))892 return cxl_fw_do_cancel(fwl);893 894 /*895 * Slot numbers are 1-indexed896 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')897 * Check for rollover using modulo, and 1-index it by adding 1898 */899 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;900 901 /* Do the transfer via mailbox cmd */902 transfer = kzalloc(size_in, GFP_KERNEL);903 if (!transfer)904 return FW_UPLOAD_ERR_RW_ERROR;905 906 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);907 memcpy(transfer->data, data + offset, cur_size);908 if (mds->fw.oneshot) {909 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;910 transfer->slot = mds->fw.next_slot;911 } else {912 if (offset == 0) {913 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;914 } else if (remaining == 0) {915 transfer->action = CXL_FW_TRANSFER_ACTION_END;916 transfer->slot = mds->fw.next_slot;917 } else {918 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;919 }920 }921 922 mbox_cmd = (struct cxl_mbox_cmd) {923 .opcode = CXL_MBOX_OP_TRANSFER_FW,924 .size_in = size_in,925 .payload_in = transfer,926 .poll_interval_ms = 1000,927 .poll_count = 30,928 };929 930 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);931 if (rc < 0) {932 rc = FW_UPLOAD_ERR_RW_ERROR;933 goto out_free;934 }935 936 *written = cur_size;937 938 /* Activate FW if oneshot or if the last slice was written */939 if (mds->fw.oneshot || remaining == 0) {940 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",941 mds->fw.next_slot);942 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);943 if (rc < 0) {944 dev_err(&cxlmd->dev, "Error activating firmware: %d\n",945 rc);946 rc = FW_UPLOAD_ERR_HW_ERROR;947 goto out_free;948 }949 }950 951 rc = FW_UPLOAD_ERR_NONE;952 953out_free:954 kfree(transfer);955 return rc;956}957 958static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)959{960 struct cxl_memdev_state *mds = fwl->dd_handle;961 962 /*963 * cxl_internal_send_cmd() handles background operations synchronously.964 * No need to wait for completions here - any errors would've been965 * reported and handled during the ->write() call(s).966 * Just check if a cancel request was received, and return success.967 */968 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))969 return cxl_fw_do_cancel(fwl);970 971 return FW_UPLOAD_ERR_NONE;972}973 974static void cxl_fw_cancel(struct fw_upload *fwl)975{976 struct cxl_memdev_state *mds = fwl->dd_handle;977 978 set_bit(CXL_FW_CANCEL, mds->fw.state);979}980 981static const struct fw_upload_ops cxl_memdev_fw_ops = {982 .prepare = cxl_fw_prepare,983 .write = cxl_fw_write,984 .poll_complete = cxl_fw_poll_complete,985 .cancel = cxl_fw_cancel,986 .cleanup = cxl_fw_cleanup,987};988 989static void cxl_remove_fw_upload(void *fwl)990{991 firmware_upload_unregister(fwl);992}993 994int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)995{996 struct cxl_dev_state *cxlds = &mds->cxlds;997 struct device *dev = &cxlds->cxlmd->dev;998 struct fw_upload *fwl;999 1000 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds))1001 return 0;1002 1003 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),1004 &cxl_memdev_fw_ops, mds);1005 if (IS_ERR(fwl))1006 return PTR_ERR(fwl);1007 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);1008}1009EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, CXL);1010 1011static const struct file_operations cxl_memdev_fops = {1012 .owner = THIS_MODULE,1013 .unlocked_ioctl = cxl_memdev_ioctl,1014 .open = cxl_memdev_open,1015 .release = cxl_memdev_release_file,1016 .compat_ioctl = compat_ptr_ioctl,1017 .llseek = noop_llseek,1018};1019 1020struct cxl_memdev *devm_cxl_add_memdev(struct device *host,1021 struct cxl_dev_state *cxlds)1022{1023 struct cxl_memdev *cxlmd;1024 struct device *dev;1025 struct cdev *cdev;1026 int rc;1027 1028 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);1029 if (IS_ERR(cxlmd))1030 return cxlmd;1031 1032 dev = &cxlmd->dev;1033 rc = dev_set_name(dev, "mem%d", cxlmd->id);1034 if (rc)1035 goto err;1036 1037 /*1038 * Activate ioctl operations, no cxl_memdev_rwsem manipulation1039 * needed as this is ordered with cdev_add() publishing the device.1040 */1041 cxlmd->cxlds = cxlds;1042 cxlds->cxlmd = cxlmd;1043 1044 cdev = &cxlmd->cdev;1045 rc = cdev_device_add(cdev, dev);1046 if (rc)1047 goto err;1048 1049 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);1050 if (rc)1051 return ERR_PTR(rc);1052 return cxlmd;1053 1054err:1055 /*1056 * The cdev was briefly live, shutdown any ioctl operations that1057 * saw that state.1058 */1059 cxl_memdev_shutdown(dev);1060 put_device(dev);1061 return ERR_PTR(rc);1062}1063EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);1064 1065static void sanitize_teardown_notifier(void *data)1066{1067 struct cxl_memdev_state *mds = data;1068 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;1069 struct kernfs_node *state;1070 1071 /*1072 * Prevent new irq triggered invocations of the workqueue and1073 * flush inflight invocations.1074 */1075 mutex_lock(&cxl_mbox->mbox_mutex);1076 state = mds->security.sanitize_node;1077 mds->security.sanitize_node = NULL;1078 mutex_unlock(&cxl_mbox->mbox_mutex);1079 1080 cancel_delayed_work_sync(&mds->security.poll_dwork);1081 sysfs_put(state);1082}1083 1084int devm_cxl_sanitize_setup_notifier(struct device *host,1085 struct cxl_memdev *cxlmd)1086{1087 struct cxl_dev_state *cxlds = cxlmd->cxlds;1088 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);1089 struct kernfs_node *sec;1090 1091 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))1092 return 0;1093 1094 /*1095 * Note, the expectation is that @cxlmd would have failed to be1096 * created if these sysfs_get_dirent calls fail.1097 */1098 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security");1099 if (!sec)1100 return -ENOENT;1101 mds->security.sanitize_node = sysfs_get_dirent(sec, "state");1102 sysfs_put(sec);1103 if (!mds->security.sanitize_node)1104 return -ENOENT;1105 1106 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);1107}1108EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, CXL);1109 1110__init int cxl_memdev_init(void)1111{1112 dev_t devt;1113 int rc;1114 1115 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");1116 if (rc)1117 return rc;1118 1119 cxl_mem_major = MAJOR(devt);1120 1121 return 0;1122}1123 1124void cxl_memdev_exit(void)1125{1126 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);1127}1128