466 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2021 Intel Corporation. All rights reserved. */3#include <linux/libnvdimm.h>4#include <linux/unaligned.h>5#include <linux/device.h>6#include <linux/module.h>7#include <linux/ndctl.h>8#include <linux/async.h>9#include <linux/slab.h>10#include <linux/nd.h>11#include "cxlmem.h"12#include "cxl.h"13 14static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);15 16static void clear_exclusive(void *mds)17{18 clear_exclusive_cxl_commands(mds, exclusive_cmds);19}20 21static void unregister_nvdimm(void *nvdimm)22{23 nvdimm_delete(nvdimm);24}25 26static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)27{28 struct nvdimm *nvdimm = to_nvdimm(dev);29 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);30 31 return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));32}33static DEVICE_ATTR_RO(provider);34 35static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)36{37 struct nvdimm *nvdimm = to_nvdimm(dev);38 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);39 struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;40 41 return sysfs_emit(buf, "%lld\n", cxlds->serial);42}43static DEVICE_ATTR_RO(id);44 45static struct attribute *cxl_dimm_attributes[] = {46 &dev_attr_id.attr,47 &dev_attr_provider.attr,48 NULL49};50 51static const struct attribute_group cxl_dimm_attribute_group = {52 .name = "cxl",53 .attrs = cxl_dimm_attributes,54};55 56static const struct attribute_group *cxl_dimm_attribute_groups[] = {57 &cxl_dimm_attribute_group,58 NULL59};60 61static int cxl_nvdimm_probe(struct device *dev)62{63 struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);64 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;65 struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;66 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);67 unsigned long flags = 0, cmd_mask = 0;68 struct nvdimm *nvdimm;69 int rc;70 71 set_exclusive_cxl_commands(mds, exclusive_cmds);72 rc = devm_add_action_or_reset(dev, clear_exclusive, mds);73 if (rc)74 return rc;75 76 set_bit(NDD_LABELING, &flags);77 set_bit(NDD_REGISTER_SYNC, &flags);78 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);79 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);80 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);81 nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,82 cxl_dimm_attribute_groups, flags,83 cmd_mask, 0, NULL, cxl_nvd->dev_id,84 cxl_security_ops, NULL);85 if (!nvdimm)86 return -ENOMEM;87 88 dev_set_drvdata(dev, nvdimm);89 return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);90}91 92static struct cxl_driver cxl_nvdimm_driver = {93 .name = "cxl_nvdimm",94 .probe = cxl_nvdimm_probe,95 .id = CXL_DEVICE_NVDIMM,96 .drv = {97 .suppress_bind_attrs = true,98 },99};100 101static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds,102 struct nd_cmd_get_config_size *cmd,103 unsigned int buf_len)104{105 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;106 107 if (sizeof(*cmd) > buf_len)108 return -EINVAL;109 110 *cmd = (struct nd_cmd_get_config_size){111 .config_size = mds->lsa_size,112 .max_xfer =113 cxl_mbox->payload_size - sizeof(struct cxl_mbox_set_lsa),114 };115 116 return 0;117}118 119static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds,120 struct nd_cmd_get_config_data_hdr *cmd,121 unsigned int buf_len)122{123 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;124 struct cxl_mbox_get_lsa get_lsa;125 struct cxl_mbox_cmd mbox_cmd;126 int rc;127 128 if (sizeof(*cmd) > buf_len)129 return -EINVAL;130 if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)131 return -EINVAL;132 133 get_lsa = (struct cxl_mbox_get_lsa) {134 .offset = cpu_to_le32(cmd->in_offset),135 .length = cpu_to_le32(cmd->in_length),136 };137 mbox_cmd = (struct cxl_mbox_cmd) {138 .opcode = CXL_MBOX_OP_GET_LSA,139 .payload_in = &get_lsa,140 .size_in = sizeof(get_lsa),141 .size_out = cmd->in_length,142 .payload_out = cmd->out_buf,143 };144 145 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);146 cmd->status = 0;147 148 return rc;149}150 151static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds,152 struct nd_cmd_set_config_hdr *cmd,153 unsigned int buf_len)154{155 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;156 struct cxl_mbox_set_lsa *set_lsa;157 struct cxl_mbox_cmd mbox_cmd;158 int rc;159 160 if (sizeof(*cmd) > buf_len)161 return -EINVAL;162 163 /* 4-byte status follows the input data in the payload */164 if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)165 return -EINVAL;166 167 set_lsa =168 kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);169 if (!set_lsa)170 return -ENOMEM;171 172 *set_lsa = (struct cxl_mbox_set_lsa) {173 .offset = cpu_to_le32(cmd->in_offset),174 };175 memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);176 mbox_cmd = (struct cxl_mbox_cmd) {177 .opcode = CXL_MBOX_OP_SET_LSA,178 .payload_in = set_lsa,179 .size_in = struct_size(set_lsa, data, cmd->in_length),180 };181 182 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);183 184 /*185 * Set "firmware" status (4-packed bytes at the end of the input186 * payload.187 */188 put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);189 kvfree(set_lsa);190 191 return rc;192}193 194static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,195 void *buf, unsigned int buf_len)196{197 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);198 unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);199 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;200 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);201 202 if (!test_bit(cmd, &cmd_mask))203 return -ENOTTY;204 205 switch (cmd) {206 case ND_CMD_GET_CONFIG_SIZE:207 return cxl_pmem_get_config_size(mds, buf, buf_len);208 case ND_CMD_GET_CONFIG_DATA:209 return cxl_pmem_get_config_data(mds, buf, buf_len);210 case ND_CMD_SET_CONFIG_DATA:211 return cxl_pmem_set_config_data(mds, buf, buf_len);212 default:213 return -ENOTTY;214 }215}216 217static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,218 struct nvdimm *nvdimm, unsigned int cmd, void *buf,219 unsigned int buf_len, int *cmd_rc)220{221 /*222 * No firmware response to translate, let the transport error223 * code take precedence.224 */225 *cmd_rc = 0;226 227 if (!nvdimm)228 return -ENOTTY;229 return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);230}231 232static int detach_nvdimm(struct device *dev, void *data)233{234 struct cxl_nvdimm *cxl_nvd;235 bool release = false;236 237 if (!is_cxl_nvdimm(dev))238 return 0;239 240 scoped_guard(device, dev) {241 if (dev->driver) {242 cxl_nvd = to_cxl_nvdimm(dev);243 if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)244 release = true;245 }246 }247 if (release)248 device_release_driver(dev);249 return 0;250}251 252static void unregister_nvdimm_bus(void *_cxl_nvb)253{254 struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;255 struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;256 257 bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);258 259 cxl_nvb->nvdimm_bus = NULL;260 nvdimm_bus_unregister(nvdimm_bus);261}262 263static int cxl_nvdimm_bridge_probe(struct device *dev)264{265 struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);266 267 cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {268 .provider_name = "CXL",269 .module = THIS_MODULE,270 .ndctl = cxl_pmem_ctl,271 };272 273 cxl_nvb->nvdimm_bus =274 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);275 276 if (!cxl_nvb->nvdimm_bus)277 return -ENOMEM;278 279 return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);280}281 282static struct cxl_driver cxl_nvdimm_bridge_driver = {283 .name = "cxl_nvdimm_bridge",284 .probe = cxl_nvdimm_bridge_probe,285 .id = CXL_DEVICE_NVDIMM_BRIDGE,286 .drv = {287 .suppress_bind_attrs = true,288 },289};290 291static void unregister_nvdimm_region(void *nd_region)292{293 nvdimm_region_delete(nd_region);294}295 296static void cxlr_pmem_remove_resource(void *res)297{298 remove_resource(res);299}300 301struct cxl_pmem_region_info {302 u64 offset;303 u64 serial;304};305 306static int cxl_pmem_region_probe(struct device *dev)307{308 struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];309 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);310 struct cxl_region *cxlr = cxlr_pmem->cxlr;311 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;312 struct cxl_pmem_region_info *info = NULL;313 struct nd_interleave_set *nd_set;314 struct nd_region_desc ndr_desc;315 struct cxl_nvdimm *cxl_nvd;316 struct nvdimm *nvdimm;317 struct resource *res;318 int rc, i = 0;319 320 memset(&mappings, 0, sizeof(mappings));321 memset(&ndr_desc, 0, sizeof(ndr_desc));322 323 res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);324 if (!res)325 return -ENOMEM;326 327 res->name = "Persistent Memory";328 res->start = cxlr_pmem->hpa_range.start;329 res->end = cxlr_pmem->hpa_range.end;330 res->flags = IORESOURCE_MEM;331 res->desc = IORES_DESC_PERSISTENT_MEMORY;332 333 rc = insert_resource(&iomem_resource, res);334 if (rc)335 return rc;336 337 rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);338 if (rc)339 return rc;340 341 ndr_desc.res = res;342 ndr_desc.provider_data = cxlr_pmem;343 344 ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);345 ndr_desc.target_node = phys_to_target_node(res->start);346 if (ndr_desc.target_node == NUMA_NO_NODE) {347 ndr_desc.target_node = ndr_desc.numa_node;348 dev_dbg(&cxlr->dev, "changing target node from %d to %d",349 NUMA_NO_NODE, ndr_desc.target_node);350 }351 352 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);353 if (!nd_set)354 return -ENOMEM;355 356 ndr_desc.memregion = cxlr->id;357 set_bit(ND_REGION_CXL, &ndr_desc.flags);358 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);359 360 info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);361 if (!info)362 return -ENOMEM;363 364 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {365 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];366 struct cxl_memdev *cxlmd = m->cxlmd;367 struct cxl_dev_state *cxlds = cxlmd->cxlds;368 369 cxl_nvd = cxlmd->cxl_nvd;370 nvdimm = dev_get_drvdata(&cxl_nvd->dev);371 if (!nvdimm) {372 dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,373 dev_name(&cxlmd->dev));374 rc = -ENODEV;375 goto out_nvd;376 }377 378 m->cxl_nvd = cxl_nvd;379 mappings[i] = (struct nd_mapping_desc) {380 .nvdimm = nvdimm,381 .start = m->start,382 .size = m->size,383 .position = i,384 };385 info[i].offset = m->start;386 info[i].serial = cxlds->serial;387 }388 ndr_desc.num_mappings = cxlr_pmem->nr_mappings;389 ndr_desc.mapping = mappings;390 391 /*392 * TODO enable CXL labels which skip the need for 'interleave-set cookie'393 */394 nd_set->cookie1 =395 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);396 nd_set->cookie2 = nd_set->cookie1;397 ndr_desc.nd_set = nd_set;398 399 cxlr_pmem->nd_region =400 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);401 if (!cxlr_pmem->nd_region) {402 rc = -ENOMEM;403 goto out_nvd;404 }405 406 rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,407 cxlr_pmem->nd_region);408out_nvd:409 kfree(info);410 411 return rc;412}413 414static struct cxl_driver cxl_pmem_region_driver = {415 .name = "cxl_pmem_region",416 .probe = cxl_pmem_region_probe,417 .id = CXL_DEVICE_PMEM_REGION,418 .drv = {419 .suppress_bind_attrs = true,420 },421};422 423static __init int cxl_pmem_init(void)424{425 int rc;426 427 set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);428 set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);429 430 rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);431 if (rc)432 return rc;433 434 rc = cxl_driver_register(&cxl_nvdimm_driver);435 if (rc)436 goto err_nvdimm;437 438 rc = cxl_driver_register(&cxl_pmem_region_driver);439 if (rc)440 goto err_region;441 442 return 0;443 444err_region:445 cxl_driver_unregister(&cxl_nvdimm_driver);446err_nvdimm:447 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);448 return rc;449}450 451static __exit void cxl_pmem_exit(void)452{453 cxl_driver_unregister(&cxl_pmem_region_driver);454 cxl_driver_unregister(&cxl_nvdimm_driver);455 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);456}457 458MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support");459MODULE_LICENSE("GPL v2");460module_init(cxl_pmem_init);461module_exit(cxl_pmem_exit);462MODULE_IMPORT_NS(CXL);463MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);464MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);465MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);466