1468 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2020 Intel Corporation. All rights reserved. */3#include <linux/security.h>4#include <linux/debugfs.h>5#include <linux/ktime.h>6#include <linux/mutex.h>7#include <linux/unaligned.h>8#include <cxlpci.h>9#include <cxlmem.h>10#include <cxl.h>11 12#include "core.h"13#include "trace.h"14 15static bool cxl_raw_allow_all;16 17/**18 * DOC: cxl mbox19 *20 * Core implementation of the CXL 2.0 Type-3 Memory Device Mailbox. The21 * implementation is used by the cxl_pci driver to initialize the device22 * and implement the cxl_mem.h IOCTL UAPI. It also implements the23 * backend of the cxl_pmem_ctl() transport for LIBNVDIMM.24 */25 26#define cxl_for_each_cmd(cmd) \27 for ((cmd) = &cxl_mem_commands[0]; \28 ((cmd) - cxl_mem_commands) < ARRAY_SIZE(cxl_mem_commands); (cmd)++)29 30#define CXL_CMD(_id, sin, sout, _flags) \31 [CXL_MEM_COMMAND_ID_##_id] = { \32 .info = { \33 .id = CXL_MEM_COMMAND_ID_##_id, \34 .size_in = sin, \35 .size_out = sout, \36 }, \37 .opcode = CXL_MBOX_OP_##_id, \38 .flags = _flags, \39 }40 41#define CXL_VARIABLE_PAYLOAD ~0U42/*43 * This table defines the supported mailbox commands for the driver. This table44 * is made up of a UAPI structure. Non-negative values as parameters in the45 * table will be validated against the user's input. For example, if size_in is46 * 0, and the user passed in 1, it is an error.47 */48static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {49 CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),50#ifdef CONFIG_CXL_MEM_RAW_COMMANDS51 CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),52#endif53 CXL_CMD(GET_SUPPORTED_LOGS, 0, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),54 CXL_CMD(GET_FW_INFO, 0, 0x50, 0),55 CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),56 CXL_CMD(GET_LSA, 0x8, CXL_VARIABLE_PAYLOAD, 0),57 CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),58 CXL_CMD(GET_LOG, 0x18, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),59 CXL_CMD(GET_LOG_CAPS, 0x10, 0x4, 0),60 CXL_CMD(CLEAR_LOG, 0x10, 0, 0),61 CXL_CMD(GET_SUP_LOG_SUBLIST, 0x2, CXL_VARIABLE_PAYLOAD, 0),62 CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),63 CXL_CMD(SET_LSA, CXL_VARIABLE_PAYLOAD, 0, 0),64 CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),65 CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),66 CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),67 CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),68 CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),69 CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0),70};71 72/*73 * Commands that RAW doesn't permit. The rationale for each:74 *75 * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment /76 * coordination of transaction timeout values at the root bridge level.77 *78 * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live79 * and needs to be coordinated with HDM updates.80 *81 * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the82 * driver and any writes from userspace invalidates those contents.83 *84 * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes85 * to the device after it is marked clean, userspace can not make that86 * assertion.87 *88 * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that89 * is kept up to date with patrol notifications and error management.90 *91 * CXL_MBOX_OP_[GET_,INJECT_,CLEAR_]POISON: These commands require kernel92 * driver orchestration for safety.93 */94static u16 cxl_disabled_raw_commands[] = {95 CXL_MBOX_OP_ACTIVATE_FW,96 CXL_MBOX_OP_SET_PARTITION_INFO,97 CXL_MBOX_OP_SET_LSA,98 CXL_MBOX_OP_SET_SHUTDOWN_STATE,99 CXL_MBOX_OP_SCAN_MEDIA,100 CXL_MBOX_OP_GET_SCAN_MEDIA,101 CXL_MBOX_OP_GET_POISON,102 CXL_MBOX_OP_INJECT_POISON,103 CXL_MBOX_OP_CLEAR_POISON,104};105 106/*107 * Command sets that RAW doesn't permit. All opcodes in this set are108 * disabled because they pass plain text security payloads over the109 * user/kernel boundary. This functionality is intended to be wrapped110 * behind the keys ABI which allows for encrypted payloads in the UAPI111 */112static u8 security_command_sets[] = {113 0x44, /* Sanitize */114 0x45, /* Persistent Memory Data-at-rest Security */115 0x46, /* Security Passthrough */116};117 118static bool cxl_is_security_command(u16 opcode)119{120 int i;121 122 for (i = 0; i < ARRAY_SIZE(security_command_sets); i++)123 if (security_command_sets[i] == (opcode >> 8))124 return true;125 return false;126}127 128static void cxl_set_security_cmd_enabled(struct cxl_security_state *security,129 u16 opcode)130{131 switch (opcode) {132 case CXL_MBOX_OP_SANITIZE:133 set_bit(CXL_SEC_ENABLED_SANITIZE, security->enabled_cmds);134 break;135 case CXL_MBOX_OP_SECURE_ERASE:136 set_bit(CXL_SEC_ENABLED_SECURE_ERASE,137 security->enabled_cmds);138 break;139 case CXL_MBOX_OP_GET_SECURITY_STATE:140 set_bit(CXL_SEC_ENABLED_GET_SECURITY_STATE,141 security->enabled_cmds);142 break;143 case CXL_MBOX_OP_SET_PASSPHRASE:144 set_bit(CXL_SEC_ENABLED_SET_PASSPHRASE,145 security->enabled_cmds);146 break;147 case CXL_MBOX_OP_DISABLE_PASSPHRASE:148 set_bit(CXL_SEC_ENABLED_DISABLE_PASSPHRASE,149 security->enabled_cmds);150 break;151 case CXL_MBOX_OP_UNLOCK:152 set_bit(CXL_SEC_ENABLED_UNLOCK, security->enabled_cmds);153 break;154 case CXL_MBOX_OP_FREEZE_SECURITY:155 set_bit(CXL_SEC_ENABLED_FREEZE_SECURITY,156 security->enabled_cmds);157 break;158 case CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE:159 set_bit(CXL_SEC_ENABLED_PASSPHRASE_SECURE_ERASE,160 security->enabled_cmds);161 break;162 default:163 break;164 }165}166 167static bool cxl_is_poison_command(u16 opcode)168{169#define CXL_MBOX_OP_POISON_CMDS 0x43170 171 if ((opcode >> 8) == CXL_MBOX_OP_POISON_CMDS)172 return true;173 174 return false;175}176 177static void cxl_set_poison_cmd_enabled(struct cxl_poison_state *poison,178 u16 opcode)179{180 switch (opcode) {181 case CXL_MBOX_OP_GET_POISON:182 set_bit(CXL_POISON_ENABLED_LIST, poison->enabled_cmds);183 break;184 case CXL_MBOX_OP_INJECT_POISON:185 set_bit(CXL_POISON_ENABLED_INJECT, poison->enabled_cmds);186 break;187 case CXL_MBOX_OP_CLEAR_POISON:188 set_bit(CXL_POISON_ENABLED_CLEAR, poison->enabled_cmds);189 break;190 case CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS:191 set_bit(CXL_POISON_ENABLED_SCAN_CAPS, poison->enabled_cmds);192 break;193 case CXL_MBOX_OP_SCAN_MEDIA:194 set_bit(CXL_POISON_ENABLED_SCAN_MEDIA, poison->enabled_cmds);195 break;196 case CXL_MBOX_OP_GET_SCAN_MEDIA:197 set_bit(CXL_POISON_ENABLED_SCAN_RESULTS, poison->enabled_cmds);198 break;199 default:200 break;201 }202}203 204static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)205{206 struct cxl_mem_command *c;207 208 cxl_for_each_cmd(c)209 if (c->opcode == opcode)210 return c;211 212 return NULL;213}214 215static const char *cxl_mem_opcode_to_name(u16 opcode)216{217 struct cxl_mem_command *c;218 219 c = cxl_mem_find_command(opcode);220 if (!c)221 return NULL;222 223 return cxl_command_names[c->info.id].name;224}225 226/**227 * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command228 * @cxl_mbox: CXL mailbox context229 * @mbox_cmd: initialized command to execute230 *231 * Context: Any context.232 * Return:233 * * %>=0 - Number of bytes returned in @out.234 * * %-E2BIG - Payload is too large for hardware.235 * * %-EBUSY - Couldn't acquire exclusive mailbox access.236 * * %-EFAULT - Hardware error occurred.237 * * %-ENXIO - Command completed, but device reported an error.238 * * %-EIO - Unexpected output size.239 *240 * Mailbox commands may execute successfully yet the device itself reported an241 * error. While this distinction can be useful for commands from userspace, the242 * kernel will only be able to use results when both are successful.243 */244int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,245 struct cxl_mbox_cmd *mbox_cmd)246{247 size_t out_size, min_out;248 int rc;249 250 if (mbox_cmd->size_in > cxl_mbox->payload_size ||251 mbox_cmd->size_out > cxl_mbox->payload_size)252 return -E2BIG;253 254 out_size = mbox_cmd->size_out;255 min_out = mbox_cmd->min_out;256 rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);257 /*258 * EIO is reserved for a payload size mismatch and mbox_send()259 * may not return this error.260 */261 if (WARN_ONCE(rc == -EIO, "Bad return code: -EIO"))262 return -ENXIO;263 if (rc)264 return rc;265 266 if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS &&267 mbox_cmd->return_code != CXL_MBOX_CMD_RC_BACKGROUND)268 return cxl_mbox_cmd_rc2errno(mbox_cmd);269 270 if (!out_size)271 return 0;272 273 /*274 * Variable sized output needs to at least satisfy the caller's275 * minimum if not the fully requested size.276 */277 if (min_out == 0)278 min_out = out_size;279 280 if (mbox_cmd->size_out < min_out)281 return -EIO;282 return 0;283}284EXPORT_SYMBOL_NS_GPL(cxl_internal_send_cmd, CXL);285 286static bool cxl_mem_raw_command_allowed(u16 opcode)287{288 int i;289 290 if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS))291 return false;292 293 if (security_locked_down(LOCKDOWN_PCI_ACCESS))294 return false;295 296 if (cxl_raw_allow_all)297 return true;298 299 if (cxl_is_security_command(opcode))300 return false;301 302 for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++)303 if (cxl_disabled_raw_commands[i] == opcode)304 return false;305 306 return true;307}308 309/**310 * cxl_payload_from_user_allowed() - Check contents of in_payload.311 * @opcode: The mailbox command opcode.312 * @payload_in: Pointer to the input payload passed in from user space.313 *314 * Return:315 * * true - payload_in passes check for @opcode.316 * * false - payload_in contains invalid or unsupported values.317 *318 * The driver may inspect payload contents before sending a mailbox319 * command from user space to the device. The intent is to reject320 * commands with input payloads that are known to be unsafe. This321 * check is not intended to replace the users careful selection of322 * mailbox command parameters and makes no guarantee that the user323 * command will succeed, nor that it is appropriate.324 *325 * The specific checks are determined by the opcode.326 */327static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in)328{329 switch (opcode) {330 case CXL_MBOX_OP_SET_PARTITION_INFO: {331 struct cxl_mbox_set_partition_info *pi = payload_in;332 333 if (pi->flags & CXL_SET_PARTITION_IMMEDIATE_FLAG)334 return false;335 break;336 }337 case CXL_MBOX_OP_CLEAR_LOG: {338 const uuid_t *uuid = (uuid_t *)payload_in;339 340 /*341 * Restrict the ‘Clear log’ action to only apply to342 * Vendor debug logs.343 */344 return uuid_equal(uuid, &DEFINE_CXL_VENDOR_DEBUG_UUID);345 }346 default:347 break;348 }349 return true;350}351 352static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,353 struct cxl_memdev_state *mds, u16 opcode,354 size_t in_size, size_t out_size, u64 in_payload)355{356 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;357 *mbox = (struct cxl_mbox_cmd) {358 .opcode = opcode,359 .size_in = in_size,360 };361 362 if (in_size) {363 mbox->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),364 in_size);365 if (IS_ERR(mbox->payload_in))366 return PTR_ERR(mbox->payload_in);367 368 if (!cxl_payload_from_user_allowed(opcode, mbox->payload_in)) {369 dev_dbg(mds->cxlds.dev, "%s: input payload not allowed\n",370 cxl_mem_opcode_to_name(opcode));371 kvfree(mbox->payload_in);372 return -EBUSY;373 }374 }375 376 /* Prepare to handle a full payload for variable sized output */377 if (out_size == CXL_VARIABLE_PAYLOAD)378 mbox->size_out = cxl_mbox->payload_size;379 else380 mbox->size_out = out_size;381 382 if (mbox->size_out) {383 mbox->payload_out = kvzalloc(mbox->size_out, GFP_KERNEL);384 if (!mbox->payload_out) {385 kvfree(mbox->payload_in);386 return -ENOMEM;387 }388 }389 return 0;390}391 392static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)393{394 kvfree(mbox->payload_in);395 kvfree(mbox->payload_out);396}397 398static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,399 const struct cxl_send_command *send_cmd,400 struct cxl_memdev_state *mds)401{402 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;403 404 if (send_cmd->raw.rsvd)405 return -EINVAL;406 407 /*408 * Unlike supported commands, the output size of RAW commands409 * gets passed along without further checking, so it must be410 * validated here.411 */412 if (send_cmd->out.size > cxl_mbox->payload_size)413 return -EINVAL;414 415 if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))416 return -EPERM;417 418 dev_WARN_ONCE(mds->cxlds.dev, true, "raw command path used\n");419 420 *mem_cmd = (struct cxl_mem_command) {421 .info = {422 .id = CXL_MEM_COMMAND_ID_RAW,423 .size_in = send_cmd->in.size,424 .size_out = send_cmd->out.size,425 },426 .opcode = send_cmd->raw.opcode427 };428 429 return 0;430}431 432static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,433 const struct cxl_send_command *send_cmd,434 struct cxl_memdev_state *mds)435{436 struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];437 const struct cxl_command_info *info = &c->info;438 439 if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)440 return -EINVAL;441 442 if (send_cmd->rsvd)443 return -EINVAL;444 445 if (send_cmd->in.rsvd || send_cmd->out.rsvd)446 return -EINVAL;447 448 /* Check that the command is enabled for hardware */449 if (!test_bit(info->id, mds->enabled_cmds))450 return -ENOTTY;451 452 /* Check that the command is not claimed for exclusive kernel use */453 if (test_bit(info->id, mds->exclusive_cmds))454 return -EBUSY;455 456 /* Check the input buffer is the expected size */457 if ((info->size_in != CXL_VARIABLE_PAYLOAD) &&458 (info->size_in != send_cmd->in.size))459 return -ENOMEM;460 461 /* Check the output buffer is at least large enough */462 if ((info->size_out != CXL_VARIABLE_PAYLOAD) &&463 (send_cmd->out.size < info->size_out))464 return -ENOMEM;465 466 *mem_cmd = (struct cxl_mem_command) {467 .info = {468 .id = info->id,469 .flags = info->flags,470 .size_in = send_cmd->in.size,471 .size_out = send_cmd->out.size,472 },473 .opcode = c->opcode474 };475 476 return 0;477}478 479/**480 * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.481 * @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.482 * @mds: The driver data for the operation483 * @send_cmd: &struct cxl_send_command copied in from userspace.484 *485 * Return:486 * * %0 - @out_cmd is ready to send.487 * * %-ENOTTY - Invalid command specified.488 * * %-EINVAL - Reserved fields or invalid values were used.489 * * %-ENOMEM - Input or output buffer wasn't sized properly.490 * * %-EPERM - Attempted to use a protected command.491 * * %-EBUSY - Kernel has claimed exclusive access to this opcode492 *493 * The result of this command is a fully validated command in @mbox_cmd that is494 * safe to send to the hardware.495 */496static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,497 struct cxl_memdev_state *mds,498 const struct cxl_send_command *send_cmd)499{500 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;501 struct cxl_mem_command mem_cmd;502 int rc;503 504 if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)505 return -ENOTTY;506 507 /*508 * The user can never specify an input payload larger than what hardware509 * supports, but output can be arbitrarily large (simply write out as510 * much data as the hardware provides).511 */512 if (send_cmd->in.size > cxl_mbox->payload_size)513 return -EINVAL;514 515 /* Sanitize and construct a cxl_mem_command */516 if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)517 rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, mds);518 else519 rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, mds);520 521 if (rc)522 return rc;523 524 /* Sanitize and construct a cxl_mbox_cmd */525 return cxl_mbox_cmd_ctor(mbox_cmd, mds, mem_cmd.opcode,526 mem_cmd.info.size_in, mem_cmd.info.size_out,527 send_cmd->in.payload);528}529 530int cxl_query_cmd(struct cxl_memdev *cxlmd,531 struct cxl_mem_query_commands __user *q)532{533 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);534 struct device *dev = &cxlmd->dev;535 struct cxl_mem_command *cmd;536 u32 n_commands;537 int j = 0;538 539 dev_dbg(dev, "Query IOCTL\n");540 541 if (get_user(n_commands, &q->n_commands))542 return -EFAULT;543 544 /* returns the total number if 0 elements are requested. */545 if (n_commands == 0)546 return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);547 548 /*549 * otherwise, return min(n_commands, total commands) cxl_command_info550 * structures.551 */552 cxl_for_each_cmd(cmd) {553 struct cxl_command_info info = cmd->info;554 555 if (test_bit(info.id, mds->enabled_cmds))556 info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;557 if (test_bit(info.id, mds->exclusive_cmds))558 info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;559 560 if (copy_to_user(&q->commands[j++], &info, sizeof(info)))561 return -EFAULT;562 563 if (j == n_commands)564 break;565 }566 567 return 0;568}569 570/**571 * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.572 * @mds: The driver data for the operation573 * @mbox_cmd: The validated mailbox command.574 * @out_payload: Pointer to userspace's output payload.575 * @size_out: (Input) Max payload size to copy out.576 * (Output) Payload size hardware generated.577 * @retval: Hardware generated return code from the operation.578 *579 * Return:580 * * %0 - Mailbox transaction succeeded. This implies the mailbox581 * protocol completed successfully not that the operation itself582 * was successful.583 * * %-ENOMEM - Couldn't allocate a bounce buffer.584 * * %-EFAULT - Something happened with copy_to/from_user.585 * * %-EINTR - Mailbox acquisition interrupted.586 * * %-EXXX - Transaction level failures.587 *588 * Dispatches a mailbox command on behalf of a userspace request.589 * The output payload is copied to userspace.590 *591 * See cxl_send_cmd().592 */593static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,594 struct cxl_mbox_cmd *mbox_cmd,595 u64 out_payload, s32 *size_out,596 u32 *retval)597{598 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;599 struct device *dev = mds->cxlds.dev;600 int rc;601 602 dev_dbg(dev,603 "Submitting %s command for user\n"604 "\topcode: %x\n"605 "\tsize: %zx\n",606 cxl_mem_opcode_to_name(mbox_cmd->opcode),607 mbox_cmd->opcode, mbox_cmd->size_in);608 609 rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);610 if (rc)611 goto out;612 613 /*614 * @size_out contains the max size that's allowed to be written back out615 * to userspace. While the payload may have written more output than616 * this it will have to be ignored.617 */618 if (mbox_cmd->size_out) {619 dev_WARN_ONCE(dev, mbox_cmd->size_out > *size_out,620 "Invalid return size\n");621 if (copy_to_user(u64_to_user_ptr(out_payload),622 mbox_cmd->payload_out, mbox_cmd->size_out)) {623 rc = -EFAULT;624 goto out;625 }626 }627 628 *size_out = mbox_cmd->size_out;629 *retval = mbox_cmd->return_code;630 631out:632 cxl_mbox_cmd_dtor(mbox_cmd);633 return rc;634}635 636int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)637{638 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);639 struct device *dev = &cxlmd->dev;640 struct cxl_send_command send;641 struct cxl_mbox_cmd mbox_cmd;642 int rc;643 644 dev_dbg(dev, "Send IOCTL\n");645 646 if (copy_from_user(&send, s, sizeof(send)))647 return -EFAULT;648 649 rc = cxl_validate_cmd_from_user(&mbox_cmd, mds, &send);650 if (rc)651 return rc;652 653 rc = handle_mailbox_cmd_from_user(mds, &mbox_cmd, send.out.payload,654 &send.out.size, &send.retval);655 if (rc)656 return rc;657 658 if (copy_to_user(s, &send, sizeof(send)))659 return -EFAULT;660 661 return 0;662}663 664static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,665 u32 *size, u8 *out)666{667 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;668 u32 remaining = *size;669 u32 offset = 0;670 671 while (remaining) {672 u32 xfer_size = min_t(u32, remaining, cxl_mbox->payload_size);673 struct cxl_mbox_cmd mbox_cmd;674 struct cxl_mbox_get_log log;675 int rc;676 677 log = (struct cxl_mbox_get_log) {678 .uuid = *uuid,679 .offset = cpu_to_le32(offset),680 .length = cpu_to_le32(xfer_size),681 };682 683 mbox_cmd = (struct cxl_mbox_cmd) {684 .opcode = CXL_MBOX_OP_GET_LOG,685 .size_in = sizeof(log),686 .payload_in = &log,687 .size_out = xfer_size,688 .payload_out = out,689 };690 691 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);692 693 /*694 * The output payload length that indicates the number695 * of valid bytes can be smaller than the Log buffer696 * size.697 */698 if (rc == -EIO && mbox_cmd.size_out < xfer_size) {699 offset += mbox_cmd.size_out;700 break;701 }702 703 if (rc < 0)704 return rc;705 706 out += xfer_size;707 remaining -= xfer_size;708 offset += xfer_size;709 }710 711 *size = offset;712 713 return 0;714}715 716/**717 * cxl_walk_cel() - Walk through the Command Effects Log.718 * @mds: The driver data for the operation719 * @size: Length of the Command Effects Log.720 * @cel: CEL721 *722 * Iterate over each entry in the CEL and determine if the driver supports the723 * command. If so, the command is enabled for the device and can be used later.724 */725static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)726{727 struct cxl_cel_entry *cel_entry;728 const int cel_entries = size / sizeof(*cel_entry);729 struct device *dev = mds->cxlds.dev;730 int i;731 732 cel_entry = (struct cxl_cel_entry *) cel;733 734 for (i = 0; i < cel_entries; i++) {735 u16 opcode = le16_to_cpu(cel_entry[i].opcode);736 struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);737 int enabled = 0;738 739 if (cmd) {740 set_bit(cmd->info.id, mds->enabled_cmds);741 enabled++;742 }743 744 if (cxl_is_poison_command(opcode)) {745 cxl_set_poison_cmd_enabled(&mds->poison, opcode);746 enabled++;747 }748 749 if (cxl_is_security_command(opcode)) {750 cxl_set_security_cmd_enabled(&mds->security, opcode);751 enabled++;752 }753 754 dev_dbg(dev, "Opcode 0x%04x %s\n", opcode,755 enabled ? "enabled" : "unsupported by driver");756 }757}758 759static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)760{761 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;762 struct cxl_mbox_get_supported_logs *ret;763 struct cxl_mbox_cmd mbox_cmd;764 int rc;765 766 ret = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);767 if (!ret)768 return ERR_PTR(-ENOMEM);769 770 mbox_cmd = (struct cxl_mbox_cmd) {771 .opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,772 .size_out = cxl_mbox->payload_size,773 .payload_out = ret,774 /* At least the record number field must be valid */775 .min_out = 2,776 };777 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);778 if (rc < 0) {779 kvfree(ret);780 return ERR_PTR(rc);781 }782 783 784 return ret;785}786 787enum {788 CEL_UUID,789 VENDOR_DEBUG_UUID,790};791 792/* See CXL 2.0 Table 170. Get Log Input Payload */793static const uuid_t log_uuid[] = {794 [CEL_UUID] = DEFINE_CXL_CEL_UUID,795 [VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID,796};797 798/**799 * cxl_enumerate_cmds() - Enumerate commands for a device.800 * @mds: The driver data for the operation801 *802 * Returns 0 if enumerate completed successfully.803 *804 * CXL devices have optional support for certain commands. This function will805 * determine the set of supported commands for the hardware and update the806 * enabled_cmds bitmap in the @mds.807 */808int cxl_enumerate_cmds(struct cxl_memdev_state *mds)809{810 struct cxl_mbox_get_supported_logs *gsl;811 struct device *dev = mds->cxlds.dev;812 struct cxl_mem_command *cmd;813 int i, rc;814 815 gsl = cxl_get_gsl(mds);816 if (IS_ERR(gsl))817 return PTR_ERR(gsl);818 819 rc = -ENOENT;820 for (i = 0; i < le16_to_cpu(gsl->entries); i++) {821 u32 size = le32_to_cpu(gsl->entry[i].size);822 uuid_t uuid = gsl->entry[i].uuid;823 u8 *log;824 825 dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size);826 827 if (!uuid_equal(&uuid, &log_uuid[CEL_UUID]))828 continue;829 830 log = kvmalloc(size, GFP_KERNEL);831 if (!log) {832 rc = -ENOMEM;833 goto out;834 }835 836 rc = cxl_xfer_log(mds, &uuid, &size, log);837 if (rc) {838 kvfree(log);839 goto out;840 }841 842 cxl_walk_cel(mds, size, log);843 kvfree(log);844 845 /* In case CEL was bogus, enable some default commands. */846 cxl_for_each_cmd(cmd)847 if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)848 set_bit(cmd->info.id, mds->enabled_cmds);849 850 /* Found the required CEL */851 rc = 0;852 }853out:854 kvfree(gsl);855 return rc;856}857EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL);858 859void cxl_event_trace_record(const struct cxl_memdev *cxlmd,860 enum cxl_event_log_type type,861 enum cxl_event_type event_type,862 const uuid_t *uuid, union cxl_event *evt)863{864 if (event_type == CXL_CPER_EVENT_MEM_MODULE) {865 trace_cxl_memory_module(cxlmd, type, &evt->mem_module);866 return;867 }868 if (event_type == CXL_CPER_EVENT_GENERIC) {869 trace_cxl_generic_event(cxlmd, type, uuid, &evt->generic);870 return;871 }872 873 if (trace_cxl_general_media_enabled() || trace_cxl_dram_enabled()) {874 u64 dpa, hpa = ULLONG_MAX;875 struct cxl_region *cxlr;876 877 /*878 * These trace points are annotated with HPA and region879 * translations. Take topology mutation locks and lookup880 * { HPA, REGION } from { DPA, MEMDEV } in the event record.881 */882 guard(rwsem_read)(&cxl_region_rwsem);883 guard(rwsem_read)(&cxl_dpa_rwsem);884 885 dpa = le64_to_cpu(evt->media_hdr.phys_addr) & CXL_DPA_MASK;886 cxlr = cxl_dpa_to_region(cxlmd, dpa);887 if (cxlr)888 hpa = cxl_dpa_to_hpa(cxlr, cxlmd, dpa);889 890 if (event_type == CXL_CPER_EVENT_GEN_MEDIA)891 trace_cxl_general_media(cxlmd, type, cxlr, hpa,892 &evt->gen_media);893 else if (event_type == CXL_CPER_EVENT_DRAM)894 trace_cxl_dram(cxlmd, type, cxlr, hpa, &evt->dram);895 }896}897EXPORT_SYMBOL_NS_GPL(cxl_event_trace_record, CXL);898 899static void __cxl_event_trace_record(const struct cxl_memdev *cxlmd,900 enum cxl_event_log_type type,901 struct cxl_event_record_raw *record)902{903 enum cxl_event_type ev_type = CXL_CPER_EVENT_GENERIC;904 const uuid_t *uuid = &record->id;905 906 if (uuid_equal(uuid, &CXL_EVENT_GEN_MEDIA_UUID))907 ev_type = CXL_CPER_EVENT_GEN_MEDIA;908 else if (uuid_equal(uuid, &CXL_EVENT_DRAM_UUID))909 ev_type = CXL_CPER_EVENT_DRAM;910 else if (uuid_equal(uuid, &CXL_EVENT_MEM_MODULE_UUID))911 ev_type = CXL_CPER_EVENT_MEM_MODULE;912 913 cxl_event_trace_record(cxlmd, type, ev_type, uuid, &record->event);914}915 916static int cxl_clear_event_record(struct cxl_memdev_state *mds,917 enum cxl_event_log_type log,918 struct cxl_get_event_payload *get_pl)919{920 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;921 struct cxl_mbox_clear_event_payload *payload;922 u16 total = le16_to_cpu(get_pl->record_count);923 u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES;924 size_t pl_size = struct_size(payload, handles, max_handles);925 struct cxl_mbox_cmd mbox_cmd;926 u16 cnt;927 int rc = 0;928 int i;929 930 /* Payload size may limit the max handles */931 if (pl_size > cxl_mbox->payload_size) {932 max_handles = (cxl_mbox->payload_size - sizeof(*payload)) /933 sizeof(__le16);934 pl_size = struct_size(payload, handles, max_handles);935 }936 937 payload = kvzalloc(pl_size, GFP_KERNEL);938 if (!payload)939 return -ENOMEM;940 941 *payload = (struct cxl_mbox_clear_event_payload) {942 .event_log = log,943 };944 945 mbox_cmd = (struct cxl_mbox_cmd) {946 .opcode = CXL_MBOX_OP_CLEAR_EVENT_RECORD,947 .payload_in = payload,948 .size_in = pl_size,949 };950 951 /*952 * Clear Event Records uses u8 for the handle cnt while Get Event953 * Record can return up to 0xffff records.954 */955 i = 0;956 for (cnt = 0; cnt < total; cnt++) {957 struct cxl_event_record_raw *raw = &get_pl->records[cnt];958 struct cxl_event_generic *gen = &raw->event.generic;959 960 payload->handles[i++] = gen->hdr.handle;961 dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log,962 le16_to_cpu(payload->handles[i - 1]));963 964 if (i == max_handles) {965 payload->nr_recs = i;966 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);967 if (rc)968 goto free_pl;969 i = 0;970 }971 }972 973 /* Clear what is left if any */974 if (i) {975 payload->nr_recs = i;976 mbox_cmd.size_in = struct_size(payload, handles, i);977 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);978 if (rc)979 goto free_pl;980 }981 982free_pl:983 kvfree(payload);984 return rc;985}986 987static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,988 enum cxl_event_log_type type)989{990 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;991 struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;992 struct device *dev = mds->cxlds.dev;993 struct cxl_get_event_payload *payload;994 u8 log_type = type;995 u16 nr_rec;996 997 mutex_lock(&mds->event.log_lock);998 payload = mds->event.buf;999 1000 do {1001 int rc, i;1002 struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd) {1003 .opcode = CXL_MBOX_OP_GET_EVENT_RECORD,1004 .payload_in = &log_type,1005 .size_in = sizeof(log_type),1006 .payload_out = payload,1007 .size_out = cxl_mbox->payload_size,1008 .min_out = struct_size(payload, records, 0),1009 };1010 1011 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);1012 if (rc) {1013 dev_err_ratelimited(dev,1014 "Event log '%d': Failed to query event records : %d",1015 type, rc);1016 break;1017 }1018 1019 nr_rec = le16_to_cpu(payload->record_count);1020 if (!nr_rec)1021 break;1022 1023 for (i = 0; i < nr_rec; i++)1024 __cxl_event_trace_record(cxlmd, type,1025 &payload->records[i]);1026 1027 if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW)1028 trace_cxl_overflow(cxlmd, type, payload);1029 1030 rc = cxl_clear_event_record(mds, type, payload);1031 if (rc) {1032 dev_err_ratelimited(dev,1033 "Event log '%d': Failed to clear events : %d",1034 type, rc);1035 break;1036 }1037 } while (nr_rec);1038 1039 mutex_unlock(&mds->event.log_lock);1040}1041 1042/**1043 * cxl_mem_get_event_records - Get Event Records from the device1044 * @mds: The driver data for the operation1045 * @status: Event Status register value identifying which events are available.1046 *1047 * Retrieve all event records available on the device, report them as trace1048 * events, and clear them.1049 *1050 * See CXL rev 3.0 @8.2.9.2.2 Get Event Records1051 * See CXL rev 3.0 @8.2.9.2.3 Clear Event Records1052 */1053void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status)1054{1055 dev_dbg(mds->cxlds.dev, "Reading event logs: %x\n", status);1056 1057 if (status & CXLDEV_EVENT_STATUS_FATAL)1058 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FATAL);1059 if (status & CXLDEV_EVENT_STATUS_FAIL)1060 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FAIL);1061 if (status & CXLDEV_EVENT_STATUS_WARN)1062 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_WARN);1063 if (status & CXLDEV_EVENT_STATUS_INFO)1064 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_INFO);1065}1066EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL);1067 1068/**1069 * cxl_mem_get_partition_info - Get partition info1070 * @mds: The driver data for the operation1071 *1072 * Retrieve the current partition info for the device specified. The active1073 * values are the current capacity in bytes. If not 0, the 'next' values are1074 * the pending values, in bytes, which take affect on next cold reset.1075 *1076 * Return: 0 if no error: or the result of the mailbox command.1077 *1078 * See CXL @8.2.9.5.2.1 Get Partition Info1079 */1080static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)1081{1082 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;1083 struct cxl_mbox_get_partition_info pi;1084 struct cxl_mbox_cmd mbox_cmd;1085 int rc;1086 1087 mbox_cmd = (struct cxl_mbox_cmd) {1088 .opcode = CXL_MBOX_OP_GET_PARTITION_INFO,1089 .size_out = sizeof(pi),1090 .payload_out = &pi,1091 };1092 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);1093 if (rc)1094 return rc;1095 1096 mds->active_volatile_bytes =1097 le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER;1098 mds->active_persistent_bytes =1099 le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER;1100 mds->next_volatile_bytes =1101 le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;1102 mds->next_persistent_bytes =1103 le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;1104 1105 return 0;1106}1107 1108/**1109 * cxl_dev_state_identify() - Send the IDENTIFY command to the device.1110 * @mds: The driver data for the operation1111 *1112 * Return: 0 if identify was executed successfully or media not ready.1113 *1114 * This will dispatch the identify command to the device and on success populate1115 * structures to be exported to sysfs.1116 */1117int cxl_dev_state_identify(struct cxl_memdev_state *mds)1118{1119 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;1120 /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */1121 struct cxl_mbox_identify id;1122 struct cxl_mbox_cmd mbox_cmd;1123 u32 val;1124 int rc;1125 1126 if (!mds->cxlds.media_ready)1127 return 0;1128 1129 mbox_cmd = (struct cxl_mbox_cmd) {1130 .opcode = CXL_MBOX_OP_IDENTIFY,1131 .size_out = sizeof(id),1132 .payload_out = &id,1133 };1134 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);1135 if (rc < 0)1136 return rc;1137 1138 mds->total_bytes =1139 le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER;1140 mds->volatile_only_bytes =1141 le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER;1142 mds->persistent_only_bytes =1143 le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER;1144 mds->partition_align_bytes =1145 le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;1146 1147 mds->lsa_size = le32_to_cpu(id.lsa_size);1148 memcpy(mds->firmware_version, id.fw_revision,1149 sizeof(id.fw_revision));1150 1151 if (test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) {1152 val = get_unaligned_le24(id.poison_list_max_mer);1153 mds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX);1154 }1155 1156 return 0;1157}1158EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL);1159 1160static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)1161{1162 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;1163 int rc;1164 u32 sec_out = 0;1165 struct cxl_get_security_output {1166 __le32 flags;1167 } out;1168 struct cxl_mbox_cmd sec_cmd = {1169 .opcode = CXL_MBOX_OP_GET_SECURITY_STATE,1170 .payload_out = &out,1171 .size_out = sizeof(out),1172 };1173 struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd };1174 1175 if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE)1176 return -EINVAL;1177 1178 rc = cxl_internal_send_cmd(cxl_mbox, &sec_cmd);1179 if (rc < 0) {1180 dev_err(cxl_mbox->host, "Failed to get security state : %d", rc);1181 return rc;1182 }1183 1184 /*1185 * Prior to using these commands, any security applied to1186 * the user data areas of the device shall be DISABLED (or1187 * UNLOCKED for secure erase case).1188 */1189 sec_out = le32_to_cpu(out.flags);1190 if (sec_out & CXL_PMEM_SEC_STATE_USER_PASS_SET)1191 return -EINVAL;1192 1193 if (cmd == CXL_MBOX_OP_SECURE_ERASE &&1194 sec_out & CXL_PMEM_SEC_STATE_LOCKED)1195 return -EINVAL;1196 1197 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);1198 if (rc < 0) {1199 dev_err(cxl_mbox->host, "Failed to sanitize device : %d", rc);1200 return rc;1201 }1202 1203 return 0;1204}1205 1206 1207/**1208 * cxl_mem_sanitize() - Send a sanitization command to the device.1209 * @cxlmd: The device for the operation1210 * @cmd: The specific sanitization command opcode1211 *1212 * Return: 0 if the command was executed successfully, regardless of1213 * whether or not the actual security operation is done in the background,1214 * such as for the Sanitize case.1215 * Error return values can be the result of the mailbox command, -EINVAL1216 * when security requirements are not met or invalid contexts, or -EBUSY1217 * if the sanitize operation is already in flight.1218 *1219 * See CXL 3.0 @8.2.9.8.5.1 Sanitize and @8.2.9.8.5.2 Secure Erase.1220 */1221int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)1222{1223 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);1224 struct cxl_port *endpoint;1225 int rc;1226 1227 /* synchronize with cxl_mem_probe() and decoder write operations */1228 guard(device)(&cxlmd->dev);1229 endpoint = cxlmd->endpoint;1230 down_read(&cxl_region_rwsem);1231 /*1232 * Require an endpoint to be safe otherwise the driver can not1233 * be sure that the device is unmapped.1234 */1235 if (endpoint && cxl_num_decoders_committed(endpoint) == 0)1236 rc = __cxl_mem_sanitize(mds, cmd);1237 else1238 rc = -EBUSY;1239 up_read(&cxl_region_rwsem);1240 1241 return rc;1242}1243 1244static int add_dpa_res(struct device *dev, struct resource *parent,1245 struct resource *res, resource_size_t start,1246 resource_size_t size, const char *type)1247{1248 int rc;1249 1250 res->name = type;1251 res->start = start;1252 res->end = start + size - 1;1253 res->flags = IORESOURCE_MEM;1254 if (resource_size(res) == 0) {1255 dev_dbg(dev, "DPA(%s): no capacity\n", res->name);1256 return 0;1257 }1258 rc = request_resource(parent, res);1259 if (rc) {1260 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,1261 res, rc);1262 return rc;1263 }1264 1265 dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);1266 1267 return 0;1268}1269 1270int cxl_mem_create_range_info(struct cxl_memdev_state *mds)1271{1272 struct cxl_dev_state *cxlds = &mds->cxlds;1273 struct device *dev = cxlds->dev;1274 int rc;1275 1276 if (!cxlds->media_ready) {1277 cxlds->dpa_res = DEFINE_RES_MEM(0, 0);1278 cxlds->ram_res = DEFINE_RES_MEM(0, 0);1279 cxlds->pmem_res = DEFINE_RES_MEM(0, 0);1280 return 0;1281 }1282 1283 cxlds->dpa_res = DEFINE_RES_MEM(0, mds->total_bytes);1284 1285 if (mds->partition_align_bytes == 0) {1286 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,1287 mds->volatile_only_bytes, "ram");1288 if (rc)1289 return rc;1290 return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,1291 mds->volatile_only_bytes,1292 mds->persistent_only_bytes, "pmem");1293 }1294 1295 rc = cxl_mem_get_partition_info(mds);1296 if (rc) {1297 dev_err(dev, "Failed to query partition information\n");1298 return rc;1299 }1300 1301 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,1302 mds->active_volatile_bytes, "ram");1303 if (rc)1304 return rc;1305 return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,1306 mds->active_volatile_bytes,1307 mds->active_persistent_bytes, "pmem");1308}1309EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);1310 1311int cxl_set_timestamp(struct cxl_memdev_state *mds)1312{1313 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;1314 struct cxl_mbox_cmd mbox_cmd;1315 struct cxl_mbox_set_timestamp_in pi;1316 int rc;1317 1318 pi.timestamp = cpu_to_le64(ktime_get_real_ns());1319 mbox_cmd = (struct cxl_mbox_cmd) {1320 .opcode = CXL_MBOX_OP_SET_TIMESTAMP,1321 .size_in = sizeof(pi),1322 .payload_in = &pi,1323 };1324 1325 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);1326 /*1327 * Command is optional. Devices may have another way of providing1328 * a timestamp, or may return all 0s in timestamp fields.1329 * Don't report an error if this command isn't supported1330 */1331 if (rc && (mbox_cmd.return_code != CXL_MBOX_CMD_RC_UNSUPPORTED))1332 return rc;1333 1334 return 0;1335}1336EXPORT_SYMBOL_NS_GPL(cxl_set_timestamp, CXL);1337 1338int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,1339 struct cxl_region *cxlr)1340{1341 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);1342 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;1343 struct cxl_mbox_poison_out *po;1344 struct cxl_mbox_poison_in pi;1345 int nr_records = 0;1346 int rc;1347 1348 rc = mutex_lock_interruptible(&mds->poison.lock);1349 if (rc)1350 return rc;1351 1352 po = mds->poison.list_out;1353 pi.offset = cpu_to_le64(offset);1354 pi.length = cpu_to_le64(len / CXL_POISON_LEN_MULT);1355 1356 do {1357 struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd){1358 .opcode = CXL_MBOX_OP_GET_POISON,1359 .size_in = sizeof(pi),1360 .payload_in = &pi,1361 .size_out = cxl_mbox->payload_size,1362 .payload_out = po,1363 .min_out = struct_size(po, record, 0),1364 };1365 1366 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);1367 if (rc)1368 break;1369 1370 for (int i = 0; i < le16_to_cpu(po->count); i++)1371 trace_cxl_poison(cxlmd, cxlr, &po->record[i],1372 po->flags, po->overflow_ts,1373 CXL_POISON_TRACE_LIST);1374 1375 /* Protect against an uncleared _FLAG_MORE */1376 nr_records = nr_records + le16_to_cpu(po->count);1377 if (nr_records >= mds->poison.max_errors) {1378 dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n",1379 nr_records);1380 break;1381 }1382 } while (po->flags & CXL_POISON_FLAG_MORE);1383 1384 mutex_unlock(&mds->poison.lock);1385 return rc;1386}1387EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, CXL);1388 1389static void free_poison_buf(void *buf)1390{1391 kvfree(buf);1392}1393 1394/* Get Poison List output buffer is protected by mds->poison.lock */1395static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)1396{1397 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;1398 1399 mds->poison.list_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);1400 if (!mds->poison.list_out)1401 return -ENOMEM;1402 1403 return devm_add_action_or_reset(mds->cxlds.dev, free_poison_buf,1404 mds->poison.list_out);1405}1406 1407int cxl_poison_state_init(struct cxl_memdev_state *mds)1408{1409 int rc;1410 1411 if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds))1412 return 0;1413 1414 rc = cxl_poison_alloc_buf(mds);1415 if (rc) {1416 clear_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds);1417 return rc;1418 }1419 1420 mutex_init(&mds->poison.lock);1421 return 0;1422}1423EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL);1424 1425int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host)1426{1427 if (!cxl_mbox || !host)1428 return -EINVAL;1429 1430 cxl_mbox->host = host;1431 mutex_init(&cxl_mbox->mbox_mutex);1432 rcuwait_init(&cxl_mbox->mbox_wait);1433 1434 return 0;1435}1436EXPORT_SYMBOL_NS_GPL(cxl_mailbox_init, CXL);1437 1438struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)1439{1440 struct cxl_memdev_state *mds;1441 1442 mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);1443 if (!mds) {1444 dev_err(dev, "No memory available\n");1445 return ERR_PTR(-ENOMEM);1446 }1447 1448 mutex_init(&mds->event.log_lock);1449 mds->cxlds.dev = dev;1450 mds->cxlds.reg_map.host = dev;1451 mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE;1452 mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;1453 mds->ram_perf.qos_class = CXL_QOS_CLASS_INVALID;1454 mds->pmem_perf.qos_class = CXL_QOS_CLASS_INVALID;1455 1456 return mds;1457}1458EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, CXL);1459 1460void __init cxl_mbox_init(void)1461{1462 struct dentry *mbox_debugfs;1463 1464 mbox_debugfs = cxl_debugfs_create_dir("mbox");1465 debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,1466 &cxl_raw_allow_all);1467}1468