2219 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * CXL Flash Device Driver4 *5 * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation6 * Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation7 *8 * Copyright (C) 2015 IBM Corporation9 */10 11#include <linux/delay.h>12#include <linux/file.h>13#include <linux/interrupt.h>14#include <linux/pci.h>15#include <linux/syscalls.h>16#include <linux/unaligned.h>17 18#include <scsi/scsi.h>19#include <scsi/scsi_host.h>20#include <scsi/scsi_cmnd.h>21#include <scsi/scsi_eh.h>22#include <uapi/scsi/cxlflash_ioctl.h>23 24#include "sislite.h"25#include "common.h"26#include "vlun.h"27#include "superpipe.h"28 29struct cxlflash_global global;30 31/**32 * marshal_rele_to_resize() - translate release to resize structure33 * @release: Source structure from which to translate/copy.34 * @resize: Destination structure for the translate/copy.35 */36static void marshal_rele_to_resize(struct dk_cxlflash_release *release,37 struct dk_cxlflash_resize *resize)38{39 resize->hdr = release->hdr;40 resize->context_id = release->context_id;41 resize->rsrc_handle = release->rsrc_handle;42}43 44/**45 * marshal_det_to_rele() - translate detach to release structure46 * @detach: Destination structure for the translate/copy.47 * @release: Source structure from which to translate/copy.48 */49static void marshal_det_to_rele(struct dk_cxlflash_detach *detach,50 struct dk_cxlflash_release *release)51{52 release->hdr = detach->hdr;53 release->context_id = detach->context_id;54}55 56/**57 * marshal_udir_to_rele() - translate udirect to release structure58 * @udirect: Source structure from which to translate/copy.59 * @release: Destination structure for the translate/copy.60 */61static void marshal_udir_to_rele(struct dk_cxlflash_udirect *udirect,62 struct dk_cxlflash_release *release)63{64 release->hdr = udirect->hdr;65 release->context_id = udirect->context_id;66 release->rsrc_handle = udirect->rsrc_handle;67}68 69/**70 * cxlflash_free_errpage() - frees resources associated with global error page71 */72void cxlflash_free_errpage(void)73{74 75 mutex_lock(&global.mutex);76 if (global.err_page) {77 __free_page(global.err_page);78 global.err_page = NULL;79 }80 mutex_unlock(&global.mutex);81}82 83/**84 * cxlflash_stop_term_user_contexts() - stops/terminates known user contexts85 * @cfg: Internal structure associated with the host.86 *87 * When the host needs to go down, all users must be quiesced and their88 * memory freed. This is accomplished by putting the contexts in error89 * state which will notify the user and let them 'drive' the tear down.90 * Meanwhile, this routine camps until all user contexts have been removed.91 *92 * Note that the main loop in this routine will always execute at least once93 * to flush the reset_waitq.94 */95void cxlflash_stop_term_user_contexts(struct cxlflash_cfg *cfg)96{97 struct device *dev = &cfg->dev->dev;98 int i, found = true;99 100 cxlflash_mark_contexts_error(cfg);101 102 while (true) {103 for (i = 0; i < MAX_CONTEXT; i++)104 if (cfg->ctx_tbl[i]) {105 found = true;106 break;107 }108 109 if (!found && list_empty(&cfg->ctx_err_recovery))110 return;111 112 dev_dbg(dev, "%s: Wait for user contexts to quiesce...\n",113 __func__);114 wake_up_all(&cfg->reset_waitq);115 ssleep(1);116 found = false;117 }118}119 120/**121 * find_error_context() - locates a context by cookie on the error recovery list122 * @cfg: Internal structure associated with the host.123 * @rctxid: Desired context by id.124 * @file: Desired context by file.125 *126 * Return: Found context on success, NULL on failure127 */128static struct ctx_info *find_error_context(struct cxlflash_cfg *cfg, u64 rctxid,129 struct file *file)130{131 struct ctx_info *ctxi;132 133 list_for_each_entry(ctxi, &cfg->ctx_err_recovery, list)134 if ((ctxi->ctxid == rctxid) || (ctxi->file == file))135 return ctxi;136 137 return NULL;138}139 140/**141 * get_context() - obtains a validated and locked context reference142 * @cfg: Internal structure associated with the host.143 * @rctxid: Desired context (raw, un-decoded format).144 * @arg: LUN information or file associated with request.145 * @ctx_ctrl: Control information to 'steer' desired lookup.146 *147 * NOTE: despite the name pid, in linux, current->pid actually refers148 * to the lightweight process id (tid) and can change if the process is149 * multi threaded. The tgid remains constant for the process and only changes150 * when the process of fork. For all intents and purposes, think of tgid151 * as a pid in the traditional sense.152 *153 * Return: Validated context on success, NULL on failure154 */155struct ctx_info *get_context(struct cxlflash_cfg *cfg, u64 rctxid,156 void *arg, enum ctx_ctrl ctx_ctrl)157{158 struct device *dev = &cfg->dev->dev;159 struct ctx_info *ctxi = NULL;160 struct lun_access *lun_access = NULL;161 struct file *file = NULL;162 struct llun_info *lli = arg;163 u64 ctxid = DECODE_CTXID(rctxid);164 int rc;165 pid_t pid = task_tgid_nr(current), ctxpid = 0;166 167 if (ctx_ctrl & CTX_CTRL_FILE) {168 lli = NULL;169 file = (struct file *)arg;170 }171 172 if (ctx_ctrl & CTX_CTRL_CLONE)173 pid = task_ppid_nr(current);174 175 if (likely(ctxid < MAX_CONTEXT)) {176 while (true) {177 mutex_lock(&cfg->ctx_tbl_list_mutex);178 ctxi = cfg->ctx_tbl[ctxid];179 if (ctxi)180 if ((file && (ctxi->file != file)) ||181 (!file && (ctxi->ctxid != rctxid)))182 ctxi = NULL;183 184 if ((ctx_ctrl & CTX_CTRL_ERR) ||185 (!ctxi && (ctx_ctrl & CTX_CTRL_ERR_FALLBACK)))186 ctxi = find_error_context(cfg, rctxid, file);187 if (!ctxi) {188 mutex_unlock(&cfg->ctx_tbl_list_mutex);189 goto out;190 }191 192 /*193 * Need to acquire ownership of the context while still194 * under the table/list lock to serialize with a remove195 * thread. Use the 'try' to avoid stalling the196 * table/list lock for a single context.197 *198 * Note that the lock order is:199 *200 * cfg->ctx_tbl_list_mutex -> ctxi->mutex201 *202 * Therefore release ctx_tbl_list_mutex before retrying.203 */204 rc = mutex_trylock(&ctxi->mutex);205 mutex_unlock(&cfg->ctx_tbl_list_mutex);206 if (rc)207 break; /* got the context's lock! */208 }209 210 if (ctxi->unavail)211 goto denied;212 213 ctxpid = ctxi->pid;214 if (likely(!(ctx_ctrl & CTX_CTRL_NOPID)))215 if (pid != ctxpid)216 goto denied;217 218 if (lli) {219 list_for_each_entry(lun_access, &ctxi->luns, list)220 if (lun_access->lli == lli)221 goto out;222 goto denied;223 }224 }225 226out:227 dev_dbg(dev, "%s: rctxid=%016llx ctxinfo=%p ctxpid=%u pid=%u "228 "ctx_ctrl=%u\n", __func__, rctxid, ctxi, ctxpid, pid,229 ctx_ctrl);230 231 return ctxi;232 233denied:234 mutex_unlock(&ctxi->mutex);235 ctxi = NULL;236 goto out;237}238 239/**240 * put_context() - release a context that was retrieved from get_context()241 * @ctxi: Context to release.242 *243 * For now, releasing the context equates to unlocking it's mutex.244 */245void put_context(struct ctx_info *ctxi)246{247 mutex_unlock(&ctxi->mutex);248}249 250/**251 * afu_attach() - attach a context to the AFU252 * @cfg: Internal structure associated with the host.253 * @ctxi: Context to attach.254 *255 * Upon setting the context capabilities, they must be confirmed with256 * a read back operation as the context might have been closed since257 * the mailbox was unlocked. When this occurs, registration is failed.258 *259 * Return: 0 on success, -errno on failure260 */261static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)262{263 struct device *dev = &cfg->dev->dev;264 struct afu *afu = cfg->afu;265 struct sisl_ctrl_map __iomem *ctrl_map = ctxi->ctrl_map;266 int rc = 0;267 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);268 u64 val;269 int i;270 271 /* Unlock cap and restrict user to read/write cmds in translated mode */272 readq_be(&ctrl_map->mbox_r);273 val = (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD);274 writeq_be(val, &ctrl_map->ctx_cap);275 val = readq_be(&ctrl_map->ctx_cap);276 if (val != (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD)) {277 dev_err(dev, "%s: ctx may be closed val=%016llx\n",278 __func__, val);279 rc = -EAGAIN;280 goto out;281 }282 283 if (afu_is_ocxl_lisn(afu)) {284 /* Set up the LISN effective address for each interrupt */285 for (i = 0; i < ctxi->irqs; i++) {286 val = cfg->ops->get_irq_objhndl(ctxi->ctx, i);287 writeq_be(val, &ctrl_map->lisn_ea[i]);288 }289 290 /* Use primary HWQ PASID as identifier for all interrupts */291 val = hwq->ctx_hndl;292 writeq_be(SISL_LISN_PASID(val, val), &ctrl_map->lisn_pasid[0]);293 writeq_be(SISL_LISN_PASID(0UL, val), &ctrl_map->lisn_pasid[1]);294 }295 296 /* Set up MMIO registers pointing to the RHT */297 writeq_be((u64)ctxi->rht_start, &ctrl_map->rht_start);298 val = SISL_RHT_CNT_ID((u64)MAX_RHT_PER_CONTEXT, (u64)(hwq->ctx_hndl));299 writeq_be(val, &ctrl_map->rht_cnt_id);300out:301 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);302 return rc;303}304 305/**306 * read_cap16() - issues a SCSI READ_CAP16 command307 * @sdev: SCSI device associated with LUN.308 * @lli: LUN destined for capacity request.309 *310 * The READ_CAP16 can take quite a while to complete. Should an EEH occur while311 * in scsi_execute_cmd(), the EEH handler will attempt to recover. As part of312 * the recovery, the handler drains all currently running ioctls, waiting until313 * they have completed before proceeding with a reset. As this routine is used314 * on the ioctl path, this can create a condition where the EEH handler becomes315 * stuck, infinitely waiting for this ioctl thread. To avoid this behavior,316 * temporarily unmark this thread as an ioctl thread by releasing the ioctl317 * read semaphore. This will allow the EEH handler to proceed with a recovery318 * while this thread is still running. Once the scsi_execute_cmd() returns,319 * reacquire the ioctl read semaphore and check the adapter state in case it320 * changed while inside of scsi_execute_cmd(). The state check will wait if the321 * adapter is still being recovered or return a failure if the recovery failed.322 * In the event that the adapter reset failed, simply return the failure as the323 * ioctl would be unable to continue.324 *325 * Note that the above puts a requirement on this routine to only be called on326 * an ioctl thread.327 *328 * Return: 0 on success, -errno on failure329 */330static int read_cap16(struct scsi_device *sdev, struct llun_info *lli)331{332 struct cxlflash_cfg *cfg = shost_priv(sdev->host);333 struct device *dev = &cfg->dev->dev;334 struct glun_info *gli = lli->parent;335 struct scsi_sense_hdr sshdr;336 const struct scsi_exec_args exec_args = {337 .sshdr = &sshdr,338 };339 u8 *cmd_buf = NULL;340 u8 *scsi_cmd = NULL;341 int rc = 0;342 int result = 0;343 int retry_cnt = 0;344 u32 to = CMD_TIMEOUT * HZ;345 346retry:347 cmd_buf = kzalloc(CMD_BUFSIZE, GFP_KERNEL);348 scsi_cmd = kzalloc(MAX_COMMAND_SIZE, GFP_KERNEL);349 if (unlikely(!cmd_buf || !scsi_cmd)) {350 rc = -ENOMEM;351 goto out;352 }353 354 scsi_cmd[0] = SERVICE_ACTION_IN_16; /* read cap(16) */355 scsi_cmd[1] = SAI_READ_CAPACITY_16; /* service action */356 put_unaligned_be32(CMD_BUFSIZE, &scsi_cmd[10]);357 358 dev_dbg(dev, "%s: %ssending cmd(%02x)\n", __func__,359 retry_cnt ? "re" : "", scsi_cmd[0]);360 361 /* Drop the ioctl read semaphore across lengthy call */362 up_read(&cfg->ioctl_rwsem);363 result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_IN, cmd_buf,364 CMD_BUFSIZE, to, CMD_RETRIES, &exec_args);365 down_read(&cfg->ioctl_rwsem);366 rc = check_state(cfg);367 if (rc) {368 dev_err(dev, "%s: Failed state result=%08x\n",369 __func__, result);370 rc = -ENODEV;371 goto out;372 }373 374 if (result > 0 && scsi_sense_valid(&sshdr)) {375 if (result & SAM_STAT_CHECK_CONDITION) {376 switch (sshdr.sense_key) {377 case NO_SENSE:378 case RECOVERED_ERROR:379 case NOT_READY:380 result &= ~SAM_STAT_CHECK_CONDITION;381 break;382 case UNIT_ATTENTION:383 switch (sshdr.asc) {384 case 0x29: /* Power on Reset or Device Reset */385 fallthrough;386 case 0x2A: /* Device capacity changed */387 case 0x3F: /* Report LUNs changed */388 /* Retry the command once more */389 if (retry_cnt++ < 1) {390 kfree(cmd_buf);391 kfree(scsi_cmd);392 goto retry;393 }394 }395 break;396 default:397 break;398 }399 }400 }401 402 if (result) {403 dev_err(dev, "%s: command failed, result=%08x\n",404 __func__, result);405 rc = -EIO;406 goto out;407 }408 409 /*410 * Read cap was successful, grab values from the buffer;411 * note that we don't need to worry about unaligned access412 * as the buffer is allocated on an aligned boundary.413 */414 mutex_lock(&gli->mutex);415 gli->max_lba = be64_to_cpu(*((__be64 *)&cmd_buf[0]));416 gli->blk_len = be32_to_cpu(*((__be32 *)&cmd_buf[8]));417 mutex_unlock(&gli->mutex);418 419out:420 kfree(cmd_buf);421 kfree(scsi_cmd);422 423 dev_dbg(dev, "%s: maxlba=%lld blklen=%d rc=%d\n",424 __func__, gli->max_lba, gli->blk_len, rc);425 return rc;426}427 428/**429 * get_rhte() - obtains validated resource handle table entry reference430 * @ctxi: Context owning the resource handle.431 * @rhndl: Resource handle associated with entry.432 * @lli: LUN associated with request.433 *434 * Return: Validated RHTE on success, NULL on failure435 */436struct sisl_rht_entry *get_rhte(struct ctx_info *ctxi, res_hndl_t rhndl,437 struct llun_info *lli)438{439 struct cxlflash_cfg *cfg = ctxi->cfg;440 struct device *dev = &cfg->dev->dev;441 struct sisl_rht_entry *rhte = NULL;442 443 if (unlikely(!ctxi->rht_start)) {444 dev_dbg(dev, "%s: Context does not have allocated RHT\n",445 __func__);446 goto out;447 }448 449 if (unlikely(rhndl >= MAX_RHT_PER_CONTEXT)) {450 dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",451 __func__, rhndl);452 goto out;453 }454 455 if (unlikely(ctxi->rht_lun[rhndl] != lli)) {456 dev_dbg(dev, "%s: Bad resource handle LUN rhndl=%d\n",457 __func__, rhndl);458 goto out;459 }460 461 rhte = &ctxi->rht_start[rhndl];462 if (unlikely(rhte->nmask == 0)) {463 dev_dbg(dev, "%s: Unopened resource handle rhndl=%d\n",464 __func__, rhndl);465 rhte = NULL;466 goto out;467 }468 469out:470 return rhte;471}472 473/**474 * rhte_checkout() - obtains free/empty resource handle table entry475 * @ctxi: Context owning the resource handle.476 * @lli: LUN associated with request.477 *478 * Return: Free RHTE on success, NULL on failure479 */480struct sisl_rht_entry *rhte_checkout(struct ctx_info *ctxi,481 struct llun_info *lli)482{483 struct cxlflash_cfg *cfg = ctxi->cfg;484 struct device *dev = &cfg->dev->dev;485 struct sisl_rht_entry *rhte = NULL;486 int i;487 488 /* Find a free RHT entry */489 for (i = 0; i < MAX_RHT_PER_CONTEXT; i++)490 if (ctxi->rht_start[i].nmask == 0) {491 rhte = &ctxi->rht_start[i];492 ctxi->rht_out++;493 break;494 }495 496 if (likely(rhte))497 ctxi->rht_lun[i] = lli;498 499 dev_dbg(dev, "%s: returning rhte=%p index=%d\n", __func__, rhte, i);500 return rhte;501}502 503/**504 * rhte_checkin() - releases a resource handle table entry505 * @ctxi: Context owning the resource handle.506 * @rhte: RHTE to release.507 */508void rhte_checkin(struct ctx_info *ctxi,509 struct sisl_rht_entry *rhte)510{511 u32 rsrc_handle = rhte - ctxi->rht_start;512 513 rhte->nmask = 0;514 rhte->fp = 0;515 ctxi->rht_out--;516 ctxi->rht_lun[rsrc_handle] = NULL;517 ctxi->rht_needs_ws[rsrc_handle] = false;518}519 520/**521 * rht_format1() - populates a RHTE for format 1522 * @rhte: RHTE to populate.523 * @lun_id: LUN ID of LUN associated with RHTE.524 * @perm: Desired permissions for RHTE.525 * @port_sel: Port selection mask526 */527static void rht_format1(struct sisl_rht_entry *rhte, u64 lun_id, u32 perm,528 u32 port_sel)529{530 /*531 * Populate the Format 1 RHT entry for direct access (physical532 * LUN) using the synchronization sequence defined in the533 * SISLite specification.534 */535 struct sisl_rht_entry_f1 dummy = { 0 };536 struct sisl_rht_entry_f1 *rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;537 538 memset(rhte_f1, 0, sizeof(*rhte_f1));539 rhte_f1->fp = SISL_RHT_FP(1U, 0);540 dma_wmb(); /* Make setting of format bit visible */541 542 rhte_f1->lun_id = lun_id;543 dma_wmb(); /* Make setting of LUN id visible */544 545 /*546 * Use a dummy RHT Format 1 entry to build the second dword547 * of the entry that must be populated in a single write when548 * enabled (valid bit set to TRUE).549 */550 dummy.valid = 0x80;551 dummy.fp = SISL_RHT_FP(1U, perm);552 dummy.port_sel = port_sel;553 rhte_f1->dw = dummy.dw;554 555 dma_wmb(); /* Make remaining RHT entry fields visible */556}557 558/**559 * cxlflash_lun_attach() - attaches a user to a LUN and manages the LUN's mode560 * @gli: LUN to attach.561 * @mode: Desired mode of the LUN.562 * @locked: Mutex status on current thread.563 *564 * Return: 0 on success, -errno on failure565 */566int cxlflash_lun_attach(struct glun_info *gli, enum lun_mode mode, bool locked)567{568 int rc = 0;569 570 if (!locked)571 mutex_lock(&gli->mutex);572 573 if (gli->mode == MODE_NONE)574 gli->mode = mode;575 else if (gli->mode != mode) {576 pr_debug("%s: gli_mode=%d requested_mode=%d\n",577 __func__, gli->mode, mode);578 rc = -EINVAL;579 goto out;580 }581 582 gli->users++;583 WARN_ON(gli->users <= 0);584out:585 pr_debug("%s: Returning rc=%d gli->mode=%u gli->users=%u\n",586 __func__, rc, gli->mode, gli->users);587 if (!locked)588 mutex_unlock(&gli->mutex);589 return rc;590}591 592/**593 * cxlflash_lun_detach() - detaches a user from a LUN and resets the LUN's mode594 * @gli: LUN to detach.595 *596 * When resetting the mode, terminate block allocation resources as they597 * are no longer required (service is safe to call even when block allocation598 * resources were not present - such as when transitioning from physical mode).599 * These resources will be reallocated when needed (subsequent transition to600 * virtual mode).601 */602void cxlflash_lun_detach(struct glun_info *gli)603{604 mutex_lock(&gli->mutex);605 WARN_ON(gli->mode == MODE_NONE);606 if (--gli->users == 0) {607 gli->mode = MODE_NONE;608 cxlflash_ba_terminate(&gli->blka.ba_lun);609 }610 pr_debug("%s: gli->users=%u\n", __func__, gli->users);611 WARN_ON(gli->users < 0);612 mutex_unlock(&gli->mutex);613}614 615/**616 * _cxlflash_disk_release() - releases the specified resource entry617 * @sdev: SCSI device associated with LUN.618 * @ctxi: Context owning resources.619 * @release: Release ioctl data structure.620 *621 * For LUNs in virtual mode, the virtual LUN associated with the specified622 * resource handle is resized to 0 prior to releasing the RHTE. Note that the623 * AFU sync should _not_ be performed when the context is sitting on the error624 * recovery list. A context on the error recovery list is not known to the AFU625 * due to reset. When the context is recovered, it will be reattached and made626 * known again to the AFU.627 *628 * Return: 0 on success, -errno on failure629 */630int _cxlflash_disk_release(struct scsi_device *sdev,631 struct ctx_info *ctxi,632 struct dk_cxlflash_release *release)633{634 struct cxlflash_cfg *cfg = shost_priv(sdev->host);635 struct device *dev = &cfg->dev->dev;636 struct llun_info *lli = sdev->hostdata;637 struct glun_info *gli = lli->parent;638 struct afu *afu = cfg->afu;639 bool put_ctx = false;640 641 struct dk_cxlflash_resize size;642 res_hndl_t rhndl = release->rsrc_handle;643 644 int rc = 0;645 int rcr = 0;646 u64 ctxid = DECODE_CTXID(release->context_id),647 rctxid = release->context_id;648 649 struct sisl_rht_entry *rhte;650 struct sisl_rht_entry_f1 *rhte_f1;651 652 dev_dbg(dev, "%s: ctxid=%llu rhndl=%llu gli->mode=%u gli->users=%u\n",653 __func__, ctxid, release->rsrc_handle, gli->mode, gli->users);654 655 if (!ctxi) {656 ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);657 if (unlikely(!ctxi)) {658 dev_dbg(dev, "%s: Bad context ctxid=%llu\n",659 __func__, ctxid);660 rc = -EINVAL;661 goto out;662 }663 664 put_ctx = true;665 }666 667 rhte = get_rhte(ctxi, rhndl, lli);668 if (unlikely(!rhte)) {669 dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",670 __func__, rhndl);671 rc = -EINVAL;672 goto out;673 }674 675 /*676 * Resize to 0 for virtual LUNS by setting the size677 * to 0. This will clear LXT_START and LXT_CNT fields678 * in the RHT entry and properly sync with the AFU.679 *680 * Afterwards we clear the remaining fields.681 */682 switch (gli->mode) {683 case MODE_VIRTUAL:684 marshal_rele_to_resize(release, &size);685 size.req_size = 0;686 rc = _cxlflash_vlun_resize(sdev, ctxi, &size);687 if (rc) {688 dev_dbg(dev, "%s: resize failed rc %d\n", __func__, rc);689 goto out;690 }691 692 break;693 case MODE_PHYSICAL:694 /*695 * Clear the Format 1 RHT entry for direct access696 * (physical LUN) using the synchronization sequence697 * defined in the SISLite specification.698 */699 rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;700 701 rhte_f1->valid = 0;702 dma_wmb(); /* Make revocation of RHT entry visible */703 704 rhte_f1->lun_id = 0;705 dma_wmb(); /* Make clearing of LUN id visible */706 707 rhte_f1->dw = 0;708 dma_wmb(); /* Make RHT entry bottom-half clearing visible */709 710 if (!ctxi->err_recovery_active) {711 rcr = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);712 if (unlikely(rcr))713 dev_dbg(dev, "%s: AFU sync failed rc=%d\n",714 __func__, rcr);715 }716 break;717 default:718 WARN(1, "Unsupported LUN mode!");719 goto out;720 }721 722 rhte_checkin(ctxi, rhte);723 cxlflash_lun_detach(gli);724 725out:726 if (put_ctx)727 put_context(ctxi);728 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);729 return rc;730}731 732int cxlflash_disk_release(struct scsi_device *sdev, void *release)733{734 return _cxlflash_disk_release(sdev, NULL, release);735}736 737/**738 * destroy_context() - releases a context739 * @cfg: Internal structure associated with the host.740 * @ctxi: Context to release.741 *742 * This routine is safe to be called with a a non-initialized context.743 * Also note that the routine conditionally checks for the existence744 * of the context control map before clearing the RHT registers and745 * context capabilities because it is possible to destroy a context746 * while the context is in the error state (previous mapping was747 * removed [so there is no need to worry about clearing] and context748 * is waiting for a new mapping).749 */750static void destroy_context(struct cxlflash_cfg *cfg,751 struct ctx_info *ctxi)752{753 struct afu *afu = cfg->afu;754 755 if (ctxi->initialized) {756 WARN_ON(!list_empty(&ctxi->luns));757 758 /* Clear RHT registers and drop all capabilities for context */759 if (afu->afu_map && ctxi->ctrl_map) {760 writeq_be(0, &ctxi->ctrl_map->rht_start);761 writeq_be(0, &ctxi->ctrl_map->rht_cnt_id);762 writeq_be(0, &ctxi->ctrl_map->ctx_cap);763 }764 }765 766 /* Free memory associated with context */767 free_page((ulong)ctxi->rht_start);768 kfree(ctxi->rht_needs_ws);769 kfree(ctxi->rht_lun);770 kfree(ctxi);771}772 773/**774 * create_context() - allocates and initializes a context775 * @cfg: Internal structure associated with the host.776 *777 * Return: Allocated context on success, NULL on failure778 */779static struct ctx_info *create_context(struct cxlflash_cfg *cfg)780{781 struct device *dev = &cfg->dev->dev;782 struct ctx_info *ctxi = NULL;783 struct llun_info **lli = NULL;784 u8 *ws = NULL;785 struct sisl_rht_entry *rhte;786 787 ctxi = kzalloc(sizeof(*ctxi), GFP_KERNEL);788 lli = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*lli)), GFP_KERNEL);789 ws = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*ws)), GFP_KERNEL);790 if (unlikely(!ctxi || !lli || !ws)) {791 dev_err(dev, "%s: Unable to allocate context\n", __func__);792 goto err;793 }794 795 rhte = (struct sisl_rht_entry *)get_zeroed_page(GFP_KERNEL);796 if (unlikely(!rhte)) {797 dev_err(dev, "%s: Unable to allocate RHT\n", __func__);798 goto err;799 }800 801 ctxi->rht_lun = lli;802 ctxi->rht_needs_ws = ws;803 ctxi->rht_start = rhte;804out:805 return ctxi;806 807err:808 kfree(ws);809 kfree(lli);810 kfree(ctxi);811 ctxi = NULL;812 goto out;813}814 815/**816 * init_context() - initializes a previously allocated context817 * @ctxi: Previously allocated context818 * @cfg: Internal structure associated with the host.819 * @ctx: Previously obtained context cookie.820 * @ctxid: Previously obtained process element associated with CXL context.821 * @file: Previously obtained file associated with CXL context.822 * @perms: User-specified permissions.823 * @irqs: User-specified number of interrupts.824 */825static void init_context(struct ctx_info *ctxi, struct cxlflash_cfg *cfg,826 void *ctx, int ctxid, struct file *file, u32 perms,827 u64 irqs)828{829 struct afu *afu = cfg->afu;830 831 ctxi->rht_perms = perms;832 ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;833 ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);834 ctxi->irqs = irqs;835 ctxi->pid = task_tgid_nr(current); /* tgid = pid */836 ctxi->ctx = ctx;837 ctxi->cfg = cfg;838 ctxi->file = file;839 ctxi->initialized = true;840 mutex_init(&ctxi->mutex);841 kref_init(&ctxi->kref);842 INIT_LIST_HEAD(&ctxi->luns);843 INIT_LIST_HEAD(&ctxi->list); /* initialize for list_empty() */844}845 846/**847 * remove_context() - context kref release handler848 * @kref: Kernel reference associated with context to be removed.849 *850 * When a context no longer has any references it can safely be removed851 * from global access and destroyed. Note that it is assumed the thread852 * relinquishing access to the context holds its mutex.853 */854static void remove_context(struct kref *kref)855{856 struct ctx_info *ctxi = container_of(kref, struct ctx_info, kref);857 struct cxlflash_cfg *cfg = ctxi->cfg;858 u64 ctxid = DECODE_CTXID(ctxi->ctxid);859 860 /* Remove context from table/error list */861 WARN_ON(!mutex_is_locked(&ctxi->mutex));862 ctxi->unavail = true;863 mutex_unlock(&ctxi->mutex);864 mutex_lock(&cfg->ctx_tbl_list_mutex);865 mutex_lock(&ctxi->mutex);866 867 if (!list_empty(&ctxi->list))868 list_del(&ctxi->list);869 cfg->ctx_tbl[ctxid] = NULL;870 mutex_unlock(&cfg->ctx_tbl_list_mutex);871 mutex_unlock(&ctxi->mutex);872 873 /* Context now completely uncoupled/unreachable */874 destroy_context(cfg, ctxi);875}876 877/**878 * _cxlflash_disk_detach() - detaches a LUN from a context879 * @sdev: SCSI device associated with LUN.880 * @ctxi: Context owning resources.881 * @detach: Detach ioctl data structure.882 *883 * As part of the detach, all per-context resources associated with the LUN884 * are cleaned up. When detaching the last LUN for a context, the context885 * itself is cleaned up and released.886 *887 * Return: 0 on success, -errno on failure888 */889static int _cxlflash_disk_detach(struct scsi_device *sdev,890 struct ctx_info *ctxi,891 struct dk_cxlflash_detach *detach)892{893 struct cxlflash_cfg *cfg = shost_priv(sdev->host);894 struct device *dev = &cfg->dev->dev;895 struct llun_info *lli = sdev->hostdata;896 struct lun_access *lun_access, *t;897 struct dk_cxlflash_release rel;898 bool put_ctx = false;899 900 int i;901 int rc = 0;902 u64 ctxid = DECODE_CTXID(detach->context_id),903 rctxid = detach->context_id;904 905 dev_dbg(dev, "%s: ctxid=%llu\n", __func__, ctxid);906 907 if (!ctxi) {908 ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);909 if (unlikely(!ctxi)) {910 dev_dbg(dev, "%s: Bad context ctxid=%llu\n",911 __func__, ctxid);912 rc = -EINVAL;913 goto out;914 }915 916 put_ctx = true;917 }918 919 /* Cleanup outstanding resources tied to this LUN */920 if (ctxi->rht_out) {921 marshal_det_to_rele(detach, &rel);922 for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) {923 if (ctxi->rht_lun[i] == lli) {924 rel.rsrc_handle = i;925 _cxlflash_disk_release(sdev, ctxi, &rel);926 }927 928 /* No need to loop further if we're done */929 if (ctxi->rht_out == 0)930 break;931 }932 }933 934 /* Take our LUN out of context, free the node */935 list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)936 if (lun_access->lli == lli) {937 list_del(&lun_access->list);938 kfree(lun_access);939 lun_access = NULL;940 break;941 }942 943 /*944 * Release the context reference and the sdev reference that945 * bound this LUN to the context.946 */947 if (kref_put(&ctxi->kref, remove_context))948 put_ctx = false;949 scsi_device_put(sdev);950out:951 if (put_ctx)952 put_context(ctxi);953 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);954 return rc;955}956 957static int cxlflash_disk_detach(struct scsi_device *sdev, void *detach)958{959 return _cxlflash_disk_detach(sdev, NULL, detach);960}961 962/**963 * cxlflash_cxl_release() - release handler for adapter file descriptor964 * @inode: File-system inode associated with fd.965 * @file: File installed with adapter file descriptor.966 *967 * This routine is the release handler for the fops registered with968 * the CXL services on an initial attach for a context. It is called969 * when a close (explicity by the user or as part of a process tear970 * down) is performed on the adapter file descriptor returned to the971 * user. The user should be aware that explicitly performing a close972 * considered catastrophic and subsequent usage of the superpipe API973 * with previously saved off tokens will fail.974 *975 * This routine derives the context reference and calls detach for976 * each LUN associated with the context.The final detach operation977 * causes the context itself to be freed. With exception to when the978 * CXL process element (context id) lookup fails (a case that should979 * theoretically never occur), every call into this routine results980 * in a complete freeing of a context.981 *982 * Detaching the LUN is typically an ioctl() operation and the underlying983 * code assumes that ioctl_rwsem has been acquired as a reader. To support984 * that design point, the semaphore is acquired and released around detach.985 *986 * Return: 0 on success987 */988static int cxlflash_cxl_release(struct inode *inode, struct file *file)989{990 struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,991 cxl_fops);992 void *ctx = cfg->ops->fops_get_context(file);993 struct device *dev = &cfg->dev->dev;994 struct ctx_info *ctxi = NULL;995 struct dk_cxlflash_detach detach = { { 0 }, 0 };996 struct lun_access *lun_access, *t;997 enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;998 int ctxid;999 1000 ctxid = cfg->ops->process_element(ctx);1001 if (unlikely(ctxid < 0)) {1002 dev_err(dev, "%s: Context %p was closed ctxid=%d\n",1003 __func__, ctx, ctxid);1004 goto out;1005 }1006 1007 ctxi = get_context(cfg, ctxid, file, ctrl);1008 if (unlikely(!ctxi)) {1009 ctxi = get_context(cfg, ctxid, file, ctrl | CTX_CTRL_CLONE);1010 if (!ctxi) {1011 dev_dbg(dev, "%s: ctxid=%d already free\n",1012 __func__, ctxid);1013 goto out_release;1014 }1015 1016 dev_dbg(dev, "%s: Another process owns ctxid=%d\n",1017 __func__, ctxid);1018 put_context(ctxi);1019 goto out;1020 }1021 1022 dev_dbg(dev, "%s: close for ctxid=%d\n", __func__, ctxid);1023 1024 down_read(&cfg->ioctl_rwsem);1025 detach.context_id = ctxi->ctxid;1026 list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)1027 _cxlflash_disk_detach(lun_access->sdev, ctxi, &detach);1028 up_read(&cfg->ioctl_rwsem);1029out_release:1030 cfg->ops->fd_release(inode, file);1031out:1032 dev_dbg(dev, "%s: returning\n", __func__);1033 return 0;1034}1035 1036/**1037 * unmap_context() - clears a previously established mapping1038 * @ctxi: Context owning the mapping.1039 *1040 * This routine is used to switch between the error notification page1041 * (dummy page of all 1's) and the real mapping (established by the CXL1042 * fault handler).1043 */1044static void unmap_context(struct ctx_info *ctxi)1045{1046 unmap_mapping_range(ctxi->file->f_mapping, 0, 0, 1);1047}1048 1049/**1050 * get_err_page() - obtains and allocates the error notification page1051 * @cfg: Internal structure associated with the host.1052 *1053 * Return: error notification page on success, NULL on failure1054 */1055static struct page *get_err_page(struct cxlflash_cfg *cfg)1056{1057 struct page *err_page = global.err_page;1058 struct device *dev = &cfg->dev->dev;1059 1060 if (unlikely(!err_page)) {1061 err_page = alloc_page(GFP_KERNEL);1062 if (unlikely(!err_page)) {1063 dev_err(dev, "%s: Unable to allocate err_page\n",1064 __func__);1065 goto out;1066 }1067 1068 memset(page_address(err_page), -1, PAGE_SIZE);1069 1070 /* Serialize update w/ other threads to avoid a leak */1071 mutex_lock(&global.mutex);1072 if (likely(!global.err_page))1073 global.err_page = err_page;1074 else {1075 __free_page(err_page);1076 err_page = global.err_page;1077 }1078 mutex_unlock(&global.mutex);1079 }1080 1081out:1082 dev_dbg(dev, "%s: returning err_page=%p\n", __func__, err_page);1083 return err_page;1084}1085 1086/**1087 * cxlflash_mmap_fault() - mmap fault handler for adapter file descriptor1088 * @vmf: VM fault associated with current fault.1089 *1090 * To support error notification via MMIO, faults are 'caught' by this routine1091 * that was inserted before passing back the adapter file descriptor on attach.1092 * When a fault occurs, this routine evaluates if error recovery is active and1093 * if so, installs the error page to 'notify' the user about the error state.1094 * During normal operation, the fault is simply handled by the original fault1095 * handler that was installed by CXL services as part of initializing the1096 * adapter file descriptor. The VMA's page protection bits are toggled to1097 * indicate cached/not-cached depending on the memory backing the fault.1098 *1099 * Return: 0 on success, VM_FAULT_SIGBUS on failure1100 */1101static vm_fault_t cxlflash_mmap_fault(struct vm_fault *vmf)1102{1103 struct vm_area_struct *vma = vmf->vma;1104 struct file *file = vma->vm_file;1105 struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,1106 cxl_fops);1107 void *ctx = cfg->ops->fops_get_context(file);1108 struct device *dev = &cfg->dev->dev;1109 struct ctx_info *ctxi = NULL;1110 struct page *err_page = NULL;1111 enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;1112 vm_fault_t rc = 0;1113 int ctxid;1114 1115 ctxid = cfg->ops->process_element(ctx);1116 if (unlikely(ctxid < 0)) {1117 dev_err(dev, "%s: Context %p was closed ctxid=%d\n",1118 __func__, ctx, ctxid);1119 goto err;1120 }1121 1122 ctxi = get_context(cfg, ctxid, file, ctrl);1123 if (unlikely(!ctxi)) {1124 dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);1125 goto err;1126 }1127 1128 dev_dbg(dev, "%s: fault for context %d\n", __func__, ctxid);1129 1130 if (likely(!ctxi->err_recovery_active)) {1131 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);1132 rc = ctxi->cxl_mmap_vmops->fault(vmf);1133 } else {1134 dev_dbg(dev, "%s: err recovery active, use err_page\n",1135 __func__);1136 1137 err_page = get_err_page(cfg);1138 if (unlikely(!err_page)) {1139 dev_err(dev, "%s: Could not get err_page\n", __func__);1140 rc = VM_FAULT_RETRY;1141 goto out;1142 }1143 1144 get_page(err_page);1145 vmf->page = err_page;1146 vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);1147 }1148 1149out:1150 if (likely(ctxi))1151 put_context(ctxi);1152 dev_dbg(dev, "%s: returning rc=%x\n", __func__, rc);1153 return rc;1154 1155err:1156 rc = VM_FAULT_SIGBUS;1157 goto out;1158}1159 1160/*1161 * Local MMAP vmops to 'catch' faults1162 */1163static const struct vm_operations_struct cxlflash_mmap_vmops = {1164 .fault = cxlflash_mmap_fault,1165};1166 1167/**1168 * cxlflash_cxl_mmap() - mmap handler for adapter file descriptor1169 * @file: File installed with adapter file descriptor.1170 * @vma: VM area associated with mapping.1171 *1172 * Installs local mmap vmops to 'catch' faults for error notification support.1173 *1174 * Return: 0 on success, -errno on failure1175 */1176static int cxlflash_cxl_mmap(struct file *file, struct vm_area_struct *vma)1177{1178 struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,1179 cxl_fops);1180 void *ctx = cfg->ops->fops_get_context(file);1181 struct device *dev = &cfg->dev->dev;1182 struct ctx_info *ctxi = NULL;1183 enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;1184 int ctxid;1185 int rc = 0;1186 1187 ctxid = cfg->ops->process_element(ctx);1188 if (unlikely(ctxid < 0)) {1189 dev_err(dev, "%s: Context %p was closed ctxid=%d\n",1190 __func__, ctx, ctxid);1191 rc = -EIO;1192 goto out;1193 }1194 1195 ctxi = get_context(cfg, ctxid, file, ctrl);1196 if (unlikely(!ctxi)) {1197 dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);1198 rc = -EIO;1199 goto out;1200 }1201 1202 dev_dbg(dev, "%s: mmap for context %d\n", __func__, ctxid);1203 1204 rc = cfg->ops->fd_mmap(file, vma);1205 if (likely(!rc)) {1206 /* Insert ourself in the mmap fault handler path */1207 ctxi->cxl_mmap_vmops = vma->vm_ops;1208 vma->vm_ops = &cxlflash_mmap_vmops;1209 }1210 1211out:1212 if (likely(ctxi))1213 put_context(ctxi);1214 return rc;1215}1216 1217const struct file_operations cxlflash_cxl_fops = {1218 .owner = THIS_MODULE,1219 .mmap = cxlflash_cxl_mmap,1220 .release = cxlflash_cxl_release,1221};1222 1223/**1224 * cxlflash_mark_contexts_error() - move contexts to error state and list1225 * @cfg: Internal structure associated with the host.1226 *1227 * A context is only moved over to the error list when there are no outstanding1228 * references to it. This ensures that a running operation has completed.1229 *1230 * Return: 0 on success, -errno on failure1231 */1232int cxlflash_mark_contexts_error(struct cxlflash_cfg *cfg)1233{1234 int i, rc = 0;1235 struct ctx_info *ctxi = NULL;1236 1237 mutex_lock(&cfg->ctx_tbl_list_mutex);1238 1239 for (i = 0; i < MAX_CONTEXT; i++) {1240 ctxi = cfg->ctx_tbl[i];1241 if (ctxi) {1242 mutex_lock(&ctxi->mutex);1243 cfg->ctx_tbl[i] = NULL;1244 list_add(&ctxi->list, &cfg->ctx_err_recovery);1245 ctxi->err_recovery_active = true;1246 ctxi->ctrl_map = NULL;1247 unmap_context(ctxi);1248 mutex_unlock(&ctxi->mutex);1249 }1250 }1251 1252 mutex_unlock(&cfg->ctx_tbl_list_mutex);1253 return rc;1254}1255 1256/*1257 * Dummy NULL fops1258 */1259static const struct file_operations null_fops = {1260 .owner = THIS_MODULE,1261};1262 1263/**1264 * check_state() - checks and responds to the current adapter state1265 * @cfg: Internal structure associated with the host.1266 *1267 * This routine can block and should only be used on process context.1268 * It assumes that the caller is an ioctl thread and holding the ioctl1269 * read semaphore. This is temporarily let up across the wait to allow1270 * for draining actively running ioctls. Also note that when waking up1271 * from waiting in reset, the state is unknown and must be checked again1272 * before proceeding.1273 *1274 * Return: 0 on success, -errno on failure1275 */1276int check_state(struct cxlflash_cfg *cfg)1277{1278 struct device *dev = &cfg->dev->dev;1279 int rc = 0;1280 1281retry:1282 switch (cfg->state) {1283 case STATE_RESET:1284 dev_dbg(dev, "%s: Reset state, going to wait...\n", __func__);1285 up_read(&cfg->ioctl_rwsem);1286 rc = wait_event_interruptible(cfg->reset_waitq,1287 cfg->state != STATE_RESET);1288 down_read(&cfg->ioctl_rwsem);1289 if (unlikely(rc))1290 break;1291 goto retry;1292 case STATE_FAILTERM:1293 dev_dbg(dev, "%s: Failed/Terminating\n", __func__);1294 rc = -ENODEV;1295 break;1296 default:1297 break;1298 }1299 1300 return rc;1301}1302 1303/**1304 * cxlflash_disk_attach() - attach a LUN to a context1305 * @sdev: SCSI device associated with LUN.1306 * @arg: Attach ioctl data structure.1307 *1308 * Creates a context and attaches LUN to it. A LUN can only be attached1309 * one time to a context (subsequent attaches for the same context/LUN pair1310 * are not supported). Additional LUNs can be attached to a context by1311 * specifying the 'reuse' flag defined in the cxlflash_ioctl.h header.1312 *1313 * Return: 0 on success, -errno on failure1314 */1315static int cxlflash_disk_attach(struct scsi_device *sdev, void *arg)1316{1317 struct dk_cxlflash_attach *attach = arg;1318 struct cxlflash_cfg *cfg = shost_priv(sdev->host);1319 struct device *dev = &cfg->dev->dev;1320 struct afu *afu = cfg->afu;1321 struct llun_info *lli = sdev->hostdata;1322 struct glun_info *gli = lli->parent;1323 struct ctx_info *ctxi = NULL;1324 struct lun_access *lun_access = NULL;1325 int rc = 0;1326 u32 perms;1327 int ctxid = -1;1328 u64 irqs = attach->num_interrupts;1329 u64 flags = 0UL;1330 u64 rctxid = 0UL;1331 struct file *file = NULL;1332 1333 void *ctx = NULL;1334 1335 int fd = -1;1336 1337 if (irqs > 4) {1338 dev_dbg(dev, "%s: Cannot support this many interrupts %llu\n",1339 __func__, irqs);1340 rc = -EINVAL;1341 goto out;1342 }1343 1344 if (gli->max_lba == 0) {1345 dev_dbg(dev, "%s: No capacity info for LUN=%016llx\n",1346 __func__, lli->lun_id[sdev->channel]);1347 rc = read_cap16(sdev, lli);1348 if (rc) {1349 dev_err(dev, "%s: Invalid device rc=%d\n",1350 __func__, rc);1351 rc = -ENODEV;1352 goto out;1353 }1354 dev_dbg(dev, "%s: LBA = %016llx\n", __func__, gli->max_lba);1355 dev_dbg(dev, "%s: BLK_LEN = %08x\n", __func__, gli->blk_len);1356 }1357 1358 if (attach->hdr.flags & DK_CXLFLASH_ATTACH_REUSE_CONTEXT) {1359 rctxid = attach->context_id;1360 ctxi = get_context(cfg, rctxid, NULL, 0);1361 if (!ctxi) {1362 dev_dbg(dev, "%s: Bad context rctxid=%016llx\n",1363 __func__, rctxid);1364 rc = -EINVAL;1365 goto out;1366 }1367 1368 list_for_each_entry(lun_access, &ctxi->luns, list)1369 if (lun_access->lli == lli) {1370 dev_dbg(dev, "%s: Already attached\n",1371 __func__);1372 rc = -EINVAL;1373 goto out;1374 }1375 }1376 1377 rc = scsi_device_get(sdev);1378 if (unlikely(rc)) {1379 dev_err(dev, "%s: Unable to get sdev reference\n", __func__);1380 goto out;1381 }1382 1383 lun_access = kzalloc(sizeof(*lun_access), GFP_KERNEL);1384 if (unlikely(!lun_access)) {1385 dev_err(dev, "%s: Unable to allocate lun_access\n", __func__);1386 rc = -ENOMEM;1387 goto err;1388 }1389 1390 lun_access->lli = lli;1391 lun_access->sdev = sdev;1392 1393 /* Non-NULL context indicates reuse (another context reference) */1394 if (ctxi) {1395 dev_dbg(dev, "%s: Reusing context for LUN rctxid=%016llx\n",1396 __func__, rctxid);1397 kref_get(&ctxi->kref);1398 list_add(&lun_access->list, &ctxi->luns);1399 goto out_attach;1400 }1401 1402 ctxi = create_context(cfg);1403 if (unlikely(!ctxi)) {1404 dev_err(dev, "%s: Failed to create context ctxid=%d\n",1405 __func__, ctxid);1406 rc = -ENOMEM;1407 goto err;1408 }1409 1410 ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);1411 if (IS_ERR_OR_NULL(ctx)) {1412 dev_err(dev, "%s: Could not initialize context %p\n",1413 __func__, ctx);1414 rc = -ENODEV;1415 goto err;1416 }1417 1418 rc = cfg->ops->start_work(ctx, irqs);1419 if (unlikely(rc)) {1420 dev_dbg(dev, "%s: Could not start context rc=%d\n",1421 __func__, rc);1422 goto err;1423 }1424 1425 ctxid = cfg->ops->process_element(ctx);1426 if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {1427 dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);1428 rc = -EPERM;1429 goto err;1430 }1431 1432 file = cfg->ops->get_fd(ctx, &cfg->cxl_fops, &fd);1433 if (unlikely(fd < 0)) {1434 rc = -ENODEV;1435 dev_err(dev, "%s: Could not get file descriptor\n", __func__);1436 goto err;1437 }1438 1439 /* Translate read/write O_* flags from fcntl.h to AFU permission bits */1440 perms = SISL_RHT_PERM(attach->hdr.flags + 1);1441 1442 /* Context mutex is locked upon return */1443 init_context(ctxi, cfg, ctx, ctxid, file, perms, irqs);1444 1445 rc = afu_attach(cfg, ctxi);1446 if (unlikely(rc)) {1447 dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);1448 goto err;1449 }1450 1451 /*1452 * No error paths after this point. Once the fd is installed it's1453 * visible to user space and can't be undone safely on this thread.1454 * There is no need to worry about a deadlock here because no one1455 * knows about us yet; we can be the only one holding our mutex.1456 */1457 list_add(&lun_access->list, &ctxi->luns);1458 mutex_lock(&cfg->ctx_tbl_list_mutex);1459 mutex_lock(&ctxi->mutex);1460 cfg->ctx_tbl[ctxid] = ctxi;1461 mutex_unlock(&cfg->ctx_tbl_list_mutex);1462 fd_install(fd, file);1463 1464out_attach:1465 if (fd != -1)1466 flags |= DK_CXLFLASH_APP_CLOSE_ADAP_FD;1467 if (afu_is_sq_cmd_mode(afu))1468 flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;1469 1470 attach->hdr.return_flags = flags;1471 attach->context_id = ctxi->ctxid;1472 attach->block_size = gli->blk_len;1473 attach->mmio_size = sizeof(afu->afu_map->hosts[0].harea);1474 attach->last_lba = gli->max_lba;1475 attach->max_xfer = sdev->host->max_sectors * MAX_SECTOR_UNIT;1476 attach->max_xfer /= gli->blk_len;1477 1478out:1479 attach->adap_fd = fd;1480 1481 if (ctxi)1482 put_context(ctxi);1483 1484 dev_dbg(dev, "%s: returning ctxid=%d fd=%d bs=%lld rc=%d llba=%lld\n",1485 __func__, ctxid, fd, attach->block_size, rc, attach->last_lba);1486 return rc;1487 1488err:1489 /* Cleanup CXL context; okay to 'stop' even if it was not started */1490 if (!IS_ERR_OR_NULL(ctx)) {1491 cfg->ops->stop_context(ctx);1492 cfg->ops->release_context(ctx);1493 ctx = NULL;1494 }1495 1496 /*1497 * Here, we're overriding the fops with a dummy all-NULL fops because1498 * fput() calls the release fop, which will cause us to mistakenly1499 * call into the CXL code. Rather than try to add yet more complexity1500 * to that routine (cxlflash_cxl_release) we should try to fix the1501 * issue here.1502 */1503 if (fd > 0) {1504 file->f_op = &null_fops;1505 fput(file);1506 put_unused_fd(fd);1507 fd = -1;1508 file = NULL;1509 }1510 1511 /* Cleanup our context */1512 if (ctxi) {1513 destroy_context(cfg, ctxi);1514 ctxi = NULL;1515 }1516 1517 kfree(lun_access);1518 scsi_device_put(sdev);1519 goto out;1520}1521 1522/**1523 * recover_context() - recovers a context in error1524 * @cfg: Internal structure associated with the host.1525 * @ctxi: Context to release.1526 * @adap_fd: Adapter file descriptor associated with new/recovered context.1527 *1528 * Restablishes the state for a context-in-error.1529 *1530 * Return: 0 on success, -errno on failure1531 */1532static int recover_context(struct cxlflash_cfg *cfg,1533 struct ctx_info *ctxi,1534 int *adap_fd)1535{1536 struct device *dev = &cfg->dev->dev;1537 int rc = 0;1538 int fd = -1;1539 int ctxid = -1;1540 struct file *file;1541 void *ctx;1542 struct afu *afu = cfg->afu;1543 1544 ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);1545 if (IS_ERR_OR_NULL(ctx)) {1546 dev_err(dev, "%s: Could not initialize context %p\n",1547 __func__, ctx);1548 rc = -ENODEV;1549 goto out;1550 }1551 1552 rc = cfg->ops->start_work(ctx, ctxi->irqs);1553 if (unlikely(rc)) {1554 dev_dbg(dev, "%s: Could not start context rc=%d\n",1555 __func__, rc);1556 goto err1;1557 }1558 1559 ctxid = cfg->ops->process_element(ctx);1560 if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {1561 dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);1562 rc = -EPERM;1563 goto err2;1564 }1565 1566 file = cfg->ops->get_fd(ctx, &cfg->cxl_fops, &fd);1567 if (unlikely(fd < 0)) {1568 rc = -ENODEV;1569 dev_err(dev, "%s: Could not get file descriptor\n", __func__);1570 goto err2;1571 }1572 1573 /* Update with new MMIO area based on updated context id */1574 ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;1575 1576 rc = afu_attach(cfg, ctxi);1577 if (rc) {1578 dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);1579 goto err3;1580 }1581 1582 /*1583 * No error paths after this point. Once the fd is installed it's1584 * visible to user space and can't be undone safely on this thread.1585 */1586 ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);1587 ctxi->ctx = ctx;1588 ctxi->file = file;1589 1590 /*1591 * Put context back in table (note the reinit of the context list);1592 * we must first drop the context's mutex and then acquire it in1593 * order with the table/list mutex to avoid a deadlock - safe to do1594 * here because no one can find us at this moment in time.1595 */1596 mutex_unlock(&ctxi->mutex);1597 mutex_lock(&cfg->ctx_tbl_list_mutex);1598 mutex_lock(&ctxi->mutex);1599 list_del_init(&ctxi->list);1600 cfg->ctx_tbl[ctxid] = ctxi;1601 mutex_unlock(&cfg->ctx_tbl_list_mutex);1602 fd_install(fd, file);1603 *adap_fd = fd;1604out:1605 dev_dbg(dev, "%s: returning ctxid=%d fd=%d rc=%d\n",1606 __func__, ctxid, fd, rc);1607 return rc;1608 1609err3:1610 fput(file);1611 put_unused_fd(fd);1612err2:1613 cfg->ops->stop_context(ctx);1614err1:1615 cfg->ops->release_context(ctx);1616 goto out;1617}1618 1619/**1620 * cxlflash_afu_recover() - initiates AFU recovery1621 * @sdev: SCSI device associated with LUN.1622 * @arg: Recover ioctl data structure.1623 *1624 * Only a single recovery is allowed at a time to avoid exhausting CXL1625 * resources (leading to recovery failure) in the event that we're up1626 * against the maximum number of contexts limit. For similar reasons,1627 * a context recovery is retried if there are multiple recoveries taking1628 * place at the same time and the failure was due to CXL services being1629 * unable to keep up.1630 *1631 * As this routine is called on ioctl context, it holds the ioctl r/w1632 * semaphore that is used to drain ioctls in recovery scenarios. The1633 * implementation to achieve the pacing described above (a local mutex)1634 * requires that the ioctl r/w semaphore be dropped and reacquired to1635 * avoid a 3-way deadlock when multiple process recoveries operate in1636 * parallel.1637 *1638 * Because a user can detect an error condition before the kernel, it is1639 * quite possible for this routine to act as the kernel's EEH detection1640 * source (MMIO read of mbox_r). Because of this, there is a window of1641 * time where an EEH might have been detected but not yet 'serviced'1642 * (callback invoked, causing the device to enter reset state). To avoid1643 * looping in this routine during that window, a 1 second sleep is in place1644 * between the time the MMIO failure is detected and the time a wait on the1645 * reset wait queue is attempted via check_state().1646 *1647 * Return: 0 on success, -errno on failure1648 */1649static int cxlflash_afu_recover(struct scsi_device *sdev, void *arg)1650{1651 struct dk_cxlflash_recover_afu *recover = arg;1652 struct cxlflash_cfg *cfg = shost_priv(sdev->host);1653 struct device *dev = &cfg->dev->dev;1654 struct llun_info *lli = sdev->hostdata;1655 struct afu *afu = cfg->afu;1656 struct ctx_info *ctxi = NULL;1657 struct mutex *mutex = &cfg->ctx_recovery_mutex;1658 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);1659 u64 flags;1660 u64 ctxid = DECODE_CTXID(recover->context_id),1661 rctxid = recover->context_id;1662 long reg;1663 bool locked = true;1664 int lretry = 20; /* up to 2 seconds */1665 int new_adap_fd = -1;1666 int rc = 0;1667 1668 atomic_inc(&cfg->recovery_threads);1669 up_read(&cfg->ioctl_rwsem);1670 rc = mutex_lock_interruptible(mutex);1671 down_read(&cfg->ioctl_rwsem);1672 if (rc) {1673 locked = false;1674 goto out;1675 }1676 1677 rc = check_state(cfg);1678 if (rc) {1679 dev_err(dev, "%s: Failed state rc=%d\n", __func__, rc);1680 rc = -ENODEV;1681 goto out;1682 }1683 1684 dev_dbg(dev, "%s: reason=%016llx rctxid=%016llx\n",1685 __func__, recover->reason, rctxid);1686 1687retry:1688 /* Ensure that this process is attached to the context */1689 ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);1690 if (unlikely(!ctxi)) {1691 dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);1692 rc = -EINVAL;1693 goto out;1694 }1695 1696 if (ctxi->err_recovery_active) {1697retry_recover:1698 rc = recover_context(cfg, ctxi, &new_adap_fd);1699 if (unlikely(rc)) {1700 dev_err(dev, "%s: Recovery failed ctxid=%llu rc=%d\n",1701 __func__, ctxid, rc);1702 if ((rc == -ENODEV) &&1703 ((atomic_read(&cfg->recovery_threads) > 1) ||1704 (lretry--))) {1705 dev_dbg(dev, "%s: Going to try again\n",1706 __func__);1707 mutex_unlock(mutex);1708 msleep(100);1709 rc = mutex_lock_interruptible(mutex);1710 if (rc) {1711 locked = false;1712 goto out;1713 }1714 goto retry_recover;1715 }1716 1717 goto out;1718 }1719 1720 ctxi->err_recovery_active = false;1721 1722 flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD |1723 DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET;1724 if (afu_is_sq_cmd_mode(afu))1725 flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;1726 1727 recover->hdr.return_flags = flags;1728 recover->context_id = ctxi->ctxid;1729 recover->adap_fd = new_adap_fd;1730 recover->mmio_size = sizeof(afu->afu_map->hosts[0].harea);1731 goto out;1732 }1733 1734 /* Test if in error state */1735 reg = readq_be(&hwq->ctrl_map->mbox_r);1736 if (reg == -1) {1737 dev_dbg(dev, "%s: MMIO fail, wait for recovery.\n", __func__);1738 1739 /*1740 * Before checking the state, put back the context obtained with1741 * get_context() as it is no longer needed and sleep for a short1742 * period of time (see prolog notes).1743 */1744 put_context(ctxi);1745 ctxi = NULL;1746 ssleep(1);1747 rc = check_state(cfg);1748 if (unlikely(rc))1749 goto out;1750 goto retry;1751 }1752 1753 dev_dbg(dev, "%s: MMIO working, no recovery required\n", __func__);1754out:1755 if (likely(ctxi))1756 put_context(ctxi);1757 if (locked)1758 mutex_unlock(mutex);1759 atomic_dec_if_positive(&cfg->recovery_threads);1760 return rc;1761}1762 1763/**1764 * process_sense() - evaluates and processes sense data1765 * @sdev: SCSI device associated with LUN.1766 * @verify: Verify ioctl data structure.1767 *1768 * Return: 0 on success, -errno on failure1769 */1770static int process_sense(struct scsi_device *sdev,1771 struct dk_cxlflash_verify *verify)1772{1773 struct cxlflash_cfg *cfg = shost_priv(sdev->host);1774 struct device *dev = &cfg->dev->dev;1775 struct llun_info *lli = sdev->hostdata;1776 struct glun_info *gli = lli->parent;1777 u64 prev_lba = gli->max_lba;1778 struct scsi_sense_hdr sshdr = { 0 };1779 int rc = 0;1780 1781 rc = scsi_normalize_sense((const u8 *)&verify->sense_data,1782 DK_CXLFLASH_VERIFY_SENSE_LEN, &sshdr);1783 if (!rc) {1784 dev_err(dev, "%s: Failed to normalize sense data\n", __func__);1785 rc = -EINVAL;1786 goto out;1787 }1788 1789 switch (sshdr.sense_key) {1790 case NO_SENSE:1791 case RECOVERED_ERROR:1792 case NOT_READY:1793 break;1794 case UNIT_ATTENTION:1795 switch (sshdr.asc) {1796 case 0x29: /* Power on Reset or Device Reset */1797 fallthrough;1798 case 0x2A: /* Device settings/capacity changed */1799 rc = read_cap16(sdev, lli);1800 if (rc) {1801 rc = -ENODEV;1802 break;1803 }1804 if (prev_lba != gli->max_lba)1805 dev_dbg(dev, "%s: Capacity changed old=%lld "1806 "new=%lld\n", __func__, prev_lba,1807 gli->max_lba);1808 break;1809 case 0x3F: /* Report LUNs changed, Rescan. */1810 scsi_scan_host(cfg->host);1811 break;1812 default:1813 rc = -EIO;1814 break;1815 }1816 break;1817 default:1818 rc = -EIO;1819 break;1820 }1821out:1822 dev_dbg(dev, "%s: sense_key %x asc %x ascq %x rc %d\n", __func__,1823 sshdr.sense_key, sshdr.asc, sshdr.ascq, rc);1824 return rc;1825}1826 1827/**1828 * cxlflash_disk_verify() - verifies a LUN is the same and handle size changes1829 * @sdev: SCSI device associated with LUN.1830 * @arg: Verify ioctl data structure.1831 *1832 * Return: 0 on success, -errno on failure1833 */1834static int cxlflash_disk_verify(struct scsi_device *sdev, void *arg)1835{1836 struct dk_cxlflash_verify *verify = arg;1837 int rc = 0;1838 struct ctx_info *ctxi = NULL;1839 struct cxlflash_cfg *cfg = shost_priv(sdev->host);1840 struct device *dev = &cfg->dev->dev;1841 struct llun_info *lli = sdev->hostdata;1842 struct glun_info *gli = lli->parent;1843 struct sisl_rht_entry *rhte = NULL;1844 res_hndl_t rhndl = verify->rsrc_handle;1845 u64 ctxid = DECODE_CTXID(verify->context_id),1846 rctxid = verify->context_id;1847 u64 last_lba = 0;1848 1849 dev_dbg(dev, "%s: ctxid=%llu rhndl=%016llx, hint=%016llx, "1850 "flags=%016llx\n", __func__, ctxid, verify->rsrc_handle,1851 verify->hint, verify->hdr.flags);1852 1853 ctxi = get_context(cfg, rctxid, lli, 0);1854 if (unlikely(!ctxi)) {1855 dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);1856 rc = -EINVAL;1857 goto out;1858 }1859 1860 rhte = get_rhte(ctxi, rhndl, lli);1861 if (unlikely(!rhte)) {1862 dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",1863 __func__, rhndl);1864 rc = -EINVAL;1865 goto out;1866 }1867 1868 /*1869 * Look at the hint/sense to see if it requires us to redrive1870 * inquiry (i.e. the Unit attention is due to the WWN changing).1871 */1872 if (verify->hint & DK_CXLFLASH_VERIFY_HINT_SENSE) {1873 /* Can't hold mutex across process_sense/read_cap16,1874 * since we could have an intervening EEH event.1875 */1876 ctxi->unavail = true;1877 mutex_unlock(&ctxi->mutex);1878 rc = process_sense(sdev, verify);1879 if (unlikely(rc)) {1880 dev_err(dev, "%s: Failed to validate sense data (%d)\n",1881 __func__, rc);1882 mutex_lock(&ctxi->mutex);1883 ctxi->unavail = false;1884 goto out;1885 }1886 mutex_lock(&ctxi->mutex);1887 ctxi->unavail = false;1888 }1889 1890 switch (gli->mode) {1891 case MODE_PHYSICAL:1892 last_lba = gli->max_lba;1893 break;1894 case MODE_VIRTUAL:1895 /* Cast lxt_cnt to u64 for multiply to be treated as 64bit op */1896 last_lba = ((u64)rhte->lxt_cnt * MC_CHUNK_SIZE * gli->blk_len);1897 last_lba /= CXLFLASH_BLOCK_SIZE;1898 last_lba--;1899 break;1900 default:1901 WARN(1, "Unsupported LUN mode!");1902 }1903 1904 verify->last_lba = last_lba;1905 1906out:1907 if (likely(ctxi))1908 put_context(ctxi);1909 dev_dbg(dev, "%s: returning rc=%d llba=%llx\n",1910 __func__, rc, verify->last_lba);1911 return rc;1912}1913 1914/**1915 * decode_ioctl() - translates an encoded ioctl to an easily identifiable string1916 * @cmd: The ioctl command to decode.1917 *1918 * Return: A string identifying the decoded ioctl.1919 */1920static char *decode_ioctl(unsigned int cmd)1921{1922 switch (cmd) {1923 case DK_CXLFLASH_ATTACH:1924 return __stringify_1(DK_CXLFLASH_ATTACH);1925 case DK_CXLFLASH_USER_DIRECT:1926 return __stringify_1(DK_CXLFLASH_USER_DIRECT);1927 case DK_CXLFLASH_USER_VIRTUAL:1928 return __stringify_1(DK_CXLFLASH_USER_VIRTUAL);1929 case DK_CXLFLASH_VLUN_RESIZE:1930 return __stringify_1(DK_CXLFLASH_VLUN_RESIZE);1931 case DK_CXLFLASH_RELEASE:1932 return __stringify_1(DK_CXLFLASH_RELEASE);1933 case DK_CXLFLASH_DETACH:1934 return __stringify_1(DK_CXLFLASH_DETACH);1935 case DK_CXLFLASH_VERIFY:1936 return __stringify_1(DK_CXLFLASH_VERIFY);1937 case DK_CXLFLASH_VLUN_CLONE:1938 return __stringify_1(DK_CXLFLASH_VLUN_CLONE);1939 case DK_CXLFLASH_RECOVER_AFU:1940 return __stringify_1(DK_CXLFLASH_RECOVER_AFU);1941 case DK_CXLFLASH_MANAGE_LUN:1942 return __stringify_1(DK_CXLFLASH_MANAGE_LUN);1943 }1944 1945 return "UNKNOWN";1946}1947 1948/**1949 * cxlflash_disk_direct_open() - opens a direct (physical) disk1950 * @sdev: SCSI device associated with LUN.1951 * @arg: UDirect ioctl data structure.1952 *1953 * On successful return, the user is informed of the resource handle1954 * to be used to identify the direct lun and the size (in blocks) of1955 * the direct lun in last LBA format.1956 *1957 * Return: 0 on success, -errno on failure1958 */1959static int cxlflash_disk_direct_open(struct scsi_device *sdev, void *arg)1960{1961 struct cxlflash_cfg *cfg = shost_priv(sdev->host);1962 struct device *dev = &cfg->dev->dev;1963 struct afu *afu = cfg->afu;1964 struct llun_info *lli = sdev->hostdata;1965 struct glun_info *gli = lli->parent;1966 struct dk_cxlflash_release rel = { { 0 }, 0 };1967 1968 struct dk_cxlflash_udirect *pphys = (struct dk_cxlflash_udirect *)arg;1969 1970 u64 ctxid = DECODE_CTXID(pphys->context_id),1971 rctxid = pphys->context_id;1972 u64 lun_size = 0;1973 u64 last_lba = 0;1974 u64 rsrc_handle = -1;1975 u32 port = CHAN2PORTMASK(sdev->channel);1976 1977 int rc = 0;1978 1979 struct ctx_info *ctxi = NULL;1980 struct sisl_rht_entry *rhte = NULL;1981 1982 dev_dbg(dev, "%s: ctxid=%llu ls=%llu\n", __func__, ctxid, lun_size);1983 1984 rc = cxlflash_lun_attach(gli, MODE_PHYSICAL, false);1985 if (unlikely(rc)) {1986 dev_dbg(dev, "%s: Failed attach to LUN (PHYSICAL)\n", __func__);1987 goto out;1988 }1989 1990 ctxi = get_context(cfg, rctxid, lli, 0);1991 if (unlikely(!ctxi)) {1992 dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);1993 rc = -EINVAL;1994 goto err1;1995 }1996 1997 rhte = rhte_checkout(ctxi, lli);1998 if (unlikely(!rhte)) {1999 dev_dbg(dev, "%s: Too many opens ctxid=%lld\n",2000 __func__, ctxid);2001 rc = -EMFILE; /* too many opens */2002 goto err1;2003 }2004 2005 rsrc_handle = (rhte - ctxi->rht_start);2006 2007 rht_format1(rhte, lli->lun_id[sdev->channel], ctxi->rht_perms, port);2008 2009 last_lba = gli->max_lba;2010 pphys->hdr.return_flags = 0;2011 pphys->last_lba = last_lba;2012 pphys->rsrc_handle = rsrc_handle;2013 2014 rc = cxlflash_afu_sync(afu, ctxid, rsrc_handle, AFU_LW_SYNC);2015 if (unlikely(rc)) {2016 dev_dbg(dev, "%s: AFU sync failed rc=%d\n", __func__, rc);2017 goto err2;2018 }2019 2020out:2021 if (likely(ctxi))2022 put_context(ctxi);2023 dev_dbg(dev, "%s: returning handle=%llu rc=%d llba=%llu\n",2024 __func__, rsrc_handle, rc, last_lba);2025 return rc;2026 2027err2:2028 marshal_udir_to_rele(pphys, &rel);2029 _cxlflash_disk_release(sdev, ctxi, &rel);2030 goto out;2031err1:2032 cxlflash_lun_detach(gli);2033 goto out;2034}2035 2036/**2037 * ioctl_common() - common IOCTL handler for driver2038 * @sdev: SCSI device associated with LUN.2039 * @cmd: IOCTL command.2040 *2041 * Handles common fencing operations that are valid for multiple ioctls. Always2042 * allow through ioctls that are cleanup oriented in nature, even when operating2043 * in a failed/terminating state.2044 *2045 * Return: 0 on success, -errno on failure2046 */2047static int ioctl_common(struct scsi_device *sdev, unsigned int cmd)2048{2049 struct cxlflash_cfg *cfg = shost_priv(sdev->host);2050 struct device *dev = &cfg->dev->dev;2051 struct llun_info *lli = sdev->hostdata;2052 int rc = 0;2053 2054 if (unlikely(!lli)) {2055 dev_dbg(dev, "%s: Unknown LUN\n", __func__);2056 rc = -EINVAL;2057 goto out;2058 }2059 2060 rc = check_state(cfg);2061 if (unlikely(rc) && (cfg->state == STATE_FAILTERM)) {2062 switch (cmd) {2063 case DK_CXLFLASH_VLUN_RESIZE:2064 case DK_CXLFLASH_RELEASE:2065 case DK_CXLFLASH_DETACH:2066 dev_dbg(dev, "%s: Command override rc=%d\n",2067 __func__, rc);2068 rc = 0;2069 break;2070 }2071 }2072out:2073 return rc;2074}2075 2076/**2077 * cxlflash_ioctl() - IOCTL handler for driver2078 * @sdev: SCSI device associated with LUN.2079 * @cmd: IOCTL command.2080 * @arg: Userspace ioctl data structure.2081 *2082 * A read/write semaphore is used to implement a 'drain' of currently2083 * running ioctls. The read semaphore is taken at the beginning of each2084 * ioctl thread and released upon concluding execution. Additionally the2085 * semaphore should be released and then reacquired in any ioctl execution2086 * path which will wait for an event to occur that is outside the scope of2087 * the ioctl (i.e. an adapter reset). To drain the ioctls currently running,2088 * a thread simply needs to acquire the write semaphore.2089 *2090 * Return: 0 on success, -errno on failure2091 */2092int cxlflash_ioctl(struct scsi_device *sdev, unsigned int cmd, void __user *arg)2093{2094 typedef int (*sioctl) (struct scsi_device *, void *);2095 2096 struct cxlflash_cfg *cfg = shost_priv(sdev->host);2097 struct device *dev = &cfg->dev->dev;2098 struct afu *afu = cfg->afu;2099 struct dk_cxlflash_hdr *hdr;2100 char buf[sizeof(union cxlflash_ioctls)];2101 size_t size = 0;2102 bool known_ioctl = false;2103 int idx;2104 int rc = 0;2105 struct Scsi_Host *shost = sdev->host;2106 sioctl do_ioctl = NULL;2107 2108 static const struct {2109 size_t size;2110 sioctl ioctl;2111 } ioctl_tbl[] = { /* NOTE: order matters here */2112 {sizeof(struct dk_cxlflash_attach), cxlflash_disk_attach},2113 {sizeof(struct dk_cxlflash_udirect), cxlflash_disk_direct_open},2114 {sizeof(struct dk_cxlflash_release), cxlflash_disk_release},2115 {sizeof(struct dk_cxlflash_detach), cxlflash_disk_detach},2116 {sizeof(struct dk_cxlflash_verify), cxlflash_disk_verify},2117 {sizeof(struct dk_cxlflash_recover_afu), cxlflash_afu_recover},2118 {sizeof(struct dk_cxlflash_manage_lun), cxlflash_manage_lun},2119 {sizeof(struct dk_cxlflash_uvirtual), cxlflash_disk_virtual_open},2120 {sizeof(struct dk_cxlflash_resize), cxlflash_vlun_resize},2121 {sizeof(struct dk_cxlflash_clone), cxlflash_disk_clone},2122 };2123 2124 /* Hold read semaphore so we can drain if needed */2125 down_read(&cfg->ioctl_rwsem);2126 2127 /* Restrict command set to physical support only for internal LUN */2128 if (afu->internal_lun)2129 switch (cmd) {2130 case DK_CXLFLASH_RELEASE:2131 case DK_CXLFLASH_USER_VIRTUAL:2132 case DK_CXLFLASH_VLUN_RESIZE:2133 case DK_CXLFLASH_VLUN_CLONE:2134 dev_dbg(dev, "%s: %s not supported for lun_mode=%d\n",2135 __func__, decode_ioctl(cmd), afu->internal_lun);2136 rc = -EINVAL;2137 goto cxlflash_ioctl_exit;2138 }2139 2140 switch (cmd) {2141 case DK_CXLFLASH_ATTACH:2142 case DK_CXLFLASH_USER_DIRECT:2143 case DK_CXLFLASH_RELEASE:2144 case DK_CXLFLASH_DETACH:2145 case DK_CXLFLASH_VERIFY:2146 case DK_CXLFLASH_RECOVER_AFU:2147 case DK_CXLFLASH_USER_VIRTUAL:2148 case DK_CXLFLASH_VLUN_RESIZE:2149 case DK_CXLFLASH_VLUN_CLONE:2150 dev_dbg(dev, "%s: %s (%08X) on dev(%d/%d/%d/%llu)\n",2151 __func__, decode_ioctl(cmd), cmd, shost->host_no,2152 sdev->channel, sdev->id, sdev->lun);2153 rc = ioctl_common(sdev, cmd);2154 if (unlikely(rc))2155 goto cxlflash_ioctl_exit;2156 2157 fallthrough;2158 2159 case DK_CXLFLASH_MANAGE_LUN:2160 known_ioctl = true;2161 idx = _IOC_NR(cmd) - _IOC_NR(DK_CXLFLASH_ATTACH);2162 size = ioctl_tbl[idx].size;2163 do_ioctl = ioctl_tbl[idx].ioctl;2164 2165 if (likely(do_ioctl))2166 break;2167 2168 fallthrough;2169 default:2170 rc = -EINVAL;2171 goto cxlflash_ioctl_exit;2172 }2173 2174 if (unlikely(copy_from_user(&buf, arg, size))) {2175 dev_err(dev, "%s: copy_from_user() fail size=%lu cmd=%u (%s) arg=%p\n",2176 __func__, size, cmd, decode_ioctl(cmd), arg);2177 rc = -EFAULT;2178 goto cxlflash_ioctl_exit;2179 }2180 2181 hdr = (struct dk_cxlflash_hdr *)&buf;2182 if (hdr->version != DK_CXLFLASH_VERSION_0) {2183 dev_dbg(dev, "%s: Version %u not supported for %s\n",2184 __func__, hdr->version, decode_ioctl(cmd));2185 rc = -EINVAL;2186 goto cxlflash_ioctl_exit;2187 }2188 2189 if (hdr->rsvd[0] || hdr->rsvd[1] || hdr->rsvd[2] || hdr->return_flags) {2190 dev_dbg(dev, "%s: Reserved/rflags populated\n", __func__);2191 rc = -EINVAL;2192 goto cxlflash_ioctl_exit;2193 }2194 2195 rc = do_ioctl(sdev, (void *)&buf);2196 if (likely(!rc))2197 if (unlikely(copy_to_user(arg, &buf, size))) {2198 dev_err(dev, "%s: copy_to_user() fail size=%lu cmd=%u (%s) arg=%p\n",2199 __func__, size, cmd, decode_ioctl(cmd), arg);2200 rc = -EFAULT;2201 }2202 2203 /* fall through to exit */2204 2205cxlflash_ioctl_exit:2206 up_read(&cfg->ioctl_rwsem);2207 if (unlikely(rc && known_ioctl))2208 dev_err(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "2209 "returned rc %d\n", __func__,2210 decode_ioctl(cmd), cmd, shost->host_no,2211 sdev->channel, sdev->id, sdev->lun, rc);2212 else2213 dev_dbg(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "2214 "returned rc %d\n", __func__, decode_ioctl(cmd),2215 cmd, shost->host_no, sdev->channel, sdev->id,2216 sdev->lun, rc);2217 return rc;2218}2219