1337 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * CXL Flash Device Driver4 *5 * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation6 * Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation7 *8 * Copyright (C) 2015 IBM Corporation9 */10 11#include <linux/interrupt.h>12#include <linux/pci.h>13#include <linux/syscalls.h>14#include <linux/unaligned.h>15#include <asm/bitsperlong.h>16 17#include <scsi/scsi_cmnd.h>18#include <scsi/scsi_host.h>19#include <uapi/scsi/cxlflash_ioctl.h>20 21#include "sislite.h"22#include "common.h"23#include "vlun.h"24#include "superpipe.h"25 26/**27 * marshal_virt_to_resize() - translate uvirtual to resize structure28 * @virt: Source structure from which to translate/copy.29 * @resize: Destination structure for the translate/copy.30 */31static void marshal_virt_to_resize(struct dk_cxlflash_uvirtual *virt,32 struct dk_cxlflash_resize *resize)33{34 resize->hdr = virt->hdr;35 resize->context_id = virt->context_id;36 resize->rsrc_handle = virt->rsrc_handle;37 resize->req_size = virt->lun_size;38 resize->last_lba = virt->last_lba;39}40 41/**42 * marshal_clone_to_rele() - translate clone to release structure43 * @clone: Source structure from which to translate/copy.44 * @release: Destination structure for the translate/copy.45 */46static void marshal_clone_to_rele(struct dk_cxlflash_clone *clone,47 struct dk_cxlflash_release *release)48{49 release->hdr = clone->hdr;50 release->context_id = clone->context_id_dst;51}52 53/**54 * ba_init() - initializes a block allocator55 * @ba_lun: Block allocator to initialize.56 *57 * Return: 0 on success, -errno on failure58 */59static int ba_init(struct ba_lun *ba_lun)60{61 struct ba_lun_info *bali = NULL;62 int lun_size_au = 0, i = 0;63 int last_word_underflow = 0;64 u64 *lam;65 66 pr_debug("%s: Initializing LUN: lun_id=%016llx "67 "ba_lun->lsize=%lx ba_lun->au_size=%lX\n",68 __func__, ba_lun->lun_id, ba_lun->lsize, ba_lun->au_size);69 70 /* Calculate bit map size */71 lun_size_au = ba_lun->lsize / ba_lun->au_size;72 if (lun_size_au == 0) {73 pr_debug("%s: Requested LUN size of 0!\n", __func__);74 return -EINVAL;75 }76 77 /* Allocate lun information container */78 bali = kzalloc(sizeof(struct ba_lun_info), GFP_KERNEL);79 if (unlikely(!bali)) {80 pr_err("%s: Failed to allocate lun_info lun_id=%016llx\n",81 __func__, ba_lun->lun_id);82 return -ENOMEM;83 }84 85 bali->total_aus = lun_size_au;86 bali->lun_bmap_size = lun_size_au / BITS_PER_LONG;87 88 if (lun_size_au % BITS_PER_LONG)89 bali->lun_bmap_size++;90 91 /* Allocate bitmap space */92 bali->lun_alloc_map = kzalloc((bali->lun_bmap_size * sizeof(u64)),93 GFP_KERNEL);94 if (unlikely(!bali->lun_alloc_map)) {95 pr_err("%s: Failed to allocate lun allocation map: "96 "lun_id=%016llx\n", __func__, ba_lun->lun_id);97 kfree(bali);98 return -ENOMEM;99 }100 101 /* Initialize the bit map size and set all bits to '1' */102 bali->free_aun_cnt = lun_size_au;103 104 for (i = 0; i < bali->lun_bmap_size; i++)105 bali->lun_alloc_map[i] = 0xFFFFFFFFFFFFFFFFULL;106 107 /* If the last word not fully utilized, mark extra bits as allocated */108 last_word_underflow = (bali->lun_bmap_size * BITS_PER_LONG);109 last_word_underflow -= bali->free_aun_cnt;110 if (last_word_underflow > 0) {111 lam = &bali->lun_alloc_map[bali->lun_bmap_size - 1];112 for (i = (HIBIT - last_word_underflow + 1);113 i < BITS_PER_LONG;114 i++)115 clear_bit(i, (ulong *)lam);116 }117 118 /* Initialize high elevator index, low/curr already at 0 from kzalloc */119 bali->free_high_idx = bali->lun_bmap_size;120 121 /* Allocate clone map */122 bali->aun_clone_map = kzalloc((bali->total_aus * sizeof(u8)),123 GFP_KERNEL);124 if (unlikely(!bali->aun_clone_map)) {125 pr_err("%s: Failed to allocate clone map: lun_id=%016llx\n",126 __func__, ba_lun->lun_id);127 kfree(bali->lun_alloc_map);128 kfree(bali);129 return -ENOMEM;130 }131 132 /* Pass the allocated LUN info as a handle to the user */133 ba_lun->ba_lun_handle = bali;134 135 pr_debug("%s: Successfully initialized the LUN: "136 "lun_id=%016llx bitmap size=%x, free_aun_cnt=%llx\n",137 __func__, ba_lun->lun_id, bali->lun_bmap_size,138 bali->free_aun_cnt);139 return 0;140}141 142/**143 * find_free_range() - locates a free bit within the block allocator144 * @low: First word in block allocator to start search.145 * @high: Last word in block allocator to search.146 * @bali: LUN information structure owning the block allocator to search.147 * @bit_word: Passes back the word in the block allocator owning the free bit.148 *149 * Return: The bit position within the passed back word, -1 on failure150 */151static int find_free_range(u32 low,152 u32 high,153 struct ba_lun_info *bali, int *bit_word)154{155 int i;156 u64 bit_pos = -1;157 ulong *lam, num_bits;158 159 for (i = low; i < high; i++)160 if (bali->lun_alloc_map[i] != 0) {161 lam = (ulong *)&bali->lun_alloc_map[i];162 num_bits = (sizeof(*lam) * BITS_PER_BYTE);163 bit_pos = find_first_bit(lam, num_bits);164 165 pr_devel("%s: Found free bit %llu in LUN "166 "map entry %016llx at bitmap index = %d\n",167 __func__, bit_pos, bali->lun_alloc_map[i], i);168 169 *bit_word = i;170 bali->free_aun_cnt--;171 clear_bit(bit_pos, lam);172 break;173 }174 175 return bit_pos;176}177 178/**179 * ba_alloc() - allocates a block from the block allocator180 * @ba_lun: Block allocator from which to allocate a block.181 *182 * Return: The allocated block, -1 on failure183 */184static u64 ba_alloc(struct ba_lun *ba_lun)185{186 u64 bit_pos = -1;187 int bit_word = 0;188 struct ba_lun_info *bali = NULL;189 190 bali = ba_lun->ba_lun_handle;191 192 pr_debug("%s: Received block allocation request: "193 "lun_id=%016llx free_aun_cnt=%llx\n",194 __func__, ba_lun->lun_id, bali->free_aun_cnt);195 196 if (bali->free_aun_cnt == 0) {197 pr_debug("%s: No space left on LUN: lun_id=%016llx\n",198 __func__, ba_lun->lun_id);199 return -1ULL;200 }201 202 /* Search to find a free entry, curr->high then low->curr */203 bit_pos = find_free_range(bali->free_curr_idx,204 bali->free_high_idx, bali, &bit_word);205 if (bit_pos == -1) {206 bit_pos = find_free_range(bali->free_low_idx,207 bali->free_curr_idx,208 bali, &bit_word);209 if (bit_pos == -1) {210 pr_debug("%s: Could not find an allocation unit on LUN:"211 " lun_id=%016llx\n", __func__, ba_lun->lun_id);212 return -1ULL;213 }214 }215 216 /* Update the free_curr_idx */217 if (bit_pos == HIBIT)218 bali->free_curr_idx = bit_word + 1;219 else220 bali->free_curr_idx = bit_word;221 222 pr_debug("%s: Allocating AU number=%llx lun_id=%016llx "223 "free_aun_cnt=%llx\n", __func__,224 ((bit_word * BITS_PER_LONG) + bit_pos), ba_lun->lun_id,225 bali->free_aun_cnt);226 227 return (u64) ((bit_word * BITS_PER_LONG) + bit_pos);228}229 230/**231 * validate_alloc() - validates the specified block has been allocated232 * @bali: LUN info owning the block allocator.233 * @aun: Block to validate.234 *235 * Return: 0 on success, -1 on failure236 */237static int validate_alloc(struct ba_lun_info *bali, u64 aun)238{239 int idx = 0, bit_pos = 0;240 241 idx = aun / BITS_PER_LONG;242 bit_pos = aun % BITS_PER_LONG;243 244 if (test_bit(bit_pos, (ulong *)&bali->lun_alloc_map[idx]))245 return -1;246 247 return 0;248}249 250/**251 * ba_free() - frees a block from the block allocator252 * @ba_lun: Block allocator from which to allocate a block.253 * @to_free: Block to free.254 *255 * Return: 0 on success, -1 on failure256 */257static int ba_free(struct ba_lun *ba_lun, u64 to_free)258{259 int idx = 0, bit_pos = 0;260 struct ba_lun_info *bali = NULL;261 262 bali = ba_lun->ba_lun_handle;263 264 if (validate_alloc(bali, to_free)) {265 pr_debug("%s: AUN %llx is not allocated on lun_id=%016llx\n",266 __func__, to_free, ba_lun->lun_id);267 return -1;268 }269 270 pr_debug("%s: Received a request to free AU=%llx lun_id=%016llx "271 "free_aun_cnt=%llx\n", __func__, to_free, ba_lun->lun_id,272 bali->free_aun_cnt);273 274 if (bali->aun_clone_map[to_free] > 0) {275 pr_debug("%s: AUN %llx lun_id=%016llx cloned. Clone count=%x\n",276 __func__, to_free, ba_lun->lun_id,277 bali->aun_clone_map[to_free]);278 bali->aun_clone_map[to_free]--;279 return 0;280 }281 282 idx = to_free / BITS_PER_LONG;283 bit_pos = to_free % BITS_PER_LONG;284 285 set_bit(bit_pos, (ulong *)&bali->lun_alloc_map[idx]);286 bali->free_aun_cnt++;287 288 if (idx < bali->free_low_idx)289 bali->free_low_idx = idx;290 else if (idx > bali->free_high_idx)291 bali->free_high_idx = idx;292 293 pr_debug("%s: Successfully freed AU bit_pos=%x bit map index=%x "294 "lun_id=%016llx free_aun_cnt=%llx\n", __func__, bit_pos, idx,295 ba_lun->lun_id, bali->free_aun_cnt);296 297 return 0;298}299 300/**301 * ba_clone() - Clone a chunk of the block allocation table302 * @ba_lun: Block allocator from which to allocate a block.303 * @to_clone: Block to clone.304 *305 * Return: 0 on success, -1 on failure306 */307static int ba_clone(struct ba_lun *ba_lun, u64 to_clone)308{309 struct ba_lun_info *bali = ba_lun->ba_lun_handle;310 311 if (validate_alloc(bali, to_clone)) {312 pr_debug("%s: AUN=%llx not allocated on lun_id=%016llx\n",313 __func__, to_clone, ba_lun->lun_id);314 return -1;315 }316 317 pr_debug("%s: Received a request to clone AUN %llx on lun_id=%016llx\n",318 __func__, to_clone, ba_lun->lun_id);319 320 if (bali->aun_clone_map[to_clone] == MAX_AUN_CLONE_CNT) {321 pr_debug("%s: AUN %llx on lun_id=%016llx hit max clones already\n",322 __func__, to_clone, ba_lun->lun_id);323 return -1;324 }325 326 bali->aun_clone_map[to_clone]++;327 328 return 0;329}330 331/**332 * ba_space() - returns the amount of free space left in the block allocator333 * @ba_lun: Block allocator.334 *335 * Return: Amount of free space in block allocator336 */337static u64 ba_space(struct ba_lun *ba_lun)338{339 struct ba_lun_info *bali = ba_lun->ba_lun_handle;340 341 return bali->free_aun_cnt;342}343 344/**345 * cxlflash_ba_terminate() - frees resources associated with the block allocator346 * @ba_lun: Block allocator.347 *348 * Safe to call in a partially allocated state.349 */350void cxlflash_ba_terminate(struct ba_lun *ba_lun)351{352 struct ba_lun_info *bali = ba_lun->ba_lun_handle;353 354 if (bali) {355 kfree(bali->aun_clone_map);356 kfree(bali->lun_alloc_map);357 kfree(bali);358 ba_lun->ba_lun_handle = NULL;359 }360}361 362/**363 * init_vlun() - initializes a LUN for virtual use364 * @lli: LUN information structure that owns the block allocator.365 *366 * Return: 0 on success, -errno on failure367 */368static int init_vlun(struct llun_info *lli)369{370 int rc = 0;371 struct glun_info *gli = lli->parent;372 struct blka *blka = &gli->blka;373 374 memset(blka, 0, sizeof(*blka));375 mutex_init(&blka->mutex);376 377 /* LUN IDs are unique per port, save the index instead */378 blka->ba_lun.lun_id = lli->lun_index;379 blka->ba_lun.lsize = gli->max_lba + 1;380 blka->ba_lun.lba_size = gli->blk_len;381 382 blka->ba_lun.au_size = MC_CHUNK_SIZE;383 blka->nchunk = blka->ba_lun.lsize / MC_CHUNK_SIZE;384 385 rc = ba_init(&blka->ba_lun);386 if (unlikely(rc))387 pr_debug("%s: cannot init block_alloc, rc=%d\n", __func__, rc);388 389 pr_debug("%s: returning rc=%d lli=%p\n", __func__, rc, lli);390 return rc;391}392 393/**394 * write_same16() - sends a SCSI WRITE_SAME16 (0) command to specified LUN395 * @sdev: SCSI device associated with LUN.396 * @lba: Logical block address to start write same.397 * @nblks: Number of logical blocks to write same.398 *399 * The SCSI WRITE_SAME16 can take quite a while to complete. Should an EEH occur400 * while in scsi_execute_cmd(), the EEH handler will attempt to recover. As401 * part of the recovery, the handler drains all currently running ioctls,402 * waiting until they have completed before proceeding with a reset. As this403 * routine is used on the ioctl path, this can create a condition where the404 * EEH handler becomes stuck, infinitely waiting for this ioctl thread. To405 * avoid this behavior, temporarily unmark this thread as an ioctl thread by406 * releasing the ioctl read semaphore. This will allow the EEH handler to407 * proceed with a recovery while this thread is still running. Once the408 * scsi_execute_cmd() returns, reacquire the ioctl read semaphore and check the409 * adapter state in case it changed while inside of scsi_execute_cmd(). The410 * state check will wait if the adapter is still being recovered or return a411 * failure if the recovery failed. In the event that the adapter reset failed,412 * simply return the failure as the ioctl would be unable to continue.413 *414 * Note that the above puts a requirement on this routine to only be called on415 * an ioctl thread.416 *417 * Return: 0 on success, -errno on failure418 */419static int write_same16(struct scsi_device *sdev,420 u64 lba,421 u32 nblks)422{423 u8 *cmd_buf = NULL;424 u8 *scsi_cmd = NULL;425 int rc = 0;426 int result = 0;427 u64 offset = lba;428 int left = nblks;429 struct cxlflash_cfg *cfg = shost_priv(sdev->host);430 struct device *dev = &cfg->dev->dev;431 const u32 s = ilog2(sdev->sector_size) - 9;432 const u32 to = sdev->request_queue->rq_timeout;433 const u32 ws_limit =434 sdev->request_queue->limits.max_write_zeroes_sectors >> s;435 436 cmd_buf = kzalloc(CMD_BUFSIZE, GFP_KERNEL);437 scsi_cmd = kzalloc(MAX_COMMAND_SIZE, GFP_KERNEL);438 if (unlikely(!cmd_buf || !scsi_cmd)) {439 rc = -ENOMEM;440 goto out;441 }442 443 while (left > 0) {444 445 scsi_cmd[0] = WRITE_SAME_16;446 scsi_cmd[1] = cfg->ws_unmap ? 0x8 : 0;447 put_unaligned_be64(offset, &scsi_cmd[2]);448 put_unaligned_be32(ws_limit < left ? ws_limit : left,449 &scsi_cmd[10]);450 451 /* Drop the ioctl read semaphore across lengthy call */452 up_read(&cfg->ioctl_rwsem);453 result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_OUT,454 cmd_buf, CMD_BUFSIZE, to,455 CMD_RETRIES, NULL);456 down_read(&cfg->ioctl_rwsem);457 rc = check_state(cfg);458 if (rc) {459 dev_err(dev, "%s: Failed state result=%08x\n",460 __func__, result);461 rc = -ENODEV;462 goto out;463 }464 465 if (result) {466 dev_err_ratelimited(dev, "%s: command failed for "467 "offset=%lld result=%08x\n",468 __func__, offset, result);469 rc = -EIO;470 goto out;471 }472 left -= ws_limit;473 offset += ws_limit;474 }475 476out:477 kfree(cmd_buf);478 kfree(scsi_cmd);479 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);480 return rc;481}482 483/**484 * grow_lxt() - expands the translation table associated with the specified RHTE485 * @afu: AFU associated with the host.486 * @sdev: SCSI device associated with LUN.487 * @ctxid: Context ID of context owning the RHTE.488 * @rhndl: Resource handle associated with the RHTE.489 * @rhte: Resource handle entry (RHTE).490 * @new_size: Number of translation entries associated with RHTE.491 *492 * By design, this routine employs a 'best attempt' allocation and will493 * truncate the requested size down if there is not sufficient space in494 * the block allocator to satisfy the request but there does exist some495 * amount of space. The user is made aware of this by returning the size496 * allocated.497 *498 * Return: 0 on success, -errno on failure499 */500static int grow_lxt(struct afu *afu,501 struct scsi_device *sdev,502 ctx_hndl_t ctxid,503 res_hndl_t rhndl,504 struct sisl_rht_entry *rhte,505 u64 *new_size)506{507 struct cxlflash_cfg *cfg = shost_priv(sdev->host);508 struct device *dev = &cfg->dev->dev;509 struct sisl_lxt_entry *lxt = NULL, *lxt_old = NULL;510 struct llun_info *lli = sdev->hostdata;511 struct glun_info *gli = lli->parent;512 struct blka *blka = &gli->blka;513 u32 av_size;514 u32 ngrps, ngrps_old;515 u64 aun; /* chunk# allocated by block allocator */516 u64 delta = *new_size - rhte->lxt_cnt;517 u64 my_new_size;518 int i, rc = 0;519 520 /*521 * Check what is available in the block allocator before re-allocating522 * LXT array. This is done up front under the mutex which must not be523 * released until after allocation is complete.524 */525 mutex_lock(&blka->mutex);526 av_size = ba_space(&blka->ba_lun);527 if (unlikely(av_size <= 0)) {528 dev_dbg(dev, "%s: ba_space error av_size=%d\n",529 __func__, av_size);530 mutex_unlock(&blka->mutex);531 rc = -ENOSPC;532 goto out;533 }534 535 if (av_size < delta)536 delta = av_size;537 538 lxt_old = rhte->lxt_start;539 ngrps_old = LXT_NUM_GROUPS(rhte->lxt_cnt);540 ngrps = LXT_NUM_GROUPS(rhte->lxt_cnt + delta);541 542 if (ngrps != ngrps_old) {543 /* reallocate to fit new size */544 lxt = kzalloc((sizeof(*lxt) * LXT_GROUP_SIZE * ngrps),545 GFP_KERNEL);546 if (unlikely(!lxt)) {547 mutex_unlock(&blka->mutex);548 rc = -ENOMEM;549 goto out;550 }551 552 /* copy over all old entries */553 memcpy(lxt, lxt_old, (sizeof(*lxt) * rhte->lxt_cnt));554 } else555 lxt = lxt_old;556 557 /* nothing can fail from now on */558 my_new_size = rhte->lxt_cnt + delta;559 560 /* add new entries to the end */561 for (i = rhte->lxt_cnt; i < my_new_size; i++) {562 /*563 * Due to the earlier check of available space, ba_alloc564 * cannot fail here. If it did due to internal error,565 * leave a rlba_base of -1u which will likely be a566 * invalid LUN (too large).567 */568 aun = ba_alloc(&blka->ba_lun);569 if ((aun == -1ULL) || (aun >= blka->nchunk))570 dev_dbg(dev, "%s: ba_alloc error allocated chunk=%llu "571 "max=%llu\n", __func__, aun, blka->nchunk - 1);572 573 /* select both ports, use r/w perms from RHT */574 lxt[i].rlba_base = ((aun << MC_CHUNK_SHIFT) |575 (lli->lun_index << LXT_LUNIDX_SHIFT) |576 (RHT_PERM_RW << LXT_PERM_SHIFT |577 lli->port_sel));578 }579 580 mutex_unlock(&blka->mutex);581 582 /*583 * The following sequence is prescribed in the SISlite spec584 * for syncing up with the AFU when adding LXT entries.585 */586 dma_wmb(); /* Make LXT updates are visible */587 588 rhte->lxt_start = lxt;589 dma_wmb(); /* Make RHT entry's LXT table update visible */590 591 rhte->lxt_cnt = my_new_size;592 dma_wmb(); /* Make RHT entry's LXT table size update visible */593 594 rc = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_LW_SYNC);595 if (unlikely(rc))596 rc = -EAGAIN;597 598 /* free old lxt if reallocated */599 if (lxt != lxt_old)600 kfree(lxt_old);601 *new_size = my_new_size;602out:603 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);604 return rc;605}606 607/**608 * shrink_lxt() - reduces translation table associated with the specified RHTE609 * @afu: AFU associated with the host.610 * @sdev: SCSI device associated with LUN.611 * @rhndl: Resource handle associated with the RHTE.612 * @rhte: Resource handle entry (RHTE).613 * @ctxi: Context owning resources.614 * @new_size: Number of translation entries associated with RHTE.615 *616 * Return: 0 on success, -errno on failure617 */618static int shrink_lxt(struct afu *afu,619 struct scsi_device *sdev,620 res_hndl_t rhndl,621 struct sisl_rht_entry *rhte,622 struct ctx_info *ctxi,623 u64 *new_size)624{625 struct cxlflash_cfg *cfg = shost_priv(sdev->host);626 struct device *dev = &cfg->dev->dev;627 struct sisl_lxt_entry *lxt, *lxt_old;628 struct llun_info *lli = sdev->hostdata;629 struct glun_info *gli = lli->parent;630 struct blka *blka = &gli->blka;631 ctx_hndl_t ctxid = DECODE_CTXID(ctxi->ctxid);632 bool needs_ws = ctxi->rht_needs_ws[rhndl];633 bool needs_sync = !ctxi->err_recovery_active;634 u32 ngrps, ngrps_old;635 u64 aun; /* chunk# allocated by block allocator */636 u64 delta = rhte->lxt_cnt - *new_size;637 u64 my_new_size;638 int i, rc = 0;639 640 lxt_old = rhte->lxt_start;641 ngrps_old = LXT_NUM_GROUPS(rhte->lxt_cnt);642 ngrps = LXT_NUM_GROUPS(rhte->lxt_cnt - delta);643 644 if (ngrps != ngrps_old) {645 /* Reallocate to fit new size unless new size is 0 */646 if (ngrps) {647 lxt = kzalloc((sizeof(*lxt) * LXT_GROUP_SIZE * ngrps),648 GFP_KERNEL);649 if (unlikely(!lxt)) {650 rc = -ENOMEM;651 goto out;652 }653 654 /* Copy over old entries that will remain */655 memcpy(lxt, lxt_old,656 (sizeof(*lxt) * (rhte->lxt_cnt - delta)));657 } else658 lxt = NULL;659 } else660 lxt = lxt_old;661 662 /* Nothing can fail from now on */663 my_new_size = rhte->lxt_cnt - delta;664 665 /*666 * The following sequence is prescribed in the SISlite spec667 * for syncing up with the AFU when removing LXT entries.668 */669 rhte->lxt_cnt = my_new_size;670 dma_wmb(); /* Make RHT entry's LXT table size update visible */671 672 rhte->lxt_start = lxt;673 dma_wmb(); /* Make RHT entry's LXT table update visible */674 675 if (needs_sync) {676 rc = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);677 if (unlikely(rc))678 rc = -EAGAIN;679 }680 681 if (needs_ws) {682 /*683 * Mark the context as unavailable, so that we can release684 * the mutex safely.685 */686 ctxi->unavail = true;687 mutex_unlock(&ctxi->mutex);688 }689 690 /* Free LBAs allocated to freed chunks */691 mutex_lock(&blka->mutex);692 for (i = delta - 1; i >= 0; i--) {693 aun = lxt_old[my_new_size + i].rlba_base >> MC_CHUNK_SHIFT;694 if (needs_ws)695 write_same16(sdev, aun, MC_CHUNK_SIZE);696 ba_free(&blka->ba_lun, aun);697 }698 mutex_unlock(&blka->mutex);699 700 if (needs_ws) {701 /* Make the context visible again */702 mutex_lock(&ctxi->mutex);703 ctxi->unavail = false;704 }705 706 /* Free old lxt if reallocated */707 if (lxt != lxt_old)708 kfree(lxt_old);709 *new_size = my_new_size;710out:711 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);712 return rc;713}714 715/**716 * _cxlflash_vlun_resize() - changes the size of a virtual LUN717 * @sdev: SCSI device associated with LUN owning virtual LUN.718 * @ctxi: Context owning resources.719 * @resize: Resize ioctl data structure.720 *721 * On successful return, the user is informed of the new size (in blocks)722 * of the virtual LUN in last LBA format. When the size of the virtual723 * LUN is zero, the last LBA is reflected as -1. See comment in the724 * prologue for _cxlflash_disk_release() regarding AFU syncs and contexts725 * on the error recovery list.726 *727 * Return: 0 on success, -errno on failure728 */729int _cxlflash_vlun_resize(struct scsi_device *sdev,730 struct ctx_info *ctxi,731 struct dk_cxlflash_resize *resize)732{733 struct cxlflash_cfg *cfg = shost_priv(sdev->host);734 struct device *dev = &cfg->dev->dev;735 struct llun_info *lli = sdev->hostdata;736 struct glun_info *gli = lli->parent;737 struct afu *afu = cfg->afu;738 bool put_ctx = false;739 740 res_hndl_t rhndl = resize->rsrc_handle;741 u64 new_size;742 u64 nsectors;743 u64 ctxid = DECODE_CTXID(resize->context_id),744 rctxid = resize->context_id;745 746 struct sisl_rht_entry *rhte;747 748 int rc = 0;749 750 /*751 * The requested size (req_size) is always assumed to be in 4k blocks,752 * so we have to convert it here from 4k to chunk size.753 */754 nsectors = (resize->req_size * CXLFLASH_BLOCK_SIZE) / gli->blk_len;755 new_size = DIV_ROUND_UP(nsectors, MC_CHUNK_SIZE);756 757 dev_dbg(dev, "%s: ctxid=%llu rhndl=%llu req_size=%llu new_size=%llu\n",758 __func__, ctxid, resize->rsrc_handle, resize->req_size,759 new_size);760 761 if (unlikely(gli->mode != MODE_VIRTUAL)) {762 dev_dbg(dev, "%s: LUN mode does not support resize mode=%d\n",763 __func__, gli->mode);764 rc = -EINVAL;765 goto out;766 767 }768 769 if (!ctxi) {770 ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);771 if (unlikely(!ctxi)) {772 dev_dbg(dev, "%s: Bad context ctxid=%llu\n",773 __func__, ctxid);774 rc = -EINVAL;775 goto out;776 }777 778 put_ctx = true;779 }780 781 rhte = get_rhte(ctxi, rhndl, lli);782 if (unlikely(!rhte)) {783 dev_dbg(dev, "%s: Bad resource handle rhndl=%u\n",784 __func__, rhndl);785 rc = -EINVAL;786 goto out;787 }788 789 if (new_size > rhte->lxt_cnt)790 rc = grow_lxt(afu, sdev, ctxid, rhndl, rhte, &new_size);791 else if (new_size < rhte->lxt_cnt)792 rc = shrink_lxt(afu, sdev, rhndl, rhte, ctxi, &new_size);793 else {794 /*795 * Rare case where there is already sufficient space, just796 * need to perform a translation sync with the AFU. This797 * scenario likely follows a previous sync failure during798 * a resize operation. Accordingly, perform the heavyweight799 * form of translation sync as it is unknown which type of800 * resize failed previously.801 */802 rc = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);803 if (unlikely(rc)) {804 rc = -EAGAIN;805 goto out;806 }807 }808 809 resize->hdr.return_flags = 0;810 resize->last_lba = (new_size * MC_CHUNK_SIZE * gli->blk_len);811 resize->last_lba /= CXLFLASH_BLOCK_SIZE;812 resize->last_lba--;813 814out:815 if (put_ctx)816 put_context(ctxi);817 dev_dbg(dev, "%s: resized to %llu returning rc=%d\n",818 __func__, resize->last_lba, rc);819 return rc;820}821 822int cxlflash_vlun_resize(struct scsi_device *sdev, void *resize)823{824 return _cxlflash_vlun_resize(sdev, NULL, resize);825}826 827/**828 * cxlflash_restore_luntable() - Restore LUN table to prior state829 * @cfg: Internal structure associated with the host.830 */831void cxlflash_restore_luntable(struct cxlflash_cfg *cfg)832{833 struct llun_info *lli, *temp;834 u32 lind;835 int k;836 struct device *dev = &cfg->dev->dev;837 __be64 __iomem *fc_port_luns;838 839 mutex_lock(&global.mutex);840 841 list_for_each_entry_safe(lli, temp, &cfg->lluns, list) {842 if (!lli->in_table)843 continue;844 845 lind = lli->lun_index;846 dev_dbg(dev, "%s: Virtual LUNs on slot %d:\n", __func__, lind);847 848 for (k = 0; k < cfg->num_fc_ports; k++)849 if (lli->port_sel & (1 << k)) {850 fc_port_luns = get_fc_port_luns(cfg, k);851 writeq_be(lli->lun_id[k], &fc_port_luns[lind]);852 dev_dbg(dev, "\t%d=%llx\n", k, lli->lun_id[k]);853 }854 }855 856 mutex_unlock(&global.mutex);857}858 859/**860 * get_num_ports() - compute number of ports from port selection mask861 * @psm: Port selection mask.862 *863 * Return: Population count of port selection mask864 */865static inline u8 get_num_ports(u32 psm)866{867 static const u8 bits[16] = { 0, 1, 1, 2, 1, 2, 2, 3,868 1, 2, 2, 3, 2, 3, 3, 4 };869 870 return bits[psm & 0xf];871}872 873/**874 * init_luntable() - write an entry in the LUN table875 * @cfg: Internal structure associated with the host.876 * @lli: Per adapter LUN information structure.877 *878 * On successful return, a LUN table entry is created:879 * - at the top for LUNs visible on multiple ports.880 * - at the bottom for LUNs visible only on one port.881 *882 * Return: 0 on success, -errno on failure883 */884static int init_luntable(struct cxlflash_cfg *cfg, struct llun_info *lli)885{886 u32 chan;887 u32 lind;888 u32 nports;889 int rc = 0;890 int k;891 struct device *dev = &cfg->dev->dev;892 __be64 __iomem *fc_port_luns;893 894 mutex_lock(&global.mutex);895 896 if (lli->in_table)897 goto out;898 899 nports = get_num_ports(lli->port_sel);900 if (nports == 0 || nports > cfg->num_fc_ports) {901 WARN(1, "Unsupported port configuration nports=%u", nports);902 rc = -EIO;903 goto out;904 }905 906 if (nports > 1) {907 /*908 * When LUN is visible from multiple ports, we will put909 * it in the top half of the LUN table.910 */911 for (k = 0; k < cfg->num_fc_ports; k++) {912 if (!(lli->port_sel & (1 << k)))913 continue;914 915 if (cfg->promote_lun_index == cfg->last_lun_index[k]) {916 rc = -ENOSPC;917 goto out;918 }919 }920 921 lind = lli->lun_index = cfg->promote_lun_index;922 dev_dbg(dev, "%s: Virtual LUNs on slot %d:\n", __func__, lind);923 924 for (k = 0; k < cfg->num_fc_ports; k++) {925 if (!(lli->port_sel & (1 << k)))926 continue;927 928 fc_port_luns = get_fc_port_luns(cfg, k);929 writeq_be(lli->lun_id[k], &fc_port_luns[lind]);930 dev_dbg(dev, "\t%d=%llx\n", k, lli->lun_id[k]);931 }932 933 cfg->promote_lun_index++;934 } else {935 /*936 * When LUN is visible only from one port, we will put937 * it in the bottom half of the LUN table.938 */939 chan = PORTMASK2CHAN(lli->port_sel);940 if (cfg->promote_lun_index == cfg->last_lun_index[chan]) {941 rc = -ENOSPC;942 goto out;943 }944 945 lind = lli->lun_index = cfg->last_lun_index[chan];946 fc_port_luns = get_fc_port_luns(cfg, chan);947 writeq_be(lli->lun_id[chan], &fc_port_luns[lind]);948 cfg->last_lun_index[chan]--;949 dev_dbg(dev, "%s: Virtual LUNs on slot %d:\n\t%d=%llx\n",950 __func__, lind, chan, lli->lun_id[chan]);951 }952 953 lli->in_table = true;954out:955 mutex_unlock(&global.mutex);956 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);957 return rc;958}959 960/**961 * cxlflash_disk_virtual_open() - open a virtual disk of specified size962 * @sdev: SCSI device associated with LUN owning virtual LUN.963 * @arg: UVirtual ioctl data structure.964 *965 * On successful return, the user is informed of the resource handle966 * to be used to identify the virtual LUN and the size (in blocks) of967 * the virtual LUN in last LBA format. When the size of the virtual LUN968 * is zero, the last LBA is reflected as -1.969 *970 * Return: 0 on success, -errno on failure971 */972int cxlflash_disk_virtual_open(struct scsi_device *sdev, void *arg)973{974 struct cxlflash_cfg *cfg = shost_priv(sdev->host);975 struct device *dev = &cfg->dev->dev;976 struct llun_info *lli = sdev->hostdata;977 struct glun_info *gli = lli->parent;978 979 struct dk_cxlflash_uvirtual *virt = (struct dk_cxlflash_uvirtual *)arg;980 struct dk_cxlflash_resize resize;981 982 u64 ctxid = DECODE_CTXID(virt->context_id),983 rctxid = virt->context_id;984 u64 lun_size = virt->lun_size;985 u64 last_lba = 0;986 u64 rsrc_handle = -1;987 988 int rc = 0;989 990 struct ctx_info *ctxi = NULL;991 struct sisl_rht_entry *rhte = NULL;992 993 dev_dbg(dev, "%s: ctxid=%llu ls=%llu\n", __func__, ctxid, lun_size);994 995 /* Setup the LUNs block allocator on first call */996 mutex_lock(&gli->mutex);997 if (gli->mode == MODE_NONE) {998 rc = init_vlun(lli);999 if (rc) {1000 dev_err(dev, "%s: init_vlun failed rc=%d\n",1001 __func__, rc);1002 rc = -ENOMEM;1003 goto err0;1004 }1005 }1006 1007 rc = cxlflash_lun_attach(gli, MODE_VIRTUAL, true);1008 if (unlikely(rc)) {1009 dev_err(dev, "%s: Failed attach to LUN (VIRTUAL)\n", __func__);1010 goto err0;1011 }1012 mutex_unlock(&gli->mutex);1013 1014 rc = init_luntable(cfg, lli);1015 if (rc) {1016 dev_err(dev, "%s: init_luntable failed rc=%d\n", __func__, rc);1017 goto err1;1018 }1019 1020 ctxi = get_context(cfg, rctxid, lli, 0);1021 if (unlikely(!ctxi)) {1022 dev_err(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);1023 rc = -EINVAL;1024 goto err1;1025 }1026 1027 rhte = rhte_checkout(ctxi, lli);1028 if (unlikely(!rhte)) {1029 dev_err(dev, "%s: too many opens ctxid=%llu\n",1030 __func__, ctxid);1031 rc = -EMFILE; /* too many opens */1032 goto err1;1033 }1034 1035 rsrc_handle = (rhte - ctxi->rht_start);1036 1037 /* Populate RHT format 0 */1038 rhte->nmask = MC_RHT_NMASK;1039 rhte->fp = SISL_RHT_FP(0U, ctxi->rht_perms);1040 1041 /* Resize even if requested size is 0 */1042 marshal_virt_to_resize(virt, &resize);1043 resize.rsrc_handle = rsrc_handle;1044 rc = _cxlflash_vlun_resize(sdev, ctxi, &resize);1045 if (rc) {1046 dev_err(dev, "%s: resize failed rc=%d\n", __func__, rc);1047 goto err2;1048 }1049 last_lba = resize.last_lba;1050 1051 if (virt->hdr.flags & DK_CXLFLASH_UVIRTUAL_NEED_WRITE_SAME)1052 ctxi->rht_needs_ws[rsrc_handle] = true;1053 1054 virt->hdr.return_flags = 0;1055 virt->last_lba = last_lba;1056 virt->rsrc_handle = rsrc_handle;1057 1058 if (get_num_ports(lli->port_sel) > 1)1059 virt->hdr.return_flags |= DK_CXLFLASH_ALL_PORTS_ACTIVE;1060out:1061 if (likely(ctxi))1062 put_context(ctxi);1063 dev_dbg(dev, "%s: returning handle=%llu rc=%d llba=%llu\n",1064 __func__, rsrc_handle, rc, last_lba);1065 return rc;1066 1067err2:1068 rhte_checkin(ctxi, rhte);1069err1:1070 cxlflash_lun_detach(gli);1071 goto out;1072err0:1073 /* Special common cleanup prior to successful LUN attach */1074 cxlflash_ba_terminate(&gli->blka.ba_lun);1075 mutex_unlock(&gli->mutex);1076 goto out;1077}1078 1079/**1080 * clone_lxt() - copies translation tables from source to destination RHTE1081 * @afu: AFU associated with the host.1082 * @blka: Block allocator associated with LUN.1083 * @ctxid: Context ID of context owning the RHTE.1084 * @rhndl: Resource handle associated with the RHTE.1085 * @rhte: Destination resource handle entry (RHTE).1086 * @rhte_src: Source resource handle entry (RHTE).1087 *1088 * Return: 0 on success, -errno on failure1089 */1090static int clone_lxt(struct afu *afu,1091 struct blka *blka,1092 ctx_hndl_t ctxid,1093 res_hndl_t rhndl,1094 struct sisl_rht_entry *rhte,1095 struct sisl_rht_entry *rhte_src)1096{1097 struct cxlflash_cfg *cfg = afu->parent;1098 struct device *dev = &cfg->dev->dev;1099 struct sisl_lxt_entry *lxt = NULL;1100 bool locked = false;1101 u32 ngrps;1102 u64 aun; /* chunk# allocated by block allocator */1103 int j;1104 int i = 0;1105 int rc = 0;1106 1107 ngrps = LXT_NUM_GROUPS(rhte_src->lxt_cnt);1108 1109 if (ngrps) {1110 /* allocate new LXTs for clone */1111 lxt = kzalloc((sizeof(*lxt) * LXT_GROUP_SIZE * ngrps),1112 GFP_KERNEL);1113 if (unlikely(!lxt)) {1114 rc = -ENOMEM;1115 goto out;1116 }1117 1118 /* copy over */1119 memcpy(lxt, rhte_src->lxt_start,1120 (sizeof(*lxt) * rhte_src->lxt_cnt));1121 1122 /* clone the LBAs in block allocator via ref_cnt, note that the1123 * block allocator mutex must be held until it is established1124 * that this routine will complete without the need for a1125 * cleanup.1126 */1127 mutex_lock(&blka->mutex);1128 locked = true;1129 for (i = 0; i < rhte_src->lxt_cnt; i++) {1130 aun = (lxt[i].rlba_base >> MC_CHUNK_SHIFT);1131 if (ba_clone(&blka->ba_lun, aun) == -1ULL) {1132 rc = -EIO;1133 goto err;1134 }1135 }1136 }1137 1138 /*1139 * The following sequence is prescribed in the SISlite spec1140 * for syncing up with the AFU when adding LXT entries.1141 */1142 dma_wmb(); /* Make LXT updates are visible */1143 1144 rhte->lxt_start = lxt;1145 dma_wmb(); /* Make RHT entry's LXT table update visible */1146 1147 rhte->lxt_cnt = rhte_src->lxt_cnt;1148 dma_wmb(); /* Make RHT entry's LXT table size update visible */1149 1150 rc = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_LW_SYNC);1151 if (unlikely(rc)) {1152 rc = -EAGAIN;1153 goto err2;1154 }1155 1156out:1157 if (locked)1158 mutex_unlock(&blka->mutex);1159 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);1160 return rc;1161err2:1162 /* Reset the RHTE */1163 rhte->lxt_cnt = 0;1164 dma_wmb();1165 rhte->lxt_start = NULL;1166 dma_wmb();1167err:1168 /* free the clones already made */1169 for (j = 0; j < i; j++) {1170 aun = (lxt[j].rlba_base >> MC_CHUNK_SHIFT);1171 ba_free(&blka->ba_lun, aun);1172 }1173 kfree(lxt);1174 goto out;1175}1176 1177/**1178 * cxlflash_disk_clone() - clone a context by making snapshot of another1179 * @sdev: SCSI device associated with LUN owning virtual LUN.1180 * @arg: Clone ioctl data structure.1181 *1182 * This routine effectively performs cxlflash_disk_open operation for each1183 * in-use virtual resource in the source context. Note that the destination1184 * context must be in pristine state and cannot have any resource handles1185 * open at the time of the clone.1186 *1187 * Return: 0 on success, -errno on failure1188 */1189int cxlflash_disk_clone(struct scsi_device *sdev, void *arg)1190{1191 struct dk_cxlflash_clone *clone = arg;1192 struct cxlflash_cfg *cfg = shost_priv(sdev->host);1193 struct device *dev = &cfg->dev->dev;1194 struct llun_info *lli = sdev->hostdata;1195 struct glun_info *gli = lli->parent;1196 struct blka *blka = &gli->blka;1197 struct afu *afu = cfg->afu;1198 struct dk_cxlflash_release release = { { 0 }, 0 };1199 1200 struct ctx_info *ctxi_src = NULL,1201 *ctxi_dst = NULL;1202 struct lun_access *lun_access_src, *lun_access_dst;1203 u32 perms;1204 u64 ctxid_src = DECODE_CTXID(clone->context_id_src),1205 ctxid_dst = DECODE_CTXID(clone->context_id_dst),1206 rctxid_src = clone->context_id_src,1207 rctxid_dst = clone->context_id_dst;1208 int i, j;1209 int rc = 0;1210 bool found;1211 LIST_HEAD(sidecar);1212 1213 dev_dbg(dev, "%s: ctxid_src=%llu ctxid_dst=%llu\n",1214 __func__, ctxid_src, ctxid_dst);1215 1216 /* Do not clone yourself */1217 if (unlikely(rctxid_src == rctxid_dst)) {1218 rc = -EINVAL;1219 goto out;1220 }1221 1222 if (unlikely(gli->mode != MODE_VIRTUAL)) {1223 rc = -EINVAL;1224 dev_dbg(dev, "%s: Only supported on virtual LUNs mode=%u\n",1225 __func__, gli->mode);1226 goto out;1227 }1228 1229 ctxi_src = get_context(cfg, rctxid_src, lli, CTX_CTRL_CLONE);1230 ctxi_dst = get_context(cfg, rctxid_dst, lli, 0);1231 if (unlikely(!ctxi_src || !ctxi_dst)) {1232 dev_dbg(dev, "%s: Bad context ctxid_src=%llu ctxid_dst=%llu\n",1233 __func__, ctxid_src, ctxid_dst);1234 rc = -EINVAL;1235 goto out;1236 }1237 1238 /* Verify there is no open resource handle in the destination context */1239 for (i = 0; i < MAX_RHT_PER_CONTEXT; i++)1240 if (ctxi_dst->rht_start[i].nmask != 0) {1241 rc = -EINVAL;1242 goto out;1243 }1244 1245 /* Clone LUN access list */1246 list_for_each_entry(lun_access_src, &ctxi_src->luns, list) {1247 found = false;1248 list_for_each_entry(lun_access_dst, &ctxi_dst->luns, list)1249 if (lun_access_dst->sdev == lun_access_src->sdev) {1250 found = true;1251 break;1252 }1253 1254 if (!found) {1255 lun_access_dst = kzalloc(sizeof(*lun_access_dst),1256 GFP_KERNEL);1257 if (unlikely(!lun_access_dst)) {1258 dev_err(dev, "%s: lun_access allocation fail\n",1259 __func__);1260 rc = -ENOMEM;1261 goto out;1262 }1263 1264 *lun_access_dst = *lun_access_src;1265 list_add(&lun_access_dst->list, &sidecar);1266 }1267 }1268 1269 if (unlikely(!ctxi_src->rht_out)) {1270 dev_dbg(dev, "%s: Nothing to clone\n", __func__);1271 goto out_success;1272 }1273 1274 /* User specified permission on attach */1275 perms = ctxi_dst->rht_perms;1276 1277 /*1278 * Copy over checked-out RHT (and their associated LXT) entries by1279 * hand, stopping after we've copied all outstanding entries and1280 * cleaning up if the clone fails.1281 *1282 * Note: This loop is equivalent to performing cxlflash_disk_open and1283 * cxlflash_vlun_resize. As such, LUN accounting needs to be taken into1284 * account by attaching after each successful RHT entry clone. In the1285 * event that a clone failure is experienced, the LUN detach is handled1286 * via the cleanup performed by _cxlflash_disk_release.1287 */1288 for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) {1289 if (ctxi_src->rht_out == ctxi_dst->rht_out)1290 break;1291 if (ctxi_src->rht_start[i].nmask == 0)1292 continue;1293 1294 /* Consume a destination RHT entry */1295 ctxi_dst->rht_out++;1296 ctxi_dst->rht_start[i].nmask = ctxi_src->rht_start[i].nmask;1297 ctxi_dst->rht_start[i].fp =1298 SISL_RHT_FP_CLONE(ctxi_src->rht_start[i].fp, perms);1299 ctxi_dst->rht_lun[i] = ctxi_src->rht_lun[i];1300 1301 rc = clone_lxt(afu, blka, ctxid_dst, i,1302 &ctxi_dst->rht_start[i],1303 &ctxi_src->rht_start[i]);1304 if (rc) {1305 marshal_clone_to_rele(clone, &release);1306 for (j = 0; j < i; j++) {1307 release.rsrc_handle = j;1308 _cxlflash_disk_release(sdev, ctxi_dst,1309 &release);1310 }1311 1312 /* Put back the one we failed on */1313 rhte_checkin(ctxi_dst, &ctxi_dst->rht_start[i]);1314 goto err;1315 }1316 1317 cxlflash_lun_attach(gli, gli->mode, false);1318 }1319 1320out_success:1321 list_splice(&sidecar, &ctxi_dst->luns);1322 1323 /* fall through */1324out:1325 if (ctxi_src)1326 put_context(ctxi_src);1327 if (ctxi_dst)1328 put_context(ctxi_dst);1329 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);1330 return rc;1331 1332err:1333 list_for_each_entry_safe(lun_access_src, lun_access_dst, &sidecar, list)1334 kfree(lun_access_src);1335 goto out;1336}1337