585 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Block driver for s390 storage class memory.4 *5 * Copyright IBM Corp. 20126 * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>7 */8 9#define KMSG_COMPONENT "scm_block"10#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt11 12#include <linux/interrupt.h>13#include <linux/spinlock.h>14#include <linux/mempool.h>15#include <linux/module.h>16#include <linux/blkdev.h>17#include <linux/blk-mq.h>18#include <linux/slab.h>19#include <linux/list.h>20#include <linux/io.h>21#include <asm/eadm.h>22#include "scm_blk.h"23 24debug_info_t *scm_debug;25static int scm_major;26static mempool_t *aidaw_pool;27static DEFINE_SPINLOCK(list_lock);28static LIST_HEAD(inactive_requests);29static unsigned int nr_requests = 64;30static unsigned int nr_requests_per_io = 8;31static atomic_t nr_devices = ATOMIC_INIT(0);32module_param(nr_requests, uint, S_IRUGO);33MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");34 35module_param(nr_requests_per_io, uint, S_IRUGO);36MODULE_PARM_DESC(nr_requests_per_io, "Number of requests per IO.");37 38MODULE_DESCRIPTION("Block driver for s390 storage class memory.");39MODULE_LICENSE("GPL");40MODULE_ALIAS("scm:scmdev*");41 42static void __scm_free_rq(struct scm_request *scmrq)43{44 struct aob_rq_header *aobrq = to_aobrq(scmrq);45 46 free_page((unsigned long) scmrq->aob);47 kfree(scmrq->request);48 kfree(aobrq);49}50 51static void scm_free_rqs(void)52{53 struct list_head *iter, *safe;54 struct scm_request *scmrq;55 56 spin_lock_irq(&list_lock);57 list_for_each_safe(iter, safe, &inactive_requests) {58 scmrq = list_entry(iter, struct scm_request, list);59 list_del(&scmrq->list);60 __scm_free_rq(scmrq);61 }62 spin_unlock_irq(&list_lock);63 64 mempool_destroy(aidaw_pool);65}66 67static int __scm_alloc_rq(void)68{69 struct aob_rq_header *aobrq;70 struct scm_request *scmrq;71 72 aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);73 if (!aobrq)74 return -ENOMEM;75 76 scmrq = (void *) aobrq->data;77 scmrq->aob = (void *) get_zeroed_page(GFP_DMA);78 if (!scmrq->aob)79 goto free;80 81 scmrq->request = kcalloc(nr_requests_per_io, sizeof(scmrq->request[0]),82 GFP_KERNEL);83 if (!scmrq->request)84 goto free;85 86 INIT_LIST_HEAD(&scmrq->list);87 spin_lock_irq(&list_lock);88 list_add(&scmrq->list, &inactive_requests);89 spin_unlock_irq(&list_lock);90 91 return 0;92free:93 __scm_free_rq(scmrq);94 return -ENOMEM;95}96 97static int scm_alloc_rqs(unsigned int nrqs)98{99 int ret = 0;100 101 aidaw_pool = mempool_create_page_pool(max(nrqs/8, 1U), 0);102 if (!aidaw_pool)103 return -ENOMEM;104 105 while (nrqs-- && !ret)106 ret = __scm_alloc_rq();107 108 return ret;109}110 111static struct scm_request *scm_request_fetch(void)112{113 struct scm_request *scmrq = NULL;114 115 spin_lock_irq(&list_lock);116 if (list_empty(&inactive_requests))117 goto out;118 scmrq = list_first_entry(&inactive_requests, struct scm_request, list);119 list_del(&scmrq->list);120out:121 spin_unlock_irq(&list_lock);122 return scmrq;123}124 125static void scm_request_done(struct scm_request *scmrq)126{127 unsigned long flags;128 struct msb *msb;129 u64 aidaw;130 int i;131 132 for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {133 msb = &scmrq->aob->msb[i];134 aidaw = (u64)dma64_to_virt(msb->data_addr);135 136 if ((msb->flags & MSB_FLAG_IDA) && aidaw &&137 IS_ALIGNED(aidaw, PAGE_SIZE))138 mempool_free(virt_to_page((void *)aidaw), aidaw_pool);139 }140 141 spin_lock_irqsave(&list_lock, flags);142 list_add(&scmrq->list, &inactive_requests);143 spin_unlock_irqrestore(&list_lock, flags);144}145 146static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)147{148 return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;149}150 151static inline struct aidaw *scm_aidaw_alloc(void)152{153 struct page *page = mempool_alloc(aidaw_pool, GFP_ATOMIC);154 155 return page ? page_address(page) : NULL;156}157 158static inline unsigned long scm_aidaw_bytes(struct aidaw *aidaw)159{160 unsigned long _aidaw = (unsigned long) aidaw;161 unsigned long bytes = ALIGN(_aidaw, PAGE_SIZE) - _aidaw;162 163 return (bytes / sizeof(*aidaw)) * PAGE_SIZE;164}165 166struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes)167{168 struct aidaw *aidaw;169 170 if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)171 return scmrq->next_aidaw;172 173 aidaw = scm_aidaw_alloc();174 if (aidaw)175 memset(aidaw, 0, PAGE_SIZE);176 return aidaw;177}178 179static int scm_request_prepare(struct scm_request *scmrq)180{181 struct scm_blk_dev *bdev = scmrq->bdev;182 struct scm_device *scmdev = bdev->gendisk->private_data;183 int pos = scmrq->aob->request.msb_count;184 struct msb *msb = &scmrq->aob->msb[pos];185 struct request *req = scmrq->request[pos];186 struct req_iterator iter;187 struct aidaw *aidaw;188 struct bio_vec bv;189 190 aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));191 if (!aidaw)192 return -ENOMEM;193 194 msb->bs = MSB_BS_4K;195 scmrq->aob->request.msb_count++;196 msb->scm_addr = scmdev->address + ((u64) blk_rq_pos(req) << 9);197 msb->oc = (rq_data_dir(req) == READ) ? MSB_OC_READ : MSB_OC_WRITE;198 msb->flags |= MSB_FLAG_IDA;199 msb->data_addr = virt_to_dma64(aidaw);200 201 rq_for_each_segment(bv, req, iter) {202 WARN_ON(bv.bv_offset);203 msb->blk_count += bv.bv_len >> 12;204 aidaw->data_addr = virt_to_dma64(page_address(bv.bv_page));205 aidaw++;206 }207 208 scmrq->next_aidaw = aidaw;209 return 0;210}211 212static inline void scm_request_set(struct scm_request *scmrq,213 struct request *req)214{215 scmrq->request[scmrq->aob->request.msb_count] = req;216}217 218static inline void scm_request_init(struct scm_blk_dev *bdev,219 struct scm_request *scmrq)220{221 struct aob_rq_header *aobrq = to_aobrq(scmrq);222 struct aob *aob = scmrq->aob;223 224 memset(scmrq->request, 0,225 nr_requests_per_io * sizeof(scmrq->request[0]));226 memset(aob, 0, sizeof(*aob));227 aobrq->scmdev = bdev->scmdev;228 aob->request.cmd_code = ARQB_CMD_MOVE;229 aob->request.data = (u64) aobrq;230 scmrq->bdev = bdev;231 scmrq->retries = 4;232 scmrq->error = BLK_STS_OK;233 /* We don't use all msbs - place aidaws at the end of the aob page. */234 scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io];235}236 237static void scm_request_requeue(struct scm_request *scmrq)238{239 struct scm_blk_dev *bdev = scmrq->bdev;240 int i;241 242 for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)243 blk_mq_requeue_request(scmrq->request[i], false);244 245 atomic_dec(&bdev->queued_reqs);246 scm_request_done(scmrq);247 blk_mq_kick_requeue_list(bdev->rq);248}249 250static void scm_request_finish(struct scm_request *scmrq)251{252 struct scm_blk_dev *bdev = scmrq->bdev;253 blk_status_t *error;254 int i;255 256 for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {257 error = blk_mq_rq_to_pdu(scmrq->request[i]);258 *error = scmrq->error;259 if (likely(!blk_should_fake_timeout(scmrq->request[i]->q)))260 blk_mq_complete_request(scmrq->request[i]);261 }262 263 atomic_dec(&bdev->queued_reqs);264 scm_request_done(scmrq);265}266 267static void scm_request_start(struct scm_request *scmrq)268{269 struct scm_blk_dev *bdev = scmrq->bdev;270 271 atomic_inc(&bdev->queued_reqs);272 if (eadm_start_aob(scmrq->aob)) {273 SCM_LOG(5, "no subchannel");274 scm_request_requeue(scmrq);275 }276}277 278struct scm_queue {279 struct scm_request *scmrq;280 spinlock_t lock;281};282 283static blk_status_t scm_blk_request(struct blk_mq_hw_ctx *hctx,284 const struct blk_mq_queue_data *qd)285{286 struct scm_device *scmdev = hctx->queue->queuedata;287 struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);288 struct scm_queue *sq = hctx->driver_data;289 struct request *req = qd->rq;290 struct scm_request *scmrq;291 292 spin_lock(&sq->lock);293 if (!scm_permit_request(bdev, req)) {294 spin_unlock(&sq->lock);295 return BLK_STS_RESOURCE;296 }297 298 scmrq = sq->scmrq;299 if (!scmrq) {300 scmrq = scm_request_fetch();301 if (!scmrq) {302 SCM_LOG(5, "no request");303 spin_unlock(&sq->lock);304 return BLK_STS_RESOURCE;305 }306 scm_request_init(bdev, scmrq);307 sq->scmrq = scmrq;308 }309 scm_request_set(scmrq, req);310 311 if (scm_request_prepare(scmrq)) {312 SCM_LOG(5, "aidaw alloc failed");313 scm_request_set(scmrq, NULL);314 315 if (scmrq->aob->request.msb_count)316 scm_request_start(scmrq);317 318 sq->scmrq = NULL;319 spin_unlock(&sq->lock);320 return BLK_STS_RESOURCE;321 }322 blk_mq_start_request(req);323 324 if (qd->last || scmrq->aob->request.msb_count == nr_requests_per_io) {325 scm_request_start(scmrq);326 sq->scmrq = NULL;327 }328 spin_unlock(&sq->lock);329 return BLK_STS_OK;330}331 332static int scm_blk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,333 unsigned int idx)334{335 struct scm_queue *qd = kzalloc(sizeof(*qd), GFP_KERNEL);336 337 if (!qd)338 return -ENOMEM;339 340 spin_lock_init(&qd->lock);341 hctx->driver_data = qd;342 343 return 0;344}345 346static void scm_blk_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int idx)347{348 struct scm_queue *qd = hctx->driver_data;349 350 WARN_ON(qd->scmrq);351 kfree(hctx->driver_data);352 hctx->driver_data = NULL;353}354 355static void __scmrq_log_error(struct scm_request *scmrq)356{357 struct aob *aob = scmrq->aob;358 359 if (scmrq->error == BLK_STS_TIMEOUT)360 SCM_LOG(1, "Request timeout");361 else {362 SCM_LOG(1, "Request error");363 SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));364 }365 if (scmrq->retries)366 SCM_LOG(1, "Retry request");367 else368 pr_err("An I/O operation to SCM failed with rc=%d\n",369 scmrq->error);370}371 372static void scm_blk_handle_error(struct scm_request *scmrq)373{374 struct scm_blk_dev *bdev = scmrq->bdev;375 unsigned long flags;376 377 if (scmrq->error != BLK_STS_IOERR)378 goto restart;379 380 /* For -EIO the response block is valid. */381 switch (scmrq->aob->response.eqc) {382 case EQC_WR_PROHIBIT:383 spin_lock_irqsave(&bdev->lock, flags);384 if (bdev->state != SCM_WR_PROHIBIT)385 pr_info("%lx: Write access to the SCM increment is suspended\n",386 (unsigned long) bdev->scmdev->address);387 bdev->state = SCM_WR_PROHIBIT;388 spin_unlock_irqrestore(&bdev->lock, flags);389 goto requeue;390 default:391 break;392 }393 394restart:395 if (!eadm_start_aob(scmrq->aob))396 return;397 398requeue:399 scm_request_requeue(scmrq);400}401 402void scm_blk_irq(struct scm_device *scmdev, void *data, blk_status_t error)403{404 struct scm_request *scmrq = data;405 406 scmrq->error = error;407 if (error) {408 __scmrq_log_error(scmrq);409 if (scmrq->retries-- > 0) {410 scm_blk_handle_error(scmrq);411 return;412 }413 }414 415 scm_request_finish(scmrq);416}417 418static void scm_blk_request_done(struct request *req)419{420 blk_status_t *error = blk_mq_rq_to_pdu(req);421 422 blk_mq_end_request(req, *error);423}424 425static const struct block_device_operations scm_blk_devops = {426 .owner = THIS_MODULE,427};428 429static const struct blk_mq_ops scm_mq_ops = {430 .queue_rq = scm_blk_request,431 .complete = scm_blk_request_done,432 .init_hctx = scm_blk_init_hctx,433 .exit_hctx = scm_blk_exit_hctx,434};435 436int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)437{438 struct queue_limits lim = {439 .logical_block_size = 1 << 12,440 };441 unsigned int devindex;442 int len, ret;443 444 lim.max_segments = min(scmdev->nr_max_block,445 (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));446 lim.max_hw_sectors = lim.max_segments << 3; /* 8 * 512 = blk_size */447 448 devindex = atomic_inc_return(&nr_devices) - 1;449 /* scma..scmz + scmaa..scmzz */450 if (devindex > 701) {451 ret = -ENODEV;452 goto out;453 }454 455 bdev->scmdev = scmdev;456 bdev->state = SCM_OPER;457 spin_lock_init(&bdev->lock);458 atomic_set(&bdev->queued_reqs, 0);459 460 bdev->tag_set.ops = &scm_mq_ops;461 bdev->tag_set.cmd_size = sizeof(blk_status_t);462 bdev->tag_set.nr_hw_queues = nr_requests;463 bdev->tag_set.queue_depth = nr_requests_per_io * nr_requests;464 bdev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;465 bdev->tag_set.numa_node = NUMA_NO_NODE;466 467 ret = blk_mq_alloc_tag_set(&bdev->tag_set);468 if (ret)469 goto out;470 471 bdev->gendisk = blk_mq_alloc_disk(&bdev->tag_set, &lim, scmdev);472 if (IS_ERR(bdev->gendisk)) {473 ret = PTR_ERR(bdev->gendisk);474 goto out_tag;475 }476 bdev->gendisk->private_data = scmdev;477 bdev->gendisk->fops = &scm_blk_devops;478 bdev->gendisk->major = scm_major;479 bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;480 bdev->gendisk->minors = SCM_NR_PARTS;481 482 len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");483 if (devindex > 25) {484 len += snprintf(bdev->gendisk->disk_name + len,485 DISK_NAME_LEN - len, "%c",486 'a' + (devindex / 26) - 1);487 devindex = devindex % 26;488 }489 snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",490 'a' + devindex);491 492 /* 512 byte sectors */493 set_capacity(bdev->gendisk, scmdev->size >> 9);494 ret = device_add_disk(&scmdev->dev, bdev->gendisk, NULL);495 if (ret)496 goto out_cleanup_disk;497 498 return 0;499 500out_cleanup_disk:501 put_disk(bdev->gendisk);502out_tag:503 blk_mq_free_tag_set(&bdev->tag_set);504out:505 atomic_dec(&nr_devices);506 return ret;507}508 509void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)510{511 del_gendisk(bdev->gendisk);512 put_disk(bdev->gendisk);513 blk_mq_free_tag_set(&bdev->tag_set);514}515 516void scm_blk_set_available(struct scm_blk_dev *bdev)517{518 unsigned long flags;519 520 spin_lock_irqsave(&bdev->lock, flags);521 if (bdev->state == SCM_WR_PROHIBIT)522 pr_info("%lx: Write access to the SCM increment is restored\n",523 (unsigned long) bdev->scmdev->address);524 bdev->state = SCM_OPER;525 spin_unlock_irqrestore(&bdev->lock, flags);526}527 528static bool __init scm_blk_params_valid(void)529{530 if (!nr_requests_per_io || nr_requests_per_io > 64)531 return false;532 533 return true;534}535 536static int __init scm_blk_init(void)537{538 int ret = -EINVAL;539 540 if (!scm_blk_params_valid())541 goto out;542 543 ret = register_blkdev(0, "scm");544 if (ret < 0)545 goto out;546 547 scm_major = ret;548 ret = scm_alloc_rqs(nr_requests);549 if (ret)550 goto out_free;551 552 scm_debug = debug_register("scm_log", 16, 1, 16);553 if (!scm_debug) {554 ret = -ENOMEM;555 goto out_free;556 }557 558 debug_register_view(scm_debug, &debug_hex_ascii_view);559 debug_set_level(scm_debug, 2);560 561 ret = scm_drv_init();562 if (ret)563 goto out_dbf;564 565 return ret;566 567out_dbf:568 debug_unregister(scm_debug);569out_free:570 scm_free_rqs();571 unregister_blkdev(scm_major, "scm");572out:573 return ret;574}575module_init(scm_blk_init);576 577static void __exit scm_blk_cleanup(void)578{579 scm_drv_cleanup();580 debug_unregister(scm_debug);581 scm_free_rqs();582 unregister_blkdev(scm_major, "scm");583}584module_exit(scm_blk_cleanup);585