3435 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 1999 Eric Youngdale4 * Copyright (C) 2014 Christoph Hellwig5 *6 * SCSI queueing library.7 * Initial versions: Eric Youngdale (eric@andante.org).8 * Based upon conversations with large numbers9 * of people at Linux Expo.10 */11 12#include <linux/bio.h>13#include <linux/bitops.h>14#include <linux/blkdev.h>15#include <linux/completion.h>16#include <linux/kernel.h>17#include <linux/export.h>18#include <linux/init.h>19#include <linux/pci.h>20#include <linux/delay.h>21#include <linux/hardirq.h>22#include <linux/scatterlist.h>23#include <linux/blk-mq.h>24#include <linux/blk-integrity.h>25#include <linux/ratelimit.h>26#include <linux/unaligned.h>27 28#include <scsi/scsi.h>29#include <scsi/scsi_cmnd.h>30#include <scsi/scsi_dbg.h>31#include <scsi/scsi_device.h>32#include <scsi/scsi_driver.h>33#include <scsi/scsi_eh.h>34#include <scsi/scsi_host.h>35#include <scsi/scsi_transport.h> /* scsi_init_limits() */36#include <scsi/scsi_dh.h>37 38#include <trace/events/scsi.h>39 40#include "scsi_debugfs.h"41#include "scsi_priv.h"42#include "scsi_logging.h"43 44/*45 * Size of integrity metadata is usually small, 1 inline sg should46 * cover normal cases.47 */48#ifdef CONFIG_ARCH_NO_SG_CHAIN49#define SCSI_INLINE_PROT_SG_CNT 050#define SCSI_INLINE_SG_CNT 051#else52#define SCSI_INLINE_PROT_SG_CNT 153#define SCSI_INLINE_SG_CNT 254#endif55 56static struct kmem_cache *scsi_sense_cache;57static DEFINE_MUTEX(scsi_sense_cache_mutex);58 59static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd);60 61int scsi_init_sense_cache(struct Scsi_Host *shost)62{63 int ret = 0;64 65 mutex_lock(&scsi_sense_cache_mutex);66 if (!scsi_sense_cache) {67 scsi_sense_cache =68 kmem_cache_create_usercopy("scsi_sense_cache",69 SCSI_SENSE_BUFFERSIZE, 0, SLAB_HWCACHE_ALIGN,70 0, SCSI_SENSE_BUFFERSIZE, NULL);71 if (!scsi_sense_cache)72 ret = -ENOMEM;73 }74 mutex_unlock(&scsi_sense_cache_mutex);75 return ret;76}77 78static void79scsi_set_blocked(struct scsi_cmnd *cmd, int reason)80{81 struct Scsi_Host *host = cmd->device->host;82 struct scsi_device *device = cmd->device;83 struct scsi_target *starget = scsi_target(device);84 85 /*86 * Set the appropriate busy bit for the device/host.87 *88 * If the host/device isn't busy, assume that something actually89 * completed, and that we should be able to queue a command now.90 *91 * Note that the prior mid-layer assumption that any host could92 * always queue at least one command is now broken. The mid-layer93 * will implement a user specifiable stall (see94 * scsi_host.max_host_blocked and scsi_device.max_device_blocked)95 * if a command is requeued with no other commands outstanding96 * either for the device or for the host.97 */98 switch (reason) {99 case SCSI_MLQUEUE_HOST_BUSY:100 atomic_set(&host->host_blocked, host->max_host_blocked);101 break;102 case SCSI_MLQUEUE_DEVICE_BUSY:103 case SCSI_MLQUEUE_EH_RETRY:104 atomic_set(&device->device_blocked,105 device->max_device_blocked);106 break;107 case SCSI_MLQUEUE_TARGET_BUSY:108 atomic_set(&starget->target_blocked,109 starget->max_target_blocked);110 break;111 }112}113 114static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd, unsigned long msecs)115{116 struct request *rq = scsi_cmd_to_rq(cmd);117 118 if (rq->rq_flags & RQF_DONTPREP) {119 rq->rq_flags &= ~RQF_DONTPREP;120 scsi_mq_uninit_cmd(cmd);121 } else {122 WARN_ON_ONCE(true);123 }124 125 blk_mq_requeue_request(rq, false);126 if (!scsi_host_in_recovery(cmd->device->host))127 blk_mq_delay_kick_requeue_list(rq->q, msecs);128}129 130/**131 * __scsi_queue_insert - private queue insertion132 * @cmd: The SCSI command being requeued133 * @reason: The reason for the requeue134 * @unbusy: Whether the queue should be unbusied135 *136 * This is a private queue insertion. The public interface137 * scsi_queue_insert() always assumes the queue should be unbusied138 * because it's always called before the completion. This function is139 * for a requeue after completion, which should only occur in this140 * file.141 */142static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)143{144 struct scsi_device *device = cmd->device;145 146 SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,147 "Inserting command %p into mlqueue\n", cmd));148 149 scsi_set_blocked(cmd, reason);150 151 /*152 * Decrement the counters, since these commands are no longer153 * active on the host/device.154 */155 if (unbusy)156 scsi_device_unbusy(device, cmd);157 158 /*159 * Requeue this command. It will go before all other commands160 * that are already in the queue. Schedule requeue work under161 * lock such that the kblockd_schedule_work() call happens162 * before blk_mq_destroy_queue() finishes.163 */164 cmd->result = 0;165 166 blk_mq_requeue_request(scsi_cmd_to_rq(cmd),167 !scsi_host_in_recovery(cmd->device->host));168}169 170/**171 * scsi_queue_insert - Reinsert a command in the queue.172 * @cmd: command that we are adding to queue.173 * @reason: why we are inserting command to queue.174 *175 * We do this for one of two cases. Either the host is busy and it cannot accept176 * any more commands for the time being, or the device returned QUEUE_FULL and177 * can accept no more commands.178 *179 * Context: This could be called either from an interrupt context or a normal180 * process context.181 */182void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)183{184 __scsi_queue_insert(cmd, reason, true);185}186 187void scsi_failures_reset_retries(struct scsi_failures *failures)188{189 struct scsi_failure *failure;190 191 failures->total_retries = 0;192 193 for (failure = failures->failure_definitions; failure->result;194 failure++)195 failure->retries = 0;196}197EXPORT_SYMBOL_GPL(scsi_failures_reset_retries);198 199/**200 * scsi_check_passthrough - Determine if passthrough scsi_cmnd needs a retry.201 * @scmd: scsi_cmnd to check.202 * @failures: scsi_failures struct that lists failures to check for.203 *204 * Returns -EAGAIN if the caller should retry else 0.205 */206static int scsi_check_passthrough(struct scsi_cmnd *scmd,207 struct scsi_failures *failures)208{209 struct scsi_failure *failure;210 struct scsi_sense_hdr sshdr;211 enum sam_status status;212 213 if (!failures)214 return 0;215 216 for (failure = failures->failure_definitions; failure->result;217 failure++) {218 if (failure->result == SCMD_FAILURE_RESULT_ANY)219 goto maybe_retry;220 221 if (host_byte(scmd->result) &&222 host_byte(scmd->result) == host_byte(failure->result))223 goto maybe_retry;224 225 status = status_byte(scmd->result);226 if (!status)227 continue;228 229 if (failure->result == SCMD_FAILURE_STAT_ANY &&230 !scsi_status_is_good(scmd->result))231 goto maybe_retry;232 233 if (status != status_byte(failure->result))234 continue;235 236 if (status_byte(failure->result) != SAM_STAT_CHECK_CONDITION ||237 failure->sense == SCMD_FAILURE_SENSE_ANY)238 goto maybe_retry;239 240 if (!scsi_command_normalize_sense(scmd, &sshdr))241 return 0;242 243 if (failure->sense != sshdr.sense_key)244 continue;245 246 if (failure->asc == SCMD_FAILURE_ASC_ANY)247 goto maybe_retry;248 249 if (failure->asc != sshdr.asc)250 continue;251 252 if (failure->ascq == SCMD_FAILURE_ASCQ_ANY ||253 failure->ascq == sshdr.ascq)254 goto maybe_retry;255 }256 257 return 0;258 259maybe_retry:260 if (failure->allowed) {261 if (failure->allowed == SCMD_FAILURE_NO_LIMIT ||262 ++failure->retries <= failure->allowed)263 return -EAGAIN;264 } else {265 if (failures->total_allowed == SCMD_FAILURE_NO_LIMIT ||266 ++failures->total_retries <= failures->total_allowed)267 return -EAGAIN;268 }269 270 return 0;271}272 273/**274 * scsi_execute_cmd - insert request and wait for the result275 * @sdev: scsi_device276 * @cmd: scsi command277 * @opf: block layer request cmd_flags278 * @buffer: data buffer279 * @bufflen: len of buffer280 * @timeout: request timeout in HZ281 * @ml_retries: number of times SCSI midlayer will retry request282 * @args: Optional args. See struct definition for field descriptions283 *284 * Returns the scsi_cmnd result field if a command was executed, or a negative285 * Linux error code if we didn't get that far.286 */287int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd,288 blk_opf_t opf, void *buffer, unsigned int bufflen,289 int timeout, int ml_retries,290 const struct scsi_exec_args *args)291{292 static const struct scsi_exec_args default_args;293 struct request *req;294 struct scsi_cmnd *scmd;295 int ret;296 297 if (!args)298 args = &default_args;299 else if (WARN_ON_ONCE(args->sense &&300 args->sense_len != SCSI_SENSE_BUFFERSIZE))301 return -EINVAL;302 303retry:304 req = scsi_alloc_request(sdev->request_queue, opf, args->req_flags);305 if (IS_ERR(req))306 return PTR_ERR(req);307 308 if (bufflen) {309 ret = blk_rq_map_kern(sdev->request_queue, req,310 buffer, bufflen, GFP_NOIO);311 if (ret)312 goto out;313 }314 scmd = blk_mq_rq_to_pdu(req);315 scmd->cmd_len = COMMAND_SIZE(cmd[0]);316 memcpy(scmd->cmnd, cmd, scmd->cmd_len);317 scmd->allowed = ml_retries;318 scmd->flags |= args->scmd_flags;319 req->timeout = timeout;320 req->rq_flags |= RQF_QUIET;321 322 /*323 * head injection *required* here otherwise quiesce won't work324 */325 blk_execute_rq(req, true);326 327 if (scsi_check_passthrough(scmd, args->failures) == -EAGAIN) {328 blk_mq_free_request(req);329 goto retry;330 }331 332 /*333 * Some devices (USB mass-storage in particular) may transfer334 * garbage data together with a residue indicating that the data335 * is invalid. Prevent the garbage from being misinterpreted336 * and prevent security leaks by zeroing out the excess data.337 */338 if (unlikely(scmd->resid_len > 0 && scmd->resid_len <= bufflen))339 memset(buffer + bufflen - scmd->resid_len, 0, scmd->resid_len);340 341 if (args->resid)342 *args->resid = scmd->resid_len;343 if (args->sense)344 memcpy(args->sense, scmd->sense_buffer, SCSI_SENSE_BUFFERSIZE);345 if (args->sshdr)346 scsi_normalize_sense(scmd->sense_buffer, scmd->sense_len,347 args->sshdr);348 349 ret = scmd->result;350 out:351 blk_mq_free_request(req);352 353 return ret;354}355EXPORT_SYMBOL(scsi_execute_cmd);356 357/*358 * Wake up the error handler if necessary. Avoid as follows that the error359 * handler is not woken up if host in-flight requests number ==360 * shost->host_failed: use call_rcu() in scsi_eh_scmd_add() in combination361 * with an RCU read lock in this function to ensure that this function in362 * its entirety either finishes before scsi_eh_scmd_add() increases the363 * host_failed counter or that it notices the shost state change made by364 * scsi_eh_scmd_add().365 */366static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd)367{368 unsigned long flags;369 370 rcu_read_lock();371 __clear_bit(SCMD_STATE_INFLIGHT, &cmd->state);372 if (unlikely(scsi_host_in_recovery(shost))) {373 unsigned int busy = scsi_host_busy(shost);374 375 spin_lock_irqsave(shost->host_lock, flags);376 if (shost->host_failed || shost->host_eh_scheduled)377 scsi_eh_wakeup(shost, busy);378 spin_unlock_irqrestore(shost->host_lock, flags);379 }380 rcu_read_unlock();381}382 383void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)384{385 struct Scsi_Host *shost = sdev->host;386 struct scsi_target *starget = scsi_target(sdev);387 388 scsi_dec_host_busy(shost, cmd);389 390 if (starget->can_queue > 0)391 atomic_dec(&starget->target_busy);392 393 sbitmap_put(&sdev->budget_map, cmd->budget_token);394 cmd->budget_token = -1;395}396 397/*398 * Kick the queue of SCSI device @sdev if @sdev != current_sdev. Called with399 * interrupts disabled.400 */401static void scsi_kick_sdev_queue(struct scsi_device *sdev, void *data)402{403 struct scsi_device *current_sdev = data;404 405 if (sdev != current_sdev)406 blk_mq_run_hw_queues(sdev->request_queue, true);407}408 409/*410 * Called for single_lun devices on IO completion. Clear starget_sdev_user,411 * and call blk_run_queue for all the scsi_devices on the target -412 * including current_sdev first.413 *414 * Called with *no* scsi locks held.415 */416static void scsi_single_lun_run(struct scsi_device *current_sdev)417{418 struct Scsi_Host *shost = current_sdev->host;419 struct scsi_target *starget = scsi_target(current_sdev);420 unsigned long flags;421 422 spin_lock_irqsave(shost->host_lock, flags);423 starget->starget_sdev_user = NULL;424 spin_unlock_irqrestore(shost->host_lock, flags);425 426 /*427 * Call blk_run_queue for all LUNs on the target, starting with428 * current_sdev. We race with others (to set starget_sdev_user),429 * but in most cases, we will be first. Ideally, each LU on the430 * target would get some limited time or requests on the target.431 */432 blk_mq_run_hw_queues(current_sdev->request_queue,433 shost->queuecommand_may_block);434 435 spin_lock_irqsave(shost->host_lock, flags);436 if (!starget->starget_sdev_user)437 __starget_for_each_device(starget, current_sdev,438 scsi_kick_sdev_queue);439 spin_unlock_irqrestore(shost->host_lock, flags);440}441 442static inline bool scsi_device_is_busy(struct scsi_device *sdev)443{444 if (scsi_device_busy(sdev) >= sdev->queue_depth)445 return true;446 if (atomic_read(&sdev->device_blocked) > 0)447 return true;448 return false;449}450 451static inline bool scsi_target_is_busy(struct scsi_target *starget)452{453 if (starget->can_queue > 0) {454 if (atomic_read(&starget->target_busy) >= starget->can_queue)455 return true;456 if (atomic_read(&starget->target_blocked) > 0)457 return true;458 }459 return false;460}461 462static inline bool scsi_host_is_busy(struct Scsi_Host *shost)463{464 if (atomic_read(&shost->host_blocked) > 0)465 return true;466 if (shost->host_self_blocked)467 return true;468 return false;469}470 471static void scsi_starved_list_run(struct Scsi_Host *shost)472{473 LIST_HEAD(starved_list);474 struct scsi_device *sdev;475 unsigned long flags;476 477 spin_lock_irqsave(shost->host_lock, flags);478 list_splice_init(&shost->starved_list, &starved_list);479 480 while (!list_empty(&starved_list)) {481 struct request_queue *slq;482 483 /*484 * As long as shost is accepting commands and we have485 * starved queues, call blk_run_queue. scsi_request_fn486 * drops the queue_lock and can add us back to the487 * starved_list.488 *489 * host_lock protects the starved_list and starved_entry.490 * scsi_request_fn must get the host_lock before checking491 * or modifying starved_list or starved_entry.492 */493 if (scsi_host_is_busy(shost))494 break;495 496 sdev = list_entry(starved_list.next,497 struct scsi_device, starved_entry);498 list_del_init(&sdev->starved_entry);499 if (scsi_target_is_busy(scsi_target(sdev))) {500 list_move_tail(&sdev->starved_entry,501 &shost->starved_list);502 continue;503 }504 505 /*506 * Once we drop the host lock, a racing scsi_remove_device()507 * call may remove the sdev from the starved list and destroy508 * it and the queue. Mitigate by taking a reference to the509 * queue and never touching the sdev again after we drop the510 * host lock. Note: if __scsi_remove_device() invokes511 * blk_mq_destroy_queue() before the queue is run from this512 * function then blk_run_queue() will return immediately since513 * blk_mq_destroy_queue() marks the queue with QUEUE_FLAG_DYING.514 */515 slq = sdev->request_queue;516 if (!blk_get_queue(slq))517 continue;518 spin_unlock_irqrestore(shost->host_lock, flags);519 520 blk_mq_run_hw_queues(slq, false);521 blk_put_queue(slq);522 523 spin_lock_irqsave(shost->host_lock, flags);524 }525 /* put any unprocessed entries back */526 list_splice(&starved_list, &shost->starved_list);527 spin_unlock_irqrestore(shost->host_lock, flags);528}529 530/**531 * scsi_run_queue - Select a proper request queue to serve next.532 * @q: last request's queue533 *534 * The previous command was completely finished, start a new one if possible.535 */536static void scsi_run_queue(struct request_queue *q)537{538 struct scsi_device *sdev = q->queuedata;539 540 if (scsi_target(sdev)->single_lun)541 scsi_single_lun_run(sdev);542 if (!list_empty(&sdev->host->starved_list))543 scsi_starved_list_run(sdev->host);544 545 /* Note: blk_mq_kick_requeue_list() runs the queue asynchronously. */546 blk_mq_kick_requeue_list(q);547}548 549void scsi_requeue_run_queue(struct work_struct *work)550{551 struct scsi_device *sdev;552 struct request_queue *q;553 554 sdev = container_of(work, struct scsi_device, requeue_work);555 q = sdev->request_queue;556 scsi_run_queue(q);557}558 559void scsi_run_host_queues(struct Scsi_Host *shost)560{561 struct scsi_device *sdev;562 563 shost_for_each_device(sdev, shost)564 scsi_run_queue(sdev->request_queue);565}566 567static void scsi_uninit_cmd(struct scsi_cmnd *cmd)568{569 if (!blk_rq_is_passthrough(scsi_cmd_to_rq(cmd))) {570 struct scsi_driver *drv = scsi_cmd_to_driver(cmd);571 572 if (drv->uninit_command)573 drv->uninit_command(cmd);574 }575}576 577void scsi_free_sgtables(struct scsi_cmnd *cmd)578{579 if (cmd->sdb.table.nents)580 sg_free_table_chained(&cmd->sdb.table,581 SCSI_INLINE_SG_CNT);582 if (scsi_prot_sg_count(cmd))583 sg_free_table_chained(&cmd->prot_sdb->table,584 SCSI_INLINE_PROT_SG_CNT);585}586EXPORT_SYMBOL_GPL(scsi_free_sgtables);587 588static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)589{590 scsi_free_sgtables(cmd);591 scsi_uninit_cmd(cmd);592}593 594static void scsi_run_queue_async(struct scsi_device *sdev)595{596 if (scsi_host_in_recovery(sdev->host))597 return;598 599 if (scsi_target(sdev)->single_lun ||600 !list_empty(&sdev->host->starved_list)) {601 kblockd_schedule_work(&sdev->requeue_work);602 } else {603 /*604 * smp_mb() present in sbitmap_queue_clear() or implied in605 * .end_io is for ordering writing .device_busy in606 * scsi_device_unbusy() and reading sdev->restarts.607 */608 int old = atomic_read(&sdev->restarts);609 610 /*611 * ->restarts has to be kept as non-zero if new budget612 * contention occurs.613 *614 * No need to run queue when either another re-run615 * queue wins in updating ->restarts or a new budget616 * contention occurs.617 */618 if (old && atomic_cmpxchg(&sdev->restarts, old, 0) == old)619 blk_mq_run_hw_queues(sdev->request_queue, true);620 }621}622 623/* Returns false when no more bytes to process, true if there are more */624static bool scsi_end_request(struct request *req, blk_status_t error,625 unsigned int bytes)626{627 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);628 struct scsi_device *sdev = cmd->device;629 struct request_queue *q = sdev->request_queue;630 631 if (blk_update_request(req, error, bytes))632 return true;633 634 if (q->limits.features & BLK_FEAT_ADD_RANDOM)635 add_disk_randomness(req->q->disk);636 637 WARN_ON_ONCE(!blk_rq_is_passthrough(req) &&638 !(cmd->flags & SCMD_INITIALIZED));639 cmd->flags = 0;640 641 /*642 * Calling rcu_barrier() is not necessary here because the643 * SCSI error handler guarantees that the function called by644 * call_rcu() has been called before scsi_end_request() is645 * called.646 */647 destroy_rcu_head(&cmd->rcu);648 649 /*650 * In the MQ case the command gets freed by __blk_mq_end_request,651 * so we have to do all cleanup that depends on it earlier.652 *653 * We also can't kick the queues from irq context, so we654 * will have to defer it to a workqueue.655 */656 scsi_mq_uninit_cmd(cmd);657 658 /*659 * queue is still alive, so grab the ref for preventing it660 * from being cleaned up during running queue.661 */662 percpu_ref_get(&q->q_usage_counter);663 664 __blk_mq_end_request(req, error);665 666 scsi_run_queue_async(sdev);667 668 percpu_ref_put(&q->q_usage_counter);669 return false;670}671 672/**673 * scsi_result_to_blk_status - translate a SCSI result code into blk_status_t674 * @result: scsi error code675 *676 * Translate a SCSI result code into a blk_status_t value.677 */678static blk_status_t scsi_result_to_blk_status(int result)679{680 /*681 * Check the scsi-ml byte first in case we converted a host or status682 * byte.683 */684 switch (scsi_ml_byte(result)) {685 case SCSIML_STAT_OK:686 break;687 case SCSIML_STAT_RESV_CONFLICT:688 return BLK_STS_RESV_CONFLICT;689 case SCSIML_STAT_NOSPC:690 return BLK_STS_NOSPC;691 case SCSIML_STAT_MED_ERROR:692 return BLK_STS_MEDIUM;693 case SCSIML_STAT_TGT_FAILURE:694 return BLK_STS_TARGET;695 case SCSIML_STAT_DL_TIMEOUT:696 return BLK_STS_DURATION_LIMIT;697 }698 699 switch (host_byte(result)) {700 case DID_OK:701 if (scsi_status_is_good(result))702 return BLK_STS_OK;703 return BLK_STS_IOERR;704 case DID_TRANSPORT_FAILFAST:705 case DID_TRANSPORT_MARGINAL:706 return BLK_STS_TRANSPORT;707 default:708 return BLK_STS_IOERR;709 }710}711 712/**713 * scsi_rq_err_bytes - determine number of bytes till the next failure boundary714 * @rq: request to examine715 *716 * Description:717 * A request could be merge of IOs which require different failure718 * handling. This function determines the number of bytes which719 * can be failed from the beginning of the request without720 * crossing into area which need to be retried further.721 *722 * Return:723 * The number of bytes to fail.724 */725static unsigned int scsi_rq_err_bytes(const struct request *rq)726{727 blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;728 unsigned int bytes = 0;729 struct bio *bio;730 731 if (!(rq->rq_flags & RQF_MIXED_MERGE))732 return blk_rq_bytes(rq);733 734 /*735 * Currently the only 'mixing' which can happen is between736 * different fastfail types. We can safely fail portions737 * which have all the failfast bits that the first one has -738 * the ones which are at least as eager to fail as the first739 * one.740 */741 for (bio = rq->bio; bio; bio = bio->bi_next) {742 if ((bio->bi_opf & ff) != ff)743 break;744 bytes += bio->bi_iter.bi_size;745 }746 747 /* this could lead to infinite loop */748 BUG_ON(blk_rq_bytes(rq) && !bytes);749 return bytes;750}751 752static bool scsi_cmd_runtime_exceeced(struct scsi_cmnd *cmd)753{754 struct request *req = scsi_cmd_to_rq(cmd);755 unsigned long wait_for;756 757 if (cmd->allowed == SCSI_CMD_RETRIES_NO_LIMIT)758 return false;759 760 wait_for = (cmd->allowed + 1) * req->timeout;761 if (time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {762 scmd_printk(KERN_ERR, cmd, "timing out command, waited %lus\n",763 wait_for/HZ);764 return true;765 }766 return false;767}768 769/*770 * When ALUA transition state is returned, reprep the cmd to771 * use the ALUA handler's transition timeout. Delay the reprep772 * 1 sec to avoid aggressive retries of the target in that773 * state.774 */775#define ALUA_TRANSITION_REPREP_DELAY 1000776 777/* Helper for scsi_io_completion() when special action required. */778static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)779{780 struct request *req = scsi_cmd_to_rq(cmd);781 int level = 0;782 enum {ACTION_FAIL, ACTION_REPREP, ACTION_DELAYED_REPREP,783 ACTION_RETRY, ACTION_DELAYED_RETRY} action;784 struct scsi_sense_hdr sshdr;785 bool sense_valid;786 bool sense_current = true; /* false implies "deferred sense" */787 blk_status_t blk_stat;788 789 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);790 if (sense_valid)791 sense_current = !scsi_sense_is_deferred(&sshdr);792 793 blk_stat = scsi_result_to_blk_status(result);794 795 if (host_byte(result) == DID_RESET) {796 /* Third party bus reset or reset for error recovery797 * reasons. Just retry the command and see what798 * happens.799 */800 action = ACTION_RETRY;801 } else if (sense_valid && sense_current) {802 switch (sshdr.sense_key) {803 case UNIT_ATTENTION:804 if (cmd->device->removable) {805 /* Detected disc change. Set a bit806 * and quietly refuse further access.807 */808 cmd->device->changed = 1;809 action = ACTION_FAIL;810 } else {811 /* Must have been a power glitch, or a812 * bus reset. Could not have been a813 * media change, so we just retry the814 * command and see what happens.815 */816 action = ACTION_RETRY;817 }818 break;819 case ILLEGAL_REQUEST:820 /* If we had an ILLEGAL REQUEST returned, then821 * we may have performed an unsupported822 * command. The only thing this should be823 * would be a ten byte read where only a six824 * byte read was supported. Also, on a system825 * where READ CAPACITY failed, we may have826 * read past the end of the disk.827 */828 if ((cmd->device->use_10_for_rw &&829 sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&830 (cmd->cmnd[0] == READ_10 ||831 cmd->cmnd[0] == WRITE_10)) {832 /* This will issue a new 6-byte command. */833 cmd->device->use_10_for_rw = 0;834 action = ACTION_REPREP;835 } else if (sshdr.asc == 0x10) /* DIX */ {836 action = ACTION_FAIL;837 blk_stat = BLK_STS_PROTECTION;838 /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */839 } else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {840 action = ACTION_FAIL;841 blk_stat = BLK_STS_TARGET;842 } else843 action = ACTION_FAIL;844 break;845 case ABORTED_COMMAND:846 action = ACTION_FAIL;847 if (sshdr.asc == 0x10) /* DIF */848 blk_stat = BLK_STS_PROTECTION;849 break;850 case NOT_READY:851 /* If the device is in the process of becoming852 * ready, or has a temporary blockage, retry.853 */854 if (sshdr.asc == 0x04) {855 switch (sshdr.ascq) {856 case 0x01: /* becoming ready */857 case 0x04: /* format in progress */858 case 0x05: /* rebuild in progress */859 case 0x06: /* recalculation in progress */860 case 0x07: /* operation in progress */861 case 0x08: /* Long write in progress */862 case 0x09: /* self test in progress */863 case 0x11: /* notify (enable spinup) required */864 case 0x14: /* space allocation in progress */865 case 0x1a: /* start stop unit in progress */866 case 0x1b: /* sanitize in progress */867 case 0x1d: /* configuration in progress */868 case 0x24: /* depopulation in progress */869 case 0x25: /* depopulation restore in progress */870 action = ACTION_DELAYED_RETRY;871 break;872 case 0x0a: /* ALUA state transition */873 action = ACTION_DELAYED_REPREP;874 break;875 default:876 action = ACTION_FAIL;877 break;878 }879 } else880 action = ACTION_FAIL;881 break;882 case VOLUME_OVERFLOW:883 /* See SSC3rXX or current. */884 action = ACTION_FAIL;885 break;886 case DATA_PROTECT:887 action = ACTION_FAIL;888 if ((sshdr.asc == 0x0C && sshdr.ascq == 0x12) ||889 (sshdr.asc == 0x55 &&890 (sshdr.ascq == 0x0E || sshdr.ascq == 0x0F))) {891 /* Insufficient zone resources */892 blk_stat = BLK_STS_ZONE_OPEN_RESOURCE;893 }894 break;895 case COMPLETED:896 fallthrough;897 default:898 action = ACTION_FAIL;899 break;900 }901 } else902 action = ACTION_FAIL;903 904 if (action != ACTION_FAIL && scsi_cmd_runtime_exceeced(cmd))905 action = ACTION_FAIL;906 907 switch (action) {908 case ACTION_FAIL:909 /* Give up and fail the remainder of the request */910 if (!(req->rq_flags & RQF_QUIET)) {911 static DEFINE_RATELIMIT_STATE(_rs,912 DEFAULT_RATELIMIT_INTERVAL,913 DEFAULT_RATELIMIT_BURST);914 915 if (unlikely(scsi_logging_level))916 level =917 SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT,918 SCSI_LOG_MLCOMPLETE_BITS);919 920 /*921 * if logging is enabled the failure will be printed922 * in scsi_log_completion(), so avoid duplicate messages923 */924 if (!level && __ratelimit(&_rs)) {925 scsi_print_result(cmd, NULL, FAILED);926 if (sense_valid)927 scsi_print_sense(cmd);928 scsi_print_command(cmd);929 }930 }931 if (!scsi_end_request(req, blk_stat, scsi_rq_err_bytes(req)))932 return;933 fallthrough;934 case ACTION_REPREP:935 scsi_mq_requeue_cmd(cmd, 0);936 break;937 case ACTION_DELAYED_REPREP:938 scsi_mq_requeue_cmd(cmd, ALUA_TRANSITION_REPREP_DELAY);939 break;940 case ACTION_RETRY:941 /* Retry the same command immediately */942 __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, false);943 break;944 case ACTION_DELAYED_RETRY:945 /* Retry the same command after a delay */946 __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, false);947 break;948 }949}950 951/*952 * Helper for scsi_io_completion() when cmd->result is non-zero. Returns a953 * new result that may suppress further error checking. Also modifies954 * *blk_statp in some cases.955 */956static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result,957 blk_status_t *blk_statp)958{959 bool sense_valid;960 bool sense_current = true; /* false implies "deferred sense" */961 struct request *req = scsi_cmd_to_rq(cmd);962 struct scsi_sense_hdr sshdr;963 964 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);965 if (sense_valid)966 sense_current = !scsi_sense_is_deferred(&sshdr);967 968 if (blk_rq_is_passthrough(req)) {969 if (sense_valid) {970 /*971 * SG_IO wants current and deferred errors972 */973 cmd->sense_len = min(8 + cmd->sense_buffer[7],974 SCSI_SENSE_BUFFERSIZE);975 }976 if (sense_current)977 *blk_statp = scsi_result_to_blk_status(result);978 } else if (blk_rq_bytes(req) == 0 && sense_current) {979 /*980 * Flush commands do not transfers any data, and thus cannot use981 * good_bytes != blk_rq_bytes(req) as the signal for an error.982 * This sets *blk_statp explicitly for the problem case.983 */984 *blk_statp = scsi_result_to_blk_status(result);985 }986 /*987 * Recovered errors need reporting, but they're always treated as988 * success, so fiddle the result code here. For passthrough requests989 * we already took a copy of the original into sreq->result which990 * is what gets returned to the user991 */992 if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {993 bool do_print = true;994 /*995 * if ATA PASS-THROUGH INFORMATION AVAILABLE [0x0, 0x1d]996 * skip print since caller wants ATA registers. Only occurs997 * on SCSI ATA PASS_THROUGH commands when CK_COND=1998 */999 if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))1000 do_print = false;1001 else if (req->rq_flags & RQF_QUIET)1002 do_print = false;1003 if (do_print)1004 scsi_print_sense(cmd);1005 result = 0;1006 /* for passthrough, *blk_statp may be set */1007 *blk_statp = BLK_STS_OK;1008 }1009 /*1010 * Another corner case: the SCSI status byte is non-zero but 'good'.1011 * Example: PRE-FETCH command returns SAM_STAT_CONDITION_MET when1012 * it is able to fit nominated LBs in its cache (and SAM_STAT_GOOD1013 * if it can't fit). Treat SAM_STAT_CONDITION_MET and the related1014 * intermediate statuses (both obsolete in SAM-4) as good.1015 */1016 if ((result & 0xff) && scsi_status_is_good(result)) {1017 result = 0;1018 *blk_statp = BLK_STS_OK;1019 }1020 return result;1021}1022 1023/**1024 * scsi_io_completion - Completion processing for SCSI commands.1025 * @cmd: command that is finished.1026 * @good_bytes: number of processed bytes.1027 *1028 * We will finish off the specified number of sectors. If we are done, the1029 * command block will be released and the queue function will be goosed. If we1030 * are not done then we have to figure out what to do next:1031 *1032 * a) We can call scsi_mq_requeue_cmd(). The request will be1033 * unprepared and put back on the queue. Then a new command will1034 * be created for it. This should be used if we made forward1035 * progress, or if we want to switch from READ(10) to READ(6) for1036 * example.1037 *1038 * b) We can call scsi_io_completion_action(). The request will be1039 * put back on the queue and retried using the same command as1040 * before, possibly after a delay.1041 *1042 * c) We can call scsi_end_request() with blk_stat other than1043 * BLK_STS_OK, to fail the remainder of the request.1044 */1045void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)1046{1047 int result = cmd->result;1048 struct request *req = scsi_cmd_to_rq(cmd);1049 blk_status_t blk_stat = BLK_STS_OK;1050 1051 if (unlikely(result)) /* a nz result may or may not be an error */1052 result = scsi_io_completion_nz_result(cmd, result, &blk_stat);1053 1054 /*1055 * Next deal with any sectors which we were able to correctly1056 * handle.1057 */1058 SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd,1059 "%u sectors total, %d bytes done.\n",1060 blk_rq_sectors(req), good_bytes));1061 1062 /*1063 * Failed, zero length commands always need to drop down1064 * to retry code. Fast path should return in this block.1065 */1066 if (likely(blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK)) {1067 if (likely(!scsi_end_request(req, blk_stat, good_bytes)))1068 return; /* no bytes remaining */1069 }1070 1071 /* Kill remainder if no retries. */1072 if (unlikely(blk_stat && scsi_noretry_cmd(cmd))) {1073 if (scsi_end_request(req, blk_stat, blk_rq_bytes(req)))1074 WARN_ONCE(true,1075 "Bytes remaining after failed, no-retry command");1076 return;1077 }1078 1079 /*1080 * If there had been no error, but we have leftover bytes in the1081 * request just queue the command up again.1082 */1083 if (likely(result == 0))1084 scsi_mq_requeue_cmd(cmd, 0);1085 else1086 scsi_io_completion_action(cmd, result);1087}1088 1089static inline bool scsi_cmd_needs_dma_drain(struct scsi_device *sdev,1090 struct request *rq)1091{1092 return sdev->dma_drain_len && blk_rq_is_passthrough(rq) &&1093 !op_is_write(req_op(rq)) &&1094 sdev->host->hostt->dma_need_drain(rq);1095}1096 1097/**1098 * scsi_alloc_sgtables - Allocate and initialize data and integrity scatterlists1099 * @cmd: SCSI command data structure to initialize.1100 *1101 * Initializes @cmd->sdb and also @cmd->prot_sdb if data integrity is enabled1102 * for @cmd.1103 *1104 * Returns:1105 * * BLK_STS_OK - on success1106 * * BLK_STS_RESOURCE - if the failure is retryable1107 * * BLK_STS_IOERR - if the failure is fatal1108 */1109blk_status_t scsi_alloc_sgtables(struct scsi_cmnd *cmd)1110{1111 struct scsi_device *sdev = cmd->device;1112 struct request *rq = scsi_cmd_to_rq(cmd);1113 unsigned short nr_segs = blk_rq_nr_phys_segments(rq);1114 struct scatterlist *last_sg = NULL;1115 blk_status_t ret;1116 bool need_drain = scsi_cmd_needs_dma_drain(sdev, rq);1117 int count;1118 1119 if (WARN_ON_ONCE(!nr_segs))1120 return BLK_STS_IOERR;1121 1122 /*1123 * Make sure there is space for the drain. The driver must adjust1124 * max_hw_segments to be prepared for this.1125 */1126 if (need_drain)1127 nr_segs++;1128 1129 /*1130 * If sg table allocation fails, requeue request later.1131 */1132 if (unlikely(sg_alloc_table_chained(&cmd->sdb.table, nr_segs,1133 cmd->sdb.table.sgl, SCSI_INLINE_SG_CNT)))1134 return BLK_STS_RESOURCE;1135 1136 /*1137 * Next, walk the list, and fill in the addresses and sizes of1138 * each segment.1139 */1140 count = __blk_rq_map_sg(rq->q, rq, cmd->sdb.table.sgl, &last_sg);1141 1142 if (blk_rq_bytes(rq) & rq->q->limits.dma_pad_mask) {1143 unsigned int pad_len =1144 (rq->q->limits.dma_pad_mask & ~blk_rq_bytes(rq)) + 1;1145 1146 last_sg->length += pad_len;1147 cmd->extra_len += pad_len;1148 }1149 1150 if (need_drain) {1151 sg_unmark_end(last_sg);1152 last_sg = sg_next(last_sg);1153 sg_set_buf(last_sg, sdev->dma_drain_buf, sdev->dma_drain_len);1154 sg_mark_end(last_sg);1155 1156 cmd->extra_len += sdev->dma_drain_len;1157 count++;1158 }1159 1160 BUG_ON(count > cmd->sdb.table.nents);1161 cmd->sdb.table.nents = count;1162 cmd->sdb.length = blk_rq_payload_bytes(rq);1163 1164 if (blk_integrity_rq(rq)) {1165 struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;1166 1167 if (WARN_ON_ONCE(!prot_sdb)) {1168 /*1169 * This can happen if someone (e.g. multipath)1170 * queues a command to a device on an adapter1171 * that does not support DIX.1172 */1173 ret = BLK_STS_IOERR;1174 goto out_free_sgtables;1175 }1176 1177 if (sg_alloc_table_chained(&prot_sdb->table,1178 rq->nr_integrity_segments,1179 prot_sdb->table.sgl,1180 SCSI_INLINE_PROT_SG_CNT)) {1181 ret = BLK_STS_RESOURCE;1182 goto out_free_sgtables;1183 }1184 1185 count = blk_rq_map_integrity_sg(rq, prot_sdb->table.sgl);1186 cmd->prot_sdb = prot_sdb;1187 cmd->prot_sdb->table.nents = count;1188 }1189 1190 return BLK_STS_OK;1191out_free_sgtables:1192 scsi_free_sgtables(cmd);1193 return ret;1194}1195EXPORT_SYMBOL(scsi_alloc_sgtables);1196 1197/**1198 * scsi_initialize_rq - initialize struct scsi_cmnd partially1199 * @rq: Request associated with the SCSI command to be initialized.1200 *1201 * This function initializes the members of struct scsi_cmnd that must be1202 * initialized before request processing starts and that won't be1203 * reinitialized if a SCSI command is requeued.1204 */1205static void scsi_initialize_rq(struct request *rq)1206{1207 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);1208 1209 memset(cmd->cmnd, 0, sizeof(cmd->cmnd));1210 cmd->cmd_len = MAX_COMMAND_SIZE;1211 cmd->sense_len = 0;1212 init_rcu_head(&cmd->rcu);1213 cmd->jiffies_at_alloc = jiffies;1214 cmd->retries = 0;1215}1216 1217struct request *scsi_alloc_request(struct request_queue *q, blk_opf_t opf,1218 blk_mq_req_flags_t flags)1219{1220 struct request *rq;1221 1222 rq = blk_mq_alloc_request(q, opf, flags);1223 if (!IS_ERR(rq))1224 scsi_initialize_rq(rq);1225 return rq;1226}1227EXPORT_SYMBOL_GPL(scsi_alloc_request);1228 1229/*1230 * Only called when the request isn't completed by SCSI, and not freed by1231 * SCSI1232 */1233static void scsi_cleanup_rq(struct request *rq)1234{1235 if (rq->rq_flags & RQF_DONTPREP) {1236 scsi_mq_uninit_cmd(blk_mq_rq_to_pdu(rq));1237 rq->rq_flags &= ~RQF_DONTPREP;1238 }1239}1240 1241/* Called before a request is prepared. See also scsi_mq_prep_fn(). */1242void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)1243{1244 struct request *rq = scsi_cmd_to_rq(cmd);1245 1246 if (!blk_rq_is_passthrough(rq) && !(cmd->flags & SCMD_INITIALIZED)) {1247 cmd->flags |= SCMD_INITIALIZED;1248 scsi_initialize_rq(rq);1249 }1250 1251 cmd->device = dev;1252 INIT_LIST_HEAD(&cmd->eh_entry);1253 INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);1254}1255 1256static blk_status_t scsi_setup_scsi_cmnd(struct scsi_device *sdev,1257 struct request *req)1258{1259 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);1260 1261 /*1262 * Passthrough requests may transfer data, in which case they must1263 * a bio attached to them. Or they might contain a SCSI command1264 * that does not transfer data, in which case they may optionally1265 * submit a request without an attached bio.1266 */1267 if (req->bio) {1268 blk_status_t ret = scsi_alloc_sgtables(cmd);1269 if (unlikely(ret != BLK_STS_OK))1270 return ret;1271 } else {1272 BUG_ON(blk_rq_bytes(req));1273 1274 memset(&cmd->sdb, 0, sizeof(cmd->sdb));1275 }1276 1277 cmd->transfersize = blk_rq_bytes(req);1278 return BLK_STS_OK;1279}1280 1281static blk_status_t1282scsi_device_state_check(struct scsi_device *sdev, struct request *req)1283{1284 switch (sdev->sdev_state) {1285 case SDEV_CREATED:1286 return BLK_STS_OK;1287 case SDEV_OFFLINE:1288 case SDEV_TRANSPORT_OFFLINE:1289 /*1290 * If the device is offline we refuse to process any1291 * commands. The device must be brought online1292 * before trying any recovery commands.1293 */1294 if (!sdev->offline_already) {1295 sdev->offline_already = true;1296 sdev_printk(KERN_ERR, sdev,1297 "rejecting I/O to offline device\n");1298 }1299 return BLK_STS_IOERR;1300 case SDEV_DEL:1301 /*1302 * If the device is fully deleted, we refuse to1303 * process any commands as well.1304 */1305 sdev_printk(KERN_ERR, sdev,1306 "rejecting I/O to dead device\n");1307 return BLK_STS_IOERR;1308 case SDEV_BLOCK:1309 case SDEV_CREATED_BLOCK:1310 return BLK_STS_RESOURCE;1311 case SDEV_QUIESCE:1312 /*1313 * If the device is blocked we only accept power management1314 * commands.1315 */1316 if (req && WARN_ON_ONCE(!(req->rq_flags & RQF_PM)))1317 return BLK_STS_RESOURCE;1318 return BLK_STS_OK;1319 default:1320 /*1321 * For any other not fully online state we only allow1322 * power management commands.1323 */1324 if (req && !(req->rq_flags & RQF_PM))1325 return BLK_STS_OFFLINE;1326 return BLK_STS_OK;1327 }1328}1329 1330/*1331 * scsi_dev_queue_ready: if we can send requests to sdev, assign one token1332 * and return the token else return -1.1333 */1334static inline int scsi_dev_queue_ready(struct request_queue *q,1335 struct scsi_device *sdev)1336{1337 int token;1338 1339 token = sbitmap_get(&sdev->budget_map);1340 if (token < 0)1341 return -1;1342 1343 if (!atomic_read(&sdev->device_blocked))1344 return token;1345 1346 /*1347 * Only unblock if no other commands are pending and1348 * if device_blocked has decreased to zero1349 */1350 if (scsi_device_busy(sdev) > 1 ||1351 atomic_dec_return(&sdev->device_blocked) > 0) {1352 sbitmap_put(&sdev->budget_map, token);1353 return -1;1354 }1355 1356 SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev,1357 "unblocking device at zero depth\n"));1358 1359 return token;1360}1361 1362/*1363 * scsi_target_queue_ready: checks if there we can send commands to target1364 * @sdev: scsi device on starget to check.1365 */1366static inline int scsi_target_queue_ready(struct Scsi_Host *shost,1367 struct scsi_device *sdev)1368{1369 struct scsi_target *starget = scsi_target(sdev);1370 unsigned int busy;1371 1372 if (starget->single_lun) {1373 spin_lock_irq(shost->host_lock);1374 if (starget->starget_sdev_user &&1375 starget->starget_sdev_user != sdev) {1376 spin_unlock_irq(shost->host_lock);1377 return 0;1378 }1379 starget->starget_sdev_user = sdev;1380 spin_unlock_irq(shost->host_lock);1381 }1382 1383 if (starget->can_queue <= 0)1384 return 1;1385 1386 busy = atomic_inc_return(&starget->target_busy) - 1;1387 if (atomic_read(&starget->target_blocked) > 0) {1388 if (busy)1389 goto starved;1390 1391 /*1392 * unblock after target_blocked iterates to zero1393 */1394 if (atomic_dec_return(&starget->target_blocked) > 0)1395 goto out_dec;1396 1397 SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,1398 "unblocking target at zero depth\n"));1399 }1400 1401 if (busy >= starget->can_queue)1402 goto starved;1403 1404 return 1;1405 1406starved:1407 spin_lock_irq(shost->host_lock);1408 list_move_tail(&sdev->starved_entry, &shost->starved_list);1409 spin_unlock_irq(shost->host_lock);1410out_dec:1411 if (starget->can_queue > 0)1412 atomic_dec(&starget->target_busy);1413 return 0;1414}1415 1416/*1417 * scsi_host_queue_ready: if we can send requests to shost, return 1 else1418 * return 0. We must end up running the queue again whenever 0 is1419 * returned, else IO can hang.1420 */1421static inline int scsi_host_queue_ready(struct request_queue *q,1422 struct Scsi_Host *shost,1423 struct scsi_device *sdev,1424 struct scsi_cmnd *cmd)1425{1426 if (atomic_read(&shost->host_blocked) > 0) {1427 if (scsi_host_busy(shost) > 0)1428 goto starved;1429 1430 /*1431 * unblock after host_blocked iterates to zero1432 */1433 if (atomic_dec_return(&shost->host_blocked) > 0)1434 goto out_dec;1435 1436 SCSI_LOG_MLQUEUE(3,1437 shost_printk(KERN_INFO, shost,1438 "unblocking host at zero depth\n"));1439 }1440 1441 if (shost->host_self_blocked)1442 goto starved;1443 1444 /* We're OK to process the command, so we can't be starved */1445 if (!list_empty(&sdev->starved_entry)) {1446 spin_lock_irq(shost->host_lock);1447 if (!list_empty(&sdev->starved_entry))1448 list_del_init(&sdev->starved_entry);1449 spin_unlock_irq(shost->host_lock);1450 }1451 1452 __set_bit(SCMD_STATE_INFLIGHT, &cmd->state);1453 1454 return 1;1455 1456starved:1457 spin_lock_irq(shost->host_lock);1458 if (list_empty(&sdev->starved_entry))1459 list_add_tail(&sdev->starved_entry, &shost->starved_list);1460 spin_unlock_irq(shost->host_lock);1461out_dec:1462 scsi_dec_host_busy(shost, cmd);1463 return 0;1464}1465 1466/*1467 * Busy state exporting function for request stacking drivers.1468 *1469 * For efficiency, no lock is taken to check the busy state of1470 * shost/starget/sdev, since the returned value is not guaranteed and1471 * may be changed after request stacking drivers call the function,1472 * regardless of taking lock or not.1473 *1474 * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi1475 * needs to return 'not busy'. Otherwise, request stacking drivers1476 * may hold requests forever.1477 */1478static bool scsi_mq_lld_busy(struct request_queue *q)1479{1480 struct scsi_device *sdev = q->queuedata;1481 struct Scsi_Host *shost;1482 1483 if (blk_queue_dying(q))1484 return false;1485 1486 shost = sdev->host;1487 1488 /*1489 * Ignore host/starget busy state.1490 * Since block layer does not have a concept of fairness across1491 * multiple queues, congestion of host/starget needs to be handled1492 * in SCSI layer.1493 */1494 if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev))1495 return true;1496 1497 return false;1498}1499 1500/*1501 * Block layer request completion callback. May be called from interrupt1502 * context.1503 */1504static void scsi_complete(struct request *rq)1505{1506 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);1507 enum scsi_disposition disposition;1508 1509 INIT_LIST_HEAD(&cmd->eh_entry);1510 1511 atomic_inc(&cmd->device->iodone_cnt);1512 if (cmd->result)1513 atomic_inc(&cmd->device->ioerr_cnt);1514 1515 disposition = scsi_decide_disposition(cmd);1516 if (disposition != SUCCESS && scsi_cmd_runtime_exceeced(cmd))1517 disposition = SUCCESS;1518 1519 scsi_log_completion(cmd, disposition);1520 1521 switch (disposition) {1522 case SUCCESS:1523 scsi_finish_command(cmd);1524 break;1525 case NEEDS_RETRY:1526 scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);1527 break;1528 case ADD_TO_MLQUEUE:1529 scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);1530 break;1531 default:1532 scsi_eh_scmd_add(cmd);1533 break;1534 }1535}1536 1537/**1538 * scsi_dispatch_cmd - Dispatch a command to the low-level driver.1539 * @cmd: command block we are dispatching.1540 *1541 * Return: nonzero return request was rejected and device's queue needs to be1542 * plugged.1543 */1544static int scsi_dispatch_cmd(struct scsi_cmnd *cmd)1545{1546 struct Scsi_Host *host = cmd->device->host;1547 int rtn = 0;1548 1549 atomic_inc(&cmd->device->iorequest_cnt);1550 1551 /* check if the device is still usable */1552 if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {1553 /* in SDEV_DEL we error all commands. DID_NO_CONNECT1554 * returns an immediate error upwards, and signals1555 * that the device is no longer present */1556 cmd->result = DID_NO_CONNECT << 16;1557 goto done;1558 }1559 1560 /* Check to see if the scsi lld made this device blocked. */1561 if (unlikely(scsi_device_blocked(cmd->device))) {1562 /*1563 * in blocked state, the command is just put back on1564 * the device queue. The suspend state has already1565 * blocked the queue so future requests should not1566 * occur until the device transitions out of the1567 * suspend state.1568 */1569 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,1570 "queuecommand : device blocked\n"));1571 atomic_dec(&cmd->device->iorequest_cnt);1572 return SCSI_MLQUEUE_DEVICE_BUSY;1573 }1574 1575 /* Store the LUN value in cmnd, if needed. */1576 if (cmd->device->lun_in_cdb)1577 cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) |1578 (cmd->device->lun << 5 & 0xe0);1579 1580 scsi_log_send(cmd);1581 1582 /*1583 * Before we queue this command, check if the command1584 * length exceeds what the host adapter can handle.1585 */1586 if (cmd->cmd_len > cmd->device->host->max_cmd_len) {1587 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,1588 "queuecommand : command too long. "1589 "cdb_size=%d host->max_cmd_len=%d\n",1590 cmd->cmd_len, cmd->device->host->max_cmd_len));1591 cmd->result = (DID_ABORT << 16);1592 goto done;1593 }1594 1595 if (unlikely(host->shost_state == SHOST_DEL)) {1596 cmd->result = (DID_NO_CONNECT << 16);1597 goto done;1598 1599 }1600 1601 trace_scsi_dispatch_cmd_start(cmd);1602 rtn = host->hostt->queuecommand(host, cmd);1603 if (rtn) {1604 atomic_dec(&cmd->device->iorequest_cnt);1605 trace_scsi_dispatch_cmd_error(cmd, rtn);1606 if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&1607 rtn != SCSI_MLQUEUE_TARGET_BUSY)1608 rtn = SCSI_MLQUEUE_HOST_BUSY;1609 1610 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,1611 "queuecommand : request rejected\n"));1612 }1613 1614 return rtn;1615 done:1616 scsi_done(cmd);1617 return 0;1618}1619 1620/* Size in bytes of the sg-list stored in the scsi-mq command-private data. */1621static unsigned int scsi_mq_inline_sgl_size(struct Scsi_Host *shost)1622{1623 return min_t(unsigned int, shost->sg_tablesize, SCSI_INLINE_SG_CNT) *1624 sizeof(struct scatterlist);1625}1626 1627static blk_status_t scsi_prepare_cmd(struct request *req)1628{1629 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);1630 struct scsi_device *sdev = req->q->queuedata;1631 struct Scsi_Host *shost = sdev->host;1632 bool in_flight = test_bit(SCMD_STATE_INFLIGHT, &cmd->state);1633 struct scatterlist *sg;1634 1635 scsi_init_command(sdev, cmd);1636 1637 cmd->eh_eflags = 0;1638 cmd->prot_type = 0;1639 cmd->prot_flags = 0;1640 cmd->submitter = 0;1641 memset(&cmd->sdb, 0, sizeof(cmd->sdb));1642 cmd->underflow = 0;1643 cmd->transfersize = 0;1644 cmd->host_scribble = NULL;1645 cmd->result = 0;1646 cmd->extra_len = 0;1647 cmd->state = 0;1648 if (in_flight)1649 __set_bit(SCMD_STATE_INFLIGHT, &cmd->state);1650 1651 /*1652 * Only clear the driver-private command data if the LLD does not supply1653 * a function to initialize that data.1654 */1655 if (!shost->hostt->init_cmd_priv)1656 memset(cmd + 1, 0, shost->hostt->cmd_size);1657 1658 cmd->prot_op = SCSI_PROT_NORMAL;1659 if (blk_rq_bytes(req))1660 cmd->sc_data_direction = rq_dma_dir(req);1661 else1662 cmd->sc_data_direction = DMA_NONE;1663 1664 sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;1665 cmd->sdb.table.sgl = sg;1666 1667 if (scsi_host_get_prot(shost)) {1668 memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer));1669 1670 cmd->prot_sdb->table.sgl =1671 (struct scatterlist *)(cmd->prot_sdb + 1);1672 }1673 1674 /*1675 * Special handling for passthrough commands, which don't go to the ULP1676 * at all:1677 */1678 if (blk_rq_is_passthrough(req))1679 return scsi_setup_scsi_cmnd(sdev, req);1680 1681 if (sdev->handler && sdev->handler->prep_fn) {1682 blk_status_t ret = sdev->handler->prep_fn(sdev, req);1683 1684 if (ret != BLK_STS_OK)1685 return ret;1686 }1687 1688 /* Usually overridden by the ULP */1689 cmd->allowed = 0;1690 memset(cmd->cmnd, 0, sizeof(cmd->cmnd));1691 return scsi_cmd_to_driver(cmd)->init_command(cmd);1692}1693 1694static void scsi_done_internal(struct scsi_cmnd *cmd, bool complete_directly)1695{1696 struct request *req = scsi_cmd_to_rq(cmd);1697 1698 switch (cmd->submitter) {1699 case SUBMITTED_BY_BLOCK_LAYER:1700 break;1701 case SUBMITTED_BY_SCSI_ERROR_HANDLER:1702 return scsi_eh_done(cmd);1703 case SUBMITTED_BY_SCSI_RESET_IOCTL:1704 return;1705 }1706 1707 if (unlikely(blk_should_fake_timeout(scsi_cmd_to_rq(cmd)->q)))1708 return;1709 if (unlikely(test_and_set_bit(SCMD_STATE_COMPLETE, &cmd->state)))1710 return;1711 trace_scsi_dispatch_cmd_done(cmd);1712 1713 if (complete_directly)1714 blk_mq_complete_request_direct(req, scsi_complete);1715 else1716 blk_mq_complete_request(req);1717}1718 1719void scsi_done(struct scsi_cmnd *cmd)1720{1721 scsi_done_internal(cmd, false);1722}1723EXPORT_SYMBOL(scsi_done);1724 1725void scsi_done_direct(struct scsi_cmnd *cmd)1726{1727 scsi_done_internal(cmd, true);1728}1729EXPORT_SYMBOL(scsi_done_direct);1730 1731static void scsi_mq_put_budget(struct request_queue *q, int budget_token)1732{1733 struct scsi_device *sdev = q->queuedata;1734 1735 sbitmap_put(&sdev->budget_map, budget_token);1736}1737 1738/*1739 * When to reinvoke queueing after a resource shortage. It's 3 msecs to1740 * not change behaviour from the previous unplug mechanism, experimentation1741 * may prove this needs changing.1742 */1743#define SCSI_QUEUE_DELAY 31744 1745static int scsi_mq_get_budget(struct request_queue *q)1746{1747 struct scsi_device *sdev = q->queuedata;1748 int token = scsi_dev_queue_ready(q, sdev);1749 1750 if (token >= 0)1751 return token;1752 1753 atomic_inc(&sdev->restarts);1754 1755 /*1756 * Orders atomic_inc(&sdev->restarts) and atomic_read(&sdev->device_busy).1757 * .restarts must be incremented before .device_busy is read because the1758 * code in scsi_run_queue_async() depends on the order of these operations.1759 */1760 smp_mb__after_atomic();1761 1762 /*1763 * If all in-flight requests originated from this LUN are completed1764 * before reading .device_busy, sdev->device_busy will be observed as1765 * zero, then blk_mq_delay_run_hw_queues() will dispatch this request1766 * soon. Otherwise, completion of one of these requests will observe1767 * the .restarts flag, and the request queue will be run for handling1768 * this request, see scsi_end_request().1769 */1770 if (unlikely(scsi_device_busy(sdev) == 0 &&1771 !scsi_device_blocked(sdev)))1772 blk_mq_delay_run_hw_queues(sdev->request_queue, SCSI_QUEUE_DELAY);1773 return -1;1774}1775 1776static void scsi_mq_set_rq_budget_token(struct request *req, int token)1777{1778 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);1779 1780 cmd->budget_token = token;1781}1782 1783static int scsi_mq_get_rq_budget_token(struct request *req)1784{1785 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);1786 1787 return cmd->budget_token;1788}1789 1790static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,1791 const struct blk_mq_queue_data *bd)1792{1793 struct request *req = bd->rq;1794 struct request_queue *q = req->q;1795 struct scsi_device *sdev = q->queuedata;1796 struct Scsi_Host *shost = sdev->host;1797 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);1798 blk_status_t ret;1799 int reason;1800 1801 WARN_ON_ONCE(cmd->budget_token < 0);1802 1803 /*1804 * If the device is not in running state we will reject some or all1805 * commands.1806 */1807 if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {1808 ret = scsi_device_state_check(sdev, req);1809 if (ret != BLK_STS_OK)1810 goto out_put_budget;1811 }1812 1813 ret = BLK_STS_RESOURCE;1814 if (!scsi_target_queue_ready(shost, sdev))1815 goto out_put_budget;1816 if (unlikely(scsi_host_in_recovery(shost))) {1817 if (cmd->flags & SCMD_FAIL_IF_RECOVERING)1818 ret = BLK_STS_OFFLINE;1819 goto out_dec_target_busy;1820 }1821 if (!scsi_host_queue_ready(q, shost, sdev, cmd))1822 goto out_dec_target_busy;1823 1824 if (!(req->rq_flags & RQF_DONTPREP)) {1825 ret = scsi_prepare_cmd(req);1826 if (ret != BLK_STS_OK)1827 goto out_dec_host_busy;1828 req->rq_flags |= RQF_DONTPREP;1829 } else {1830 clear_bit(SCMD_STATE_COMPLETE, &cmd->state);1831 }1832 1833 cmd->flags &= SCMD_PRESERVED_FLAGS;1834 if (sdev->simple_tags)1835 cmd->flags |= SCMD_TAGGED;1836 if (bd->last)1837 cmd->flags |= SCMD_LAST;1838 1839 scsi_set_resid(cmd, 0);1840 memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);1841 cmd->submitter = SUBMITTED_BY_BLOCK_LAYER;1842 1843 blk_mq_start_request(req);1844 reason = scsi_dispatch_cmd(cmd);1845 if (reason) {1846 scsi_set_blocked(cmd, reason);1847 ret = BLK_STS_RESOURCE;1848 goto out_dec_host_busy;1849 }1850 1851 return BLK_STS_OK;1852 1853out_dec_host_busy:1854 scsi_dec_host_busy(shost, cmd);1855out_dec_target_busy:1856 if (scsi_target(sdev)->can_queue > 0)1857 atomic_dec(&scsi_target(sdev)->target_busy);1858out_put_budget:1859 scsi_mq_put_budget(q, cmd->budget_token);1860 cmd->budget_token = -1;1861 switch (ret) {1862 case BLK_STS_OK:1863 break;1864 case BLK_STS_RESOURCE:1865 if (scsi_device_blocked(sdev))1866 ret = BLK_STS_DEV_RESOURCE;1867 break;1868 case BLK_STS_AGAIN:1869 cmd->result = DID_BUS_BUSY << 16;1870 if (req->rq_flags & RQF_DONTPREP)1871 scsi_mq_uninit_cmd(cmd);1872 break;1873 default:1874 if (unlikely(!scsi_device_online(sdev)))1875 cmd->result = DID_NO_CONNECT << 16;1876 else1877 cmd->result = DID_ERROR << 16;1878 /*1879 * Make sure to release all allocated resources when1880 * we hit an error, as we will never see this command1881 * again.1882 */1883 if (req->rq_flags & RQF_DONTPREP)1884 scsi_mq_uninit_cmd(cmd);1885 scsi_run_queue_async(sdev);1886 break;1887 }1888 return ret;1889}1890 1891static int scsi_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,1892 unsigned int hctx_idx, unsigned int numa_node)1893{1894 struct Scsi_Host *shost = set->driver_data;1895 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);1896 struct scatterlist *sg;1897 int ret = 0;1898 1899 cmd->sense_buffer =1900 kmem_cache_alloc_node(scsi_sense_cache, GFP_KERNEL, numa_node);1901 if (!cmd->sense_buffer)1902 return -ENOMEM;1903 1904 if (scsi_host_get_prot(shost)) {1905 sg = (void *)cmd + sizeof(struct scsi_cmnd) +1906 shost->hostt->cmd_size;1907 cmd->prot_sdb = (void *)sg + scsi_mq_inline_sgl_size(shost);1908 }1909 1910 if (shost->hostt->init_cmd_priv) {1911 ret = shost->hostt->init_cmd_priv(shost, cmd);1912 if (ret < 0)1913 kmem_cache_free(scsi_sense_cache, cmd->sense_buffer);1914 }1915 1916 return ret;1917}1918 1919static void scsi_mq_exit_request(struct blk_mq_tag_set *set, struct request *rq,1920 unsigned int hctx_idx)1921{1922 struct Scsi_Host *shost = set->driver_data;1923 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);1924 1925 if (shost->hostt->exit_cmd_priv)1926 shost->hostt->exit_cmd_priv(shost, cmd);1927 kmem_cache_free(scsi_sense_cache, cmd->sense_buffer);1928}1929 1930 1931static int scsi_mq_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)1932{1933 struct Scsi_Host *shost = hctx->driver_data;1934 1935 if (shost->hostt->mq_poll)1936 return shost->hostt->mq_poll(shost, hctx->queue_num);1937 1938 return 0;1939}1940 1941static int scsi_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,1942 unsigned int hctx_idx)1943{1944 struct Scsi_Host *shost = data;1945 1946 hctx->driver_data = shost;1947 return 0;1948}1949 1950static void scsi_map_queues(struct blk_mq_tag_set *set)1951{1952 struct Scsi_Host *shost = container_of(set, struct Scsi_Host, tag_set);1953 1954 if (shost->hostt->map_queues)1955 return shost->hostt->map_queues(shost);1956 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);1957}1958 1959void scsi_init_limits(struct Scsi_Host *shost, struct queue_limits *lim)1960{1961 struct device *dev = shost->dma_dev;1962 1963 memset(lim, 0, sizeof(*lim));1964 lim->max_segments =1965 min_t(unsigned short, shost->sg_tablesize, SG_MAX_SEGMENTS);1966 1967 if (scsi_host_prot_dma(shost)) {1968 shost->sg_prot_tablesize =1969 min_not_zero(shost->sg_prot_tablesize,1970 (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);1971 BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);1972 lim->max_integrity_segments = shost->sg_prot_tablesize;1973 }1974 1975 lim->max_hw_sectors = shost->max_sectors;1976 lim->seg_boundary_mask = shost->dma_boundary;1977 lim->max_segment_size = shost->max_segment_size;1978 lim->virt_boundary_mask = shost->virt_boundary_mask;1979 lim->dma_alignment = max_t(unsigned int,1980 shost->dma_alignment, dma_get_cache_alignment() - 1);1981 1982 if (shost->no_highmem)1983 lim->features |= BLK_FEAT_BOUNCE_HIGH;1984 1985 /*1986 * Propagate the DMA formation properties to the dma-mapping layer as1987 * a courtesy service to the LLDDs. This needs to check that the buses1988 * actually support the DMA API first, though.1989 */1990 if (dev->dma_parms) {1991 dma_set_seg_boundary(dev, shost->dma_boundary);1992 dma_set_max_seg_size(dev, shost->max_segment_size);1993 }1994}1995EXPORT_SYMBOL_GPL(scsi_init_limits);1996 1997static const struct blk_mq_ops scsi_mq_ops_no_commit = {1998 .get_budget = scsi_mq_get_budget,1999 .put_budget = scsi_mq_put_budget,2000 .queue_rq = scsi_queue_rq,2001 .complete = scsi_complete,2002 .timeout = scsi_timeout,2003#ifdef CONFIG_BLK_DEBUG_FS2004 .show_rq = scsi_show_rq,2005#endif2006 .init_request = scsi_mq_init_request,2007 .exit_request = scsi_mq_exit_request,2008 .cleanup_rq = scsi_cleanup_rq,2009 .busy = scsi_mq_lld_busy,2010 .map_queues = scsi_map_queues,2011 .init_hctx = scsi_init_hctx,2012 .poll = scsi_mq_poll,2013 .set_rq_budget_token = scsi_mq_set_rq_budget_token,2014 .get_rq_budget_token = scsi_mq_get_rq_budget_token,2015};2016 2017 2018static void scsi_commit_rqs(struct blk_mq_hw_ctx *hctx)2019{2020 struct Scsi_Host *shost = hctx->driver_data;2021 2022 shost->hostt->commit_rqs(shost, hctx->queue_num);2023}2024 2025static const struct blk_mq_ops scsi_mq_ops = {2026 .get_budget = scsi_mq_get_budget,2027 .put_budget = scsi_mq_put_budget,2028 .queue_rq = scsi_queue_rq,2029 .commit_rqs = scsi_commit_rqs,2030 .complete = scsi_complete,2031 .timeout = scsi_timeout,2032#ifdef CONFIG_BLK_DEBUG_FS2033 .show_rq = scsi_show_rq,2034#endif2035 .init_request = scsi_mq_init_request,2036 .exit_request = scsi_mq_exit_request,2037 .cleanup_rq = scsi_cleanup_rq,2038 .busy = scsi_mq_lld_busy,2039 .map_queues = scsi_map_queues,2040 .init_hctx = scsi_init_hctx,2041 .poll = scsi_mq_poll,2042 .set_rq_budget_token = scsi_mq_set_rq_budget_token,2043 .get_rq_budget_token = scsi_mq_get_rq_budget_token,2044};2045 2046int scsi_mq_setup_tags(struct Scsi_Host *shost)2047{2048 unsigned int cmd_size, sgl_size;2049 struct blk_mq_tag_set *tag_set = &shost->tag_set;2050 2051 sgl_size = max_t(unsigned int, sizeof(struct scatterlist),2052 scsi_mq_inline_sgl_size(shost));2053 cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size;2054 if (scsi_host_get_prot(shost))2055 cmd_size += sizeof(struct scsi_data_buffer) +2056 sizeof(struct scatterlist) * SCSI_INLINE_PROT_SG_CNT;2057 2058 memset(tag_set, 0, sizeof(*tag_set));2059 if (shost->hostt->commit_rqs)2060 tag_set->ops = &scsi_mq_ops;2061 else2062 tag_set->ops = &scsi_mq_ops_no_commit;2063 tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1;2064 tag_set->nr_maps = shost->nr_maps ? : 1;2065 tag_set->queue_depth = shost->can_queue;2066 tag_set->cmd_size = cmd_size;2067 tag_set->numa_node = dev_to_node(shost->dma_dev);2068 tag_set->flags = BLK_MQ_F_SHOULD_MERGE;2069 tag_set->flags |=2070 BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy);2071 if (shost->queuecommand_may_block)2072 tag_set->flags |= BLK_MQ_F_BLOCKING;2073 tag_set->driver_data = shost;2074 if (shost->host_tagset)2075 tag_set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;2076 2077 return blk_mq_alloc_tag_set(tag_set);2078}2079 2080void scsi_mq_free_tags(struct kref *kref)2081{2082 struct Scsi_Host *shost = container_of(kref, typeof(*shost),2083 tagset_refcnt);2084 2085 blk_mq_free_tag_set(&shost->tag_set);2086 complete(&shost->tagset_freed);2087}2088 2089/**2090 * scsi_device_from_queue - return sdev associated with a request_queue2091 * @q: The request queue to return the sdev from2092 *2093 * Return the sdev associated with a request queue or NULL if the2094 * request_queue does not reference a SCSI device.2095 */2096struct scsi_device *scsi_device_from_queue(struct request_queue *q)2097{2098 struct scsi_device *sdev = NULL;2099 2100 if (q->mq_ops == &scsi_mq_ops_no_commit ||2101 q->mq_ops == &scsi_mq_ops)2102 sdev = q->queuedata;2103 if (!sdev || !get_device(&sdev->sdev_gendev))2104 sdev = NULL;2105 2106 return sdev;2107}2108/*2109 * pktcdvd should have been integrated into the SCSI layers, but for historical2110 * reasons like the old IDE driver it isn't. This export allows it to safely2111 * probe if a given device is a SCSI one and only attach to that.2112 */2113#ifdef CONFIG_CDROM_PKTCDVD_MODULE2114EXPORT_SYMBOL_GPL(scsi_device_from_queue);2115#endif2116 2117/**2118 * scsi_block_requests - Utility function used by low-level drivers to prevent2119 * further commands from being queued to the device.2120 * @shost: host in question2121 *2122 * There is no timer nor any other means by which the requests get unblocked2123 * other than the low-level driver calling scsi_unblock_requests().2124 */2125void scsi_block_requests(struct Scsi_Host *shost)2126{2127 shost->host_self_blocked = 1;2128}2129EXPORT_SYMBOL(scsi_block_requests);2130 2131/**2132 * scsi_unblock_requests - Utility function used by low-level drivers to allow2133 * further commands to be queued to the device.2134 * @shost: host in question2135 *2136 * There is no timer nor any other means by which the requests get unblocked2137 * other than the low-level driver calling scsi_unblock_requests(). This is done2138 * as an API function so that changes to the internals of the scsi mid-layer2139 * won't require wholesale changes to drivers that use this feature.2140 */2141void scsi_unblock_requests(struct Scsi_Host *shost)2142{2143 shost->host_self_blocked = 0;2144 scsi_run_host_queues(shost);2145}2146EXPORT_SYMBOL(scsi_unblock_requests);2147 2148void scsi_exit_queue(void)2149{2150 kmem_cache_destroy(scsi_sense_cache);2151}2152 2153/**2154 * scsi_mode_select - issue a mode select2155 * @sdev: SCSI device to be queried2156 * @pf: Page format bit (1 == standard, 0 == vendor specific)2157 * @sp: Save page bit (0 == don't save, 1 == save)2158 * @buffer: request buffer (may not be smaller than eight bytes)2159 * @len: length of request buffer.2160 * @timeout: command timeout2161 * @retries: number of retries before failing2162 * @data: returns a structure abstracting the mode header data2163 * @sshdr: place to put sense data (or NULL if no sense to be collected).2164 * must be SCSI_SENSE_BUFFERSIZE big.2165 *2166 * Returns zero if successful; negative error number or scsi2167 * status on error2168 *2169 */2170int scsi_mode_select(struct scsi_device *sdev, int pf, int sp,2171 unsigned char *buffer, int len, int timeout, int retries,2172 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)2173{2174 unsigned char cmd[10];2175 unsigned char *real_buffer;2176 const struct scsi_exec_args exec_args = {2177 .sshdr = sshdr,2178 };2179 int ret;2180 2181 memset(cmd, 0, sizeof(cmd));2182 cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);2183 2184 /*2185 * Use MODE SELECT(10) if the device asked for it or if the mode page2186 * and the mode select header cannot fit within the maximumm 255 bytes2187 * of the MODE SELECT(6) command.2188 */2189 if (sdev->use_10_for_ms ||2190 len + 4 > 255 ||2191 data->block_descriptor_length > 255) {2192 if (len > 65535 - 8)2193 return -EINVAL;2194 real_buffer = kmalloc(8 + len, GFP_KERNEL);2195 if (!real_buffer)2196 return -ENOMEM;2197 memcpy(real_buffer + 8, buffer, len);2198 len += 8;2199 real_buffer[0] = 0;2200 real_buffer[1] = 0;2201 real_buffer[2] = data->medium_type;2202 real_buffer[3] = data->device_specific;2203 real_buffer[4] = data->longlba ? 0x01 : 0;2204 real_buffer[5] = 0;2205 put_unaligned_be16(data->block_descriptor_length,2206 &real_buffer[6]);2207 2208 cmd[0] = MODE_SELECT_10;2209 put_unaligned_be16(len, &cmd[7]);2210 } else {2211 if (data->longlba)2212 return -EINVAL;2213 2214 real_buffer = kmalloc(4 + len, GFP_KERNEL);2215 if (!real_buffer)2216 return -ENOMEM;2217 memcpy(real_buffer + 4, buffer, len);2218 len += 4;2219 real_buffer[0] = 0;2220 real_buffer[1] = data->medium_type;2221 real_buffer[2] = data->device_specific;2222 real_buffer[3] = data->block_descriptor_length;2223 2224 cmd[0] = MODE_SELECT;2225 cmd[4] = len;2226 }2227 2228 ret = scsi_execute_cmd(sdev, cmd, REQ_OP_DRV_OUT, real_buffer, len,2229 timeout, retries, &exec_args);2230 kfree(real_buffer);2231 return ret;2232}2233EXPORT_SYMBOL_GPL(scsi_mode_select);2234 2235/**2236 * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.2237 * @sdev: SCSI device to be queried2238 * @dbd: set to prevent mode sense from returning block descriptors2239 * @modepage: mode page being requested2240 * @subpage: sub-page of the mode page being requested2241 * @buffer: request buffer (may not be smaller than eight bytes)2242 * @len: length of request buffer.2243 * @timeout: command timeout2244 * @retries: number of retries before failing2245 * @data: returns a structure abstracting the mode header data2246 * @sshdr: place to put sense data (or NULL if no sense to be collected).2247 * must be SCSI_SENSE_BUFFERSIZE big.2248 *2249 * Returns zero if successful, or a negative error number on failure2250 */2251int2252scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, int subpage,2253 unsigned char *buffer, int len, int timeout, int retries,2254 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)2255{2256 unsigned char cmd[12];2257 int use_10_for_ms;2258 int header_length;2259 int result;2260 struct scsi_sense_hdr my_sshdr;2261 struct scsi_failure failure_defs[] = {2262 {2263 .sense = UNIT_ATTENTION,2264 .asc = SCMD_FAILURE_ASC_ANY,2265 .ascq = SCMD_FAILURE_ASCQ_ANY,2266 .allowed = retries,2267 .result = SAM_STAT_CHECK_CONDITION,2268 },2269 {}2270 };2271 struct scsi_failures failures = {2272 .failure_definitions = failure_defs,2273 };2274 const struct scsi_exec_args exec_args = {2275 /* caller might not be interested in sense, but we need it */2276 .sshdr = sshdr ? : &my_sshdr,2277 .failures = &failures,2278 };2279 2280 memset(data, 0, sizeof(*data));2281 memset(&cmd[0], 0, 12);2282 2283 dbd = sdev->set_dbd_for_ms ? 8 : dbd;2284 cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */2285 cmd[2] = modepage;2286 cmd[3] = subpage;2287 2288 sshdr = exec_args.sshdr;2289 2290 retry:2291 use_10_for_ms = sdev->use_10_for_ms || len > 255;2292 2293 if (use_10_for_ms) {2294 if (len < 8 || len > 65535)2295 return -EINVAL;2296 2297 cmd[0] = MODE_SENSE_10;2298 put_unaligned_be16(len, &cmd[7]);2299 header_length = 8;2300 } else {2301 if (len < 4)2302 return -EINVAL;2303 2304 cmd[0] = MODE_SENSE;2305 cmd[4] = len;2306 header_length = 4;2307 }2308 2309 memset(buffer, 0, len);2310 2311 result = scsi_execute_cmd(sdev, cmd, REQ_OP_DRV_IN, buffer, len,2312 timeout, retries, &exec_args);2313 if (result < 0)2314 return result;2315 2316 /* This code looks awful: what it's doing is making sure an2317 * ILLEGAL REQUEST sense return identifies the actual command2318 * byte as the problem. MODE_SENSE commands can return2319 * ILLEGAL REQUEST if the code page isn't supported */2320 2321 if (!scsi_status_is_good(result)) {2322 if (scsi_sense_valid(sshdr)) {2323 if ((sshdr->sense_key == ILLEGAL_REQUEST) &&2324 (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {2325 /*2326 * Invalid command operation code: retry using2327 * MODE SENSE(6) if this was a MODE SENSE(10)2328 * request, except if the request mode page is2329 * too large for MODE SENSE single byte2330 * allocation length field.2331 */2332 if (use_10_for_ms) {2333 if (len > 255)2334 return -EIO;2335 sdev->use_10_for_ms = 0;2336 goto retry;2337 }2338 }2339 }2340 return -EIO;2341 }2342 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&2343 (modepage == 6 || modepage == 8))) {2344 /* Initio breakage? */2345 header_length = 0;2346 data->length = 13;2347 data->medium_type = 0;2348 data->device_specific = 0;2349 data->longlba = 0;2350 data->block_descriptor_length = 0;2351 } else if (use_10_for_ms) {2352 data->length = get_unaligned_be16(&buffer[0]) + 2;2353 data->medium_type = buffer[2];2354 data->device_specific = buffer[3];2355 data->longlba = buffer[4] & 0x01;2356 data->block_descriptor_length = get_unaligned_be16(&buffer[6]);2357 } else {2358 data->length = buffer[0] + 1;2359 data->medium_type = buffer[1];2360 data->device_specific = buffer[2];2361 data->block_descriptor_length = buffer[3];2362 }2363 data->header_length = header_length;2364 2365 return 0;2366}2367EXPORT_SYMBOL(scsi_mode_sense);2368 2369/**2370 * scsi_test_unit_ready - test if unit is ready2371 * @sdev: scsi device to change the state of.2372 * @timeout: command timeout2373 * @retries: number of retries before failing2374 * @sshdr: outpout pointer for decoded sense information.2375 *2376 * Returns zero if unsuccessful or an error if TUR failed. For2377 * removable media, UNIT_ATTENTION sets ->changed flag.2378 **/2379int2380scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,2381 struct scsi_sense_hdr *sshdr)2382{2383 char cmd[] = {2384 TEST_UNIT_READY, 0, 0, 0, 0, 0,2385 };2386 const struct scsi_exec_args exec_args = {2387 .sshdr = sshdr,2388 };2389 int result;2390 2391 /* try to eat the UNIT_ATTENTION if there are enough retries */2392 do {2393 result = scsi_execute_cmd(sdev, cmd, REQ_OP_DRV_IN, NULL, 0,2394 timeout, 1, &exec_args);2395 if (sdev->removable && result > 0 && scsi_sense_valid(sshdr) &&2396 sshdr->sense_key == UNIT_ATTENTION)2397 sdev->changed = 1;2398 } while (result > 0 && scsi_sense_valid(sshdr) &&2399 sshdr->sense_key == UNIT_ATTENTION && --retries);2400 2401 return result;2402}2403EXPORT_SYMBOL(scsi_test_unit_ready);2404 2405/**2406 * scsi_device_set_state - Take the given device through the device state model.2407 * @sdev: scsi device to change the state of.2408 * @state: state to change to.2409 *2410 * Returns zero if successful or an error if the requested2411 * transition is illegal.2412 */2413int2414scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)2415{2416 enum scsi_device_state oldstate = sdev->sdev_state;2417 2418 if (state == oldstate)2419 return 0;2420 2421 switch (state) {2422 case SDEV_CREATED:2423 switch (oldstate) {2424 case SDEV_CREATED_BLOCK:2425 break;2426 default:2427 goto illegal;2428 }2429 break;2430 2431 case SDEV_RUNNING:2432 switch (oldstate) {2433 case SDEV_CREATED:2434 case SDEV_OFFLINE:2435 case SDEV_TRANSPORT_OFFLINE:2436 case SDEV_QUIESCE:2437 case SDEV_BLOCK:2438 break;2439 default:2440 goto illegal;2441 }2442 break;2443 2444 case SDEV_QUIESCE:2445 switch (oldstate) {2446 case SDEV_RUNNING:2447 case SDEV_OFFLINE:2448 case SDEV_TRANSPORT_OFFLINE:2449 break;2450 default:2451 goto illegal;2452 }2453 break;2454 2455 case SDEV_OFFLINE:2456 case SDEV_TRANSPORT_OFFLINE:2457 switch (oldstate) {2458 case SDEV_CREATED:2459 case SDEV_RUNNING:2460 case SDEV_QUIESCE:2461 case SDEV_BLOCK:2462 break;2463 default:2464 goto illegal;2465 }2466 break;2467 2468 case SDEV_BLOCK:2469 switch (oldstate) {2470 case SDEV_RUNNING:2471 case SDEV_CREATED_BLOCK:2472 case SDEV_QUIESCE:2473 case SDEV_OFFLINE:2474 break;2475 default:2476 goto illegal;2477 }2478 break;2479 2480 case SDEV_CREATED_BLOCK:2481 switch (oldstate) {2482 case SDEV_CREATED:2483 break;2484 default:2485 goto illegal;2486 }2487 break;2488 2489 case SDEV_CANCEL:2490 switch (oldstate) {2491 case SDEV_CREATED:2492 case SDEV_RUNNING:2493 case SDEV_QUIESCE:2494 case SDEV_OFFLINE:2495 case SDEV_TRANSPORT_OFFLINE:2496 break;2497 default:2498 goto illegal;2499 }2500 break;2501 2502 case SDEV_DEL:2503 switch (oldstate) {2504 case SDEV_CREATED:2505 case SDEV_RUNNING:2506 case SDEV_OFFLINE:2507 case SDEV_TRANSPORT_OFFLINE:2508 case SDEV_CANCEL:2509 case SDEV_BLOCK:2510 case SDEV_CREATED_BLOCK:2511 break;2512 default:2513 goto illegal;2514 }2515 break;2516 2517 }2518 sdev->offline_already = false;2519 sdev->sdev_state = state;2520 return 0;2521 2522 illegal:2523 SCSI_LOG_ERROR_RECOVERY(1,2524 sdev_printk(KERN_ERR, sdev,2525 "Illegal state transition %s->%s",2526 scsi_device_state_name(oldstate),2527 scsi_device_state_name(state))2528 );2529 return -EINVAL;2530}2531EXPORT_SYMBOL(scsi_device_set_state);2532 2533/**2534 * scsi_evt_emit - emit a single SCSI device uevent2535 * @sdev: associated SCSI device2536 * @evt: event to emit2537 *2538 * Send a single uevent (scsi_event) to the associated scsi_device.2539 */2540static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt)2541{2542 int idx = 0;2543 char *envp[3];2544 2545 switch (evt->evt_type) {2546 case SDEV_EVT_MEDIA_CHANGE:2547 envp[idx++] = "SDEV_MEDIA_CHANGE=1";2548 break;2549 case SDEV_EVT_INQUIRY_CHANGE_REPORTED:2550 scsi_rescan_device(sdev);2551 envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED";2552 break;2553 case SDEV_EVT_CAPACITY_CHANGE_REPORTED:2554 envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED";2555 break;2556 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:2557 envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED";2558 break;2559 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:2560 envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED";2561 break;2562 case SDEV_EVT_LUN_CHANGE_REPORTED:2563 envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED";2564 break;2565 case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED:2566 envp[idx++] = "SDEV_UA=ASYMMETRIC_ACCESS_STATE_CHANGED";2567 break;2568 case SDEV_EVT_POWER_ON_RESET_OCCURRED:2569 envp[idx++] = "SDEV_UA=POWER_ON_RESET_OCCURRED";2570 break;2571 default:2572 /* do nothing */2573 break;2574 }2575 2576 envp[idx++] = NULL;2577 2578 kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp);2579}2580 2581/**2582 * scsi_evt_thread - send a uevent for each scsi event2583 * @work: work struct for scsi_device2584 *2585 * Dispatch queued events to their associated scsi_device kobjects2586 * as uevents.2587 */2588void scsi_evt_thread(struct work_struct *work)2589{2590 struct scsi_device *sdev;2591 enum scsi_device_event evt_type;2592 LIST_HEAD(event_list);2593 2594 sdev = container_of(work, struct scsi_device, event_work);2595 2596 for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++)2597 if (test_and_clear_bit(evt_type, sdev->pending_events))2598 sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL);2599 2600 while (1) {2601 struct scsi_event *evt;2602 struct list_head *this, *tmp;2603 unsigned long flags;2604 2605 spin_lock_irqsave(&sdev->list_lock, flags);2606 list_splice_init(&sdev->event_list, &event_list);2607 spin_unlock_irqrestore(&sdev->list_lock, flags);2608 2609 if (list_empty(&event_list))2610 break;2611 2612 list_for_each_safe(this, tmp, &event_list) {2613 evt = list_entry(this, struct scsi_event, node);2614 list_del(&evt->node);2615 scsi_evt_emit(sdev, evt);2616 kfree(evt);2617 }2618 }2619}2620 2621/**2622 * sdev_evt_send - send asserted event to uevent thread2623 * @sdev: scsi_device event occurred on2624 * @evt: event to send2625 *2626 * Assert scsi device event asynchronously.2627 */2628void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)2629{2630 unsigned long flags;2631 2632#if 02633 /* FIXME: currently this check eliminates all media change events2634 * for polled devices. Need to update to discriminate between AN2635 * and polled events */2636 if (!test_bit(evt->evt_type, sdev->supported_events)) {2637 kfree(evt);2638 return;2639 }2640#endif2641 2642 spin_lock_irqsave(&sdev->list_lock, flags);2643 list_add_tail(&evt->node, &sdev->event_list);2644 schedule_work(&sdev->event_work);2645 spin_unlock_irqrestore(&sdev->list_lock, flags);2646}2647EXPORT_SYMBOL_GPL(sdev_evt_send);2648 2649/**2650 * sdev_evt_alloc - allocate a new scsi event2651 * @evt_type: type of event to allocate2652 * @gfpflags: GFP flags for allocation2653 *2654 * Allocates and returns a new scsi_event.2655 */2656struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type,2657 gfp_t gfpflags)2658{2659 struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags);2660 if (!evt)2661 return NULL;2662 2663 evt->evt_type = evt_type;2664 INIT_LIST_HEAD(&evt->node);2665 2666 /* evt_type-specific initialization, if any */2667 switch (evt_type) {2668 case SDEV_EVT_MEDIA_CHANGE:2669 case SDEV_EVT_INQUIRY_CHANGE_REPORTED:2670 case SDEV_EVT_CAPACITY_CHANGE_REPORTED:2671 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:2672 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:2673 case SDEV_EVT_LUN_CHANGE_REPORTED:2674 case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED:2675 case SDEV_EVT_POWER_ON_RESET_OCCURRED:2676 default:2677 /* do nothing */2678 break;2679 }2680 2681 return evt;2682}2683EXPORT_SYMBOL_GPL(sdev_evt_alloc);2684 2685/**2686 * sdev_evt_send_simple - send asserted event to uevent thread2687 * @sdev: scsi_device event occurred on2688 * @evt_type: type of event to send2689 * @gfpflags: GFP flags for allocation2690 *2691 * Assert scsi device event asynchronously, given an event type.2692 */2693void sdev_evt_send_simple(struct scsi_device *sdev,2694 enum scsi_device_event evt_type, gfp_t gfpflags)2695{2696 struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags);2697 if (!evt) {2698 sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n",2699 evt_type);2700 return;2701 }2702 2703 sdev_evt_send(sdev, evt);2704}2705EXPORT_SYMBOL_GPL(sdev_evt_send_simple);2706 2707/**2708 * scsi_device_quiesce - Block all commands except power management.2709 * @sdev: scsi device to quiesce.2710 *2711 * This works by trying to transition to the SDEV_QUIESCE state2712 * (which must be a legal transition). When the device is in this2713 * state, only power management requests will be accepted, all others will2714 * be deferred.2715 *2716 * Must be called with user context, may sleep.2717 *2718 * Returns zero if unsuccessful or an error if not.2719 */2720int2721scsi_device_quiesce(struct scsi_device *sdev)2722{2723 struct request_queue *q = sdev->request_queue;2724 int err;2725 2726 /*2727 * It is allowed to call scsi_device_quiesce() multiple times from2728 * the same context but concurrent scsi_device_quiesce() calls are2729 * not allowed.2730 */2731 WARN_ON_ONCE(sdev->quiesced_by && sdev->quiesced_by != current);2732 2733 if (sdev->quiesced_by == current)2734 return 0;2735 2736 blk_set_pm_only(q);2737 2738 blk_mq_freeze_queue(q);2739 /*2740 * Ensure that the effect of blk_set_pm_only() will be visible2741 * for percpu_ref_tryget() callers that occur after the queue2742 * unfreeze even if the queue was already frozen before this function2743 * was called. See also https://lwn.net/Articles/573497/.2744 */2745 synchronize_rcu();2746 blk_mq_unfreeze_queue(q);2747 2748 mutex_lock(&sdev->state_mutex);2749 err = scsi_device_set_state(sdev, SDEV_QUIESCE);2750 if (err == 0)2751 sdev->quiesced_by = current;2752 else2753 blk_clear_pm_only(q);2754 mutex_unlock(&sdev->state_mutex);2755 2756 return err;2757}2758EXPORT_SYMBOL(scsi_device_quiesce);2759 2760/**2761 * scsi_device_resume - Restart user issued commands to a quiesced device.2762 * @sdev: scsi device to resume.2763 *2764 * Moves the device from quiesced back to running and restarts the2765 * queues.2766 *2767 * Must be called with user context, may sleep.2768 */2769void scsi_device_resume(struct scsi_device *sdev)2770{2771 /* check if the device state was mutated prior to resume, and if2772 * so assume the state is being managed elsewhere (for example2773 * device deleted during suspend)2774 */2775 mutex_lock(&sdev->state_mutex);2776 if (sdev->sdev_state == SDEV_QUIESCE)2777 scsi_device_set_state(sdev, SDEV_RUNNING);2778 if (sdev->quiesced_by) {2779 sdev->quiesced_by = NULL;2780 blk_clear_pm_only(sdev->request_queue);2781 }2782 mutex_unlock(&sdev->state_mutex);2783}2784EXPORT_SYMBOL(scsi_device_resume);2785 2786static void2787device_quiesce_fn(struct scsi_device *sdev, void *data)2788{2789 scsi_device_quiesce(sdev);2790}2791 2792void2793scsi_target_quiesce(struct scsi_target *starget)2794{2795 starget_for_each_device(starget, NULL, device_quiesce_fn);2796}2797EXPORT_SYMBOL(scsi_target_quiesce);2798 2799static void2800device_resume_fn(struct scsi_device *sdev, void *data)2801{2802 scsi_device_resume(sdev);2803}2804 2805void2806scsi_target_resume(struct scsi_target *starget)2807{2808 starget_for_each_device(starget, NULL, device_resume_fn);2809}2810EXPORT_SYMBOL(scsi_target_resume);2811 2812static int __scsi_internal_device_block_nowait(struct scsi_device *sdev)2813{2814 if (scsi_device_set_state(sdev, SDEV_BLOCK))2815 return scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);2816 2817 return 0;2818}2819 2820void scsi_start_queue(struct scsi_device *sdev)2821{2822 if (cmpxchg(&sdev->queue_stopped, 1, 0))2823 blk_mq_unquiesce_queue(sdev->request_queue);2824}2825 2826static void scsi_stop_queue(struct scsi_device *sdev)2827{2828 /*2829 * The atomic variable of ->queue_stopped covers that2830 * blk_mq_quiesce_queue* is balanced with blk_mq_unquiesce_queue.2831 *2832 * The caller needs to wait until quiesce is done.2833 */2834 if (!cmpxchg(&sdev->queue_stopped, 0, 1))2835 blk_mq_quiesce_queue_nowait(sdev->request_queue);2836}2837 2838/**2839 * scsi_internal_device_block_nowait - try to transition to the SDEV_BLOCK state2840 * @sdev: device to block2841 *2842 * Pause SCSI command processing on the specified device. Does not sleep.2843 *2844 * Returns zero if successful or a negative error code upon failure.2845 *2846 * Notes:2847 * This routine transitions the device to the SDEV_BLOCK state (which must be2848 * a legal transition). When the device is in this state, command processing2849 * is paused until the device leaves the SDEV_BLOCK state. See also2850 * scsi_internal_device_unblock_nowait().2851 */2852int scsi_internal_device_block_nowait(struct scsi_device *sdev)2853{2854 int ret = __scsi_internal_device_block_nowait(sdev);2855 2856 /*2857 * The device has transitioned to SDEV_BLOCK. Stop the2858 * block layer from calling the midlayer with this device's2859 * request queue.2860 */2861 if (!ret)2862 scsi_stop_queue(sdev);2863 return ret;2864}2865EXPORT_SYMBOL_GPL(scsi_internal_device_block_nowait);2866 2867/**2868 * scsi_device_block - try to transition to the SDEV_BLOCK state2869 * @sdev: device to block2870 * @data: dummy argument, ignored2871 *2872 * Pause SCSI command processing on the specified device. Callers must wait2873 * until all ongoing scsi_queue_rq() calls have finished after this function2874 * returns.2875 *2876 * Note:2877 * This routine transitions the device to the SDEV_BLOCK state (which must be2878 * a legal transition). When the device is in this state, command processing2879 * is paused until the device leaves the SDEV_BLOCK state. See also2880 * scsi_internal_device_unblock().2881 */2882static void scsi_device_block(struct scsi_device *sdev, void *data)2883{2884 int err;2885 enum scsi_device_state state;2886 2887 mutex_lock(&sdev->state_mutex);2888 err = __scsi_internal_device_block_nowait(sdev);2889 state = sdev->sdev_state;2890 if (err == 0)2891 /*2892 * scsi_stop_queue() must be called with the state_mutex2893 * held. Otherwise a simultaneous scsi_start_queue() call2894 * might unquiesce the queue before we quiesce it.2895 */2896 scsi_stop_queue(sdev);2897 2898 mutex_unlock(&sdev->state_mutex);2899 2900 WARN_ONCE(err, "%s: failed to block %s in state %d\n",2901 __func__, dev_name(&sdev->sdev_gendev), state);2902}2903 2904/**2905 * scsi_internal_device_unblock_nowait - resume a device after a block request2906 * @sdev: device to resume2907 * @new_state: state to set the device to after unblocking2908 *2909 * Restart the device queue for a previously suspended SCSI device. Does not2910 * sleep.2911 *2912 * Returns zero if successful or a negative error code upon failure.2913 *2914 * Notes:2915 * This routine transitions the device to the SDEV_RUNNING state or to one of2916 * the offline states (which must be a legal transition) allowing the midlayer2917 * to goose the queue for this device.2918 */2919int scsi_internal_device_unblock_nowait(struct scsi_device *sdev,2920 enum scsi_device_state new_state)2921{2922 switch (new_state) {2923 case SDEV_RUNNING:2924 case SDEV_TRANSPORT_OFFLINE:2925 break;2926 default:2927 return -EINVAL;2928 }2929 2930 /*2931 * Try to transition the scsi device to SDEV_RUNNING or one of the2932 * offlined states and goose the device queue if successful.2933 */2934 switch (sdev->sdev_state) {2935 case SDEV_BLOCK:2936 case SDEV_TRANSPORT_OFFLINE:2937 sdev->sdev_state = new_state;2938 break;2939 case SDEV_CREATED_BLOCK:2940 if (new_state == SDEV_TRANSPORT_OFFLINE ||2941 new_state == SDEV_OFFLINE)2942 sdev->sdev_state = new_state;2943 else2944 sdev->sdev_state = SDEV_CREATED;2945 break;2946 case SDEV_CANCEL:2947 case SDEV_OFFLINE:2948 break;2949 default:2950 return -EINVAL;2951 }2952 scsi_start_queue(sdev);2953 2954 return 0;2955}2956EXPORT_SYMBOL_GPL(scsi_internal_device_unblock_nowait);2957 2958/**2959 * scsi_internal_device_unblock - resume a device after a block request2960 * @sdev: device to resume2961 * @new_state: state to set the device to after unblocking2962 *2963 * Restart the device queue for a previously suspended SCSI device. May sleep.2964 *2965 * Returns zero if successful or a negative error code upon failure.2966 *2967 * Notes:2968 * This routine transitions the device to the SDEV_RUNNING state or to one of2969 * the offline states (which must be a legal transition) allowing the midlayer2970 * to goose the queue for this device.2971 */2972static int scsi_internal_device_unblock(struct scsi_device *sdev,2973 enum scsi_device_state new_state)2974{2975 int ret;2976 2977 mutex_lock(&sdev->state_mutex);2978 ret = scsi_internal_device_unblock_nowait(sdev, new_state);2979 mutex_unlock(&sdev->state_mutex);2980 2981 return ret;2982}2983 2984static int2985target_block(struct device *dev, void *data)2986{2987 if (scsi_is_target_device(dev))2988 starget_for_each_device(to_scsi_target(dev), NULL,2989 scsi_device_block);2990 return 0;2991}2992 2993/**2994 * scsi_block_targets - transition all SCSI child devices to SDEV_BLOCK state2995 * @dev: a parent device of one or more scsi_target devices2996 * @shost: the Scsi_Host to which this device belongs2997 *2998 * Iterate over all children of @dev, which should be scsi_target devices,2999 * and switch all subordinate scsi devices to SDEV_BLOCK state. Wait for3000 * ongoing scsi_queue_rq() calls to finish. May sleep.3001 *3002 * Note:3003 * @dev must not itself be a scsi_target device.3004 */3005void3006scsi_block_targets(struct Scsi_Host *shost, struct device *dev)3007{3008 WARN_ON_ONCE(scsi_is_target_device(dev));3009 device_for_each_child(dev, NULL, target_block);3010 blk_mq_wait_quiesce_done(&shost->tag_set);3011}3012EXPORT_SYMBOL_GPL(scsi_block_targets);3013 3014static void3015device_unblock(struct scsi_device *sdev, void *data)3016{3017 scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data);3018}3019 3020static int3021target_unblock(struct device *dev, void *data)3022{3023 if (scsi_is_target_device(dev))3024 starget_for_each_device(to_scsi_target(dev), data,3025 device_unblock);3026 return 0;3027}3028 3029void3030scsi_target_unblock(struct device *dev, enum scsi_device_state new_state)3031{3032 if (scsi_is_target_device(dev))3033 starget_for_each_device(to_scsi_target(dev), &new_state,3034 device_unblock);3035 else3036 device_for_each_child(dev, &new_state, target_unblock);3037}3038EXPORT_SYMBOL_GPL(scsi_target_unblock);3039 3040/**3041 * scsi_host_block - Try to transition all logical units to the SDEV_BLOCK state3042 * @shost: device to block3043 *3044 * Pause SCSI command processing for all logical units associated with the SCSI3045 * host and wait until pending scsi_queue_rq() calls have finished.3046 *3047 * Returns zero if successful or a negative error code upon failure.3048 */3049int3050scsi_host_block(struct Scsi_Host *shost)3051{3052 struct scsi_device *sdev;3053 int ret;3054 3055 /*3056 * Call scsi_internal_device_block_nowait so we can avoid3057 * calling synchronize_rcu() for each LUN.3058 */3059 shost_for_each_device(sdev, shost) {3060 mutex_lock(&sdev->state_mutex);3061 ret = scsi_internal_device_block_nowait(sdev);3062 mutex_unlock(&sdev->state_mutex);3063 if (ret) {3064 scsi_device_put(sdev);3065 return ret;3066 }3067 }3068 3069 /* Wait for ongoing scsi_queue_rq() calls to finish. */3070 blk_mq_wait_quiesce_done(&shost->tag_set);3071 3072 return 0;3073}3074EXPORT_SYMBOL_GPL(scsi_host_block);3075 3076int3077scsi_host_unblock(struct Scsi_Host *shost, int new_state)3078{3079 struct scsi_device *sdev;3080 int ret = 0;3081 3082 shost_for_each_device(sdev, shost) {3083 ret = scsi_internal_device_unblock(sdev, new_state);3084 if (ret) {3085 scsi_device_put(sdev);3086 break;3087 }3088 }3089 return ret;3090}3091EXPORT_SYMBOL_GPL(scsi_host_unblock);3092 3093/**3094 * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt3095 * @sgl: scatter-gather list3096 * @sg_count: number of segments in sg3097 * @offset: offset in bytes into sg, on return offset into the mapped area3098 * @len: bytes to map, on return number of bytes mapped3099 *3100 * Returns virtual address of the start of the mapped page3101 */3102void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,3103 size_t *offset, size_t *len)3104{3105 int i;3106 size_t sg_len = 0, len_complete = 0;3107 struct scatterlist *sg;3108 struct page *page;3109 3110 WARN_ON(!irqs_disabled());3111 3112 for_each_sg(sgl, sg, sg_count, i) {3113 len_complete = sg_len; /* Complete sg-entries */3114 sg_len += sg->length;3115 if (sg_len > *offset)3116 break;3117 }3118 3119 if (unlikely(i == sg_count)) {3120 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "3121 "elements %d\n",3122 __func__, sg_len, *offset, sg_count);3123 WARN_ON(1);3124 return NULL;3125 }3126 3127 /* Offset starting from the beginning of first page in this sg-entry */3128 *offset = *offset - len_complete + sg->offset;3129 3130 /* Assumption: contiguous pages can be accessed as "page + i" */3131 page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT));3132 *offset &= ~PAGE_MASK;3133 3134 /* Bytes in this sg-entry from *offset to the end of the page */3135 sg_len = PAGE_SIZE - *offset;3136 if (*len > sg_len)3137 *len = sg_len;3138 3139 return kmap_atomic(page);3140}3141EXPORT_SYMBOL(scsi_kmap_atomic_sg);3142 3143/**3144 * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg3145 * @virt: virtual address to be unmapped3146 */3147void scsi_kunmap_atomic_sg(void *virt)3148{3149 kunmap_atomic(virt);3150}3151EXPORT_SYMBOL(scsi_kunmap_atomic_sg);3152 3153void sdev_disable_disk_events(struct scsi_device *sdev)3154{3155 atomic_inc(&sdev->disk_events_disable_depth);3156}3157EXPORT_SYMBOL(sdev_disable_disk_events);3158 3159void sdev_enable_disk_events(struct scsi_device *sdev)3160{3161 if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0))3162 return;3163 atomic_dec(&sdev->disk_events_disable_depth);3164}3165EXPORT_SYMBOL(sdev_enable_disk_events);3166 3167static unsigned char designator_prio(const unsigned char *d)3168{3169 if (d[1] & 0x30)3170 /* not associated with LUN */3171 return 0;3172 3173 if (d[3] == 0)3174 /* invalid length */3175 return 0;3176 3177 /*3178 * Order of preference for lun descriptor:3179 * - SCSI name string3180 * - NAA IEEE Registered Extended3181 * - EUI-64 based 16-byte3182 * - EUI-64 based 12-byte3183 * - NAA IEEE Registered3184 * - NAA IEEE Extended3185 * - EUI-64 based 8-byte3186 * - SCSI name string (truncated)3187 * - T10 Vendor ID3188 * as longer descriptors reduce the likelyhood3189 * of identification clashes.3190 */3191 3192 switch (d[1] & 0xf) {3193 case 8:3194 /* SCSI name string, variable-length UTF-8 */3195 return 9;3196 case 3:3197 switch (d[4] >> 4) {3198 case 6:3199 /* NAA registered extended */3200 return 8;3201 case 5:3202 /* NAA registered */3203 return 5;3204 case 4:3205 /* NAA extended */3206 return 4;3207 case 3:3208 /* NAA locally assigned */3209 return 1;3210 default:3211 break;3212 }3213 break;3214 case 2:3215 switch (d[3]) {3216 case 16:3217 /* EUI64-based, 16 byte */3218 return 7;3219 case 12:3220 /* EUI64-based, 12 byte */3221 return 6;3222 case 8:3223 /* EUI64-based, 8 byte */3224 return 3;3225 default:3226 break;3227 }3228 break;3229 case 1:3230 /* T10 vendor ID */3231 return 1;3232 default:3233 break;3234 }3235 3236 return 0;3237}3238 3239/**3240 * scsi_vpd_lun_id - return a unique device identification3241 * @sdev: SCSI device3242 * @id: buffer for the identification3243 * @id_len: length of the buffer3244 *3245 * Copies a unique device identification into @id based3246 * on the information in the VPD page 0x83 of the device.3247 * The string will be formatted as a SCSI name string.3248 *3249 * Returns the length of the identification or error on failure.3250 * If the identifier is longer than the supplied buffer the actual3251 * identifier length is returned and the buffer is not zero-padded.3252 */3253int scsi_vpd_lun_id(struct scsi_device *sdev, char *id, size_t id_len)3254{3255 u8 cur_id_prio = 0;3256 u8 cur_id_size = 0;3257 const unsigned char *d, *cur_id_str;3258 const struct scsi_vpd *vpd_pg83;3259 int id_size = -EINVAL;3260 3261 rcu_read_lock();3262 vpd_pg83 = rcu_dereference(sdev->vpd_pg83);3263 if (!vpd_pg83) {3264 rcu_read_unlock();3265 return -ENXIO;3266 }3267 3268 /* The id string must be at least 20 bytes + terminating NULL byte */3269 if (id_len < 21) {3270 rcu_read_unlock();3271 return -EINVAL;3272 }3273 3274 memset(id, 0, id_len);3275 for (d = vpd_pg83->data + 4;3276 d < vpd_pg83->data + vpd_pg83->len;3277 d += d[3] + 4) {3278 u8 prio = designator_prio(d);3279 3280 if (prio == 0 || cur_id_prio > prio)3281 continue;3282 3283 switch (d[1] & 0xf) {3284 case 0x1:3285 /* T10 Vendor ID */3286 if (cur_id_size > d[3])3287 break;3288 cur_id_prio = prio;3289 cur_id_size = d[3];3290 if (cur_id_size + 4 > id_len)3291 cur_id_size = id_len - 4;3292 cur_id_str = d + 4;3293 id_size = snprintf(id, id_len, "t10.%*pE",3294 cur_id_size, cur_id_str);3295 break;3296 case 0x2:3297 /* EUI-64 */3298 cur_id_prio = prio;3299 cur_id_size = d[3];3300 cur_id_str = d + 4;3301 switch (cur_id_size) {3302 case 8:3303 id_size = snprintf(id, id_len,3304 "eui.%8phN",3305 cur_id_str);3306 break;3307 case 12:3308 id_size = snprintf(id, id_len,3309 "eui.%12phN",3310 cur_id_str);3311 break;3312 case 16:3313 id_size = snprintf(id, id_len,3314 "eui.%16phN",3315 cur_id_str);3316 break;3317 default:3318 break;3319 }3320 break;3321 case 0x3:3322 /* NAA */3323 cur_id_prio = prio;3324 cur_id_size = d[3];3325 cur_id_str = d + 4;3326 switch (cur_id_size) {3327 case 8:3328 id_size = snprintf(id, id_len,3329 "naa.%8phN",3330 cur_id_str);3331 break;3332 case 16:3333 id_size = snprintf(id, id_len,3334 "naa.%16phN",3335 cur_id_str);3336 break;3337 default:3338 break;3339 }3340 break;3341 case 0x8:3342 /* SCSI name string */3343 if (cur_id_size > d[3])3344 break;3345 /* Prefer others for truncated descriptor */3346 if (d[3] > id_len) {3347 prio = 2;3348 if (cur_id_prio > prio)3349 break;3350 }3351 cur_id_prio = prio;3352 cur_id_size = id_size = d[3];3353 cur_id_str = d + 4;3354 if (cur_id_size >= id_len)3355 cur_id_size = id_len - 1;3356 memcpy(id, cur_id_str, cur_id_size);3357 break;3358 default:3359 break;3360 }3361 }3362 rcu_read_unlock();3363 3364 return id_size;3365}3366EXPORT_SYMBOL(scsi_vpd_lun_id);3367 3368/*3369 * scsi_vpd_tpg_id - return a target port group identifier3370 * @sdev: SCSI device3371 *3372 * Returns the Target Port Group identifier from the information3373 * froom VPD page 0x83 of the device.3374 *3375 * Returns the identifier or error on failure.3376 */3377int scsi_vpd_tpg_id(struct scsi_device *sdev, int *rel_id)3378{3379 const unsigned char *d;3380 const struct scsi_vpd *vpd_pg83;3381 int group_id = -EAGAIN, rel_port = -1;3382 3383 rcu_read_lock();3384 vpd_pg83 = rcu_dereference(sdev->vpd_pg83);3385 if (!vpd_pg83) {3386 rcu_read_unlock();3387 return -ENXIO;3388 }3389 3390 d = vpd_pg83->data + 4;3391 while (d < vpd_pg83->data + vpd_pg83->len) {3392 switch (d[1] & 0xf) {3393 case 0x4:3394 /* Relative target port */3395 rel_port = get_unaligned_be16(&d[6]);3396 break;3397 case 0x5:3398 /* Target port group */3399 group_id = get_unaligned_be16(&d[6]);3400 break;3401 default:3402 break;3403 }3404 d += d[3] + 4;3405 }3406 rcu_read_unlock();3407 3408 if (group_id >= 0 && rel_id && rel_port != -1)3409 *rel_id = rel_port;3410 3411 return group_id;3412}3413EXPORT_SYMBOL(scsi_vpd_tpg_id);3414 3415/**3416 * scsi_build_sense - build sense data for a command3417 * @scmd: scsi command for which the sense should be formatted3418 * @desc: Sense format (non-zero == descriptor format,3419 * 0 == fixed format)3420 * @key: Sense key3421 * @asc: Additional sense code3422 * @ascq: Additional sense code qualifier3423 *3424 **/3425void scsi_build_sense(struct scsi_cmnd *scmd, int desc, u8 key, u8 asc, u8 ascq)3426{3427 scsi_build_sense_buffer(desc, scmd->sense_buffer, key, asc, ascq);3428 scmd->result = SAM_STAT_CHECK_CONDITION;3429}3430EXPORT_SYMBOL_GPL(scsi_build_sense);3431 3432#ifdef CONFIG_SCSI_LIB_KUNIT_TEST3433#include "scsi_lib_test.c"3434#endif3435