758 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * S/390 common I/O routines -- low level i/o calls4 *5 * Copyright IBM Corp. 1999, 20086 * Author(s): Ingo Adlung (adlung@de.ibm.com)7 * Cornelia Huck (cornelia.huck@de.ibm.com)8 * Arnd Bergmann (arndb@de.ibm.com)9 * Martin Schwidefsky (schwidefsky@de.ibm.com)10 */11 12#define KMSG_COMPONENT "cio"13#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt14 15#include <linux/ftrace.h>16#include <linux/module.h>17#include <linux/init.h>18#include <linux/slab.h>19#include <linux/device.h>20#include <linux/kernel_stat.h>21#include <linux/interrupt.h>22#include <linux/irq.h>23#include <asm/cio.h>24#include <asm/delay.h>25#include <asm/irq.h>26#include <asm/irq_regs.h>27#include <asm/setup.h>28#include <asm/ipl.h>29#include <asm/chpid.h>30#include <asm/airq.h>31#include <asm/isc.h>32#include <linux/sched/cputime.h>33#include <asm/fcx.h>34#include <asm/nmi.h>35#include <asm/crw.h>36#include "cio.h"37#include "css.h"38#include "chsc.h"39#include "ioasm.h"40#include "io_sch.h"41#include "blacklist.h"42#include "cio_debug.h"43#include "chp.h"44#include "trace.h"45 46debug_info_t *cio_debug_msg_id;47debug_info_t *cio_debug_trace_id;48debug_info_t *cio_debug_crw_id;49 50DEFINE_PER_CPU_ALIGNED(struct irb, cio_irb);51EXPORT_PER_CPU_SYMBOL(cio_irb);52 53/*54 * Function: cio_debug_init55 * Initializes three debug logs for common I/O:56 * - cio_msg logs generic cio messages57 * - cio_trace logs the calling of different functions58 * - cio_crw logs machine check related cio messages59 */60static int __init cio_debug_init(void)61{62 cio_debug_msg_id = debug_register("cio_msg", 16, 1, 11 * sizeof(long));63 if (!cio_debug_msg_id)64 goto out_unregister;65 debug_register_view(cio_debug_msg_id, &debug_sprintf_view);66 debug_set_level(cio_debug_msg_id, 2);67 cio_debug_trace_id = debug_register("cio_trace", 16, 1, 16);68 if (!cio_debug_trace_id)69 goto out_unregister;70 debug_register_view(cio_debug_trace_id, &debug_hex_ascii_view);71 debug_set_level(cio_debug_trace_id, 2);72 cio_debug_crw_id = debug_register("cio_crw", 8, 1, 8 * sizeof(long));73 if (!cio_debug_crw_id)74 goto out_unregister;75 debug_register_view(cio_debug_crw_id, &debug_sprintf_view);76 debug_set_level(cio_debug_crw_id, 4);77 return 0;78 79out_unregister:80 debug_unregister(cio_debug_msg_id);81 debug_unregister(cio_debug_trace_id);82 debug_unregister(cio_debug_crw_id);83 return -1;84}85 86arch_initcall (cio_debug_init);87 88int cio_set_options(struct subchannel *sch, int flags)89{90 struct io_subchannel_private *priv = to_io_private(sch);91 92 priv->options.suspend = (flags & DOIO_ALLOW_SUSPEND) != 0;93 priv->options.prefetch = (flags & DOIO_DENY_PREFETCH) != 0;94 priv->options.inter = (flags & DOIO_SUPPRESS_INTER) != 0;95 return 0;96}97 98static int99cio_start_handle_notoper(struct subchannel *sch, __u8 lpm)100{101 char dbf_text[15];102 103 if (lpm != 0)104 sch->lpm &= ~lpm;105 else106 sch->lpm = 0;107 108 CIO_MSG_EVENT(2, "cio_start: 'not oper' status for "109 "subchannel 0.%x.%04x!\n", sch->schid.ssid,110 sch->schid.sch_no);111 112 if (cio_update_schib(sch))113 return -ENODEV;114 115 sprintf(dbf_text, "no%s", dev_name(&sch->dev));116 CIO_TRACE_EVENT(0, dbf_text);117 CIO_HEX_EVENT(0, &sch->schib, sizeof (struct schib));118 119 return (sch->lpm ? -EACCES : -ENODEV);120}121 122int123cio_start_key (struct subchannel *sch, /* subchannel structure */124 struct ccw1 * cpa, /* logical channel prog addr */125 __u8 lpm, /* logical path mask */126 __u8 key) /* storage key */127{128 struct io_subchannel_private *priv = to_io_private(sch);129 union orb *orb = &priv->orb;130 int ccode;131 132 CIO_TRACE_EVENT(5, "stIO");133 CIO_TRACE_EVENT(5, dev_name(&sch->dev));134 135 memset(orb, 0, sizeof(union orb));136 /* sch is always under 2G. */137 orb->cmd.intparm = (u32)virt_to_phys(sch);138 orb->cmd.fmt = 1;139 140 orb->cmd.pfch = priv->options.prefetch == 0;141 orb->cmd.spnd = priv->options.suspend;142 orb->cmd.ssic = priv->options.suspend && priv->options.inter;143 orb->cmd.lpm = (lpm != 0) ? lpm : sch->lpm;144 /*145 * for 64 bit we always support 64 bit IDAWs with 4k page size only146 */147 orb->cmd.c64 = 1;148 orb->cmd.i2k = 0;149 orb->cmd.key = key >> 4;150 /* issue "Start Subchannel" */151 orb->cmd.cpa = virt_to_dma32(cpa);152 ccode = ssch(sch->schid, orb);153 154 /* process condition code */155 CIO_HEX_EVENT(5, &ccode, sizeof(ccode));156 157 switch (ccode) {158 case 0:159 /*160 * initialize device status information161 */162 sch->schib.scsw.cmd.actl |= SCSW_ACTL_START_PEND;163 return 0;164 case 1: /* status pending */165 case 2: /* busy */166 return -EBUSY;167 case 3: /* device/path not operational */168 return cio_start_handle_notoper(sch, lpm);169 default:170 return ccode;171 }172}173EXPORT_SYMBOL_GPL(cio_start_key);174 175int176cio_start (struct subchannel *sch, struct ccw1 *cpa, __u8 lpm)177{178 return cio_start_key(sch, cpa, lpm, PAGE_DEFAULT_KEY);179}180EXPORT_SYMBOL_GPL(cio_start);181 182/*183 * resume suspended I/O operation184 */185int186cio_resume (struct subchannel *sch)187{188 int ccode;189 190 CIO_TRACE_EVENT(4, "resIO");191 CIO_TRACE_EVENT(4, dev_name(&sch->dev));192 193 ccode = rsch (sch->schid);194 195 CIO_HEX_EVENT(4, &ccode, sizeof(ccode));196 197 switch (ccode) {198 case 0:199 sch->schib.scsw.cmd.actl |= SCSW_ACTL_RESUME_PEND;200 return 0;201 case 1:202 return -EBUSY;203 case 2:204 return -EINVAL;205 default:206 /*207 * useless to wait for request completion208 * as device is no longer operational !209 */210 return -ENODEV;211 }212}213EXPORT_SYMBOL_GPL(cio_resume);214 215/*216 * halt I/O operation217 */218int219cio_halt(struct subchannel *sch)220{221 int ccode;222 223 if (!sch)224 return -ENODEV;225 226 CIO_TRACE_EVENT(2, "haltIO");227 CIO_TRACE_EVENT(2, dev_name(&sch->dev));228 229 /*230 * Issue "Halt subchannel" and process condition code231 */232 ccode = hsch (sch->schid);233 234 CIO_HEX_EVENT(2, &ccode, sizeof(ccode));235 236 switch (ccode) {237 case 0:238 sch->schib.scsw.cmd.actl |= SCSW_ACTL_HALT_PEND;239 return 0;240 case 1: /* status pending */241 case 2: /* busy */242 return -EBUSY;243 default: /* device not operational */244 return -ENODEV;245 }246}247EXPORT_SYMBOL_GPL(cio_halt);248 249/*250 * Clear I/O operation251 */252int253cio_clear(struct subchannel *sch)254{255 int ccode;256 257 if (!sch)258 return -ENODEV;259 260 CIO_TRACE_EVENT(2, "clearIO");261 CIO_TRACE_EVENT(2, dev_name(&sch->dev));262 263 /*264 * Issue "Clear subchannel" and process condition code265 */266 ccode = csch (sch->schid);267 268 CIO_HEX_EVENT(2, &ccode, sizeof(ccode));269 270 switch (ccode) {271 case 0:272 sch->schib.scsw.cmd.actl |= SCSW_ACTL_CLEAR_PEND;273 return 0;274 default: /* device not operational */275 return -ENODEV;276 }277}278EXPORT_SYMBOL_GPL(cio_clear);279 280/*281 * Function: cio_cancel282 * Issues a "Cancel Subchannel" on the specified subchannel283 * Note: We don't need any fancy intparms and flags here284 * since xsch is executed synchronously.285 * Only for common I/O internal use as for now.286 */287int288cio_cancel (struct subchannel *sch)289{290 int ccode;291 292 if (!sch)293 return -ENODEV;294 295 CIO_TRACE_EVENT(2, "cancelIO");296 CIO_TRACE_EVENT(2, dev_name(&sch->dev));297 298 ccode = xsch (sch->schid);299 300 CIO_HEX_EVENT(2, &ccode, sizeof(ccode));301 302 switch (ccode) {303 case 0: /* success */304 /* Update information in scsw. */305 if (cio_update_schib(sch))306 return -ENODEV;307 return 0;308 case 1: /* status pending */309 return -EBUSY;310 case 2: /* not applicable */311 return -EINVAL;312 default: /* not oper */313 return -ENODEV;314 }315}316EXPORT_SYMBOL_GPL(cio_cancel);317 318/**319 * cio_cancel_halt_clear - Cancel running I/O by performing cancel, halt320 * and clear ordinally if subchannel is valid.321 * @sch: subchannel on which to perform the cancel_halt_clear operation322 * @iretry: the number of the times remained to retry the next operation323 *324 * This should be called repeatedly since halt/clear are asynchronous325 * operations. We do one try with cio_cancel, three tries with cio_halt,326 * 255 tries with cio_clear. The caller should initialize @iretry with327 * the value 255 for its first call to this, and keep using the same328 * @iretry in the subsequent calls until it gets a non -EBUSY return.329 *330 * Returns 0 if device now idle, -ENODEV for device not operational,331 * -EBUSY if an interrupt is expected (either from halt/clear or from a332 * status pending), and -EIO if out of retries.333 */334int cio_cancel_halt_clear(struct subchannel *sch, int *iretry)335{336 int ret;337 338 if (cio_update_schib(sch))339 return -ENODEV;340 if (!sch->schib.pmcw.ena)341 /* Not operational -> done. */342 return 0;343 /* Stage 1: cancel io. */344 if (!(scsw_actl(&sch->schib.scsw) & SCSW_ACTL_HALT_PEND) &&345 !(scsw_actl(&sch->schib.scsw) & SCSW_ACTL_CLEAR_PEND)) {346 if (!scsw_is_tm(&sch->schib.scsw)) {347 ret = cio_cancel(sch);348 if (ret != -EINVAL)349 return ret;350 }351 /*352 * Cancel io unsuccessful or not applicable (transport mode).353 * Continue with asynchronous instructions.354 */355 *iretry = 3; /* 3 halt retries. */356 }357 /* Stage 2: halt io. */358 if (!(scsw_actl(&sch->schib.scsw) & SCSW_ACTL_CLEAR_PEND)) {359 if (*iretry) {360 *iretry -= 1;361 ret = cio_halt(sch);362 if (ret != -EBUSY)363 return (ret == 0) ? -EBUSY : ret;364 }365 /* Halt io unsuccessful. */366 *iretry = 255; /* 255 clear retries. */367 }368 /* Stage 3: clear io. */369 if (*iretry) {370 *iretry -= 1;371 ret = cio_clear(sch);372 return (ret == 0) ? -EBUSY : ret;373 }374 /* Function was unsuccessful */375 return -EIO;376}377EXPORT_SYMBOL_GPL(cio_cancel_halt_clear);378 379static void cio_apply_config(struct subchannel *sch, struct schib *schib)380{381 schib->pmcw.intparm = sch->config.intparm;382 schib->pmcw.mbi = sch->config.mbi;383 schib->pmcw.isc = sch->config.isc;384 schib->pmcw.ena = sch->config.ena;385 schib->pmcw.mme = sch->config.mme;386 schib->pmcw.mp = sch->config.mp;387 schib->pmcw.csense = sch->config.csense;388 schib->pmcw.mbfc = sch->config.mbfc;389 if (sch->config.mbfc)390 schib->mba = sch->config.mba;391}392 393static int cio_check_config(struct subchannel *sch, struct schib *schib)394{395 return (schib->pmcw.intparm == sch->config.intparm) &&396 (schib->pmcw.mbi == sch->config.mbi) &&397 (schib->pmcw.isc == sch->config.isc) &&398 (schib->pmcw.ena == sch->config.ena) &&399 (schib->pmcw.mme == sch->config.mme) &&400 (schib->pmcw.mp == sch->config.mp) &&401 (schib->pmcw.csense == sch->config.csense) &&402 (schib->pmcw.mbfc == sch->config.mbfc) &&403 (!sch->config.mbfc || (schib->mba == sch->config.mba));404}405 406/*407 * cio_commit_config - apply configuration to the subchannel408 */409int cio_commit_config(struct subchannel *sch)410{411 int ccode, retry, ret = 0;412 struct schib schib;413 struct irb irb;414 415 if (stsch(sch->schid, &schib) || !css_sch_is_valid(&schib))416 return -ENODEV;417 418 for (retry = 0; retry < 5; retry++) {419 /* copy desired changes to local schib */420 cio_apply_config(sch, &schib);421 ccode = msch(sch->schid, &schib);422 if (ccode < 0) /* -EIO if msch gets a program check. */423 return ccode;424 switch (ccode) {425 case 0: /* successful */426 if (stsch(sch->schid, &schib) ||427 !css_sch_is_valid(&schib))428 return -ENODEV;429 if (cio_check_config(sch, &schib)) {430 /* commit changes from local schib */431 memcpy(&sch->schib, &schib, sizeof(schib));432 return 0;433 }434 ret = -EAGAIN;435 break;436 case 1: /* status pending */437 ret = -EBUSY;438 if (tsch(sch->schid, &irb))439 return ret;440 break;441 case 2: /* busy */442 udelay(100); /* allow for recovery */443 ret = -EBUSY;444 break;445 case 3: /* not operational */446 return -ENODEV;447 }448 }449 return ret;450}451EXPORT_SYMBOL_GPL(cio_commit_config);452 453/**454 * cio_update_schib - Perform stsch and update schib if subchannel is valid.455 * @sch: subchannel on which to perform stsch456 * Return zero on success, -ENODEV otherwise.457 */458int cio_update_schib(struct subchannel *sch)459{460 struct schib schib;461 462 if (stsch(sch->schid, &schib) || !css_sch_is_valid(&schib))463 return -ENODEV;464 465 memcpy(&sch->schib, &schib, sizeof(schib));466 return 0;467}468EXPORT_SYMBOL_GPL(cio_update_schib);469 470/**471 * cio_enable_subchannel - enable a subchannel.472 * @sch: subchannel to be enabled473 * @intparm: interruption parameter to set474 */475int cio_enable_subchannel(struct subchannel *sch, u32 intparm)476{477 int ret;478 479 CIO_TRACE_EVENT(2, "ensch");480 CIO_TRACE_EVENT(2, dev_name(&sch->dev));481 482 if (sch_is_pseudo_sch(sch))483 return -EINVAL;484 if (cio_update_schib(sch))485 return -ENODEV;486 487 sch->config.ena = 1;488 sch->config.isc = sch->isc;489 sch->config.intparm = intparm;490 491 ret = cio_commit_config(sch);492 if (ret == -EIO) {493 /*494 * Got a program check in msch. Try without495 * the concurrent sense bit the next time.496 */497 sch->config.csense = 0;498 ret = cio_commit_config(sch);499 }500 CIO_HEX_EVENT(2, &ret, sizeof(ret));501 return ret;502}503EXPORT_SYMBOL_GPL(cio_enable_subchannel);504 505/**506 * cio_disable_subchannel - disable a subchannel.507 * @sch: subchannel to disable508 */509int cio_disable_subchannel(struct subchannel *sch)510{511 int ret;512 513 CIO_TRACE_EVENT(2, "dissch");514 CIO_TRACE_EVENT(2, dev_name(&sch->dev));515 516 if (sch_is_pseudo_sch(sch))517 return 0;518 if (cio_update_schib(sch))519 return -ENODEV;520 521 sch->config.ena = 0;522 ret = cio_commit_config(sch);523 524 CIO_HEX_EVENT(2, &ret, sizeof(ret));525 return ret;526}527EXPORT_SYMBOL_GPL(cio_disable_subchannel);528 529/*530 * do_cio_interrupt() handles all normal I/O device IRQ's531 */532static irqreturn_t do_cio_interrupt(int irq, void *dummy)533{534 struct tpi_info *tpi_info;535 struct subchannel *sch;536 struct irb *irb;537 538 tpi_info = &get_irq_regs()->tpi_info;539 trace_s390_cio_interrupt(tpi_info);540 irb = this_cpu_ptr(&cio_irb);541 if (!tpi_info->intparm) {542 /* Clear pending interrupt condition. */543 inc_irq_stat(IRQIO_CIO);544 tsch(tpi_info->schid, irb);545 return IRQ_HANDLED;546 }547 sch = phys_to_virt(tpi_info->intparm);548 spin_lock(&sch->lock);549 /* Store interrupt response block to lowcore. */550 if (tsch(tpi_info->schid, irb) == 0) {551 /* Keep subchannel information word up to date. */552 memcpy (&sch->schib.scsw, &irb->scsw, sizeof (irb->scsw));553 /* Call interrupt handler if there is one. */554 if (sch->driver && sch->driver->irq)555 sch->driver->irq(sch);556 else557 inc_irq_stat(IRQIO_CIO);558 } else559 inc_irq_stat(IRQIO_CIO);560 spin_unlock(&sch->lock);561 562 return IRQ_HANDLED;563}564 565void __init init_cio_interrupts(void)566{567 irq_set_chip_and_handler(IO_INTERRUPT,568 &dummy_irq_chip, handle_percpu_irq);569 if (request_irq(IO_INTERRUPT, do_cio_interrupt, 0, "I/O", NULL))570 panic("Failed to register I/O interrupt\n");571}572 573#ifdef CONFIG_CCW_CONSOLE574static struct subchannel *console_sch;575static struct lock_class_key console_sch_key;576 577/*578 * Use cio_tsch to update the subchannel status and call the interrupt handler579 * if status had been pending. Called with the subchannel's lock held.580 */581void cio_tsch(struct subchannel *sch)582{583 struct irb *irb;584 int irq_context;585 586 irb = this_cpu_ptr(&cio_irb);587 /* Store interrupt response block to lowcore. */588 if (tsch(sch->schid, irb) != 0)589 /* Not status pending or not operational. */590 return;591 memcpy(&sch->schib.scsw, &irb->scsw, sizeof(union scsw));592 /* Call interrupt handler with updated status. */593 irq_context = in_interrupt();594 if (!irq_context) {595 local_bh_disable();596 irq_enter();597 }598 kstat_incr_irq_this_cpu(IO_INTERRUPT);599 if (sch->driver && sch->driver->irq)600 sch->driver->irq(sch);601 else602 inc_irq_stat(IRQIO_CIO);603 if (!irq_context) {604 irq_exit();605 _local_bh_enable();606 }607}608 609static int cio_test_for_console(struct subchannel_id schid, void *data)610{611 struct schib schib;612 613 if (stsch(schid, &schib) != 0)614 return -ENXIO;615 if ((schib.pmcw.st == SUBCHANNEL_TYPE_IO) && schib.pmcw.dnv &&616 (schib.pmcw.dev == console_devno)) {617 console_irq = schid.sch_no;618 return 1; /* found */619 }620 return 0;621}622 623static int cio_get_console_sch_no(void)624{625 struct subchannel_id schid;626 struct schib schib;627 628 init_subchannel_id(&schid);629 if (console_irq != -1) {630 /* VM provided us with the irq number of the console. */631 schid.sch_no = console_irq;632 if (stsch(schid, &schib) != 0 ||633 (schib.pmcw.st != SUBCHANNEL_TYPE_IO) || !schib.pmcw.dnv)634 return -1;635 console_devno = schib.pmcw.dev;636 } else if (console_devno != -1) {637 /* At least the console device number is known. */638 for_each_subchannel(cio_test_for_console, NULL);639 }640 return console_irq;641}642 643struct subchannel *cio_probe_console(void)644{645 struct subchannel_id schid;646 struct subchannel *sch;647 struct schib schib;648 int sch_no, ret;649 650 sch_no = cio_get_console_sch_no();651 if (sch_no == -1) {652 pr_warn("No CCW console was found\n");653 return ERR_PTR(-ENODEV);654 }655 init_subchannel_id(&schid);656 schid.sch_no = sch_no;657 ret = stsch(schid, &schib);658 if (ret)659 return ERR_PTR(-ENODEV);660 661 sch = css_alloc_subchannel(schid, &schib);662 if (IS_ERR(sch))663 return sch;664 665 lockdep_set_class(&sch->lock, &console_sch_key);666 isc_register(CONSOLE_ISC);667 sch->config.isc = CONSOLE_ISC;668 sch->config.intparm = (u32)virt_to_phys(sch);669 ret = cio_commit_config(sch);670 if (ret) {671 isc_unregister(CONSOLE_ISC);672 put_device(&sch->dev);673 return ERR_PTR(ret);674 }675 console_sch = sch;676 return sch;677}678 679int cio_is_console(struct subchannel_id schid)680{681 if (!console_sch)682 return 0;683 return schid_equal(&schid, &console_sch->schid);684}685 686void cio_register_early_subchannels(void)687{688 int ret;689 690 if (!console_sch)691 return;692 693 ret = css_register_subchannel(console_sch);694 if (ret)695 put_device(&console_sch->dev);696}697#endif /* CONFIG_CCW_CONSOLE */698 699/**700 * cio_tm_start_key - perform start function701 * @sch: subchannel on which to perform the start function702 * @tcw: transport-command word to be started703 * @lpm: mask of paths to use704 * @key: storage key to use for storage access705 *706 * Start the tcw on the given subchannel. Return zero on success, non-zero707 * otherwise.708 */709int cio_tm_start_key(struct subchannel *sch, struct tcw *tcw, u8 lpm, u8 key)710{711 int cc;712 union orb *orb = &to_io_private(sch)->orb;713 714 memset(orb, 0, sizeof(union orb));715 orb->tm.intparm = (u32)virt_to_phys(sch);716 orb->tm.key = key >> 4;717 orb->tm.b = 1;718 orb->tm.lpm = lpm ? lpm : sch->lpm;719 orb->tm.tcw = virt_to_dma32(tcw);720 cc = ssch(sch->schid, orb);721 switch (cc) {722 case 0:723 return 0;724 case 1:725 case 2:726 return -EBUSY;727 default:728 return cio_start_handle_notoper(sch, lpm);729 }730}731EXPORT_SYMBOL_GPL(cio_tm_start_key);732 733/**734 * cio_tm_intrg - perform interrogate function735 * @sch: subchannel on which to perform the interrogate function736 *737 * If the specified subchannel is running in transport-mode, perform the738 * interrogate function. Return zero on success, non-zero otherwie.739 */740int cio_tm_intrg(struct subchannel *sch)741{742 int cc;743 744 if (!to_io_private(sch)->orb.tm.b)745 return -EINVAL;746 cc = xsch(sch->schid);747 switch (cc) {748 case 0:749 case 2:750 return 0;751 case 1:752 return -EBUSY;753 default:754 return -ENODEV;755 }756}757EXPORT_SYMBOL_GPL(cio_tm_intrg);758