313 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Support for adapter interruptions4 *5 * Copyright IBM Corp. 1999, 20076 * Author(s): Ingo Adlung <adlung@de.ibm.com>7 * Cornelia Huck <cornelia.huck@de.ibm.com>8 * Arnd Bergmann <arndb@de.ibm.com>9 * Peter Oberparleiter <peter.oberparleiter@de.ibm.com>10 */11 12#include <linux/init.h>13#include <linux/irq.h>14#include <linux/kernel_stat.h>15#include <linux/module.h>16#include <linux/mutex.h>17#include <linux/rculist.h>18#include <linux/slab.h>19#include <linux/dmapool.h>20 21#include <asm/airq.h>22#include <asm/isc.h>23#include <asm/cio.h>24 25#include "cio.h"26#include "cio_debug.h"27#include "ioasm.h"28 29static DEFINE_SPINLOCK(airq_lists_lock);30static struct hlist_head airq_lists[MAX_ISC+1];31 32static struct dma_pool *airq_iv_cache;33 34/**35 * register_adapter_interrupt() - register adapter interrupt handler36 * @airq: pointer to adapter interrupt descriptor37 *38 * Returns 0 on success, or -EINVAL.39 */40int register_adapter_interrupt(struct airq_struct *airq)41{42 char dbf_txt[32];43 44 if (!airq->handler || airq->isc > MAX_ISC)45 return -EINVAL;46 if (!airq->lsi_ptr) {47 airq->lsi_ptr = cio_dma_zalloc(1);48 if (!airq->lsi_ptr)49 return -ENOMEM;50 airq->flags |= AIRQ_PTR_ALLOCATED;51 }52 snprintf(dbf_txt, sizeof(dbf_txt), "rairq:%p", airq);53 CIO_TRACE_EVENT(4, dbf_txt);54 isc_register(airq->isc);55 spin_lock(&airq_lists_lock);56 hlist_add_head_rcu(&airq->list, &airq_lists[airq->isc]);57 spin_unlock(&airq_lists_lock);58 return 0;59}60EXPORT_SYMBOL(register_adapter_interrupt);61 62/**63 * unregister_adapter_interrupt - unregister adapter interrupt handler64 * @airq: pointer to adapter interrupt descriptor65 */66void unregister_adapter_interrupt(struct airq_struct *airq)67{68 char dbf_txt[32];69 70 if (hlist_unhashed(&airq->list))71 return;72 snprintf(dbf_txt, sizeof(dbf_txt), "urairq:%p", airq);73 CIO_TRACE_EVENT(4, dbf_txt);74 spin_lock(&airq_lists_lock);75 hlist_del_rcu(&airq->list);76 spin_unlock(&airq_lists_lock);77 synchronize_rcu();78 isc_unregister(airq->isc);79 if (airq->flags & AIRQ_PTR_ALLOCATED) {80 cio_dma_free(airq->lsi_ptr, 1);81 airq->lsi_ptr = NULL;82 airq->flags &= ~AIRQ_PTR_ALLOCATED;83 }84}85EXPORT_SYMBOL(unregister_adapter_interrupt);86 87static irqreturn_t do_airq_interrupt(int irq, void *dummy)88{89 struct tpi_info *tpi_info;90 struct airq_struct *airq;91 struct hlist_head *head;92 93 tpi_info = &get_irq_regs()->tpi_info;94 trace_s390_cio_adapter_int(tpi_info);95 head = &airq_lists[tpi_info->isc];96 rcu_read_lock();97 hlist_for_each_entry_rcu(airq, head, list)98 if (*airq->lsi_ptr != 0)99 airq->handler(airq, tpi_info);100 rcu_read_unlock();101 102 return IRQ_HANDLED;103}104 105void __init init_airq_interrupts(void)106{107 irq_set_chip_and_handler(THIN_INTERRUPT,108 &dummy_irq_chip, handle_percpu_irq);109 if (request_irq(THIN_INTERRUPT, do_airq_interrupt, 0, "AIO", NULL))110 panic("Failed to register AIO interrupt\n");111}112 113static inline unsigned long iv_size(unsigned long bits)114{115 return BITS_TO_LONGS(bits) * sizeof(unsigned long);116}117 118/**119 * airq_iv_create - create an interrupt vector120 * @bits: number of bits in the interrupt vector121 * @flags: allocation flags122 * @vec: pointer to pinned guest memory if AIRQ_IV_GUESTVEC123 *124 * Returns a pointer to an interrupt vector structure125 */126struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags,127 unsigned long *vec)128{129 struct airq_iv *iv;130 unsigned long size;131 132 iv = kzalloc(sizeof(*iv), GFP_KERNEL);133 if (!iv)134 goto out;135 iv->bits = bits;136 iv->flags = flags;137 size = iv_size(bits);138 139 if (flags & AIRQ_IV_CACHELINE) {140 if ((cache_line_size() * BITS_PER_BYTE) < bits141 || !airq_iv_cache)142 goto out_free;143 144 iv->vector = dma_pool_zalloc(airq_iv_cache, GFP_KERNEL,145 &iv->vector_dma);146 if (!iv->vector)147 goto out_free;148 } else if (flags & AIRQ_IV_GUESTVEC) {149 iv->vector = vec;150 } else {151 iv->vector = cio_dma_zalloc(size);152 if (!iv->vector)153 goto out_free;154 }155 if (flags & AIRQ_IV_ALLOC) {156 iv->avail = kmalloc(size, GFP_KERNEL);157 if (!iv->avail)158 goto out_free;159 memset(iv->avail, 0xff, size);160 iv->end = 0;161 } else162 iv->end = bits;163 if (flags & AIRQ_IV_BITLOCK) {164 iv->bitlock = kzalloc(size, GFP_KERNEL);165 if (!iv->bitlock)166 goto out_free;167 }168 if (flags & AIRQ_IV_PTR) {169 size = bits * sizeof(unsigned long);170 iv->ptr = kzalloc(size, GFP_KERNEL);171 if (!iv->ptr)172 goto out_free;173 }174 if (flags & AIRQ_IV_DATA) {175 size = bits * sizeof(unsigned int);176 iv->data = kzalloc(size, GFP_KERNEL);177 if (!iv->data)178 goto out_free;179 }180 spin_lock_init(&iv->lock);181 return iv;182 183out_free:184 kfree(iv->ptr);185 kfree(iv->bitlock);186 kfree(iv->avail);187 if (iv->flags & AIRQ_IV_CACHELINE && iv->vector)188 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);189 else if (!(iv->flags & AIRQ_IV_GUESTVEC))190 cio_dma_free(iv->vector, size);191 kfree(iv);192out:193 return NULL;194}195EXPORT_SYMBOL(airq_iv_create);196 197/**198 * airq_iv_release - release an interrupt vector199 * @iv: pointer to interrupt vector structure200 */201void airq_iv_release(struct airq_iv *iv)202{203 kfree(iv->data);204 kfree(iv->ptr);205 kfree(iv->bitlock);206 if (iv->flags & AIRQ_IV_CACHELINE)207 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);208 else if (!(iv->flags & AIRQ_IV_GUESTVEC))209 cio_dma_free(iv->vector, iv_size(iv->bits));210 kfree(iv->avail);211 kfree(iv);212}213EXPORT_SYMBOL(airq_iv_release);214 215/**216 * airq_iv_alloc - allocate irq bits from an interrupt vector217 * @iv: pointer to an interrupt vector structure218 * @num: number of consecutive irq bits to allocate219 *220 * Returns the bit number of the first irq in the allocated block of irqs,221 * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been222 * specified223 */224unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num)225{226 unsigned long bit, i, flags;227 228 if (!iv->avail || num == 0)229 return -1UL;230 spin_lock_irqsave(&iv->lock, flags);231 bit = find_first_bit_inv(iv->avail, iv->bits);232 while (bit + num <= iv->bits) {233 for (i = 1; i < num; i++)234 if (!test_bit_inv(bit + i, iv->avail))235 break;236 if (i >= num) {237 /* Found a suitable block of irqs */238 for (i = 0; i < num; i++)239 clear_bit_inv(bit + i, iv->avail);240 if (bit + num >= iv->end)241 iv->end = bit + num + 1;242 break;243 }244 bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1);245 }246 if (bit + num > iv->bits)247 bit = -1UL;248 spin_unlock_irqrestore(&iv->lock, flags);249 return bit;250}251EXPORT_SYMBOL(airq_iv_alloc);252 253/**254 * airq_iv_free - free irq bits of an interrupt vector255 * @iv: pointer to interrupt vector structure256 * @bit: number of the first irq bit to free257 * @num: number of consecutive irq bits to free258 */259void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num)260{261 unsigned long i, flags;262 263 if (!iv->avail || num == 0)264 return;265 spin_lock_irqsave(&iv->lock, flags);266 for (i = 0; i < num; i++) {267 /* Clear (possibly left over) interrupt bit */268 clear_bit_inv(bit + i, iv->vector);269 /* Make the bit positions available again */270 set_bit_inv(bit + i, iv->avail);271 }272 if (bit + num >= iv->end) {273 /* Find new end of bit-field */274 while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail))275 iv->end--;276 }277 spin_unlock_irqrestore(&iv->lock, flags);278}279EXPORT_SYMBOL(airq_iv_free);280 281/**282 * airq_iv_scan - scan interrupt vector for non-zero bits283 * @iv: pointer to interrupt vector structure284 * @start: bit number to start the search285 * @end: bit number to end the search286 *287 * Returns the bit number of the next non-zero interrupt bit, or288 * -1UL if the scan completed without finding any more any non-zero bits.289 */290unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start,291 unsigned long end)292{293 unsigned long bit;294 295 /* Find non-zero bit starting from 'ivs->next'. */296 bit = find_next_bit_inv(iv->vector, end, start);297 if (bit >= end)298 return -1UL;299 clear_bit_inv(bit, iv->vector);300 return bit;301}302EXPORT_SYMBOL(airq_iv_scan);303 304int __init airq_init(void)305{306 airq_iv_cache = dma_pool_create("airq_iv_cache", cio_get_dma_css_dev(),307 cache_line_size(),308 cache_line_size(), PAGE_SIZE);309 if (!airq_iv_cache)310 return -ENOMEM;311 return 0;312}313