430 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Implement the default iomap interfaces4 *5 * (C) Copyright 2004 Linus Torvalds6 */7#include <linux/pci.h>8#include <linux/io.h>9#include <linux/kmsan-checks.h>10 11#include <linux/export.h>12 13/*14 * Read/write from/to an (offsettable) iomem cookie. It might be a PIO15 * access or a MMIO access, these functions don't care. The info is16 * encoded in the hardware mapping set up by the mapping functions17 * (or the cookie itself, depending on implementation and hw).18 *19 * The generic routines don't assume any hardware mappings, and just20 * encode the PIO/MMIO as part of the cookie. They coldly assume that21 * the MMIO IO mappings are not in the low address range.22 *23 * Architectures for which this is not true can't use this generic24 * implementation and should do their own copy.25 */26 27#ifndef HAVE_ARCH_PIO_SIZE28/*29 * We encode the physical PIO addresses (0-0xffff) into the30 * pointer by offsetting them with a constant (0x10000) and31 * assuming that all the low addresses are always PIO. That means32 * we can do some sanity checks on the low bits, and don't33 * need to just take things for granted.34 */35#define PIO_OFFSET 0x10000UL36#define PIO_MASK 0x0ffffUL37#define PIO_RESERVED 0x40000UL38#endif39 40static void bad_io_access(unsigned long port, const char *access)41{42 static int count = 10;43 if (count) {44 count--;45 WARN(1, KERN_ERR "Bad IO access at port %#lx (%s)\n", port, access);46 }47}48 49/*50 * Ugly macros are a way of life.51 */52#define IO_COND(addr, is_pio, is_mmio) do { \53 unsigned long port = (unsigned long __force)addr; \54 if (port >= PIO_RESERVED) { \55 is_mmio; \56 } else if (port > PIO_OFFSET) { \57 port &= PIO_MASK; \58 is_pio; \59 } else \60 bad_io_access(port, #is_pio ); \61} while (0)62 63#ifndef pio_read16be64#define pio_read16be(port) swab16(inw(port))65#define pio_read32be(port) swab32(inl(port))66#endif67 68#ifndef mmio_read16be69#define mmio_read16be(addr) swab16(readw(addr))70#define mmio_read32be(addr) swab32(readl(addr))71#define mmio_read64be(addr) swab64(readq(addr))72#endif73 74/*75 * Here and below, we apply __no_kmsan_checks to functions reading data from76 * hardware, to ensure that KMSAN marks their return values as initialized.77 */78__no_kmsan_checks79unsigned int ioread8(const void __iomem *addr)80{81 IO_COND(addr, return inb(port), return readb(addr));82 return 0xff;83}84__no_kmsan_checks85unsigned int ioread16(const void __iomem *addr)86{87 IO_COND(addr, return inw(port), return readw(addr));88 return 0xffff;89}90__no_kmsan_checks91unsigned int ioread16be(const void __iomem *addr)92{93 IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr));94 return 0xffff;95}96__no_kmsan_checks97unsigned int ioread32(const void __iomem *addr)98{99 IO_COND(addr, return inl(port), return readl(addr));100 return 0xffffffff;101}102__no_kmsan_checks103unsigned int ioread32be(const void __iomem *addr)104{105 IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr));106 return 0xffffffff;107}108EXPORT_SYMBOL(ioread8);109EXPORT_SYMBOL(ioread16);110EXPORT_SYMBOL(ioread16be);111EXPORT_SYMBOL(ioread32);112EXPORT_SYMBOL(ioread32be);113 114#ifdef readq115static u64 pio_read64_lo_hi(unsigned long port)116{117 u64 lo, hi;118 119 lo = inl(port);120 hi = inl(port + sizeof(u32));121 122 return lo | (hi << 32);123}124 125static u64 pio_read64_hi_lo(unsigned long port)126{127 u64 lo, hi;128 129 hi = inl(port + sizeof(u32));130 lo = inl(port);131 132 return lo | (hi << 32);133}134 135static u64 pio_read64be_lo_hi(unsigned long port)136{137 u64 lo, hi;138 139 lo = pio_read32be(port + sizeof(u32));140 hi = pio_read32be(port);141 142 return lo | (hi << 32);143}144 145static u64 pio_read64be_hi_lo(unsigned long port)146{147 u64 lo, hi;148 149 hi = pio_read32be(port);150 lo = pio_read32be(port + sizeof(u32));151 152 return lo | (hi << 32);153}154 155__no_kmsan_checks156u64 ioread64_lo_hi(const void __iomem *addr)157{158 IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr));159 return 0xffffffffffffffffULL;160}161 162__no_kmsan_checks163u64 ioread64_hi_lo(const void __iomem *addr)164{165 IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr));166 return 0xffffffffffffffffULL;167}168 169__no_kmsan_checks170u64 ioread64be_lo_hi(const void __iomem *addr)171{172 IO_COND(addr, return pio_read64be_lo_hi(port),173 return mmio_read64be(addr));174 return 0xffffffffffffffffULL;175}176 177__no_kmsan_checks178u64 ioread64be_hi_lo(const void __iomem *addr)179{180 IO_COND(addr, return pio_read64be_hi_lo(port),181 return mmio_read64be(addr));182 return 0xffffffffffffffffULL;183}184 185EXPORT_SYMBOL(ioread64_lo_hi);186EXPORT_SYMBOL(ioread64_hi_lo);187EXPORT_SYMBOL(ioread64be_lo_hi);188EXPORT_SYMBOL(ioread64be_hi_lo);189 190#endif /* readq */191 192#ifndef pio_write16be193#define pio_write16be(val,port) outw(swab16(val),port)194#define pio_write32be(val,port) outl(swab32(val),port)195#endif196 197#ifndef mmio_write16be198#define mmio_write16be(val,port) writew(swab16(val),port)199#define mmio_write32be(val,port) writel(swab32(val),port)200#define mmio_write64be(val,port) writeq(swab64(val),port)201#endif202 203void iowrite8(u8 val, void __iomem *addr)204{205 /* Make sure uninitialized memory isn't copied to devices. */206 kmsan_check_memory(&val, sizeof(val));207 IO_COND(addr, outb(val,port), writeb(val, addr));208}209void iowrite16(u16 val, void __iomem *addr)210{211 /* Make sure uninitialized memory isn't copied to devices. */212 kmsan_check_memory(&val, sizeof(val));213 IO_COND(addr, outw(val,port), writew(val, addr));214}215void iowrite16be(u16 val, void __iomem *addr)216{217 /* Make sure uninitialized memory isn't copied to devices. */218 kmsan_check_memory(&val, sizeof(val));219 IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr));220}221void iowrite32(u32 val, void __iomem *addr)222{223 /* Make sure uninitialized memory isn't copied to devices. */224 kmsan_check_memory(&val, sizeof(val));225 IO_COND(addr, outl(val,port), writel(val, addr));226}227void iowrite32be(u32 val, void __iomem *addr)228{229 /* Make sure uninitialized memory isn't copied to devices. */230 kmsan_check_memory(&val, sizeof(val));231 IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr));232}233EXPORT_SYMBOL(iowrite8);234EXPORT_SYMBOL(iowrite16);235EXPORT_SYMBOL(iowrite16be);236EXPORT_SYMBOL(iowrite32);237EXPORT_SYMBOL(iowrite32be);238 239#ifdef writeq240static void pio_write64_lo_hi(u64 val, unsigned long port)241{242 outl(val, port);243 outl(val >> 32, port + sizeof(u32));244}245 246static void pio_write64_hi_lo(u64 val, unsigned long port)247{248 outl(val >> 32, port + sizeof(u32));249 outl(val, port);250}251 252static void pio_write64be_lo_hi(u64 val, unsigned long port)253{254 pio_write32be(val, port + sizeof(u32));255 pio_write32be(val >> 32, port);256}257 258static void pio_write64be_hi_lo(u64 val, unsigned long port)259{260 pio_write32be(val >> 32, port);261 pio_write32be(val, port + sizeof(u32));262}263 264void iowrite64_lo_hi(u64 val, void __iomem *addr)265{266 /* Make sure uninitialized memory isn't copied to devices. */267 kmsan_check_memory(&val, sizeof(val));268 IO_COND(addr, pio_write64_lo_hi(val, port),269 writeq(val, addr));270}271 272void iowrite64_hi_lo(u64 val, void __iomem *addr)273{274 /* Make sure uninitialized memory isn't copied to devices. */275 kmsan_check_memory(&val, sizeof(val));276 IO_COND(addr, pio_write64_hi_lo(val, port),277 writeq(val, addr));278}279 280void iowrite64be_lo_hi(u64 val, void __iomem *addr)281{282 /* Make sure uninitialized memory isn't copied to devices. */283 kmsan_check_memory(&val, sizeof(val));284 IO_COND(addr, pio_write64be_lo_hi(val, port),285 mmio_write64be(val, addr));286}287 288void iowrite64be_hi_lo(u64 val, void __iomem *addr)289{290 /* Make sure uninitialized memory isn't copied to devices. */291 kmsan_check_memory(&val, sizeof(val));292 IO_COND(addr, pio_write64be_hi_lo(val, port),293 mmio_write64be(val, addr));294}295 296EXPORT_SYMBOL(iowrite64_lo_hi);297EXPORT_SYMBOL(iowrite64_hi_lo);298EXPORT_SYMBOL(iowrite64be_lo_hi);299EXPORT_SYMBOL(iowrite64be_hi_lo);300 301#endif /* readq */302 303/*304 * These are the "repeat MMIO read/write" functions.305 * Note the "__raw" accesses, since we don't want to306 * convert to CPU byte order. We write in "IO byte307 * order" (we also don't have IO barriers).308 */309#ifndef mmio_insb310static inline void mmio_insb(const void __iomem *addr, u8 *dst, int count)311{312 while (--count >= 0) {313 u8 data = __raw_readb(addr);314 *dst = data;315 dst++;316 }317}318static inline void mmio_insw(const void __iomem *addr, u16 *dst, int count)319{320 while (--count >= 0) {321 u16 data = __raw_readw(addr);322 *dst = data;323 dst++;324 }325}326static inline void mmio_insl(const void __iomem *addr, u32 *dst, int count)327{328 while (--count >= 0) {329 u32 data = __raw_readl(addr);330 *dst = data;331 dst++;332 }333}334#endif335 336#ifndef mmio_outsb337static inline void mmio_outsb(void __iomem *addr, const u8 *src, int count)338{339 while (--count >= 0) {340 __raw_writeb(*src, addr);341 src++;342 }343}344static inline void mmio_outsw(void __iomem *addr, const u16 *src, int count)345{346 while (--count >= 0) {347 __raw_writew(*src, addr);348 src++;349 }350}351static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count)352{353 while (--count >= 0) {354 __raw_writel(*src, addr);355 src++;356 }357}358#endif359 360void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count)361{362 IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count));363 /* KMSAN must treat values read from devices as initialized. */364 kmsan_unpoison_memory(dst, count);365}366void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count)367{368 IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count));369 /* KMSAN must treat values read from devices as initialized. */370 kmsan_unpoison_memory(dst, count * 2);371}372void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count)373{374 IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count));375 /* KMSAN must treat values read from devices as initialized. */376 kmsan_unpoison_memory(dst, count * 4);377}378EXPORT_SYMBOL(ioread8_rep);379EXPORT_SYMBOL(ioread16_rep);380EXPORT_SYMBOL(ioread32_rep);381 382void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count)383{384 /* Make sure uninitialized memory isn't copied to devices. */385 kmsan_check_memory(src, count);386 IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count));387}388void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count)389{390 /* Make sure uninitialized memory isn't copied to devices. */391 kmsan_check_memory(src, count * 2);392 IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count));393}394void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count)395{396 /* Make sure uninitialized memory isn't copied to devices. */397 kmsan_check_memory(src, count * 4);398 IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count));399}400EXPORT_SYMBOL(iowrite8_rep);401EXPORT_SYMBOL(iowrite16_rep);402EXPORT_SYMBOL(iowrite32_rep);403 404#ifdef CONFIG_HAS_IOPORT_MAP405/* Create a virtual mapping cookie for an IO port range */406void __iomem *ioport_map(unsigned long port, unsigned int nr)407{408 if (port > PIO_MASK)409 return NULL;410 return (void __iomem *) (unsigned long) (port + PIO_OFFSET);411}412 413void ioport_unmap(void __iomem *addr)414{415 /* Nothing to do */416}417EXPORT_SYMBOL(ioport_map);418EXPORT_SYMBOL(ioport_unmap);419#endif /* CONFIG_HAS_IOPORT_MAP */420 421#ifdef CONFIG_PCI422/* Hide the details if this is a MMIO or PIO address space and just do what423 * you expect in the correct way. */424void pci_iounmap(struct pci_dev *dev, void __iomem * addr)425{426 IO_COND(addr, /* nothing */, iounmap(addr));427}428EXPORT_SYMBOL(pci_iounmap);429#endif /* CONFIG_PCI */430