307 lines · c
1/* SPDX-License-Identifier: GPL-2.0-or-later */2/*3 * Generic barrier definitions.4 *5 * It should be possible to use these on really simple architectures,6 * but it serves more as a starting point for new ports.7 *8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.9 * Written by David Howells (dhowells@redhat.com)10 */11#ifndef __ASM_GENERIC_BARRIER_H12#define __ASM_GENERIC_BARRIER_H13 14#ifndef __ASSEMBLY__15 16#include <linux/compiler.h>17#include <linux/kcsan-checks.h>18#include <asm/rwonce.h>19 20#ifndef nop21#define nop() asm volatile ("nop")22#endif23 24/*25 * Architectures that want generic instrumentation can define __ prefixed26 * variants of all barriers.27 */28 29#ifdef __mb30#define mb() do { kcsan_mb(); __mb(); } while (0)31#endif32 33#ifdef __rmb34#define rmb() do { kcsan_rmb(); __rmb(); } while (0)35#endif36 37#ifdef __wmb38#define wmb() do { kcsan_wmb(); __wmb(); } while (0)39#endif40 41#ifdef __dma_mb42#define dma_mb() do { kcsan_mb(); __dma_mb(); } while (0)43#endif44 45#ifdef __dma_rmb46#define dma_rmb() do { kcsan_rmb(); __dma_rmb(); } while (0)47#endif48 49#ifdef __dma_wmb50#define dma_wmb() do { kcsan_wmb(); __dma_wmb(); } while (0)51#endif52 53/*54 * Force strict CPU ordering. And yes, this is required on UP too when we're55 * talking to devices.56 *57 * Fall back to compiler barriers if nothing better is provided.58 */59 60#ifndef mb61#define mb() barrier()62#endif63 64#ifndef rmb65#define rmb() mb()66#endif67 68#ifndef wmb69#define wmb() mb()70#endif71 72#ifndef dma_mb73#define dma_mb() mb()74#endif75 76#ifndef dma_rmb77#define dma_rmb() rmb()78#endif79 80#ifndef dma_wmb81#define dma_wmb() wmb()82#endif83 84#ifndef __smp_mb85#define __smp_mb() mb()86#endif87 88#ifndef __smp_rmb89#define __smp_rmb() rmb()90#endif91 92#ifndef __smp_wmb93#define __smp_wmb() wmb()94#endif95 96#ifdef CONFIG_SMP97 98#ifndef smp_mb99#define smp_mb() do { kcsan_mb(); __smp_mb(); } while (0)100#endif101 102#ifndef smp_rmb103#define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0)104#endif105 106#ifndef smp_wmb107#define smp_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0)108#endif109 110#else /* !CONFIG_SMP */111 112#ifndef smp_mb113#define smp_mb() barrier()114#endif115 116#ifndef smp_rmb117#define smp_rmb() barrier()118#endif119 120#ifndef smp_wmb121#define smp_wmb() barrier()122#endif123 124#endif /* CONFIG_SMP */125 126#ifndef __smp_store_mb127#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)128#endif129 130#ifndef __smp_mb__before_atomic131#define __smp_mb__before_atomic() __smp_mb()132#endif133 134#ifndef __smp_mb__after_atomic135#define __smp_mb__after_atomic() __smp_mb()136#endif137 138#ifndef __smp_store_release139#define __smp_store_release(p, v) \140do { \141 compiletime_assert_atomic_type(*p); \142 __smp_mb(); \143 WRITE_ONCE(*p, v); \144} while (0)145#endif146 147#ifndef __smp_load_acquire148#define __smp_load_acquire(p) \149({ \150 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \151 compiletime_assert_atomic_type(*p); \152 __smp_mb(); \153 (typeof(*p))___p1; \154})155#endif156 157#ifdef CONFIG_SMP158 159#ifndef smp_store_mb160#define smp_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0)161#endif162 163#ifndef smp_mb__before_atomic164#define smp_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0)165#endif166 167#ifndef smp_mb__after_atomic168#define smp_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0)169#endif170 171#ifndef smp_store_release172#define smp_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0)173#endif174 175#ifndef smp_load_acquire176#define smp_load_acquire(p) __smp_load_acquire(p)177#endif178 179#else /* !CONFIG_SMP */180 181#ifndef smp_store_mb182#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)183#endif184 185#ifndef smp_mb__before_atomic186#define smp_mb__before_atomic() barrier()187#endif188 189#ifndef smp_mb__after_atomic190#define smp_mb__after_atomic() barrier()191#endif192 193#ifndef smp_store_release194#define smp_store_release(p, v) \195do { \196 barrier(); \197 WRITE_ONCE(*p, v); \198} while (0)199#endif200 201#ifndef smp_load_acquire202#define smp_load_acquire(p) \203({ \204 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \205 barrier(); \206 (typeof(*p))___p1; \207})208#endif209 210#endif /* CONFIG_SMP */211 212/* Barriers for virtual machine guests when talking to an SMP host */213#define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0)214#define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0)215#define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0)216#define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0)217#define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0)218#define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0)219#define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0)220#define virt_load_acquire(p) __smp_load_acquire(p)221 222/**223 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency224 *225 * A control dependency provides a LOAD->STORE order, the additional RMB226 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,227 * aka. (load)-ACQUIRE.228 *229 * Architectures that do not do load speculation can have this be barrier().230 */231#ifndef smp_acquire__after_ctrl_dep232#define smp_acquire__after_ctrl_dep() smp_rmb()233#endif234 235/**236 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees237 * @ptr: pointer to the variable to wait on238 * @cond: boolean expression to wait for239 *240 * Equivalent to using READ_ONCE() on the condition variable.241 *242 * Due to C lacking lambda expressions we load the value of *ptr into a243 * pre-named variable @VAL to be used in @cond.244 */245#ifndef smp_cond_load_relaxed246#define smp_cond_load_relaxed(ptr, cond_expr) ({ \247 typeof(ptr) __PTR = (ptr); \248 __unqual_scalar_typeof(*ptr) VAL; \249 for (;;) { \250 VAL = READ_ONCE(*__PTR); \251 if (cond_expr) \252 break; \253 cpu_relax(); \254 } \255 (typeof(*ptr))VAL; \256})257#endif258 259/**260 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering261 * @ptr: pointer to the variable to wait on262 * @cond: boolean expression to wait for263 *264 * Equivalent to using smp_load_acquire() on the condition variable but employs265 * the control dependency of the wait to reduce the barrier on many platforms.266 */267#ifndef smp_cond_load_acquire268#define smp_cond_load_acquire(ptr, cond_expr) ({ \269 __unqual_scalar_typeof(*ptr) _val; \270 _val = smp_cond_load_relaxed(ptr, cond_expr); \271 smp_acquire__after_ctrl_dep(); \272 (typeof(*ptr))_val; \273})274#endif275 276/*277 * pmem_wmb() ensures that all stores for which the modification278 * are written to persistent storage by preceding instructions have279 * updated persistent storage before any data access or data transfer280 * caused by subsequent instructions is initiated.281 */282#ifndef pmem_wmb283#define pmem_wmb() wmb()284#endif285 286/*287 * ioremap_wc() maps I/O memory as memory with write-combining attributes. For288 * this kind of memory accesses, the CPU may wait for prior accesses to be289 * merged with subsequent ones. In some situation, such wait is bad for the290 * performance. io_stop_wc() can be used to prevent the merging of291 * write-combining memory accesses before this macro with those after it.292 */293#ifndef io_stop_wc294#define io_stop_wc() do { } while (0)295#endif296 297/*298 * Architectures that guarantee an implicit smp_mb() in switch_mm()299 * can override smp_mb__after_switch_mm.300 */301#ifndef smp_mb__after_switch_mm302# define smp_mb__after_switch_mm() smp_mb()303#endif304 305#endif /* !__ASSEMBLY__ */306#endif /* __ASM_GENERIC_BARRIER_H */307