4694 lines · c
1// SPDX-License-Identifier: GPL-2.02 3// Generated by scripts/atomic/gen-atomic-fallback.sh4// DO NOT MODIFY THIS FILE DIRECTLY5 6#ifndef _LINUX_ATOMIC_FALLBACK_H7#define _LINUX_ATOMIC_FALLBACK_H8 9#include <linux/compiler.h>10 11#if defined(arch_xchg)12#define raw_xchg arch_xchg13#elif defined(arch_xchg_relaxed)14#define raw_xchg(...) \15 __atomic_op_fence(arch_xchg, __VA_ARGS__)16#else17extern void raw_xchg_not_implemented(void);18#define raw_xchg(...) raw_xchg_not_implemented()19#endif20 21#if defined(arch_xchg_acquire)22#define raw_xchg_acquire arch_xchg_acquire23#elif defined(arch_xchg_relaxed)24#define raw_xchg_acquire(...) \25 __atomic_op_acquire(arch_xchg, __VA_ARGS__)26#elif defined(arch_xchg)27#define raw_xchg_acquire arch_xchg28#else29extern void raw_xchg_acquire_not_implemented(void);30#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()31#endif32 33#if defined(arch_xchg_release)34#define raw_xchg_release arch_xchg_release35#elif defined(arch_xchg_relaxed)36#define raw_xchg_release(...) \37 __atomic_op_release(arch_xchg, __VA_ARGS__)38#elif defined(arch_xchg)39#define raw_xchg_release arch_xchg40#else41extern void raw_xchg_release_not_implemented(void);42#define raw_xchg_release(...) raw_xchg_release_not_implemented()43#endif44 45#if defined(arch_xchg_relaxed)46#define raw_xchg_relaxed arch_xchg_relaxed47#elif defined(arch_xchg)48#define raw_xchg_relaxed arch_xchg49#else50extern void raw_xchg_relaxed_not_implemented(void);51#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()52#endif53 54#if defined(arch_cmpxchg)55#define raw_cmpxchg arch_cmpxchg56#elif defined(arch_cmpxchg_relaxed)57#define raw_cmpxchg(...) \58 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)59#else60extern void raw_cmpxchg_not_implemented(void);61#define raw_cmpxchg(...) raw_cmpxchg_not_implemented()62#endif63 64#if defined(arch_cmpxchg_acquire)65#define raw_cmpxchg_acquire arch_cmpxchg_acquire66#elif defined(arch_cmpxchg_relaxed)67#define raw_cmpxchg_acquire(...) \68 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)69#elif defined(arch_cmpxchg)70#define raw_cmpxchg_acquire arch_cmpxchg71#else72extern void raw_cmpxchg_acquire_not_implemented(void);73#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()74#endif75 76#if defined(arch_cmpxchg_release)77#define raw_cmpxchg_release arch_cmpxchg_release78#elif defined(arch_cmpxchg_relaxed)79#define raw_cmpxchg_release(...) \80 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)81#elif defined(arch_cmpxchg)82#define raw_cmpxchg_release arch_cmpxchg83#else84extern void raw_cmpxchg_release_not_implemented(void);85#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()86#endif87 88#if defined(arch_cmpxchg_relaxed)89#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed90#elif defined(arch_cmpxchg)91#define raw_cmpxchg_relaxed arch_cmpxchg92#else93extern void raw_cmpxchg_relaxed_not_implemented(void);94#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()95#endif96 97#if defined(arch_cmpxchg64)98#define raw_cmpxchg64 arch_cmpxchg6499#elif defined(arch_cmpxchg64_relaxed)100#define raw_cmpxchg64(...) \101 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)102#else103extern void raw_cmpxchg64_not_implemented(void);104#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()105#endif106 107#if defined(arch_cmpxchg64_acquire)108#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire109#elif defined(arch_cmpxchg64_relaxed)110#define raw_cmpxchg64_acquire(...) \111 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)112#elif defined(arch_cmpxchg64)113#define raw_cmpxchg64_acquire arch_cmpxchg64114#else115extern void raw_cmpxchg64_acquire_not_implemented(void);116#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()117#endif118 119#if defined(arch_cmpxchg64_release)120#define raw_cmpxchg64_release arch_cmpxchg64_release121#elif defined(arch_cmpxchg64_relaxed)122#define raw_cmpxchg64_release(...) \123 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)124#elif defined(arch_cmpxchg64)125#define raw_cmpxchg64_release arch_cmpxchg64126#else127extern void raw_cmpxchg64_release_not_implemented(void);128#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()129#endif130 131#if defined(arch_cmpxchg64_relaxed)132#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed133#elif defined(arch_cmpxchg64)134#define raw_cmpxchg64_relaxed arch_cmpxchg64135#else136extern void raw_cmpxchg64_relaxed_not_implemented(void);137#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()138#endif139 140#if defined(arch_cmpxchg128)141#define raw_cmpxchg128 arch_cmpxchg128142#elif defined(arch_cmpxchg128_relaxed)143#define raw_cmpxchg128(...) \144 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)145#else146extern void raw_cmpxchg128_not_implemented(void);147#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()148#endif149 150#if defined(arch_cmpxchg128_acquire)151#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire152#elif defined(arch_cmpxchg128_relaxed)153#define raw_cmpxchg128_acquire(...) \154 __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)155#elif defined(arch_cmpxchg128)156#define raw_cmpxchg128_acquire arch_cmpxchg128157#else158extern void raw_cmpxchg128_acquire_not_implemented(void);159#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()160#endif161 162#if defined(arch_cmpxchg128_release)163#define raw_cmpxchg128_release arch_cmpxchg128_release164#elif defined(arch_cmpxchg128_relaxed)165#define raw_cmpxchg128_release(...) \166 __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)167#elif defined(arch_cmpxchg128)168#define raw_cmpxchg128_release arch_cmpxchg128169#else170extern void raw_cmpxchg128_release_not_implemented(void);171#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()172#endif173 174#if defined(arch_cmpxchg128_relaxed)175#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed176#elif defined(arch_cmpxchg128)177#define raw_cmpxchg128_relaxed arch_cmpxchg128178#else179extern void raw_cmpxchg128_relaxed_not_implemented(void);180#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()181#endif182 183#if defined(arch_try_cmpxchg)184#define raw_try_cmpxchg arch_try_cmpxchg185#elif defined(arch_try_cmpxchg_relaxed)186#define raw_try_cmpxchg(...) \187 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)188#else189#define raw_try_cmpxchg(_ptr, _oldp, _new) \190({ \191 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \192 ___r = raw_cmpxchg((_ptr), ___o, (_new)); \193 if (unlikely(___r != ___o)) \194 *___op = ___r; \195 likely(___r == ___o); \196})197#endif198 199#if defined(arch_try_cmpxchg_acquire)200#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire201#elif defined(arch_try_cmpxchg_relaxed)202#define raw_try_cmpxchg_acquire(...) \203 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)204#elif defined(arch_try_cmpxchg)205#define raw_try_cmpxchg_acquire arch_try_cmpxchg206#else207#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \208({ \209 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \210 ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \211 if (unlikely(___r != ___o)) \212 *___op = ___r; \213 likely(___r == ___o); \214})215#endif216 217#if defined(arch_try_cmpxchg_release)218#define raw_try_cmpxchg_release arch_try_cmpxchg_release219#elif defined(arch_try_cmpxchg_relaxed)220#define raw_try_cmpxchg_release(...) \221 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)222#elif defined(arch_try_cmpxchg)223#define raw_try_cmpxchg_release arch_try_cmpxchg224#else225#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \226({ \227 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \228 ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \229 if (unlikely(___r != ___o)) \230 *___op = ___r; \231 likely(___r == ___o); \232})233#endif234 235#if defined(arch_try_cmpxchg_relaxed)236#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed237#elif defined(arch_try_cmpxchg)238#define raw_try_cmpxchg_relaxed arch_try_cmpxchg239#else240#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \241({ \242 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \243 ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \244 if (unlikely(___r != ___o)) \245 *___op = ___r; \246 likely(___r == ___o); \247})248#endif249 250#if defined(arch_try_cmpxchg64)251#define raw_try_cmpxchg64 arch_try_cmpxchg64252#elif defined(arch_try_cmpxchg64_relaxed)253#define raw_try_cmpxchg64(...) \254 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)255#else256#define raw_try_cmpxchg64(_ptr, _oldp, _new) \257({ \258 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \259 ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \260 if (unlikely(___r != ___o)) \261 *___op = ___r; \262 likely(___r == ___o); \263})264#endif265 266#if defined(arch_try_cmpxchg64_acquire)267#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire268#elif defined(arch_try_cmpxchg64_relaxed)269#define raw_try_cmpxchg64_acquire(...) \270 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)271#elif defined(arch_try_cmpxchg64)272#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64273#else274#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \275({ \276 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \277 ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \278 if (unlikely(___r != ___o)) \279 *___op = ___r; \280 likely(___r == ___o); \281})282#endif283 284#if defined(arch_try_cmpxchg64_release)285#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release286#elif defined(arch_try_cmpxchg64_relaxed)287#define raw_try_cmpxchg64_release(...) \288 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)289#elif defined(arch_try_cmpxchg64)290#define raw_try_cmpxchg64_release arch_try_cmpxchg64291#else292#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \293({ \294 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \295 ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \296 if (unlikely(___r != ___o)) \297 *___op = ___r; \298 likely(___r == ___o); \299})300#endif301 302#if defined(arch_try_cmpxchg64_relaxed)303#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed304#elif defined(arch_try_cmpxchg64)305#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64306#else307#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \308({ \309 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \310 ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \311 if (unlikely(___r != ___o)) \312 *___op = ___r; \313 likely(___r == ___o); \314})315#endif316 317#if defined(arch_try_cmpxchg128)318#define raw_try_cmpxchg128 arch_try_cmpxchg128319#elif defined(arch_try_cmpxchg128_relaxed)320#define raw_try_cmpxchg128(...) \321 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)322#else323#define raw_try_cmpxchg128(_ptr, _oldp, _new) \324({ \325 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \326 ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \327 if (unlikely(___r != ___o)) \328 *___op = ___r; \329 likely(___r == ___o); \330})331#endif332 333#if defined(arch_try_cmpxchg128_acquire)334#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire335#elif defined(arch_try_cmpxchg128_relaxed)336#define raw_try_cmpxchg128_acquire(...) \337 __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)338#elif defined(arch_try_cmpxchg128)339#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128340#else341#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \342({ \343 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \344 ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \345 if (unlikely(___r != ___o)) \346 *___op = ___r; \347 likely(___r == ___o); \348})349#endif350 351#if defined(arch_try_cmpxchg128_release)352#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release353#elif defined(arch_try_cmpxchg128_relaxed)354#define raw_try_cmpxchg128_release(...) \355 __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)356#elif defined(arch_try_cmpxchg128)357#define raw_try_cmpxchg128_release arch_try_cmpxchg128358#else359#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \360({ \361 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \362 ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \363 if (unlikely(___r != ___o)) \364 *___op = ___r; \365 likely(___r == ___o); \366})367#endif368 369#if defined(arch_try_cmpxchg128_relaxed)370#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed371#elif defined(arch_try_cmpxchg128)372#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128373#else374#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \375({ \376 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \377 ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \378 if (unlikely(___r != ___o)) \379 *___op = ___r; \380 likely(___r == ___o); \381})382#endif383 384#define raw_cmpxchg_local arch_cmpxchg_local385 386#ifdef arch_try_cmpxchg_local387#define raw_try_cmpxchg_local arch_try_cmpxchg_local388#else389#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \390({ \391 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \392 ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \393 if (unlikely(___r != ___o)) \394 *___op = ___r; \395 likely(___r == ___o); \396})397#endif398 399#define raw_cmpxchg64_local arch_cmpxchg64_local400 401#ifdef arch_try_cmpxchg64_local402#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local403#else404#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \405({ \406 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \407 ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \408 if (unlikely(___r != ___o)) \409 *___op = ___r; \410 likely(___r == ___o); \411})412#endif413 414#define raw_cmpxchg128_local arch_cmpxchg128_local415 416#ifdef arch_try_cmpxchg128_local417#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local418#else419#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \420({ \421 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \422 ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \423 if (unlikely(___r != ___o)) \424 *___op = ___r; \425 likely(___r == ___o); \426})427#endif428 429#define raw_sync_cmpxchg arch_sync_cmpxchg430 431#ifdef arch_sync_try_cmpxchg432#define raw_sync_try_cmpxchg arch_sync_try_cmpxchg433#else434#define raw_sync_try_cmpxchg(_ptr, _oldp, _new) \435({ \436 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \437 ___r = raw_sync_cmpxchg((_ptr), ___o, (_new)); \438 if (unlikely(___r != ___o)) \439 *___op = ___r; \440 likely(___r == ___o); \441})442#endif443 444/**445 * raw_atomic_read() - atomic load with relaxed ordering446 * @v: pointer to atomic_t447 *448 * Atomically loads the value of @v with relaxed ordering.449 *450 * Safe to use in noinstr code; prefer atomic_read() elsewhere.451 *452 * Return: The value loaded from @v.453 */454static __always_inline int455raw_atomic_read(const atomic_t *v)456{457 return arch_atomic_read(v);458}459 460/**461 * raw_atomic_read_acquire() - atomic load with acquire ordering462 * @v: pointer to atomic_t463 *464 * Atomically loads the value of @v with acquire ordering.465 *466 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.467 *468 * Return: The value loaded from @v.469 */470static __always_inline int471raw_atomic_read_acquire(const atomic_t *v)472{473#if defined(arch_atomic_read_acquire)474 return arch_atomic_read_acquire(v);475#else476 int ret;477 478 if (__native_word(atomic_t)) {479 ret = smp_load_acquire(&(v)->counter);480 } else {481 ret = raw_atomic_read(v);482 __atomic_acquire_fence();483 }484 485 return ret;486#endif487}488 489/**490 * raw_atomic_set() - atomic set with relaxed ordering491 * @v: pointer to atomic_t492 * @i: int value to assign493 *494 * Atomically sets @v to @i with relaxed ordering.495 *496 * Safe to use in noinstr code; prefer atomic_set() elsewhere.497 *498 * Return: Nothing.499 */500static __always_inline void501raw_atomic_set(atomic_t *v, int i)502{503 arch_atomic_set(v, i);504}505 506/**507 * raw_atomic_set_release() - atomic set with release ordering508 * @v: pointer to atomic_t509 * @i: int value to assign510 *511 * Atomically sets @v to @i with release ordering.512 *513 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.514 *515 * Return: Nothing.516 */517static __always_inline void518raw_atomic_set_release(atomic_t *v, int i)519{520#if defined(arch_atomic_set_release)521 arch_atomic_set_release(v, i);522#else523 if (__native_word(atomic_t)) {524 smp_store_release(&(v)->counter, i);525 } else {526 __atomic_release_fence();527 raw_atomic_set(v, i);528 }529#endif530}531 532/**533 * raw_atomic_add() - atomic add with relaxed ordering534 * @i: int value to add535 * @v: pointer to atomic_t536 *537 * Atomically updates @v to (@v + @i) with relaxed ordering.538 *539 * Safe to use in noinstr code; prefer atomic_add() elsewhere.540 *541 * Return: Nothing.542 */543static __always_inline void544raw_atomic_add(int i, atomic_t *v)545{546 arch_atomic_add(i, v);547}548 549/**550 * raw_atomic_add_return() - atomic add with full ordering551 * @i: int value to add552 * @v: pointer to atomic_t553 *554 * Atomically updates @v to (@v + @i) with full ordering.555 *556 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.557 *558 * Return: The updated value of @v.559 */560static __always_inline int561raw_atomic_add_return(int i, atomic_t *v)562{563#if defined(arch_atomic_add_return)564 return arch_atomic_add_return(i, v);565#elif defined(arch_atomic_add_return_relaxed)566 int ret;567 __atomic_pre_full_fence();568 ret = arch_atomic_add_return_relaxed(i, v);569 __atomic_post_full_fence();570 return ret;571#else572#error "Unable to define raw_atomic_add_return"573#endif574}575 576/**577 * raw_atomic_add_return_acquire() - atomic add with acquire ordering578 * @i: int value to add579 * @v: pointer to atomic_t580 *581 * Atomically updates @v to (@v + @i) with acquire ordering.582 *583 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.584 *585 * Return: The updated value of @v.586 */587static __always_inline int588raw_atomic_add_return_acquire(int i, atomic_t *v)589{590#if defined(arch_atomic_add_return_acquire)591 return arch_atomic_add_return_acquire(i, v);592#elif defined(arch_atomic_add_return_relaxed)593 int ret = arch_atomic_add_return_relaxed(i, v);594 __atomic_acquire_fence();595 return ret;596#elif defined(arch_atomic_add_return)597 return arch_atomic_add_return(i, v);598#else599#error "Unable to define raw_atomic_add_return_acquire"600#endif601}602 603/**604 * raw_atomic_add_return_release() - atomic add with release ordering605 * @i: int value to add606 * @v: pointer to atomic_t607 *608 * Atomically updates @v to (@v + @i) with release ordering.609 *610 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.611 *612 * Return: The updated value of @v.613 */614static __always_inline int615raw_atomic_add_return_release(int i, atomic_t *v)616{617#if defined(arch_atomic_add_return_release)618 return arch_atomic_add_return_release(i, v);619#elif defined(arch_atomic_add_return_relaxed)620 __atomic_release_fence();621 return arch_atomic_add_return_relaxed(i, v);622#elif defined(arch_atomic_add_return)623 return arch_atomic_add_return(i, v);624#else625#error "Unable to define raw_atomic_add_return_release"626#endif627}628 629/**630 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering631 * @i: int value to add632 * @v: pointer to atomic_t633 *634 * Atomically updates @v to (@v + @i) with relaxed ordering.635 *636 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.637 *638 * Return: The updated value of @v.639 */640static __always_inline int641raw_atomic_add_return_relaxed(int i, atomic_t *v)642{643#if defined(arch_atomic_add_return_relaxed)644 return arch_atomic_add_return_relaxed(i, v);645#elif defined(arch_atomic_add_return)646 return arch_atomic_add_return(i, v);647#else648#error "Unable to define raw_atomic_add_return_relaxed"649#endif650}651 652/**653 * raw_atomic_fetch_add() - atomic add with full ordering654 * @i: int value to add655 * @v: pointer to atomic_t656 *657 * Atomically updates @v to (@v + @i) with full ordering.658 *659 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.660 *661 * Return: The original value of @v.662 */663static __always_inline int664raw_atomic_fetch_add(int i, atomic_t *v)665{666#if defined(arch_atomic_fetch_add)667 return arch_atomic_fetch_add(i, v);668#elif defined(arch_atomic_fetch_add_relaxed)669 int ret;670 __atomic_pre_full_fence();671 ret = arch_atomic_fetch_add_relaxed(i, v);672 __atomic_post_full_fence();673 return ret;674#else675#error "Unable to define raw_atomic_fetch_add"676#endif677}678 679/**680 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering681 * @i: int value to add682 * @v: pointer to atomic_t683 *684 * Atomically updates @v to (@v + @i) with acquire ordering.685 *686 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.687 *688 * Return: The original value of @v.689 */690static __always_inline int691raw_atomic_fetch_add_acquire(int i, atomic_t *v)692{693#if defined(arch_atomic_fetch_add_acquire)694 return arch_atomic_fetch_add_acquire(i, v);695#elif defined(arch_atomic_fetch_add_relaxed)696 int ret = arch_atomic_fetch_add_relaxed(i, v);697 __atomic_acquire_fence();698 return ret;699#elif defined(arch_atomic_fetch_add)700 return arch_atomic_fetch_add(i, v);701#else702#error "Unable to define raw_atomic_fetch_add_acquire"703#endif704}705 706/**707 * raw_atomic_fetch_add_release() - atomic add with release ordering708 * @i: int value to add709 * @v: pointer to atomic_t710 *711 * Atomically updates @v to (@v + @i) with release ordering.712 *713 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.714 *715 * Return: The original value of @v.716 */717static __always_inline int718raw_atomic_fetch_add_release(int i, atomic_t *v)719{720#if defined(arch_atomic_fetch_add_release)721 return arch_atomic_fetch_add_release(i, v);722#elif defined(arch_atomic_fetch_add_relaxed)723 __atomic_release_fence();724 return arch_atomic_fetch_add_relaxed(i, v);725#elif defined(arch_atomic_fetch_add)726 return arch_atomic_fetch_add(i, v);727#else728#error "Unable to define raw_atomic_fetch_add_release"729#endif730}731 732/**733 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering734 * @i: int value to add735 * @v: pointer to atomic_t736 *737 * Atomically updates @v to (@v + @i) with relaxed ordering.738 *739 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.740 *741 * Return: The original value of @v.742 */743static __always_inline int744raw_atomic_fetch_add_relaxed(int i, atomic_t *v)745{746#if defined(arch_atomic_fetch_add_relaxed)747 return arch_atomic_fetch_add_relaxed(i, v);748#elif defined(arch_atomic_fetch_add)749 return arch_atomic_fetch_add(i, v);750#else751#error "Unable to define raw_atomic_fetch_add_relaxed"752#endif753}754 755/**756 * raw_atomic_sub() - atomic subtract with relaxed ordering757 * @i: int value to subtract758 * @v: pointer to atomic_t759 *760 * Atomically updates @v to (@v - @i) with relaxed ordering.761 *762 * Safe to use in noinstr code; prefer atomic_sub() elsewhere.763 *764 * Return: Nothing.765 */766static __always_inline void767raw_atomic_sub(int i, atomic_t *v)768{769 arch_atomic_sub(i, v);770}771 772/**773 * raw_atomic_sub_return() - atomic subtract with full ordering774 * @i: int value to subtract775 * @v: pointer to atomic_t776 *777 * Atomically updates @v to (@v - @i) with full ordering.778 *779 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.780 *781 * Return: The updated value of @v.782 */783static __always_inline int784raw_atomic_sub_return(int i, atomic_t *v)785{786#if defined(arch_atomic_sub_return)787 return arch_atomic_sub_return(i, v);788#elif defined(arch_atomic_sub_return_relaxed)789 int ret;790 __atomic_pre_full_fence();791 ret = arch_atomic_sub_return_relaxed(i, v);792 __atomic_post_full_fence();793 return ret;794#else795#error "Unable to define raw_atomic_sub_return"796#endif797}798 799/**800 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering801 * @i: int value to subtract802 * @v: pointer to atomic_t803 *804 * Atomically updates @v to (@v - @i) with acquire ordering.805 *806 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.807 *808 * Return: The updated value of @v.809 */810static __always_inline int811raw_atomic_sub_return_acquire(int i, atomic_t *v)812{813#if defined(arch_atomic_sub_return_acquire)814 return arch_atomic_sub_return_acquire(i, v);815#elif defined(arch_atomic_sub_return_relaxed)816 int ret = arch_atomic_sub_return_relaxed(i, v);817 __atomic_acquire_fence();818 return ret;819#elif defined(arch_atomic_sub_return)820 return arch_atomic_sub_return(i, v);821#else822#error "Unable to define raw_atomic_sub_return_acquire"823#endif824}825 826/**827 * raw_atomic_sub_return_release() - atomic subtract with release ordering828 * @i: int value to subtract829 * @v: pointer to atomic_t830 *831 * Atomically updates @v to (@v - @i) with release ordering.832 *833 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.834 *835 * Return: The updated value of @v.836 */837static __always_inline int838raw_atomic_sub_return_release(int i, atomic_t *v)839{840#if defined(arch_atomic_sub_return_release)841 return arch_atomic_sub_return_release(i, v);842#elif defined(arch_atomic_sub_return_relaxed)843 __atomic_release_fence();844 return arch_atomic_sub_return_relaxed(i, v);845#elif defined(arch_atomic_sub_return)846 return arch_atomic_sub_return(i, v);847#else848#error "Unable to define raw_atomic_sub_return_release"849#endif850}851 852/**853 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering854 * @i: int value to subtract855 * @v: pointer to atomic_t856 *857 * Atomically updates @v to (@v - @i) with relaxed ordering.858 *859 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.860 *861 * Return: The updated value of @v.862 */863static __always_inline int864raw_atomic_sub_return_relaxed(int i, atomic_t *v)865{866#if defined(arch_atomic_sub_return_relaxed)867 return arch_atomic_sub_return_relaxed(i, v);868#elif defined(arch_atomic_sub_return)869 return arch_atomic_sub_return(i, v);870#else871#error "Unable to define raw_atomic_sub_return_relaxed"872#endif873}874 875/**876 * raw_atomic_fetch_sub() - atomic subtract with full ordering877 * @i: int value to subtract878 * @v: pointer to atomic_t879 *880 * Atomically updates @v to (@v - @i) with full ordering.881 *882 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.883 *884 * Return: The original value of @v.885 */886static __always_inline int887raw_atomic_fetch_sub(int i, atomic_t *v)888{889#if defined(arch_atomic_fetch_sub)890 return arch_atomic_fetch_sub(i, v);891#elif defined(arch_atomic_fetch_sub_relaxed)892 int ret;893 __atomic_pre_full_fence();894 ret = arch_atomic_fetch_sub_relaxed(i, v);895 __atomic_post_full_fence();896 return ret;897#else898#error "Unable to define raw_atomic_fetch_sub"899#endif900}901 902/**903 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering904 * @i: int value to subtract905 * @v: pointer to atomic_t906 *907 * Atomically updates @v to (@v - @i) with acquire ordering.908 *909 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.910 *911 * Return: The original value of @v.912 */913static __always_inline int914raw_atomic_fetch_sub_acquire(int i, atomic_t *v)915{916#if defined(arch_atomic_fetch_sub_acquire)917 return arch_atomic_fetch_sub_acquire(i, v);918#elif defined(arch_atomic_fetch_sub_relaxed)919 int ret = arch_atomic_fetch_sub_relaxed(i, v);920 __atomic_acquire_fence();921 return ret;922#elif defined(arch_atomic_fetch_sub)923 return arch_atomic_fetch_sub(i, v);924#else925#error "Unable to define raw_atomic_fetch_sub_acquire"926#endif927}928 929/**930 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering931 * @i: int value to subtract932 * @v: pointer to atomic_t933 *934 * Atomically updates @v to (@v - @i) with release ordering.935 *936 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.937 *938 * Return: The original value of @v.939 */940static __always_inline int941raw_atomic_fetch_sub_release(int i, atomic_t *v)942{943#if defined(arch_atomic_fetch_sub_release)944 return arch_atomic_fetch_sub_release(i, v);945#elif defined(arch_atomic_fetch_sub_relaxed)946 __atomic_release_fence();947 return arch_atomic_fetch_sub_relaxed(i, v);948#elif defined(arch_atomic_fetch_sub)949 return arch_atomic_fetch_sub(i, v);950#else951#error "Unable to define raw_atomic_fetch_sub_release"952#endif953}954 955/**956 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering957 * @i: int value to subtract958 * @v: pointer to atomic_t959 *960 * Atomically updates @v to (@v - @i) with relaxed ordering.961 *962 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.963 *964 * Return: The original value of @v.965 */966static __always_inline int967raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)968{969#if defined(arch_atomic_fetch_sub_relaxed)970 return arch_atomic_fetch_sub_relaxed(i, v);971#elif defined(arch_atomic_fetch_sub)972 return arch_atomic_fetch_sub(i, v);973#else974#error "Unable to define raw_atomic_fetch_sub_relaxed"975#endif976}977 978/**979 * raw_atomic_inc() - atomic increment with relaxed ordering980 * @v: pointer to atomic_t981 *982 * Atomically updates @v to (@v + 1) with relaxed ordering.983 *984 * Safe to use in noinstr code; prefer atomic_inc() elsewhere.985 *986 * Return: Nothing.987 */988static __always_inline void989raw_atomic_inc(atomic_t *v)990{991#if defined(arch_atomic_inc)992 arch_atomic_inc(v);993#else994 raw_atomic_add(1, v);995#endif996}997 998/**999 * raw_atomic_inc_return() - atomic increment with full ordering1000 * @v: pointer to atomic_t1001 *1002 * Atomically updates @v to (@v + 1) with full ordering.1003 *1004 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.1005 *1006 * Return: The updated value of @v.1007 */1008static __always_inline int1009raw_atomic_inc_return(atomic_t *v)1010{1011#if defined(arch_atomic_inc_return)1012 return arch_atomic_inc_return(v);1013#elif defined(arch_atomic_inc_return_relaxed)1014 int ret;1015 __atomic_pre_full_fence();1016 ret = arch_atomic_inc_return_relaxed(v);1017 __atomic_post_full_fence();1018 return ret;1019#else1020 return raw_atomic_add_return(1, v);1021#endif1022}1023 1024/**1025 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering1026 * @v: pointer to atomic_t1027 *1028 * Atomically updates @v to (@v + 1) with acquire ordering.1029 *1030 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.1031 *1032 * Return: The updated value of @v.1033 */1034static __always_inline int1035raw_atomic_inc_return_acquire(atomic_t *v)1036{1037#if defined(arch_atomic_inc_return_acquire)1038 return arch_atomic_inc_return_acquire(v);1039#elif defined(arch_atomic_inc_return_relaxed)1040 int ret = arch_atomic_inc_return_relaxed(v);1041 __atomic_acquire_fence();1042 return ret;1043#elif defined(arch_atomic_inc_return)1044 return arch_atomic_inc_return(v);1045#else1046 return raw_atomic_add_return_acquire(1, v);1047#endif1048}1049 1050/**1051 * raw_atomic_inc_return_release() - atomic increment with release ordering1052 * @v: pointer to atomic_t1053 *1054 * Atomically updates @v to (@v + 1) with release ordering.1055 *1056 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.1057 *1058 * Return: The updated value of @v.1059 */1060static __always_inline int1061raw_atomic_inc_return_release(atomic_t *v)1062{1063#if defined(arch_atomic_inc_return_release)1064 return arch_atomic_inc_return_release(v);1065#elif defined(arch_atomic_inc_return_relaxed)1066 __atomic_release_fence();1067 return arch_atomic_inc_return_relaxed(v);1068#elif defined(arch_atomic_inc_return)1069 return arch_atomic_inc_return(v);1070#else1071 return raw_atomic_add_return_release(1, v);1072#endif1073}1074 1075/**1076 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering1077 * @v: pointer to atomic_t1078 *1079 * Atomically updates @v to (@v + 1) with relaxed ordering.1080 *1081 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.1082 *1083 * Return: The updated value of @v.1084 */1085static __always_inline int1086raw_atomic_inc_return_relaxed(atomic_t *v)1087{1088#if defined(arch_atomic_inc_return_relaxed)1089 return arch_atomic_inc_return_relaxed(v);1090#elif defined(arch_atomic_inc_return)1091 return arch_atomic_inc_return(v);1092#else1093 return raw_atomic_add_return_relaxed(1, v);1094#endif1095}1096 1097/**1098 * raw_atomic_fetch_inc() - atomic increment with full ordering1099 * @v: pointer to atomic_t1100 *1101 * Atomically updates @v to (@v + 1) with full ordering.1102 *1103 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.1104 *1105 * Return: The original value of @v.1106 */1107static __always_inline int1108raw_atomic_fetch_inc(atomic_t *v)1109{1110#if defined(arch_atomic_fetch_inc)1111 return arch_atomic_fetch_inc(v);1112#elif defined(arch_atomic_fetch_inc_relaxed)1113 int ret;1114 __atomic_pre_full_fence();1115 ret = arch_atomic_fetch_inc_relaxed(v);1116 __atomic_post_full_fence();1117 return ret;1118#else1119 return raw_atomic_fetch_add(1, v);1120#endif1121}1122 1123/**1124 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering1125 * @v: pointer to atomic_t1126 *1127 * Atomically updates @v to (@v + 1) with acquire ordering.1128 *1129 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.1130 *1131 * Return: The original value of @v.1132 */1133static __always_inline int1134raw_atomic_fetch_inc_acquire(atomic_t *v)1135{1136#if defined(arch_atomic_fetch_inc_acquire)1137 return arch_atomic_fetch_inc_acquire(v);1138#elif defined(arch_atomic_fetch_inc_relaxed)1139 int ret = arch_atomic_fetch_inc_relaxed(v);1140 __atomic_acquire_fence();1141 return ret;1142#elif defined(arch_atomic_fetch_inc)1143 return arch_atomic_fetch_inc(v);1144#else1145 return raw_atomic_fetch_add_acquire(1, v);1146#endif1147}1148 1149/**1150 * raw_atomic_fetch_inc_release() - atomic increment with release ordering1151 * @v: pointer to atomic_t1152 *1153 * Atomically updates @v to (@v + 1) with release ordering.1154 *1155 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.1156 *1157 * Return: The original value of @v.1158 */1159static __always_inline int1160raw_atomic_fetch_inc_release(atomic_t *v)1161{1162#if defined(arch_atomic_fetch_inc_release)1163 return arch_atomic_fetch_inc_release(v);1164#elif defined(arch_atomic_fetch_inc_relaxed)1165 __atomic_release_fence();1166 return arch_atomic_fetch_inc_relaxed(v);1167#elif defined(arch_atomic_fetch_inc)1168 return arch_atomic_fetch_inc(v);1169#else1170 return raw_atomic_fetch_add_release(1, v);1171#endif1172}1173 1174/**1175 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering1176 * @v: pointer to atomic_t1177 *1178 * Atomically updates @v to (@v + 1) with relaxed ordering.1179 *1180 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.1181 *1182 * Return: The original value of @v.1183 */1184static __always_inline int1185raw_atomic_fetch_inc_relaxed(atomic_t *v)1186{1187#if defined(arch_atomic_fetch_inc_relaxed)1188 return arch_atomic_fetch_inc_relaxed(v);1189#elif defined(arch_atomic_fetch_inc)1190 return arch_atomic_fetch_inc(v);1191#else1192 return raw_atomic_fetch_add_relaxed(1, v);1193#endif1194}1195 1196/**1197 * raw_atomic_dec() - atomic decrement with relaxed ordering1198 * @v: pointer to atomic_t1199 *1200 * Atomically updates @v to (@v - 1) with relaxed ordering.1201 *1202 * Safe to use in noinstr code; prefer atomic_dec() elsewhere.1203 *1204 * Return: Nothing.1205 */1206static __always_inline void1207raw_atomic_dec(atomic_t *v)1208{1209#if defined(arch_atomic_dec)1210 arch_atomic_dec(v);1211#else1212 raw_atomic_sub(1, v);1213#endif1214}1215 1216/**1217 * raw_atomic_dec_return() - atomic decrement with full ordering1218 * @v: pointer to atomic_t1219 *1220 * Atomically updates @v to (@v - 1) with full ordering.1221 *1222 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.1223 *1224 * Return: The updated value of @v.1225 */1226static __always_inline int1227raw_atomic_dec_return(atomic_t *v)1228{1229#if defined(arch_atomic_dec_return)1230 return arch_atomic_dec_return(v);1231#elif defined(arch_atomic_dec_return_relaxed)1232 int ret;1233 __atomic_pre_full_fence();1234 ret = arch_atomic_dec_return_relaxed(v);1235 __atomic_post_full_fence();1236 return ret;1237#else1238 return raw_atomic_sub_return(1, v);1239#endif1240}1241 1242/**1243 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering1244 * @v: pointer to atomic_t1245 *1246 * Atomically updates @v to (@v - 1) with acquire ordering.1247 *1248 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.1249 *1250 * Return: The updated value of @v.1251 */1252static __always_inline int1253raw_atomic_dec_return_acquire(atomic_t *v)1254{1255#if defined(arch_atomic_dec_return_acquire)1256 return arch_atomic_dec_return_acquire(v);1257#elif defined(arch_atomic_dec_return_relaxed)1258 int ret = arch_atomic_dec_return_relaxed(v);1259 __atomic_acquire_fence();1260 return ret;1261#elif defined(arch_atomic_dec_return)1262 return arch_atomic_dec_return(v);1263#else1264 return raw_atomic_sub_return_acquire(1, v);1265#endif1266}1267 1268/**1269 * raw_atomic_dec_return_release() - atomic decrement with release ordering1270 * @v: pointer to atomic_t1271 *1272 * Atomically updates @v to (@v - 1) with release ordering.1273 *1274 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.1275 *1276 * Return: The updated value of @v.1277 */1278static __always_inline int1279raw_atomic_dec_return_release(atomic_t *v)1280{1281#if defined(arch_atomic_dec_return_release)1282 return arch_atomic_dec_return_release(v);1283#elif defined(arch_atomic_dec_return_relaxed)1284 __atomic_release_fence();1285 return arch_atomic_dec_return_relaxed(v);1286#elif defined(arch_atomic_dec_return)1287 return arch_atomic_dec_return(v);1288#else1289 return raw_atomic_sub_return_release(1, v);1290#endif1291}1292 1293/**1294 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering1295 * @v: pointer to atomic_t1296 *1297 * Atomically updates @v to (@v - 1) with relaxed ordering.1298 *1299 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.1300 *1301 * Return: The updated value of @v.1302 */1303static __always_inline int1304raw_atomic_dec_return_relaxed(atomic_t *v)1305{1306#if defined(arch_atomic_dec_return_relaxed)1307 return arch_atomic_dec_return_relaxed(v);1308#elif defined(arch_atomic_dec_return)1309 return arch_atomic_dec_return(v);1310#else1311 return raw_atomic_sub_return_relaxed(1, v);1312#endif1313}1314 1315/**1316 * raw_atomic_fetch_dec() - atomic decrement with full ordering1317 * @v: pointer to atomic_t1318 *1319 * Atomically updates @v to (@v - 1) with full ordering.1320 *1321 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.1322 *1323 * Return: The original value of @v.1324 */1325static __always_inline int1326raw_atomic_fetch_dec(atomic_t *v)1327{1328#if defined(arch_atomic_fetch_dec)1329 return arch_atomic_fetch_dec(v);1330#elif defined(arch_atomic_fetch_dec_relaxed)1331 int ret;1332 __atomic_pre_full_fence();1333 ret = arch_atomic_fetch_dec_relaxed(v);1334 __atomic_post_full_fence();1335 return ret;1336#else1337 return raw_atomic_fetch_sub(1, v);1338#endif1339}1340 1341/**1342 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering1343 * @v: pointer to atomic_t1344 *1345 * Atomically updates @v to (@v - 1) with acquire ordering.1346 *1347 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.1348 *1349 * Return: The original value of @v.1350 */1351static __always_inline int1352raw_atomic_fetch_dec_acquire(atomic_t *v)1353{1354#if defined(arch_atomic_fetch_dec_acquire)1355 return arch_atomic_fetch_dec_acquire(v);1356#elif defined(arch_atomic_fetch_dec_relaxed)1357 int ret = arch_atomic_fetch_dec_relaxed(v);1358 __atomic_acquire_fence();1359 return ret;1360#elif defined(arch_atomic_fetch_dec)1361 return arch_atomic_fetch_dec(v);1362#else1363 return raw_atomic_fetch_sub_acquire(1, v);1364#endif1365}1366 1367/**1368 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering1369 * @v: pointer to atomic_t1370 *1371 * Atomically updates @v to (@v - 1) with release ordering.1372 *1373 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.1374 *1375 * Return: The original value of @v.1376 */1377static __always_inline int1378raw_atomic_fetch_dec_release(atomic_t *v)1379{1380#if defined(arch_atomic_fetch_dec_release)1381 return arch_atomic_fetch_dec_release(v);1382#elif defined(arch_atomic_fetch_dec_relaxed)1383 __atomic_release_fence();1384 return arch_atomic_fetch_dec_relaxed(v);1385#elif defined(arch_atomic_fetch_dec)1386 return arch_atomic_fetch_dec(v);1387#else1388 return raw_atomic_fetch_sub_release(1, v);1389#endif1390}1391 1392/**1393 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering1394 * @v: pointer to atomic_t1395 *1396 * Atomically updates @v to (@v - 1) with relaxed ordering.1397 *1398 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.1399 *1400 * Return: The original value of @v.1401 */1402static __always_inline int1403raw_atomic_fetch_dec_relaxed(atomic_t *v)1404{1405#if defined(arch_atomic_fetch_dec_relaxed)1406 return arch_atomic_fetch_dec_relaxed(v);1407#elif defined(arch_atomic_fetch_dec)1408 return arch_atomic_fetch_dec(v);1409#else1410 return raw_atomic_fetch_sub_relaxed(1, v);1411#endif1412}1413 1414/**1415 * raw_atomic_and() - atomic bitwise AND with relaxed ordering1416 * @i: int value1417 * @v: pointer to atomic_t1418 *1419 * Atomically updates @v to (@v & @i) with relaxed ordering.1420 *1421 * Safe to use in noinstr code; prefer atomic_and() elsewhere.1422 *1423 * Return: Nothing.1424 */1425static __always_inline void1426raw_atomic_and(int i, atomic_t *v)1427{1428 arch_atomic_and(i, v);1429}1430 1431/**1432 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering1433 * @i: int value1434 * @v: pointer to atomic_t1435 *1436 * Atomically updates @v to (@v & @i) with full ordering.1437 *1438 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.1439 *1440 * Return: The original value of @v.1441 */1442static __always_inline int1443raw_atomic_fetch_and(int i, atomic_t *v)1444{1445#if defined(arch_atomic_fetch_and)1446 return arch_atomic_fetch_and(i, v);1447#elif defined(arch_atomic_fetch_and_relaxed)1448 int ret;1449 __atomic_pre_full_fence();1450 ret = arch_atomic_fetch_and_relaxed(i, v);1451 __atomic_post_full_fence();1452 return ret;1453#else1454#error "Unable to define raw_atomic_fetch_and"1455#endif1456}1457 1458/**1459 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering1460 * @i: int value1461 * @v: pointer to atomic_t1462 *1463 * Atomically updates @v to (@v & @i) with acquire ordering.1464 *1465 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.1466 *1467 * Return: The original value of @v.1468 */1469static __always_inline int1470raw_atomic_fetch_and_acquire(int i, atomic_t *v)1471{1472#if defined(arch_atomic_fetch_and_acquire)1473 return arch_atomic_fetch_and_acquire(i, v);1474#elif defined(arch_atomic_fetch_and_relaxed)1475 int ret = arch_atomic_fetch_and_relaxed(i, v);1476 __atomic_acquire_fence();1477 return ret;1478#elif defined(arch_atomic_fetch_and)1479 return arch_atomic_fetch_and(i, v);1480#else1481#error "Unable to define raw_atomic_fetch_and_acquire"1482#endif1483}1484 1485/**1486 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering1487 * @i: int value1488 * @v: pointer to atomic_t1489 *1490 * Atomically updates @v to (@v & @i) with release ordering.1491 *1492 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.1493 *1494 * Return: The original value of @v.1495 */1496static __always_inline int1497raw_atomic_fetch_and_release(int i, atomic_t *v)1498{1499#if defined(arch_atomic_fetch_and_release)1500 return arch_atomic_fetch_and_release(i, v);1501#elif defined(arch_atomic_fetch_and_relaxed)1502 __atomic_release_fence();1503 return arch_atomic_fetch_and_relaxed(i, v);1504#elif defined(arch_atomic_fetch_and)1505 return arch_atomic_fetch_and(i, v);1506#else1507#error "Unable to define raw_atomic_fetch_and_release"1508#endif1509}1510 1511/**1512 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering1513 * @i: int value1514 * @v: pointer to atomic_t1515 *1516 * Atomically updates @v to (@v & @i) with relaxed ordering.1517 *1518 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.1519 *1520 * Return: The original value of @v.1521 */1522static __always_inline int1523raw_atomic_fetch_and_relaxed(int i, atomic_t *v)1524{1525#if defined(arch_atomic_fetch_and_relaxed)1526 return arch_atomic_fetch_and_relaxed(i, v);1527#elif defined(arch_atomic_fetch_and)1528 return arch_atomic_fetch_and(i, v);1529#else1530#error "Unable to define raw_atomic_fetch_and_relaxed"1531#endif1532}1533 1534/**1535 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering1536 * @i: int value1537 * @v: pointer to atomic_t1538 *1539 * Atomically updates @v to (@v & ~@i) with relaxed ordering.1540 *1541 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.1542 *1543 * Return: Nothing.1544 */1545static __always_inline void1546raw_atomic_andnot(int i, atomic_t *v)1547{1548#if defined(arch_atomic_andnot)1549 arch_atomic_andnot(i, v);1550#else1551 raw_atomic_and(~i, v);1552#endif1553}1554 1555/**1556 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering1557 * @i: int value1558 * @v: pointer to atomic_t1559 *1560 * Atomically updates @v to (@v & ~@i) with full ordering.1561 *1562 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.1563 *1564 * Return: The original value of @v.1565 */1566static __always_inline int1567raw_atomic_fetch_andnot(int i, atomic_t *v)1568{1569#if defined(arch_atomic_fetch_andnot)1570 return arch_atomic_fetch_andnot(i, v);1571#elif defined(arch_atomic_fetch_andnot_relaxed)1572 int ret;1573 __atomic_pre_full_fence();1574 ret = arch_atomic_fetch_andnot_relaxed(i, v);1575 __atomic_post_full_fence();1576 return ret;1577#else1578 return raw_atomic_fetch_and(~i, v);1579#endif1580}1581 1582/**1583 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering1584 * @i: int value1585 * @v: pointer to atomic_t1586 *1587 * Atomically updates @v to (@v & ~@i) with acquire ordering.1588 *1589 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.1590 *1591 * Return: The original value of @v.1592 */1593static __always_inline int1594raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)1595{1596#if defined(arch_atomic_fetch_andnot_acquire)1597 return arch_atomic_fetch_andnot_acquire(i, v);1598#elif defined(arch_atomic_fetch_andnot_relaxed)1599 int ret = arch_atomic_fetch_andnot_relaxed(i, v);1600 __atomic_acquire_fence();1601 return ret;1602#elif defined(arch_atomic_fetch_andnot)1603 return arch_atomic_fetch_andnot(i, v);1604#else1605 return raw_atomic_fetch_and_acquire(~i, v);1606#endif1607}1608 1609/**1610 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering1611 * @i: int value1612 * @v: pointer to atomic_t1613 *1614 * Atomically updates @v to (@v & ~@i) with release ordering.1615 *1616 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.1617 *1618 * Return: The original value of @v.1619 */1620static __always_inline int1621raw_atomic_fetch_andnot_release(int i, atomic_t *v)1622{1623#if defined(arch_atomic_fetch_andnot_release)1624 return arch_atomic_fetch_andnot_release(i, v);1625#elif defined(arch_atomic_fetch_andnot_relaxed)1626 __atomic_release_fence();1627 return arch_atomic_fetch_andnot_relaxed(i, v);1628#elif defined(arch_atomic_fetch_andnot)1629 return arch_atomic_fetch_andnot(i, v);1630#else1631 return raw_atomic_fetch_and_release(~i, v);1632#endif1633}1634 1635/**1636 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering1637 * @i: int value1638 * @v: pointer to atomic_t1639 *1640 * Atomically updates @v to (@v & ~@i) with relaxed ordering.1641 *1642 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.1643 *1644 * Return: The original value of @v.1645 */1646static __always_inline int1647raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)1648{1649#if defined(arch_atomic_fetch_andnot_relaxed)1650 return arch_atomic_fetch_andnot_relaxed(i, v);1651#elif defined(arch_atomic_fetch_andnot)1652 return arch_atomic_fetch_andnot(i, v);1653#else1654 return raw_atomic_fetch_and_relaxed(~i, v);1655#endif1656}1657 1658/**1659 * raw_atomic_or() - atomic bitwise OR with relaxed ordering1660 * @i: int value1661 * @v: pointer to atomic_t1662 *1663 * Atomically updates @v to (@v | @i) with relaxed ordering.1664 *1665 * Safe to use in noinstr code; prefer atomic_or() elsewhere.1666 *1667 * Return: Nothing.1668 */1669static __always_inline void1670raw_atomic_or(int i, atomic_t *v)1671{1672 arch_atomic_or(i, v);1673}1674 1675/**1676 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering1677 * @i: int value1678 * @v: pointer to atomic_t1679 *1680 * Atomically updates @v to (@v | @i) with full ordering.1681 *1682 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.1683 *1684 * Return: The original value of @v.1685 */1686static __always_inline int1687raw_atomic_fetch_or(int i, atomic_t *v)1688{1689#if defined(arch_atomic_fetch_or)1690 return arch_atomic_fetch_or(i, v);1691#elif defined(arch_atomic_fetch_or_relaxed)1692 int ret;1693 __atomic_pre_full_fence();1694 ret = arch_atomic_fetch_or_relaxed(i, v);1695 __atomic_post_full_fence();1696 return ret;1697#else1698#error "Unable to define raw_atomic_fetch_or"1699#endif1700}1701 1702/**1703 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering1704 * @i: int value1705 * @v: pointer to atomic_t1706 *1707 * Atomically updates @v to (@v | @i) with acquire ordering.1708 *1709 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.1710 *1711 * Return: The original value of @v.1712 */1713static __always_inline int1714raw_atomic_fetch_or_acquire(int i, atomic_t *v)1715{1716#if defined(arch_atomic_fetch_or_acquire)1717 return arch_atomic_fetch_or_acquire(i, v);1718#elif defined(arch_atomic_fetch_or_relaxed)1719 int ret = arch_atomic_fetch_or_relaxed(i, v);1720 __atomic_acquire_fence();1721 return ret;1722#elif defined(arch_atomic_fetch_or)1723 return arch_atomic_fetch_or(i, v);1724#else1725#error "Unable to define raw_atomic_fetch_or_acquire"1726#endif1727}1728 1729/**1730 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering1731 * @i: int value1732 * @v: pointer to atomic_t1733 *1734 * Atomically updates @v to (@v | @i) with release ordering.1735 *1736 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.1737 *1738 * Return: The original value of @v.1739 */1740static __always_inline int1741raw_atomic_fetch_or_release(int i, atomic_t *v)1742{1743#if defined(arch_atomic_fetch_or_release)1744 return arch_atomic_fetch_or_release(i, v);1745#elif defined(arch_atomic_fetch_or_relaxed)1746 __atomic_release_fence();1747 return arch_atomic_fetch_or_relaxed(i, v);1748#elif defined(arch_atomic_fetch_or)1749 return arch_atomic_fetch_or(i, v);1750#else1751#error "Unable to define raw_atomic_fetch_or_release"1752#endif1753}1754 1755/**1756 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering1757 * @i: int value1758 * @v: pointer to atomic_t1759 *1760 * Atomically updates @v to (@v | @i) with relaxed ordering.1761 *1762 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.1763 *1764 * Return: The original value of @v.1765 */1766static __always_inline int1767raw_atomic_fetch_or_relaxed(int i, atomic_t *v)1768{1769#if defined(arch_atomic_fetch_or_relaxed)1770 return arch_atomic_fetch_or_relaxed(i, v);1771#elif defined(arch_atomic_fetch_or)1772 return arch_atomic_fetch_or(i, v);1773#else1774#error "Unable to define raw_atomic_fetch_or_relaxed"1775#endif1776}1777 1778/**1779 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering1780 * @i: int value1781 * @v: pointer to atomic_t1782 *1783 * Atomically updates @v to (@v ^ @i) with relaxed ordering.1784 *1785 * Safe to use in noinstr code; prefer atomic_xor() elsewhere.1786 *1787 * Return: Nothing.1788 */1789static __always_inline void1790raw_atomic_xor(int i, atomic_t *v)1791{1792 arch_atomic_xor(i, v);1793}1794 1795/**1796 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering1797 * @i: int value1798 * @v: pointer to atomic_t1799 *1800 * Atomically updates @v to (@v ^ @i) with full ordering.1801 *1802 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.1803 *1804 * Return: The original value of @v.1805 */1806static __always_inline int1807raw_atomic_fetch_xor(int i, atomic_t *v)1808{1809#if defined(arch_atomic_fetch_xor)1810 return arch_atomic_fetch_xor(i, v);1811#elif defined(arch_atomic_fetch_xor_relaxed)1812 int ret;1813 __atomic_pre_full_fence();1814 ret = arch_atomic_fetch_xor_relaxed(i, v);1815 __atomic_post_full_fence();1816 return ret;1817#else1818#error "Unable to define raw_atomic_fetch_xor"1819#endif1820}1821 1822/**1823 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering1824 * @i: int value1825 * @v: pointer to atomic_t1826 *1827 * Atomically updates @v to (@v ^ @i) with acquire ordering.1828 *1829 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.1830 *1831 * Return: The original value of @v.1832 */1833static __always_inline int1834raw_atomic_fetch_xor_acquire(int i, atomic_t *v)1835{1836#if defined(arch_atomic_fetch_xor_acquire)1837 return arch_atomic_fetch_xor_acquire(i, v);1838#elif defined(arch_atomic_fetch_xor_relaxed)1839 int ret = arch_atomic_fetch_xor_relaxed(i, v);1840 __atomic_acquire_fence();1841 return ret;1842#elif defined(arch_atomic_fetch_xor)1843 return arch_atomic_fetch_xor(i, v);1844#else1845#error "Unable to define raw_atomic_fetch_xor_acquire"1846#endif1847}1848 1849/**1850 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering1851 * @i: int value1852 * @v: pointer to atomic_t1853 *1854 * Atomically updates @v to (@v ^ @i) with release ordering.1855 *1856 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.1857 *1858 * Return: The original value of @v.1859 */1860static __always_inline int1861raw_atomic_fetch_xor_release(int i, atomic_t *v)1862{1863#if defined(arch_atomic_fetch_xor_release)1864 return arch_atomic_fetch_xor_release(i, v);1865#elif defined(arch_atomic_fetch_xor_relaxed)1866 __atomic_release_fence();1867 return arch_atomic_fetch_xor_relaxed(i, v);1868#elif defined(arch_atomic_fetch_xor)1869 return arch_atomic_fetch_xor(i, v);1870#else1871#error "Unable to define raw_atomic_fetch_xor_release"1872#endif1873}1874 1875/**1876 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering1877 * @i: int value1878 * @v: pointer to atomic_t1879 *1880 * Atomically updates @v to (@v ^ @i) with relaxed ordering.1881 *1882 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.1883 *1884 * Return: The original value of @v.1885 */1886static __always_inline int1887raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)1888{1889#if defined(arch_atomic_fetch_xor_relaxed)1890 return arch_atomic_fetch_xor_relaxed(i, v);1891#elif defined(arch_atomic_fetch_xor)1892 return arch_atomic_fetch_xor(i, v);1893#else1894#error "Unable to define raw_atomic_fetch_xor_relaxed"1895#endif1896}1897 1898/**1899 * raw_atomic_xchg() - atomic exchange with full ordering1900 * @v: pointer to atomic_t1901 * @new: int value to assign1902 *1903 * Atomically updates @v to @new with full ordering.1904 *1905 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.1906 *1907 * Return: The original value of @v.1908 */1909static __always_inline int1910raw_atomic_xchg(atomic_t *v, int new)1911{1912#if defined(arch_atomic_xchg)1913 return arch_atomic_xchg(v, new);1914#elif defined(arch_atomic_xchg_relaxed)1915 int ret;1916 __atomic_pre_full_fence();1917 ret = arch_atomic_xchg_relaxed(v, new);1918 __atomic_post_full_fence();1919 return ret;1920#else1921 return raw_xchg(&v->counter, new);1922#endif1923}1924 1925/**1926 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering1927 * @v: pointer to atomic_t1928 * @new: int value to assign1929 *1930 * Atomically updates @v to @new with acquire ordering.1931 *1932 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.1933 *1934 * Return: The original value of @v.1935 */1936static __always_inline int1937raw_atomic_xchg_acquire(atomic_t *v, int new)1938{1939#if defined(arch_atomic_xchg_acquire)1940 return arch_atomic_xchg_acquire(v, new);1941#elif defined(arch_atomic_xchg_relaxed)1942 int ret = arch_atomic_xchg_relaxed(v, new);1943 __atomic_acquire_fence();1944 return ret;1945#elif defined(arch_atomic_xchg)1946 return arch_atomic_xchg(v, new);1947#else1948 return raw_xchg_acquire(&v->counter, new);1949#endif1950}1951 1952/**1953 * raw_atomic_xchg_release() - atomic exchange with release ordering1954 * @v: pointer to atomic_t1955 * @new: int value to assign1956 *1957 * Atomically updates @v to @new with release ordering.1958 *1959 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.1960 *1961 * Return: The original value of @v.1962 */1963static __always_inline int1964raw_atomic_xchg_release(atomic_t *v, int new)1965{1966#if defined(arch_atomic_xchg_release)1967 return arch_atomic_xchg_release(v, new);1968#elif defined(arch_atomic_xchg_relaxed)1969 __atomic_release_fence();1970 return arch_atomic_xchg_relaxed(v, new);1971#elif defined(arch_atomic_xchg)1972 return arch_atomic_xchg(v, new);1973#else1974 return raw_xchg_release(&v->counter, new);1975#endif1976}1977 1978/**1979 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering1980 * @v: pointer to atomic_t1981 * @new: int value to assign1982 *1983 * Atomically updates @v to @new with relaxed ordering.1984 *1985 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.1986 *1987 * Return: The original value of @v.1988 */1989static __always_inline int1990raw_atomic_xchg_relaxed(atomic_t *v, int new)1991{1992#if defined(arch_atomic_xchg_relaxed)1993 return arch_atomic_xchg_relaxed(v, new);1994#elif defined(arch_atomic_xchg)1995 return arch_atomic_xchg(v, new);1996#else1997 return raw_xchg_relaxed(&v->counter, new);1998#endif1999}2000 2001/**2002 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering2003 * @v: pointer to atomic_t2004 * @old: int value to compare with2005 * @new: int value to assign2006 *2007 * If (@v == @old), atomically updates @v to @new with full ordering.2008 * Otherwise, @v is not modified and relaxed ordering is provided.2009 *2010 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.2011 *2012 * Return: The original value of @v.2013 */2014static __always_inline int2015raw_atomic_cmpxchg(atomic_t *v, int old, int new)2016{2017#if defined(arch_atomic_cmpxchg)2018 return arch_atomic_cmpxchg(v, old, new);2019#elif defined(arch_atomic_cmpxchg_relaxed)2020 int ret;2021 __atomic_pre_full_fence();2022 ret = arch_atomic_cmpxchg_relaxed(v, old, new);2023 __atomic_post_full_fence();2024 return ret;2025#else2026 return raw_cmpxchg(&v->counter, old, new);2027#endif2028}2029 2030/**2031 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering2032 * @v: pointer to atomic_t2033 * @old: int value to compare with2034 * @new: int value to assign2035 *2036 * If (@v == @old), atomically updates @v to @new with acquire ordering.2037 * Otherwise, @v is not modified and relaxed ordering is provided.2038 *2039 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.2040 *2041 * Return: The original value of @v.2042 */2043static __always_inline int2044raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)2045{2046#if defined(arch_atomic_cmpxchg_acquire)2047 return arch_atomic_cmpxchg_acquire(v, old, new);2048#elif defined(arch_atomic_cmpxchg_relaxed)2049 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);2050 __atomic_acquire_fence();2051 return ret;2052#elif defined(arch_atomic_cmpxchg)2053 return arch_atomic_cmpxchg(v, old, new);2054#else2055 return raw_cmpxchg_acquire(&v->counter, old, new);2056#endif2057}2058 2059/**2060 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering2061 * @v: pointer to atomic_t2062 * @old: int value to compare with2063 * @new: int value to assign2064 *2065 * If (@v == @old), atomically updates @v to @new with release ordering.2066 * Otherwise, @v is not modified and relaxed ordering is provided.2067 *2068 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.2069 *2070 * Return: The original value of @v.2071 */2072static __always_inline int2073raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)2074{2075#if defined(arch_atomic_cmpxchg_release)2076 return arch_atomic_cmpxchg_release(v, old, new);2077#elif defined(arch_atomic_cmpxchg_relaxed)2078 __atomic_release_fence();2079 return arch_atomic_cmpxchg_relaxed(v, old, new);2080#elif defined(arch_atomic_cmpxchg)2081 return arch_atomic_cmpxchg(v, old, new);2082#else2083 return raw_cmpxchg_release(&v->counter, old, new);2084#endif2085}2086 2087/**2088 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering2089 * @v: pointer to atomic_t2090 * @old: int value to compare with2091 * @new: int value to assign2092 *2093 * If (@v == @old), atomically updates @v to @new with relaxed ordering.2094 * Otherwise, @v is not modified and relaxed ordering is provided.2095 *2096 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.2097 *2098 * Return: The original value of @v.2099 */2100static __always_inline int2101raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)2102{2103#if defined(arch_atomic_cmpxchg_relaxed)2104 return arch_atomic_cmpxchg_relaxed(v, old, new);2105#elif defined(arch_atomic_cmpxchg)2106 return arch_atomic_cmpxchg(v, old, new);2107#else2108 return raw_cmpxchg_relaxed(&v->counter, old, new);2109#endif2110}2111 2112/**2113 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering2114 * @v: pointer to atomic_t2115 * @old: pointer to int value to compare with2116 * @new: int value to assign2117 *2118 * If (@v == @old), atomically updates @v to @new with full ordering.2119 * Otherwise, @v is not modified, @old is updated to the current value of @v,2120 * and relaxed ordering is provided.2121 *2122 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.2123 *2124 * Return: @true if the exchange occured, @false otherwise.2125 */2126static __always_inline bool2127raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)2128{2129#if defined(arch_atomic_try_cmpxchg)2130 return arch_atomic_try_cmpxchg(v, old, new);2131#elif defined(arch_atomic_try_cmpxchg_relaxed)2132 bool ret;2133 __atomic_pre_full_fence();2134 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);2135 __atomic_post_full_fence();2136 return ret;2137#else2138 int r, o = *old;2139 r = raw_atomic_cmpxchg(v, o, new);2140 if (unlikely(r != o))2141 *old = r;2142 return likely(r == o);2143#endif2144}2145 2146/**2147 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering2148 * @v: pointer to atomic_t2149 * @old: pointer to int value to compare with2150 * @new: int value to assign2151 *2152 * If (@v == @old), atomically updates @v to @new with acquire ordering.2153 * Otherwise, @v is not modified, @old is updated to the current value of @v,2154 * and relaxed ordering is provided.2155 *2156 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.2157 *2158 * Return: @true if the exchange occured, @false otherwise.2159 */2160static __always_inline bool2161raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)2162{2163#if defined(arch_atomic_try_cmpxchg_acquire)2164 return arch_atomic_try_cmpxchg_acquire(v, old, new);2165#elif defined(arch_atomic_try_cmpxchg_relaxed)2166 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);2167 __atomic_acquire_fence();2168 return ret;2169#elif defined(arch_atomic_try_cmpxchg)2170 return arch_atomic_try_cmpxchg(v, old, new);2171#else2172 int r, o = *old;2173 r = raw_atomic_cmpxchg_acquire(v, o, new);2174 if (unlikely(r != o))2175 *old = r;2176 return likely(r == o);2177#endif2178}2179 2180/**2181 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering2182 * @v: pointer to atomic_t2183 * @old: pointer to int value to compare with2184 * @new: int value to assign2185 *2186 * If (@v == @old), atomically updates @v to @new with release ordering.2187 * Otherwise, @v is not modified, @old is updated to the current value of @v,2188 * and relaxed ordering is provided.2189 *2190 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.2191 *2192 * Return: @true if the exchange occured, @false otherwise.2193 */2194static __always_inline bool2195raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)2196{2197#if defined(arch_atomic_try_cmpxchg_release)2198 return arch_atomic_try_cmpxchg_release(v, old, new);2199#elif defined(arch_atomic_try_cmpxchg_relaxed)2200 __atomic_release_fence();2201 return arch_atomic_try_cmpxchg_relaxed(v, old, new);2202#elif defined(arch_atomic_try_cmpxchg)2203 return arch_atomic_try_cmpxchg(v, old, new);2204#else2205 int r, o = *old;2206 r = raw_atomic_cmpxchg_release(v, o, new);2207 if (unlikely(r != o))2208 *old = r;2209 return likely(r == o);2210#endif2211}2212 2213/**2214 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering2215 * @v: pointer to atomic_t2216 * @old: pointer to int value to compare with2217 * @new: int value to assign2218 *2219 * If (@v == @old), atomically updates @v to @new with relaxed ordering.2220 * Otherwise, @v is not modified, @old is updated to the current value of @v,2221 * and relaxed ordering is provided.2222 *2223 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.2224 *2225 * Return: @true if the exchange occured, @false otherwise.2226 */2227static __always_inline bool2228raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)2229{2230#if defined(arch_atomic_try_cmpxchg_relaxed)2231 return arch_atomic_try_cmpxchg_relaxed(v, old, new);2232#elif defined(arch_atomic_try_cmpxchg)2233 return arch_atomic_try_cmpxchg(v, old, new);2234#else2235 int r, o = *old;2236 r = raw_atomic_cmpxchg_relaxed(v, o, new);2237 if (unlikely(r != o))2238 *old = r;2239 return likely(r == o);2240#endif2241}2242 2243/**2244 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering2245 * @i: int value to subtract2246 * @v: pointer to atomic_t2247 *2248 * Atomically updates @v to (@v - @i) with full ordering.2249 *2250 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.2251 *2252 * Return: @true if the resulting value of @v is zero, @false otherwise.2253 */2254static __always_inline bool2255raw_atomic_sub_and_test(int i, atomic_t *v)2256{2257#if defined(arch_atomic_sub_and_test)2258 return arch_atomic_sub_and_test(i, v);2259#else2260 return raw_atomic_sub_return(i, v) == 0;2261#endif2262}2263 2264/**2265 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering2266 * @v: pointer to atomic_t2267 *2268 * Atomically updates @v to (@v - 1) with full ordering.2269 *2270 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.2271 *2272 * Return: @true if the resulting value of @v is zero, @false otherwise.2273 */2274static __always_inline bool2275raw_atomic_dec_and_test(atomic_t *v)2276{2277#if defined(arch_atomic_dec_and_test)2278 return arch_atomic_dec_and_test(v);2279#else2280 return raw_atomic_dec_return(v) == 0;2281#endif2282}2283 2284/**2285 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering2286 * @v: pointer to atomic_t2287 *2288 * Atomically updates @v to (@v + 1) with full ordering.2289 *2290 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.2291 *2292 * Return: @true if the resulting value of @v is zero, @false otherwise.2293 */2294static __always_inline bool2295raw_atomic_inc_and_test(atomic_t *v)2296{2297#if defined(arch_atomic_inc_and_test)2298 return arch_atomic_inc_and_test(v);2299#else2300 return raw_atomic_inc_return(v) == 0;2301#endif2302}2303 2304/**2305 * raw_atomic_add_negative() - atomic add and test if negative with full ordering2306 * @i: int value to add2307 * @v: pointer to atomic_t2308 *2309 * Atomically updates @v to (@v + @i) with full ordering.2310 *2311 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.2312 *2313 * Return: @true if the resulting value of @v is negative, @false otherwise.2314 */2315static __always_inline bool2316raw_atomic_add_negative(int i, atomic_t *v)2317{2318#if defined(arch_atomic_add_negative)2319 return arch_atomic_add_negative(i, v);2320#elif defined(arch_atomic_add_negative_relaxed)2321 bool ret;2322 __atomic_pre_full_fence();2323 ret = arch_atomic_add_negative_relaxed(i, v);2324 __atomic_post_full_fence();2325 return ret;2326#else2327 return raw_atomic_add_return(i, v) < 0;2328#endif2329}2330 2331/**2332 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering2333 * @i: int value to add2334 * @v: pointer to atomic_t2335 *2336 * Atomically updates @v to (@v + @i) with acquire ordering.2337 *2338 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.2339 *2340 * Return: @true if the resulting value of @v is negative, @false otherwise.2341 */2342static __always_inline bool2343raw_atomic_add_negative_acquire(int i, atomic_t *v)2344{2345#if defined(arch_atomic_add_negative_acquire)2346 return arch_atomic_add_negative_acquire(i, v);2347#elif defined(arch_atomic_add_negative_relaxed)2348 bool ret = arch_atomic_add_negative_relaxed(i, v);2349 __atomic_acquire_fence();2350 return ret;2351#elif defined(arch_atomic_add_negative)2352 return arch_atomic_add_negative(i, v);2353#else2354 return raw_atomic_add_return_acquire(i, v) < 0;2355#endif2356}2357 2358/**2359 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering2360 * @i: int value to add2361 * @v: pointer to atomic_t2362 *2363 * Atomically updates @v to (@v + @i) with release ordering.2364 *2365 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.2366 *2367 * Return: @true if the resulting value of @v is negative, @false otherwise.2368 */2369static __always_inline bool2370raw_atomic_add_negative_release(int i, atomic_t *v)2371{2372#if defined(arch_atomic_add_negative_release)2373 return arch_atomic_add_negative_release(i, v);2374#elif defined(arch_atomic_add_negative_relaxed)2375 __atomic_release_fence();2376 return arch_atomic_add_negative_relaxed(i, v);2377#elif defined(arch_atomic_add_negative)2378 return arch_atomic_add_negative(i, v);2379#else2380 return raw_atomic_add_return_release(i, v) < 0;2381#endif2382}2383 2384/**2385 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering2386 * @i: int value to add2387 * @v: pointer to atomic_t2388 *2389 * Atomically updates @v to (@v + @i) with relaxed ordering.2390 *2391 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.2392 *2393 * Return: @true if the resulting value of @v is negative, @false otherwise.2394 */2395static __always_inline bool2396raw_atomic_add_negative_relaxed(int i, atomic_t *v)2397{2398#if defined(arch_atomic_add_negative_relaxed)2399 return arch_atomic_add_negative_relaxed(i, v);2400#elif defined(arch_atomic_add_negative)2401 return arch_atomic_add_negative(i, v);2402#else2403 return raw_atomic_add_return_relaxed(i, v) < 0;2404#endif2405}2406 2407/**2408 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering2409 * @v: pointer to atomic_t2410 * @a: int value to add2411 * @u: int value to compare with2412 *2413 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.2414 * Otherwise, @v is not modified and relaxed ordering is provided.2415 *2416 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.2417 *2418 * Return: The original value of @v.2419 */2420static __always_inline int2421raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)2422{2423#if defined(arch_atomic_fetch_add_unless)2424 return arch_atomic_fetch_add_unless(v, a, u);2425#else2426 int c = raw_atomic_read(v);2427 2428 do {2429 if (unlikely(c == u))2430 break;2431 } while (!raw_atomic_try_cmpxchg(v, &c, c + a));2432 2433 return c;2434#endif2435}2436 2437/**2438 * raw_atomic_add_unless() - atomic add unless value with full ordering2439 * @v: pointer to atomic_t2440 * @a: int value to add2441 * @u: int value to compare with2442 *2443 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.2444 * Otherwise, @v is not modified and relaxed ordering is provided.2445 *2446 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.2447 *2448 * Return: @true if @v was updated, @false otherwise.2449 */2450static __always_inline bool2451raw_atomic_add_unless(atomic_t *v, int a, int u)2452{2453#if defined(arch_atomic_add_unless)2454 return arch_atomic_add_unless(v, a, u);2455#else2456 return raw_atomic_fetch_add_unless(v, a, u) != u;2457#endif2458}2459 2460/**2461 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering2462 * @v: pointer to atomic_t2463 *2464 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.2465 * Otherwise, @v is not modified and relaxed ordering is provided.2466 *2467 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.2468 *2469 * Return: @true if @v was updated, @false otherwise.2470 */2471static __always_inline bool2472raw_atomic_inc_not_zero(atomic_t *v)2473{2474#if defined(arch_atomic_inc_not_zero)2475 return arch_atomic_inc_not_zero(v);2476#else2477 return raw_atomic_add_unless(v, 1, 0);2478#endif2479}2480 2481/**2482 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering2483 * @v: pointer to atomic_t2484 *2485 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.2486 * Otherwise, @v is not modified and relaxed ordering is provided.2487 *2488 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.2489 *2490 * Return: @true if @v was updated, @false otherwise.2491 */2492static __always_inline bool2493raw_atomic_inc_unless_negative(atomic_t *v)2494{2495#if defined(arch_atomic_inc_unless_negative)2496 return arch_atomic_inc_unless_negative(v);2497#else2498 int c = raw_atomic_read(v);2499 2500 do {2501 if (unlikely(c < 0))2502 return false;2503 } while (!raw_atomic_try_cmpxchg(v, &c, c + 1));2504 2505 return true;2506#endif2507}2508 2509/**2510 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering2511 * @v: pointer to atomic_t2512 *2513 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.2514 * Otherwise, @v is not modified and relaxed ordering is provided.2515 *2516 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.2517 *2518 * Return: @true if @v was updated, @false otherwise.2519 */2520static __always_inline bool2521raw_atomic_dec_unless_positive(atomic_t *v)2522{2523#if defined(arch_atomic_dec_unless_positive)2524 return arch_atomic_dec_unless_positive(v);2525#else2526 int c = raw_atomic_read(v);2527 2528 do {2529 if (unlikely(c > 0))2530 return false;2531 } while (!raw_atomic_try_cmpxchg(v, &c, c - 1));2532 2533 return true;2534#endif2535}2536 2537/**2538 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering2539 * @v: pointer to atomic_t2540 *2541 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.2542 * Otherwise, @v is not modified and relaxed ordering is provided.2543 *2544 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.2545 *2546 * Return: The old value of (@v - 1), regardless of whether @v was updated.2547 */2548static __always_inline int2549raw_atomic_dec_if_positive(atomic_t *v)2550{2551#if defined(arch_atomic_dec_if_positive)2552 return arch_atomic_dec_if_positive(v);2553#else2554 int dec, c = raw_atomic_read(v);2555 2556 do {2557 dec = c - 1;2558 if (unlikely(dec < 0))2559 break;2560 } while (!raw_atomic_try_cmpxchg(v, &c, dec));2561 2562 return dec;2563#endif2564}2565 2566#ifdef CONFIG_GENERIC_ATOMIC642567#include <asm-generic/atomic64.h>2568#endif2569 2570/**2571 * raw_atomic64_read() - atomic load with relaxed ordering2572 * @v: pointer to atomic64_t2573 *2574 * Atomically loads the value of @v with relaxed ordering.2575 *2576 * Safe to use in noinstr code; prefer atomic64_read() elsewhere.2577 *2578 * Return: The value loaded from @v.2579 */2580static __always_inline s642581raw_atomic64_read(const atomic64_t *v)2582{2583 return arch_atomic64_read(v);2584}2585 2586/**2587 * raw_atomic64_read_acquire() - atomic load with acquire ordering2588 * @v: pointer to atomic64_t2589 *2590 * Atomically loads the value of @v with acquire ordering.2591 *2592 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.2593 *2594 * Return: The value loaded from @v.2595 */2596static __always_inline s642597raw_atomic64_read_acquire(const atomic64_t *v)2598{2599#if defined(arch_atomic64_read_acquire)2600 return arch_atomic64_read_acquire(v);2601#else2602 s64 ret;2603 2604 if (__native_word(atomic64_t)) {2605 ret = smp_load_acquire(&(v)->counter);2606 } else {2607 ret = raw_atomic64_read(v);2608 __atomic_acquire_fence();2609 }2610 2611 return ret;2612#endif2613}2614 2615/**2616 * raw_atomic64_set() - atomic set with relaxed ordering2617 * @v: pointer to atomic64_t2618 * @i: s64 value to assign2619 *2620 * Atomically sets @v to @i with relaxed ordering.2621 *2622 * Safe to use in noinstr code; prefer atomic64_set() elsewhere.2623 *2624 * Return: Nothing.2625 */2626static __always_inline void2627raw_atomic64_set(atomic64_t *v, s64 i)2628{2629 arch_atomic64_set(v, i);2630}2631 2632/**2633 * raw_atomic64_set_release() - atomic set with release ordering2634 * @v: pointer to atomic64_t2635 * @i: s64 value to assign2636 *2637 * Atomically sets @v to @i with release ordering.2638 *2639 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.2640 *2641 * Return: Nothing.2642 */2643static __always_inline void2644raw_atomic64_set_release(atomic64_t *v, s64 i)2645{2646#if defined(arch_atomic64_set_release)2647 arch_atomic64_set_release(v, i);2648#else2649 if (__native_word(atomic64_t)) {2650 smp_store_release(&(v)->counter, i);2651 } else {2652 __atomic_release_fence();2653 raw_atomic64_set(v, i);2654 }2655#endif2656}2657 2658/**2659 * raw_atomic64_add() - atomic add with relaxed ordering2660 * @i: s64 value to add2661 * @v: pointer to atomic64_t2662 *2663 * Atomically updates @v to (@v + @i) with relaxed ordering.2664 *2665 * Safe to use in noinstr code; prefer atomic64_add() elsewhere.2666 *2667 * Return: Nothing.2668 */2669static __always_inline void2670raw_atomic64_add(s64 i, atomic64_t *v)2671{2672 arch_atomic64_add(i, v);2673}2674 2675/**2676 * raw_atomic64_add_return() - atomic add with full ordering2677 * @i: s64 value to add2678 * @v: pointer to atomic64_t2679 *2680 * Atomically updates @v to (@v + @i) with full ordering.2681 *2682 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.2683 *2684 * Return: The updated value of @v.2685 */2686static __always_inline s642687raw_atomic64_add_return(s64 i, atomic64_t *v)2688{2689#if defined(arch_atomic64_add_return)2690 return arch_atomic64_add_return(i, v);2691#elif defined(arch_atomic64_add_return_relaxed)2692 s64 ret;2693 __atomic_pre_full_fence();2694 ret = arch_atomic64_add_return_relaxed(i, v);2695 __atomic_post_full_fence();2696 return ret;2697#else2698#error "Unable to define raw_atomic64_add_return"2699#endif2700}2701 2702/**2703 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering2704 * @i: s64 value to add2705 * @v: pointer to atomic64_t2706 *2707 * Atomically updates @v to (@v + @i) with acquire ordering.2708 *2709 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.2710 *2711 * Return: The updated value of @v.2712 */2713static __always_inline s642714raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)2715{2716#if defined(arch_atomic64_add_return_acquire)2717 return arch_atomic64_add_return_acquire(i, v);2718#elif defined(arch_atomic64_add_return_relaxed)2719 s64 ret = arch_atomic64_add_return_relaxed(i, v);2720 __atomic_acquire_fence();2721 return ret;2722#elif defined(arch_atomic64_add_return)2723 return arch_atomic64_add_return(i, v);2724#else2725#error "Unable to define raw_atomic64_add_return_acquire"2726#endif2727}2728 2729/**2730 * raw_atomic64_add_return_release() - atomic add with release ordering2731 * @i: s64 value to add2732 * @v: pointer to atomic64_t2733 *2734 * Atomically updates @v to (@v + @i) with release ordering.2735 *2736 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.2737 *2738 * Return: The updated value of @v.2739 */2740static __always_inline s642741raw_atomic64_add_return_release(s64 i, atomic64_t *v)2742{2743#if defined(arch_atomic64_add_return_release)2744 return arch_atomic64_add_return_release(i, v);2745#elif defined(arch_atomic64_add_return_relaxed)2746 __atomic_release_fence();2747 return arch_atomic64_add_return_relaxed(i, v);2748#elif defined(arch_atomic64_add_return)2749 return arch_atomic64_add_return(i, v);2750#else2751#error "Unable to define raw_atomic64_add_return_release"2752#endif2753}2754 2755/**2756 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering2757 * @i: s64 value to add2758 * @v: pointer to atomic64_t2759 *2760 * Atomically updates @v to (@v + @i) with relaxed ordering.2761 *2762 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.2763 *2764 * Return: The updated value of @v.2765 */2766static __always_inline s642767raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)2768{2769#if defined(arch_atomic64_add_return_relaxed)2770 return arch_atomic64_add_return_relaxed(i, v);2771#elif defined(arch_atomic64_add_return)2772 return arch_atomic64_add_return(i, v);2773#else2774#error "Unable to define raw_atomic64_add_return_relaxed"2775#endif2776}2777 2778/**2779 * raw_atomic64_fetch_add() - atomic add with full ordering2780 * @i: s64 value to add2781 * @v: pointer to atomic64_t2782 *2783 * Atomically updates @v to (@v + @i) with full ordering.2784 *2785 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.2786 *2787 * Return: The original value of @v.2788 */2789static __always_inline s642790raw_atomic64_fetch_add(s64 i, atomic64_t *v)2791{2792#if defined(arch_atomic64_fetch_add)2793 return arch_atomic64_fetch_add(i, v);2794#elif defined(arch_atomic64_fetch_add_relaxed)2795 s64 ret;2796 __atomic_pre_full_fence();2797 ret = arch_atomic64_fetch_add_relaxed(i, v);2798 __atomic_post_full_fence();2799 return ret;2800#else2801#error "Unable to define raw_atomic64_fetch_add"2802#endif2803}2804 2805/**2806 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering2807 * @i: s64 value to add2808 * @v: pointer to atomic64_t2809 *2810 * Atomically updates @v to (@v + @i) with acquire ordering.2811 *2812 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.2813 *2814 * Return: The original value of @v.2815 */2816static __always_inline s642817raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)2818{2819#if defined(arch_atomic64_fetch_add_acquire)2820 return arch_atomic64_fetch_add_acquire(i, v);2821#elif defined(arch_atomic64_fetch_add_relaxed)2822 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);2823 __atomic_acquire_fence();2824 return ret;2825#elif defined(arch_atomic64_fetch_add)2826 return arch_atomic64_fetch_add(i, v);2827#else2828#error "Unable to define raw_atomic64_fetch_add_acquire"2829#endif2830}2831 2832/**2833 * raw_atomic64_fetch_add_release() - atomic add with release ordering2834 * @i: s64 value to add2835 * @v: pointer to atomic64_t2836 *2837 * Atomically updates @v to (@v + @i) with release ordering.2838 *2839 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.2840 *2841 * Return: The original value of @v.2842 */2843static __always_inline s642844raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)2845{2846#if defined(arch_atomic64_fetch_add_release)2847 return arch_atomic64_fetch_add_release(i, v);2848#elif defined(arch_atomic64_fetch_add_relaxed)2849 __atomic_release_fence();2850 return arch_atomic64_fetch_add_relaxed(i, v);2851#elif defined(arch_atomic64_fetch_add)2852 return arch_atomic64_fetch_add(i, v);2853#else2854#error "Unable to define raw_atomic64_fetch_add_release"2855#endif2856}2857 2858/**2859 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering2860 * @i: s64 value to add2861 * @v: pointer to atomic64_t2862 *2863 * Atomically updates @v to (@v + @i) with relaxed ordering.2864 *2865 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.2866 *2867 * Return: The original value of @v.2868 */2869static __always_inline s642870raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)2871{2872#if defined(arch_atomic64_fetch_add_relaxed)2873 return arch_atomic64_fetch_add_relaxed(i, v);2874#elif defined(arch_atomic64_fetch_add)2875 return arch_atomic64_fetch_add(i, v);2876#else2877#error "Unable to define raw_atomic64_fetch_add_relaxed"2878#endif2879}2880 2881/**2882 * raw_atomic64_sub() - atomic subtract with relaxed ordering2883 * @i: s64 value to subtract2884 * @v: pointer to atomic64_t2885 *2886 * Atomically updates @v to (@v - @i) with relaxed ordering.2887 *2888 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.2889 *2890 * Return: Nothing.2891 */2892static __always_inline void2893raw_atomic64_sub(s64 i, atomic64_t *v)2894{2895 arch_atomic64_sub(i, v);2896}2897 2898/**2899 * raw_atomic64_sub_return() - atomic subtract with full ordering2900 * @i: s64 value to subtract2901 * @v: pointer to atomic64_t2902 *2903 * Atomically updates @v to (@v - @i) with full ordering.2904 *2905 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.2906 *2907 * Return: The updated value of @v.2908 */2909static __always_inline s642910raw_atomic64_sub_return(s64 i, atomic64_t *v)2911{2912#if defined(arch_atomic64_sub_return)2913 return arch_atomic64_sub_return(i, v);2914#elif defined(arch_atomic64_sub_return_relaxed)2915 s64 ret;2916 __atomic_pre_full_fence();2917 ret = arch_atomic64_sub_return_relaxed(i, v);2918 __atomic_post_full_fence();2919 return ret;2920#else2921#error "Unable to define raw_atomic64_sub_return"2922#endif2923}2924 2925/**2926 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering2927 * @i: s64 value to subtract2928 * @v: pointer to atomic64_t2929 *2930 * Atomically updates @v to (@v - @i) with acquire ordering.2931 *2932 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.2933 *2934 * Return: The updated value of @v.2935 */2936static __always_inline s642937raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)2938{2939#if defined(arch_atomic64_sub_return_acquire)2940 return arch_atomic64_sub_return_acquire(i, v);2941#elif defined(arch_atomic64_sub_return_relaxed)2942 s64 ret = arch_atomic64_sub_return_relaxed(i, v);2943 __atomic_acquire_fence();2944 return ret;2945#elif defined(arch_atomic64_sub_return)2946 return arch_atomic64_sub_return(i, v);2947#else2948#error "Unable to define raw_atomic64_sub_return_acquire"2949#endif2950}2951 2952/**2953 * raw_atomic64_sub_return_release() - atomic subtract with release ordering2954 * @i: s64 value to subtract2955 * @v: pointer to atomic64_t2956 *2957 * Atomically updates @v to (@v - @i) with release ordering.2958 *2959 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.2960 *2961 * Return: The updated value of @v.2962 */2963static __always_inline s642964raw_atomic64_sub_return_release(s64 i, atomic64_t *v)2965{2966#if defined(arch_atomic64_sub_return_release)2967 return arch_atomic64_sub_return_release(i, v);2968#elif defined(arch_atomic64_sub_return_relaxed)2969 __atomic_release_fence();2970 return arch_atomic64_sub_return_relaxed(i, v);2971#elif defined(arch_atomic64_sub_return)2972 return arch_atomic64_sub_return(i, v);2973#else2974#error "Unable to define raw_atomic64_sub_return_release"2975#endif2976}2977 2978/**2979 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering2980 * @i: s64 value to subtract2981 * @v: pointer to atomic64_t2982 *2983 * Atomically updates @v to (@v - @i) with relaxed ordering.2984 *2985 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.2986 *2987 * Return: The updated value of @v.2988 */2989static __always_inline s642990raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)2991{2992#if defined(arch_atomic64_sub_return_relaxed)2993 return arch_atomic64_sub_return_relaxed(i, v);2994#elif defined(arch_atomic64_sub_return)2995 return arch_atomic64_sub_return(i, v);2996#else2997#error "Unable to define raw_atomic64_sub_return_relaxed"2998#endif2999}3000 3001/**3002 * raw_atomic64_fetch_sub() - atomic subtract with full ordering3003 * @i: s64 value to subtract3004 * @v: pointer to atomic64_t3005 *3006 * Atomically updates @v to (@v - @i) with full ordering.3007 *3008 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.3009 *3010 * Return: The original value of @v.3011 */3012static __always_inline s643013raw_atomic64_fetch_sub(s64 i, atomic64_t *v)3014{3015#if defined(arch_atomic64_fetch_sub)3016 return arch_atomic64_fetch_sub(i, v);3017#elif defined(arch_atomic64_fetch_sub_relaxed)3018 s64 ret;3019 __atomic_pre_full_fence();3020 ret = arch_atomic64_fetch_sub_relaxed(i, v);3021 __atomic_post_full_fence();3022 return ret;3023#else3024#error "Unable to define raw_atomic64_fetch_sub"3025#endif3026}3027 3028/**3029 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering3030 * @i: s64 value to subtract3031 * @v: pointer to atomic64_t3032 *3033 * Atomically updates @v to (@v - @i) with acquire ordering.3034 *3035 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.3036 *3037 * Return: The original value of @v.3038 */3039static __always_inline s643040raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)3041{3042#if defined(arch_atomic64_fetch_sub_acquire)3043 return arch_atomic64_fetch_sub_acquire(i, v);3044#elif defined(arch_atomic64_fetch_sub_relaxed)3045 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);3046 __atomic_acquire_fence();3047 return ret;3048#elif defined(arch_atomic64_fetch_sub)3049 return arch_atomic64_fetch_sub(i, v);3050#else3051#error "Unable to define raw_atomic64_fetch_sub_acquire"3052#endif3053}3054 3055/**3056 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering3057 * @i: s64 value to subtract3058 * @v: pointer to atomic64_t3059 *3060 * Atomically updates @v to (@v - @i) with release ordering.3061 *3062 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.3063 *3064 * Return: The original value of @v.3065 */3066static __always_inline s643067raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)3068{3069#if defined(arch_atomic64_fetch_sub_release)3070 return arch_atomic64_fetch_sub_release(i, v);3071#elif defined(arch_atomic64_fetch_sub_relaxed)3072 __atomic_release_fence();3073 return arch_atomic64_fetch_sub_relaxed(i, v);3074#elif defined(arch_atomic64_fetch_sub)3075 return arch_atomic64_fetch_sub(i, v);3076#else3077#error "Unable to define raw_atomic64_fetch_sub_release"3078#endif3079}3080 3081/**3082 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering3083 * @i: s64 value to subtract3084 * @v: pointer to atomic64_t3085 *3086 * Atomically updates @v to (@v - @i) with relaxed ordering.3087 *3088 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.3089 *3090 * Return: The original value of @v.3091 */3092static __always_inline s643093raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)3094{3095#if defined(arch_atomic64_fetch_sub_relaxed)3096 return arch_atomic64_fetch_sub_relaxed(i, v);3097#elif defined(arch_atomic64_fetch_sub)3098 return arch_atomic64_fetch_sub(i, v);3099#else3100#error "Unable to define raw_atomic64_fetch_sub_relaxed"3101#endif3102}3103 3104/**3105 * raw_atomic64_inc() - atomic increment with relaxed ordering3106 * @v: pointer to atomic64_t3107 *3108 * Atomically updates @v to (@v + 1) with relaxed ordering.3109 *3110 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.3111 *3112 * Return: Nothing.3113 */3114static __always_inline void3115raw_atomic64_inc(atomic64_t *v)3116{3117#if defined(arch_atomic64_inc)3118 arch_atomic64_inc(v);3119#else3120 raw_atomic64_add(1, v);3121#endif3122}3123 3124/**3125 * raw_atomic64_inc_return() - atomic increment with full ordering3126 * @v: pointer to atomic64_t3127 *3128 * Atomically updates @v to (@v + 1) with full ordering.3129 *3130 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.3131 *3132 * Return: The updated value of @v.3133 */3134static __always_inline s643135raw_atomic64_inc_return(atomic64_t *v)3136{3137#if defined(arch_atomic64_inc_return)3138 return arch_atomic64_inc_return(v);3139#elif defined(arch_atomic64_inc_return_relaxed)3140 s64 ret;3141 __atomic_pre_full_fence();3142 ret = arch_atomic64_inc_return_relaxed(v);3143 __atomic_post_full_fence();3144 return ret;3145#else3146 return raw_atomic64_add_return(1, v);3147#endif3148}3149 3150/**3151 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering3152 * @v: pointer to atomic64_t3153 *3154 * Atomically updates @v to (@v + 1) with acquire ordering.3155 *3156 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.3157 *3158 * Return: The updated value of @v.3159 */3160static __always_inline s643161raw_atomic64_inc_return_acquire(atomic64_t *v)3162{3163#if defined(arch_atomic64_inc_return_acquire)3164 return arch_atomic64_inc_return_acquire(v);3165#elif defined(arch_atomic64_inc_return_relaxed)3166 s64 ret = arch_atomic64_inc_return_relaxed(v);3167 __atomic_acquire_fence();3168 return ret;3169#elif defined(arch_atomic64_inc_return)3170 return arch_atomic64_inc_return(v);3171#else3172 return raw_atomic64_add_return_acquire(1, v);3173#endif3174}3175 3176/**3177 * raw_atomic64_inc_return_release() - atomic increment with release ordering3178 * @v: pointer to atomic64_t3179 *3180 * Atomically updates @v to (@v + 1) with release ordering.3181 *3182 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.3183 *3184 * Return: The updated value of @v.3185 */3186static __always_inline s643187raw_atomic64_inc_return_release(atomic64_t *v)3188{3189#if defined(arch_atomic64_inc_return_release)3190 return arch_atomic64_inc_return_release(v);3191#elif defined(arch_atomic64_inc_return_relaxed)3192 __atomic_release_fence();3193 return arch_atomic64_inc_return_relaxed(v);3194#elif defined(arch_atomic64_inc_return)3195 return arch_atomic64_inc_return(v);3196#else3197 return raw_atomic64_add_return_release(1, v);3198#endif3199}3200 3201/**3202 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering3203 * @v: pointer to atomic64_t3204 *3205 * Atomically updates @v to (@v + 1) with relaxed ordering.3206 *3207 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.3208 *3209 * Return: The updated value of @v.3210 */3211static __always_inline s643212raw_atomic64_inc_return_relaxed(atomic64_t *v)3213{3214#if defined(arch_atomic64_inc_return_relaxed)3215 return arch_atomic64_inc_return_relaxed(v);3216#elif defined(arch_atomic64_inc_return)3217 return arch_atomic64_inc_return(v);3218#else3219 return raw_atomic64_add_return_relaxed(1, v);3220#endif3221}3222 3223/**3224 * raw_atomic64_fetch_inc() - atomic increment with full ordering3225 * @v: pointer to atomic64_t3226 *3227 * Atomically updates @v to (@v + 1) with full ordering.3228 *3229 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.3230 *3231 * Return: The original value of @v.3232 */3233static __always_inline s643234raw_atomic64_fetch_inc(atomic64_t *v)3235{3236#if defined(arch_atomic64_fetch_inc)3237 return arch_atomic64_fetch_inc(v);3238#elif defined(arch_atomic64_fetch_inc_relaxed)3239 s64 ret;3240 __atomic_pre_full_fence();3241 ret = arch_atomic64_fetch_inc_relaxed(v);3242 __atomic_post_full_fence();3243 return ret;3244#else3245 return raw_atomic64_fetch_add(1, v);3246#endif3247}3248 3249/**3250 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering3251 * @v: pointer to atomic64_t3252 *3253 * Atomically updates @v to (@v + 1) with acquire ordering.3254 *3255 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.3256 *3257 * Return: The original value of @v.3258 */3259static __always_inline s643260raw_atomic64_fetch_inc_acquire(atomic64_t *v)3261{3262#if defined(arch_atomic64_fetch_inc_acquire)3263 return arch_atomic64_fetch_inc_acquire(v);3264#elif defined(arch_atomic64_fetch_inc_relaxed)3265 s64 ret = arch_atomic64_fetch_inc_relaxed(v);3266 __atomic_acquire_fence();3267 return ret;3268#elif defined(arch_atomic64_fetch_inc)3269 return arch_atomic64_fetch_inc(v);3270#else3271 return raw_atomic64_fetch_add_acquire(1, v);3272#endif3273}3274 3275/**3276 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering3277 * @v: pointer to atomic64_t3278 *3279 * Atomically updates @v to (@v + 1) with release ordering.3280 *3281 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.3282 *3283 * Return: The original value of @v.3284 */3285static __always_inline s643286raw_atomic64_fetch_inc_release(atomic64_t *v)3287{3288#if defined(arch_atomic64_fetch_inc_release)3289 return arch_atomic64_fetch_inc_release(v);3290#elif defined(arch_atomic64_fetch_inc_relaxed)3291 __atomic_release_fence();3292 return arch_atomic64_fetch_inc_relaxed(v);3293#elif defined(arch_atomic64_fetch_inc)3294 return arch_atomic64_fetch_inc(v);3295#else3296 return raw_atomic64_fetch_add_release(1, v);3297#endif3298}3299 3300/**3301 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering3302 * @v: pointer to atomic64_t3303 *3304 * Atomically updates @v to (@v + 1) with relaxed ordering.3305 *3306 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.3307 *3308 * Return: The original value of @v.3309 */3310static __always_inline s643311raw_atomic64_fetch_inc_relaxed(atomic64_t *v)3312{3313#if defined(arch_atomic64_fetch_inc_relaxed)3314 return arch_atomic64_fetch_inc_relaxed(v);3315#elif defined(arch_atomic64_fetch_inc)3316 return arch_atomic64_fetch_inc(v);3317#else3318 return raw_atomic64_fetch_add_relaxed(1, v);3319#endif3320}3321 3322/**3323 * raw_atomic64_dec() - atomic decrement with relaxed ordering3324 * @v: pointer to atomic64_t3325 *3326 * Atomically updates @v to (@v - 1) with relaxed ordering.3327 *3328 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.3329 *3330 * Return: Nothing.3331 */3332static __always_inline void3333raw_atomic64_dec(atomic64_t *v)3334{3335#if defined(arch_atomic64_dec)3336 arch_atomic64_dec(v);3337#else3338 raw_atomic64_sub(1, v);3339#endif3340}3341 3342/**3343 * raw_atomic64_dec_return() - atomic decrement with full ordering3344 * @v: pointer to atomic64_t3345 *3346 * Atomically updates @v to (@v - 1) with full ordering.3347 *3348 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.3349 *3350 * Return: The updated value of @v.3351 */3352static __always_inline s643353raw_atomic64_dec_return(atomic64_t *v)3354{3355#if defined(arch_atomic64_dec_return)3356 return arch_atomic64_dec_return(v);3357#elif defined(arch_atomic64_dec_return_relaxed)3358 s64 ret;3359 __atomic_pre_full_fence();3360 ret = arch_atomic64_dec_return_relaxed(v);3361 __atomic_post_full_fence();3362 return ret;3363#else3364 return raw_atomic64_sub_return(1, v);3365#endif3366}3367 3368/**3369 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering3370 * @v: pointer to atomic64_t3371 *3372 * Atomically updates @v to (@v - 1) with acquire ordering.3373 *3374 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.3375 *3376 * Return: The updated value of @v.3377 */3378static __always_inline s643379raw_atomic64_dec_return_acquire(atomic64_t *v)3380{3381#if defined(arch_atomic64_dec_return_acquire)3382 return arch_atomic64_dec_return_acquire(v);3383#elif defined(arch_atomic64_dec_return_relaxed)3384 s64 ret = arch_atomic64_dec_return_relaxed(v);3385 __atomic_acquire_fence();3386 return ret;3387#elif defined(arch_atomic64_dec_return)3388 return arch_atomic64_dec_return(v);3389#else3390 return raw_atomic64_sub_return_acquire(1, v);3391#endif3392}3393 3394/**3395 * raw_atomic64_dec_return_release() - atomic decrement with release ordering3396 * @v: pointer to atomic64_t3397 *3398 * Atomically updates @v to (@v - 1) with release ordering.3399 *3400 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.3401 *3402 * Return: The updated value of @v.3403 */3404static __always_inline s643405raw_atomic64_dec_return_release(atomic64_t *v)3406{3407#if defined(arch_atomic64_dec_return_release)3408 return arch_atomic64_dec_return_release(v);3409#elif defined(arch_atomic64_dec_return_relaxed)3410 __atomic_release_fence();3411 return arch_atomic64_dec_return_relaxed(v);3412#elif defined(arch_atomic64_dec_return)3413 return arch_atomic64_dec_return(v);3414#else3415 return raw_atomic64_sub_return_release(1, v);3416#endif3417}3418 3419/**3420 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering3421 * @v: pointer to atomic64_t3422 *3423 * Atomically updates @v to (@v - 1) with relaxed ordering.3424 *3425 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.3426 *3427 * Return: The updated value of @v.3428 */3429static __always_inline s643430raw_atomic64_dec_return_relaxed(atomic64_t *v)3431{3432#if defined(arch_atomic64_dec_return_relaxed)3433 return arch_atomic64_dec_return_relaxed(v);3434#elif defined(arch_atomic64_dec_return)3435 return arch_atomic64_dec_return(v);3436#else3437 return raw_atomic64_sub_return_relaxed(1, v);3438#endif3439}3440 3441/**3442 * raw_atomic64_fetch_dec() - atomic decrement with full ordering3443 * @v: pointer to atomic64_t3444 *3445 * Atomically updates @v to (@v - 1) with full ordering.3446 *3447 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.3448 *3449 * Return: The original value of @v.3450 */3451static __always_inline s643452raw_atomic64_fetch_dec(atomic64_t *v)3453{3454#if defined(arch_atomic64_fetch_dec)3455 return arch_atomic64_fetch_dec(v);3456#elif defined(arch_atomic64_fetch_dec_relaxed)3457 s64 ret;3458 __atomic_pre_full_fence();3459 ret = arch_atomic64_fetch_dec_relaxed(v);3460 __atomic_post_full_fence();3461 return ret;3462#else3463 return raw_atomic64_fetch_sub(1, v);3464#endif3465}3466 3467/**3468 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering3469 * @v: pointer to atomic64_t3470 *3471 * Atomically updates @v to (@v - 1) with acquire ordering.3472 *3473 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.3474 *3475 * Return: The original value of @v.3476 */3477static __always_inline s643478raw_atomic64_fetch_dec_acquire(atomic64_t *v)3479{3480#if defined(arch_atomic64_fetch_dec_acquire)3481 return arch_atomic64_fetch_dec_acquire(v);3482#elif defined(arch_atomic64_fetch_dec_relaxed)3483 s64 ret = arch_atomic64_fetch_dec_relaxed(v);3484 __atomic_acquire_fence();3485 return ret;3486#elif defined(arch_atomic64_fetch_dec)3487 return arch_atomic64_fetch_dec(v);3488#else3489 return raw_atomic64_fetch_sub_acquire(1, v);3490#endif3491}3492 3493/**3494 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering3495 * @v: pointer to atomic64_t3496 *3497 * Atomically updates @v to (@v - 1) with release ordering.3498 *3499 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.3500 *3501 * Return: The original value of @v.3502 */3503static __always_inline s643504raw_atomic64_fetch_dec_release(atomic64_t *v)3505{3506#if defined(arch_atomic64_fetch_dec_release)3507 return arch_atomic64_fetch_dec_release(v);3508#elif defined(arch_atomic64_fetch_dec_relaxed)3509 __atomic_release_fence();3510 return arch_atomic64_fetch_dec_relaxed(v);3511#elif defined(arch_atomic64_fetch_dec)3512 return arch_atomic64_fetch_dec(v);3513#else3514 return raw_atomic64_fetch_sub_release(1, v);3515#endif3516}3517 3518/**3519 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering3520 * @v: pointer to atomic64_t3521 *3522 * Atomically updates @v to (@v - 1) with relaxed ordering.3523 *3524 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.3525 *3526 * Return: The original value of @v.3527 */3528static __always_inline s643529raw_atomic64_fetch_dec_relaxed(atomic64_t *v)3530{3531#if defined(arch_atomic64_fetch_dec_relaxed)3532 return arch_atomic64_fetch_dec_relaxed(v);3533#elif defined(arch_atomic64_fetch_dec)3534 return arch_atomic64_fetch_dec(v);3535#else3536 return raw_atomic64_fetch_sub_relaxed(1, v);3537#endif3538}3539 3540/**3541 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering3542 * @i: s64 value3543 * @v: pointer to atomic64_t3544 *3545 * Atomically updates @v to (@v & @i) with relaxed ordering.3546 *3547 * Safe to use in noinstr code; prefer atomic64_and() elsewhere.3548 *3549 * Return: Nothing.3550 */3551static __always_inline void3552raw_atomic64_and(s64 i, atomic64_t *v)3553{3554 arch_atomic64_and(i, v);3555}3556 3557/**3558 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering3559 * @i: s64 value3560 * @v: pointer to atomic64_t3561 *3562 * Atomically updates @v to (@v & @i) with full ordering.3563 *3564 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.3565 *3566 * Return: The original value of @v.3567 */3568static __always_inline s643569raw_atomic64_fetch_and(s64 i, atomic64_t *v)3570{3571#if defined(arch_atomic64_fetch_and)3572 return arch_atomic64_fetch_and(i, v);3573#elif defined(arch_atomic64_fetch_and_relaxed)3574 s64 ret;3575 __atomic_pre_full_fence();3576 ret = arch_atomic64_fetch_and_relaxed(i, v);3577 __atomic_post_full_fence();3578 return ret;3579#else3580#error "Unable to define raw_atomic64_fetch_and"3581#endif3582}3583 3584/**3585 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering3586 * @i: s64 value3587 * @v: pointer to atomic64_t3588 *3589 * Atomically updates @v to (@v & @i) with acquire ordering.3590 *3591 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.3592 *3593 * Return: The original value of @v.3594 */3595static __always_inline s643596raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)3597{3598#if defined(arch_atomic64_fetch_and_acquire)3599 return arch_atomic64_fetch_and_acquire(i, v);3600#elif defined(arch_atomic64_fetch_and_relaxed)3601 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);3602 __atomic_acquire_fence();3603 return ret;3604#elif defined(arch_atomic64_fetch_and)3605 return arch_atomic64_fetch_and(i, v);3606#else3607#error "Unable to define raw_atomic64_fetch_and_acquire"3608#endif3609}3610 3611/**3612 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering3613 * @i: s64 value3614 * @v: pointer to atomic64_t3615 *3616 * Atomically updates @v to (@v & @i) with release ordering.3617 *3618 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.3619 *3620 * Return: The original value of @v.3621 */3622static __always_inline s643623raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)3624{3625#if defined(arch_atomic64_fetch_and_release)3626 return arch_atomic64_fetch_and_release(i, v);3627#elif defined(arch_atomic64_fetch_and_relaxed)3628 __atomic_release_fence();3629 return arch_atomic64_fetch_and_relaxed(i, v);3630#elif defined(arch_atomic64_fetch_and)3631 return arch_atomic64_fetch_and(i, v);3632#else3633#error "Unable to define raw_atomic64_fetch_and_release"3634#endif3635}3636 3637/**3638 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering3639 * @i: s64 value3640 * @v: pointer to atomic64_t3641 *3642 * Atomically updates @v to (@v & @i) with relaxed ordering.3643 *3644 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.3645 *3646 * Return: The original value of @v.3647 */3648static __always_inline s643649raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)3650{3651#if defined(arch_atomic64_fetch_and_relaxed)3652 return arch_atomic64_fetch_and_relaxed(i, v);3653#elif defined(arch_atomic64_fetch_and)3654 return arch_atomic64_fetch_and(i, v);3655#else3656#error "Unable to define raw_atomic64_fetch_and_relaxed"3657#endif3658}3659 3660/**3661 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering3662 * @i: s64 value3663 * @v: pointer to atomic64_t3664 *3665 * Atomically updates @v to (@v & ~@i) with relaxed ordering.3666 *3667 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.3668 *3669 * Return: Nothing.3670 */3671static __always_inline void3672raw_atomic64_andnot(s64 i, atomic64_t *v)3673{3674#if defined(arch_atomic64_andnot)3675 arch_atomic64_andnot(i, v);3676#else3677 raw_atomic64_and(~i, v);3678#endif3679}3680 3681/**3682 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering3683 * @i: s64 value3684 * @v: pointer to atomic64_t3685 *3686 * Atomically updates @v to (@v & ~@i) with full ordering.3687 *3688 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.3689 *3690 * Return: The original value of @v.3691 */3692static __always_inline s643693raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)3694{3695#if defined(arch_atomic64_fetch_andnot)3696 return arch_atomic64_fetch_andnot(i, v);3697#elif defined(arch_atomic64_fetch_andnot_relaxed)3698 s64 ret;3699 __atomic_pre_full_fence();3700 ret = arch_atomic64_fetch_andnot_relaxed(i, v);3701 __atomic_post_full_fence();3702 return ret;3703#else3704 return raw_atomic64_fetch_and(~i, v);3705#endif3706}3707 3708/**3709 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering3710 * @i: s64 value3711 * @v: pointer to atomic64_t3712 *3713 * Atomically updates @v to (@v & ~@i) with acquire ordering.3714 *3715 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.3716 *3717 * Return: The original value of @v.3718 */3719static __always_inline s643720raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)3721{3722#if defined(arch_atomic64_fetch_andnot_acquire)3723 return arch_atomic64_fetch_andnot_acquire(i, v);3724#elif defined(arch_atomic64_fetch_andnot_relaxed)3725 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);3726 __atomic_acquire_fence();3727 return ret;3728#elif defined(arch_atomic64_fetch_andnot)3729 return arch_atomic64_fetch_andnot(i, v);3730#else3731 return raw_atomic64_fetch_and_acquire(~i, v);3732#endif3733}3734 3735/**3736 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering3737 * @i: s64 value3738 * @v: pointer to atomic64_t3739 *3740 * Atomically updates @v to (@v & ~@i) with release ordering.3741 *3742 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.3743 *3744 * Return: The original value of @v.3745 */3746static __always_inline s643747raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)3748{3749#if defined(arch_atomic64_fetch_andnot_release)3750 return arch_atomic64_fetch_andnot_release(i, v);3751#elif defined(arch_atomic64_fetch_andnot_relaxed)3752 __atomic_release_fence();3753 return arch_atomic64_fetch_andnot_relaxed(i, v);3754#elif defined(arch_atomic64_fetch_andnot)3755 return arch_atomic64_fetch_andnot(i, v);3756#else3757 return raw_atomic64_fetch_and_release(~i, v);3758#endif3759}3760 3761/**3762 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering3763 * @i: s64 value3764 * @v: pointer to atomic64_t3765 *3766 * Atomically updates @v to (@v & ~@i) with relaxed ordering.3767 *3768 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.3769 *3770 * Return: The original value of @v.3771 */3772static __always_inline s643773raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)3774{3775#if defined(arch_atomic64_fetch_andnot_relaxed)3776 return arch_atomic64_fetch_andnot_relaxed(i, v);3777#elif defined(arch_atomic64_fetch_andnot)3778 return arch_atomic64_fetch_andnot(i, v);3779#else3780 return raw_atomic64_fetch_and_relaxed(~i, v);3781#endif3782}3783 3784/**3785 * raw_atomic64_or() - atomic bitwise OR with relaxed ordering3786 * @i: s64 value3787 * @v: pointer to atomic64_t3788 *3789 * Atomically updates @v to (@v | @i) with relaxed ordering.3790 *3791 * Safe to use in noinstr code; prefer atomic64_or() elsewhere.3792 *3793 * Return: Nothing.3794 */3795static __always_inline void3796raw_atomic64_or(s64 i, atomic64_t *v)3797{3798 arch_atomic64_or(i, v);3799}3800 3801/**3802 * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering3803 * @i: s64 value3804 * @v: pointer to atomic64_t3805 *3806 * Atomically updates @v to (@v | @i) with full ordering.3807 *3808 * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.3809 *3810 * Return: The original value of @v.3811 */3812static __always_inline s643813raw_atomic64_fetch_or(s64 i, atomic64_t *v)3814{3815#if defined(arch_atomic64_fetch_or)3816 return arch_atomic64_fetch_or(i, v);3817#elif defined(arch_atomic64_fetch_or_relaxed)3818 s64 ret;3819 __atomic_pre_full_fence();3820 ret = arch_atomic64_fetch_or_relaxed(i, v);3821 __atomic_post_full_fence();3822 return ret;3823#else3824#error "Unable to define raw_atomic64_fetch_or"3825#endif3826}3827 3828/**3829 * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering3830 * @i: s64 value3831 * @v: pointer to atomic64_t3832 *3833 * Atomically updates @v to (@v | @i) with acquire ordering.3834 *3835 * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.3836 *3837 * Return: The original value of @v.3838 */3839static __always_inline s643840raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)3841{3842#if defined(arch_atomic64_fetch_or_acquire)3843 return arch_atomic64_fetch_or_acquire(i, v);3844#elif defined(arch_atomic64_fetch_or_relaxed)3845 s64 ret = arch_atomic64_fetch_or_relaxed(i, v);3846 __atomic_acquire_fence();3847 return ret;3848#elif defined(arch_atomic64_fetch_or)3849 return arch_atomic64_fetch_or(i, v);3850#else3851#error "Unable to define raw_atomic64_fetch_or_acquire"3852#endif3853}3854 3855/**3856 * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering3857 * @i: s64 value3858 * @v: pointer to atomic64_t3859 *3860 * Atomically updates @v to (@v | @i) with release ordering.3861 *3862 * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.3863 *3864 * Return: The original value of @v.3865 */3866static __always_inline s643867raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)3868{3869#if defined(arch_atomic64_fetch_or_release)3870 return arch_atomic64_fetch_or_release(i, v);3871#elif defined(arch_atomic64_fetch_or_relaxed)3872 __atomic_release_fence();3873 return arch_atomic64_fetch_or_relaxed(i, v);3874#elif defined(arch_atomic64_fetch_or)3875 return arch_atomic64_fetch_or(i, v);3876#else3877#error "Unable to define raw_atomic64_fetch_or_release"3878#endif3879}3880 3881/**3882 * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering3883 * @i: s64 value3884 * @v: pointer to atomic64_t3885 *3886 * Atomically updates @v to (@v | @i) with relaxed ordering.3887 *3888 * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.3889 *3890 * Return: The original value of @v.3891 */3892static __always_inline s643893raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)3894{3895#if defined(arch_atomic64_fetch_or_relaxed)3896 return arch_atomic64_fetch_or_relaxed(i, v);3897#elif defined(arch_atomic64_fetch_or)3898 return arch_atomic64_fetch_or(i, v);3899#else3900#error "Unable to define raw_atomic64_fetch_or_relaxed"3901#endif3902}3903 3904/**3905 * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering3906 * @i: s64 value3907 * @v: pointer to atomic64_t3908 *3909 * Atomically updates @v to (@v ^ @i) with relaxed ordering.3910 *3911 * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.3912 *3913 * Return: Nothing.3914 */3915static __always_inline void3916raw_atomic64_xor(s64 i, atomic64_t *v)3917{3918 arch_atomic64_xor(i, v);3919}3920 3921/**3922 * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering3923 * @i: s64 value3924 * @v: pointer to atomic64_t3925 *3926 * Atomically updates @v to (@v ^ @i) with full ordering.3927 *3928 * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.3929 *3930 * Return: The original value of @v.3931 */3932static __always_inline s643933raw_atomic64_fetch_xor(s64 i, atomic64_t *v)3934{3935#if defined(arch_atomic64_fetch_xor)3936 return arch_atomic64_fetch_xor(i, v);3937#elif defined(arch_atomic64_fetch_xor_relaxed)3938 s64 ret;3939 __atomic_pre_full_fence();3940 ret = arch_atomic64_fetch_xor_relaxed(i, v);3941 __atomic_post_full_fence();3942 return ret;3943#else3944#error "Unable to define raw_atomic64_fetch_xor"3945#endif3946}3947 3948/**3949 * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering3950 * @i: s64 value3951 * @v: pointer to atomic64_t3952 *3953 * Atomically updates @v to (@v ^ @i) with acquire ordering.3954 *3955 * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.3956 *3957 * Return: The original value of @v.3958 */3959static __always_inline s643960raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)3961{3962#if defined(arch_atomic64_fetch_xor_acquire)3963 return arch_atomic64_fetch_xor_acquire(i, v);3964#elif defined(arch_atomic64_fetch_xor_relaxed)3965 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);3966 __atomic_acquire_fence();3967 return ret;3968#elif defined(arch_atomic64_fetch_xor)3969 return arch_atomic64_fetch_xor(i, v);3970#else3971#error "Unable to define raw_atomic64_fetch_xor_acquire"3972#endif3973}3974 3975/**3976 * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering3977 * @i: s64 value3978 * @v: pointer to atomic64_t3979 *3980 * Atomically updates @v to (@v ^ @i) with release ordering.3981 *3982 * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.3983 *3984 * Return: The original value of @v.3985 */3986static __always_inline s643987raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)3988{3989#if defined(arch_atomic64_fetch_xor_release)3990 return arch_atomic64_fetch_xor_release(i, v);3991#elif defined(arch_atomic64_fetch_xor_relaxed)3992 __atomic_release_fence();3993 return arch_atomic64_fetch_xor_relaxed(i, v);3994#elif defined(arch_atomic64_fetch_xor)3995 return arch_atomic64_fetch_xor(i, v);3996#else3997#error "Unable to define raw_atomic64_fetch_xor_release"3998#endif3999}4000 4001/**4002 * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering4003 * @i: s64 value4004 * @v: pointer to atomic64_t4005 *4006 * Atomically updates @v to (@v ^ @i) with relaxed ordering.4007 *4008 * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.4009 *4010 * Return: The original value of @v.4011 */4012static __always_inline s644013raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)4014{4015#if defined(arch_atomic64_fetch_xor_relaxed)4016 return arch_atomic64_fetch_xor_relaxed(i, v);4017#elif defined(arch_atomic64_fetch_xor)4018 return arch_atomic64_fetch_xor(i, v);4019#else4020#error "Unable to define raw_atomic64_fetch_xor_relaxed"4021#endif4022}4023 4024/**4025 * raw_atomic64_xchg() - atomic exchange with full ordering4026 * @v: pointer to atomic64_t4027 * @new: s64 value to assign4028 *4029 * Atomically updates @v to @new with full ordering.4030 *4031 * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.4032 *4033 * Return: The original value of @v.4034 */4035static __always_inline s644036raw_atomic64_xchg(atomic64_t *v, s64 new)4037{4038#if defined(arch_atomic64_xchg)4039 return arch_atomic64_xchg(v, new);4040#elif defined(arch_atomic64_xchg_relaxed)4041 s64 ret;4042 __atomic_pre_full_fence();4043 ret = arch_atomic64_xchg_relaxed(v, new);4044 __atomic_post_full_fence();4045 return ret;4046#else4047 return raw_xchg(&v->counter, new);4048#endif4049}4050 4051/**4052 * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering4053 * @v: pointer to atomic64_t4054 * @new: s64 value to assign4055 *4056 * Atomically updates @v to @new with acquire ordering.4057 *4058 * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.4059 *4060 * Return: The original value of @v.4061 */4062static __always_inline s644063raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)4064{4065#if defined(arch_atomic64_xchg_acquire)4066 return arch_atomic64_xchg_acquire(v, new);4067#elif defined(arch_atomic64_xchg_relaxed)4068 s64 ret = arch_atomic64_xchg_relaxed(v, new);4069 __atomic_acquire_fence();4070 return ret;4071#elif defined(arch_atomic64_xchg)4072 return arch_atomic64_xchg(v, new);4073#else4074 return raw_xchg_acquire(&v->counter, new);4075#endif4076}4077 4078/**4079 * raw_atomic64_xchg_release() - atomic exchange with release ordering4080 * @v: pointer to atomic64_t4081 * @new: s64 value to assign4082 *4083 * Atomically updates @v to @new with release ordering.4084 *4085 * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.4086 *4087 * Return: The original value of @v.4088 */4089static __always_inline s644090raw_atomic64_xchg_release(atomic64_t *v, s64 new)4091{4092#if defined(arch_atomic64_xchg_release)4093 return arch_atomic64_xchg_release(v, new);4094#elif defined(arch_atomic64_xchg_relaxed)4095 __atomic_release_fence();4096 return arch_atomic64_xchg_relaxed(v, new);4097#elif defined(arch_atomic64_xchg)4098 return arch_atomic64_xchg(v, new);4099#else4100 return raw_xchg_release(&v->counter, new);4101#endif4102}4103 4104/**4105 * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering4106 * @v: pointer to atomic64_t4107 * @new: s64 value to assign4108 *4109 * Atomically updates @v to @new with relaxed ordering.4110 *4111 * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.4112 *4113 * Return: The original value of @v.4114 */4115static __always_inline s644116raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)4117{4118#if defined(arch_atomic64_xchg_relaxed)4119 return arch_atomic64_xchg_relaxed(v, new);4120#elif defined(arch_atomic64_xchg)4121 return arch_atomic64_xchg(v, new);4122#else4123 return raw_xchg_relaxed(&v->counter, new);4124#endif4125}4126 4127/**4128 * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering4129 * @v: pointer to atomic64_t4130 * @old: s64 value to compare with4131 * @new: s64 value to assign4132 *4133 * If (@v == @old), atomically updates @v to @new with full ordering.4134 * Otherwise, @v is not modified and relaxed ordering is provided.4135 *4136 * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.4137 *4138 * Return: The original value of @v.4139 */4140static __always_inline s644141raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)4142{4143#if defined(arch_atomic64_cmpxchg)4144 return arch_atomic64_cmpxchg(v, old, new);4145#elif defined(arch_atomic64_cmpxchg_relaxed)4146 s64 ret;4147 __atomic_pre_full_fence();4148 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);4149 __atomic_post_full_fence();4150 return ret;4151#else4152 return raw_cmpxchg(&v->counter, old, new);4153#endif4154}4155 4156/**4157 * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering4158 * @v: pointer to atomic64_t4159 * @old: s64 value to compare with4160 * @new: s64 value to assign4161 *4162 * If (@v == @old), atomically updates @v to @new with acquire ordering.4163 * Otherwise, @v is not modified and relaxed ordering is provided.4164 *4165 * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.4166 *4167 * Return: The original value of @v.4168 */4169static __always_inline s644170raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)4171{4172#if defined(arch_atomic64_cmpxchg_acquire)4173 return arch_atomic64_cmpxchg_acquire(v, old, new);4174#elif defined(arch_atomic64_cmpxchg_relaxed)4175 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);4176 __atomic_acquire_fence();4177 return ret;4178#elif defined(arch_atomic64_cmpxchg)4179 return arch_atomic64_cmpxchg(v, old, new);4180#else4181 return raw_cmpxchg_acquire(&v->counter, old, new);4182#endif4183}4184 4185/**4186 * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering4187 * @v: pointer to atomic64_t4188 * @old: s64 value to compare with4189 * @new: s64 value to assign4190 *4191 * If (@v == @old), atomically updates @v to @new with release ordering.4192 * Otherwise, @v is not modified and relaxed ordering is provided.4193 *4194 * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.4195 *4196 * Return: The original value of @v.4197 */4198static __always_inline s644199raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)4200{4201#if defined(arch_atomic64_cmpxchg_release)4202 return arch_atomic64_cmpxchg_release(v, old, new);4203#elif defined(arch_atomic64_cmpxchg_relaxed)4204 __atomic_release_fence();4205 return arch_atomic64_cmpxchg_relaxed(v, old, new);4206#elif defined(arch_atomic64_cmpxchg)4207 return arch_atomic64_cmpxchg(v, old, new);4208#else4209 return raw_cmpxchg_release(&v->counter, old, new);4210#endif4211}4212 4213/**4214 * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering4215 * @v: pointer to atomic64_t4216 * @old: s64 value to compare with4217 * @new: s64 value to assign4218 *4219 * If (@v == @old), atomically updates @v to @new with relaxed ordering.4220 * Otherwise, @v is not modified and relaxed ordering is provided.4221 *4222 * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.4223 *4224 * Return: The original value of @v.4225 */4226static __always_inline s644227raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)4228{4229#if defined(arch_atomic64_cmpxchg_relaxed)4230 return arch_atomic64_cmpxchg_relaxed(v, old, new);4231#elif defined(arch_atomic64_cmpxchg)4232 return arch_atomic64_cmpxchg(v, old, new);4233#else4234 return raw_cmpxchg_relaxed(&v->counter, old, new);4235#endif4236}4237 4238/**4239 * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering4240 * @v: pointer to atomic64_t4241 * @old: pointer to s64 value to compare with4242 * @new: s64 value to assign4243 *4244 * If (@v == @old), atomically updates @v to @new with full ordering.4245 * Otherwise, @v is not modified, @old is updated to the current value of @v,4246 * and relaxed ordering is provided.4247 *4248 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.4249 *4250 * Return: @true if the exchange occured, @false otherwise.4251 */4252static __always_inline bool4253raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)4254{4255#if defined(arch_atomic64_try_cmpxchg)4256 return arch_atomic64_try_cmpxchg(v, old, new);4257#elif defined(arch_atomic64_try_cmpxchg_relaxed)4258 bool ret;4259 __atomic_pre_full_fence();4260 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);4261 __atomic_post_full_fence();4262 return ret;4263#else4264 s64 r, o = *old;4265 r = raw_atomic64_cmpxchg(v, o, new);4266 if (unlikely(r != o))4267 *old = r;4268 return likely(r == o);4269#endif4270}4271 4272/**4273 * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering4274 * @v: pointer to atomic64_t4275 * @old: pointer to s64 value to compare with4276 * @new: s64 value to assign4277 *4278 * If (@v == @old), atomically updates @v to @new with acquire ordering.4279 * Otherwise, @v is not modified, @old is updated to the current value of @v,4280 * and relaxed ordering is provided.4281 *4282 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.4283 *4284 * Return: @true if the exchange occured, @false otherwise.4285 */4286static __always_inline bool4287raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)4288{4289#if defined(arch_atomic64_try_cmpxchg_acquire)4290 return arch_atomic64_try_cmpxchg_acquire(v, old, new);4291#elif defined(arch_atomic64_try_cmpxchg_relaxed)4292 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);4293 __atomic_acquire_fence();4294 return ret;4295#elif defined(arch_atomic64_try_cmpxchg)4296 return arch_atomic64_try_cmpxchg(v, old, new);4297#else4298 s64 r, o = *old;4299 r = raw_atomic64_cmpxchg_acquire(v, o, new);4300 if (unlikely(r != o))4301 *old = r;4302 return likely(r == o);4303#endif4304}4305 4306/**4307 * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering4308 * @v: pointer to atomic64_t4309 * @old: pointer to s64 value to compare with4310 * @new: s64 value to assign4311 *4312 * If (@v == @old), atomically updates @v to @new with release ordering.4313 * Otherwise, @v is not modified, @old is updated to the current value of @v,4314 * and relaxed ordering is provided.4315 *4316 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.4317 *4318 * Return: @true if the exchange occured, @false otherwise.4319 */4320static __always_inline bool4321raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)4322{4323#if defined(arch_atomic64_try_cmpxchg_release)4324 return arch_atomic64_try_cmpxchg_release(v, old, new);4325#elif defined(arch_atomic64_try_cmpxchg_relaxed)4326 __atomic_release_fence();4327 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);4328#elif defined(arch_atomic64_try_cmpxchg)4329 return arch_atomic64_try_cmpxchg(v, old, new);4330#else4331 s64 r, o = *old;4332 r = raw_atomic64_cmpxchg_release(v, o, new);4333 if (unlikely(r != o))4334 *old = r;4335 return likely(r == o);4336#endif4337}4338 4339/**4340 * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering4341 * @v: pointer to atomic64_t4342 * @old: pointer to s64 value to compare with4343 * @new: s64 value to assign4344 *4345 * If (@v == @old), atomically updates @v to @new with relaxed ordering.4346 * Otherwise, @v is not modified, @old is updated to the current value of @v,4347 * and relaxed ordering is provided.4348 *4349 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.4350 *4351 * Return: @true if the exchange occured, @false otherwise.4352 */4353static __always_inline bool4354raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)4355{4356#if defined(arch_atomic64_try_cmpxchg_relaxed)4357 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);4358#elif defined(arch_atomic64_try_cmpxchg)4359 return arch_atomic64_try_cmpxchg(v, old, new);4360#else4361 s64 r, o = *old;4362 r = raw_atomic64_cmpxchg_relaxed(v, o, new);4363 if (unlikely(r != o))4364 *old = r;4365 return likely(r == o);4366#endif4367}4368 4369/**4370 * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering4371 * @i: s64 value to subtract4372 * @v: pointer to atomic64_t4373 *4374 * Atomically updates @v to (@v - @i) with full ordering.4375 *4376 * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.4377 *4378 * Return: @true if the resulting value of @v is zero, @false otherwise.4379 */4380static __always_inline bool4381raw_atomic64_sub_and_test(s64 i, atomic64_t *v)4382{4383#if defined(arch_atomic64_sub_and_test)4384 return arch_atomic64_sub_and_test(i, v);4385#else4386 return raw_atomic64_sub_return(i, v) == 0;4387#endif4388}4389 4390/**4391 * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering4392 * @v: pointer to atomic64_t4393 *4394 * Atomically updates @v to (@v - 1) with full ordering.4395 *4396 * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.4397 *4398 * Return: @true if the resulting value of @v is zero, @false otherwise.4399 */4400static __always_inline bool4401raw_atomic64_dec_and_test(atomic64_t *v)4402{4403#if defined(arch_atomic64_dec_and_test)4404 return arch_atomic64_dec_and_test(v);4405#else4406 return raw_atomic64_dec_return(v) == 0;4407#endif4408}4409 4410/**4411 * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering4412 * @v: pointer to atomic64_t4413 *4414 * Atomically updates @v to (@v + 1) with full ordering.4415 *4416 * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.4417 *4418 * Return: @true if the resulting value of @v is zero, @false otherwise.4419 */4420static __always_inline bool4421raw_atomic64_inc_and_test(atomic64_t *v)4422{4423#if defined(arch_atomic64_inc_and_test)4424 return arch_atomic64_inc_and_test(v);4425#else4426 return raw_atomic64_inc_return(v) == 0;4427#endif4428}4429 4430/**4431 * raw_atomic64_add_negative() - atomic add and test if negative with full ordering4432 * @i: s64 value to add4433 * @v: pointer to atomic64_t4434 *4435 * Atomically updates @v to (@v + @i) with full ordering.4436 *4437 * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.4438 *4439 * Return: @true if the resulting value of @v is negative, @false otherwise.4440 */4441static __always_inline bool4442raw_atomic64_add_negative(s64 i, atomic64_t *v)4443{4444#if defined(arch_atomic64_add_negative)4445 return arch_atomic64_add_negative(i, v);4446#elif defined(arch_atomic64_add_negative_relaxed)4447 bool ret;4448 __atomic_pre_full_fence();4449 ret = arch_atomic64_add_negative_relaxed(i, v);4450 __atomic_post_full_fence();4451 return ret;4452#else4453 return raw_atomic64_add_return(i, v) < 0;4454#endif4455}4456 4457/**4458 * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering4459 * @i: s64 value to add4460 * @v: pointer to atomic64_t4461 *4462 * Atomically updates @v to (@v + @i) with acquire ordering.4463 *4464 * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.4465 *4466 * Return: @true if the resulting value of @v is negative, @false otherwise.4467 */4468static __always_inline bool4469raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)4470{4471#if defined(arch_atomic64_add_negative_acquire)4472 return arch_atomic64_add_negative_acquire(i, v);4473#elif defined(arch_atomic64_add_negative_relaxed)4474 bool ret = arch_atomic64_add_negative_relaxed(i, v);4475 __atomic_acquire_fence();4476 return ret;4477#elif defined(arch_atomic64_add_negative)4478 return arch_atomic64_add_negative(i, v);4479#else4480 return raw_atomic64_add_return_acquire(i, v) < 0;4481#endif4482}4483 4484/**4485 * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering4486 * @i: s64 value to add4487 * @v: pointer to atomic64_t4488 *4489 * Atomically updates @v to (@v + @i) with release ordering.4490 *4491 * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.4492 *4493 * Return: @true if the resulting value of @v is negative, @false otherwise.4494 */4495static __always_inline bool4496raw_atomic64_add_negative_release(s64 i, atomic64_t *v)4497{4498#if defined(arch_atomic64_add_negative_release)4499 return arch_atomic64_add_negative_release(i, v);4500#elif defined(arch_atomic64_add_negative_relaxed)4501 __atomic_release_fence();4502 return arch_atomic64_add_negative_relaxed(i, v);4503#elif defined(arch_atomic64_add_negative)4504 return arch_atomic64_add_negative(i, v);4505#else4506 return raw_atomic64_add_return_release(i, v) < 0;4507#endif4508}4509 4510/**4511 * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering4512 * @i: s64 value to add4513 * @v: pointer to atomic64_t4514 *4515 * Atomically updates @v to (@v + @i) with relaxed ordering.4516 *4517 * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.4518 *4519 * Return: @true if the resulting value of @v is negative, @false otherwise.4520 */4521static __always_inline bool4522raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)4523{4524#if defined(arch_atomic64_add_negative_relaxed)4525 return arch_atomic64_add_negative_relaxed(i, v);4526#elif defined(arch_atomic64_add_negative)4527 return arch_atomic64_add_negative(i, v);4528#else4529 return raw_atomic64_add_return_relaxed(i, v) < 0;4530#endif4531}4532 4533/**4534 * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering4535 * @v: pointer to atomic64_t4536 * @a: s64 value to add4537 * @u: s64 value to compare with4538 *4539 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.4540 * Otherwise, @v is not modified and relaxed ordering is provided.4541 *4542 * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.4543 *4544 * Return: The original value of @v.4545 */4546static __always_inline s644547raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)4548{4549#if defined(arch_atomic64_fetch_add_unless)4550 return arch_atomic64_fetch_add_unless(v, a, u);4551#else4552 s64 c = raw_atomic64_read(v);4553 4554 do {4555 if (unlikely(c == u))4556 break;4557 } while (!raw_atomic64_try_cmpxchg(v, &c, c + a));4558 4559 return c;4560#endif4561}4562 4563/**4564 * raw_atomic64_add_unless() - atomic add unless value with full ordering4565 * @v: pointer to atomic64_t4566 * @a: s64 value to add4567 * @u: s64 value to compare with4568 *4569 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.4570 * Otherwise, @v is not modified and relaxed ordering is provided.4571 *4572 * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.4573 *4574 * Return: @true if @v was updated, @false otherwise.4575 */4576static __always_inline bool4577raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)4578{4579#if defined(arch_atomic64_add_unless)4580 return arch_atomic64_add_unless(v, a, u);4581#else4582 return raw_atomic64_fetch_add_unless(v, a, u) != u;4583#endif4584}4585 4586/**4587 * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering4588 * @v: pointer to atomic64_t4589 *4590 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.4591 * Otherwise, @v is not modified and relaxed ordering is provided.4592 *4593 * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.4594 *4595 * Return: @true if @v was updated, @false otherwise.4596 */4597static __always_inline bool4598raw_atomic64_inc_not_zero(atomic64_t *v)4599{4600#if defined(arch_atomic64_inc_not_zero)4601 return arch_atomic64_inc_not_zero(v);4602#else4603 return raw_atomic64_add_unless(v, 1, 0);4604#endif4605}4606 4607/**4608 * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering4609 * @v: pointer to atomic64_t4610 *4611 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.4612 * Otherwise, @v is not modified and relaxed ordering is provided.4613 *4614 * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.4615 *4616 * Return: @true if @v was updated, @false otherwise.4617 */4618static __always_inline bool4619raw_atomic64_inc_unless_negative(atomic64_t *v)4620{4621#if defined(arch_atomic64_inc_unless_negative)4622 return arch_atomic64_inc_unless_negative(v);4623#else4624 s64 c = raw_atomic64_read(v);4625 4626 do {4627 if (unlikely(c < 0))4628 return false;4629 } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1));4630 4631 return true;4632#endif4633}4634 4635/**4636 * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering4637 * @v: pointer to atomic64_t4638 *4639 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.4640 * Otherwise, @v is not modified and relaxed ordering is provided.4641 *4642 * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.4643 *4644 * Return: @true if @v was updated, @false otherwise.4645 */4646static __always_inline bool4647raw_atomic64_dec_unless_positive(atomic64_t *v)4648{4649#if defined(arch_atomic64_dec_unless_positive)4650 return arch_atomic64_dec_unless_positive(v);4651#else4652 s64 c = raw_atomic64_read(v);4653 4654 do {4655 if (unlikely(c > 0))4656 return false;4657 } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1));4658 4659 return true;4660#endif4661}4662 4663/**4664 * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering4665 * @v: pointer to atomic64_t4666 *4667 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.4668 * Otherwise, @v is not modified and relaxed ordering is provided.4669 *4670 * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.4671 *4672 * Return: The old value of (@v - 1), regardless of whether @v was updated.4673 */4674static __always_inline s644675raw_atomic64_dec_if_positive(atomic64_t *v)4676{4677#if defined(arch_atomic64_dec_if_positive)4678 return arch_atomic64_dec_if_positive(v);4679#else4680 s64 dec, c = raw_atomic64_read(v);4681 4682 do {4683 dec = c - 1;4684 if (unlikely(dec < 0))4685 break;4686 } while (!raw_atomic64_try_cmpxchg(v, &c, dec));4687 4688 return dec;4689#endif4690}4691 4692#endif /* _LINUX_ATOMIC_FALLBACK_H */4693// b565db590afeeff0d7c9485ccbca5bb6e155749f4694