190 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Generic implementation of 64-bit atomics using spinlocks,4 * useful on processors that don't have 64-bit atomic instructions.5 *6 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>7 */8#include <linux/types.h>9#include <linux/cache.h>10#include <linux/spinlock.h>11#include <linux/init.h>12#include <linux/export.h>13#include <linux/atomic.h>14 15/*16 * We use a hashed array of spinlocks to provide exclusive access17 * to each atomic64_t variable. Since this is expected to used on18 * systems with small numbers of CPUs (<= 4 or so), we use a19 * relatively small array of 16 spinlocks to avoid wasting too much20 * memory on the spinlock array.21 */22#define NR_LOCKS 1623 24/*25 * Ensure each lock is in a separate cacheline.26 */27static union {28 raw_spinlock_t lock;29 char pad[L1_CACHE_BYTES];30} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {31 [0 ... (NR_LOCKS - 1)] = {32 .lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),33 },34};35 36static inline raw_spinlock_t *lock_addr(const atomic64_t *v)37{38 unsigned long addr = (unsigned long) v;39 40 addr >>= L1_CACHE_SHIFT;41 addr ^= (addr >> 8) ^ (addr >> 16);42 return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;43}44 45s64 generic_atomic64_read(const atomic64_t *v)46{47 unsigned long flags;48 raw_spinlock_t *lock = lock_addr(v);49 s64 val;50 51 raw_spin_lock_irqsave(lock, flags);52 val = v->counter;53 raw_spin_unlock_irqrestore(lock, flags);54 return val;55}56EXPORT_SYMBOL(generic_atomic64_read);57 58void generic_atomic64_set(atomic64_t *v, s64 i)59{60 unsigned long flags;61 raw_spinlock_t *lock = lock_addr(v);62 63 raw_spin_lock_irqsave(lock, flags);64 v->counter = i;65 raw_spin_unlock_irqrestore(lock, flags);66}67EXPORT_SYMBOL(generic_atomic64_set);68 69#define ATOMIC64_OP(op, c_op) \70void generic_atomic64_##op(s64 a, atomic64_t *v) \71{ \72 unsigned long flags; \73 raw_spinlock_t *lock = lock_addr(v); \74 \75 raw_spin_lock_irqsave(lock, flags); \76 v->counter c_op a; \77 raw_spin_unlock_irqrestore(lock, flags); \78} \79EXPORT_SYMBOL(generic_atomic64_##op);80 81#define ATOMIC64_OP_RETURN(op, c_op) \82s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v) \83{ \84 unsigned long flags; \85 raw_spinlock_t *lock = lock_addr(v); \86 s64 val; \87 \88 raw_spin_lock_irqsave(lock, flags); \89 val = (v->counter c_op a); \90 raw_spin_unlock_irqrestore(lock, flags); \91 return val; \92} \93EXPORT_SYMBOL(generic_atomic64_##op##_return);94 95#define ATOMIC64_FETCH_OP(op, c_op) \96s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v) \97{ \98 unsigned long flags; \99 raw_spinlock_t *lock = lock_addr(v); \100 s64 val; \101 \102 raw_spin_lock_irqsave(lock, flags); \103 val = v->counter; \104 v->counter c_op a; \105 raw_spin_unlock_irqrestore(lock, flags); \106 return val; \107} \108EXPORT_SYMBOL(generic_atomic64_fetch_##op);109 110#define ATOMIC64_OPS(op, c_op) \111 ATOMIC64_OP(op, c_op) \112 ATOMIC64_OP_RETURN(op, c_op) \113 ATOMIC64_FETCH_OP(op, c_op)114 115ATOMIC64_OPS(add, +=)116ATOMIC64_OPS(sub, -=)117 118#undef ATOMIC64_OPS119#define ATOMIC64_OPS(op, c_op) \120 ATOMIC64_OP(op, c_op) \121 ATOMIC64_FETCH_OP(op, c_op)122 123ATOMIC64_OPS(and, &=)124ATOMIC64_OPS(or, |=)125ATOMIC64_OPS(xor, ^=)126 127#undef ATOMIC64_OPS128#undef ATOMIC64_FETCH_OP129#undef ATOMIC64_OP130 131s64 generic_atomic64_dec_if_positive(atomic64_t *v)132{133 unsigned long flags;134 raw_spinlock_t *lock = lock_addr(v);135 s64 val;136 137 raw_spin_lock_irqsave(lock, flags);138 val = v->counter - 1;139 if (val >= 0)140 v->counter = val;141 raw_spin_unlock_irqrestore(lock, flags);142 return val;143}144EXPORT_SYMBOL(generic_atomic64_dec_if_positive);145 146s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)147{148 unsigned long flags;149 raw_spinlock_t *lock = lock_addr(v);150 s64 val;151 152 raw_spin_lock_irqsave(lock, flags);153 val = v->counter;154 if (val == o)155 v->counter = n;156 raw_spin_unlock_irqrestore(lock, flags);157 return val;158}159EXPORT_SYMBOL(generic_atomic64_cmpxchg);160 161s64 generic_atomic64_xchg(atomic64_t *v, s64 new)162{163 unsigned long flags;164 raw_spinlock_t *lock = lock_addr(v);165 s64 val;166 167 raw_spin_lock_irqsave(lock, flags);168 val = v->counter;169 v->counter = new;170 raw_spin_unlock_irqrestore(lock, flags);171 return val;172}173EXPORT_SYMBOL(generic_atomic64_xchg);174 175s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)176{177 unsigned long flags;178 raw_spinlock_t *lock = lock_addr(v);179 s64 val;180 181 raw_spin_lock_irqsave(lock, flags);182 val = v->counter;183 if (val != u)184 v->counter += a;185 raw_spin_unlock_irqrestore(lock, flags);186 187 return val;188}189EXPORT_SYMBOL(generic_atomic64_fetch_add_unless);190