192 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2#ifndef __LINUX_CACHE_H3#define __LINUX_CACHE_H4 5#include <uapi/linux/kernel.h>6#include <asm/cache.h>7 8#ifndef L1_CACHE_ALIGN9#define L1_CACHE_ALIGN(x) __ALIGN_KERNEL(x, L1_CACHE_BYTES)10#endif11 12#ifndef SMP_CACHE_BYTES13#define SMP_CACHE_BYTES L1_CACHE_BYTES14#endif15 16/**17 * SMP_CACHE_ALIGN - align a value to the L2 cacheline size18 * @x: value to align19 *20 * On some architectures, L2 ("SMP") CL size is bigger than L1, and sometimes,21 * this needs to be accounted.22 *23 * Return: aligned value.24 */25#ifndef SMP_CACHE_ALIGN26#define SMP_CACHE_ALIGN(x) ALIGN(x, SMP_CACHE_BYTES)27#endif28 29/*30 * ``__aligned_largest`` aligns a field to the value most optimal for the31 * target architecture to perform memory operations. Get the actual value32 * to be able to use it anywhere else.33 */34#ifndef __LARGEST_ALIGN35#define __LARGEST_ALIGN sizeof(struct { long x; } __aligned_largest)36#endif37 38#ifndef LARGEST_ALIGN39#define LARGEST_ALIGN(x) ALIGN(x, __LARGEST_ALIGN)40#endif41 42/*43 * __read_mostly is used to keep rarely changing variables out of frequently44 * updated cachelines. Its use should be reserved for data that is used45 * frequently in hot paths. Performance traces can help decide when to use46 * this. You want __read_mostly data to be tightly packed, so that in the47 * best case multiple frequently read variables for a hot path will be next48 * to each other in order to reduce the number of cachelines needed to49 * execute a critical path. We should be mindful and selective of its use.50 * ie: if you're going to use it please supply a *good* justification in your51 * commit log52 */53#ifndef __read_mostly54#define __read_mostly55#endif56 57/*58 * __ro_after_init is used to mark things that are read-only after init (i.e.59 * after mark_rodata_ro() has been called). These are effectively read-only,60 * but may get written to during init, so can't live in .rodata (via "const").61 */62#ifndef __ro_after_init63#define __ro_after_init __section(".data..ro_after_init")64#endif65 66#ifndef ____cacheline_aligned67#define ____cacheline_aligned __attribute__((__aligned__(SMP_CACHE_BYTES)))68#endif69 70#ifndef ____cacheline_aligned_in_smp71#ifdef CONFIG_SMP72#define ____cacheline_aligned_in_smp ____cacheline_aligned73#else74#define ____cacheline_aligned_in_smp75#endif /* CONFIG_SMP */76#endif77 78#ifndef __cacheline_aligned79#define __cacheline_aligned \80 __attribute__((__aligned__(SMP_CACHE_BYTES), \81 __section__(".data..cacheline_aligned")))82#endif /* __cacheline_aligned */83 84#ifndef __cacheline_aligned_in_smp85#ifdef CONFIG_SMP86#define __cacheline_aligned_in_smp __cacheline_aligned87#else88#define __cacheline_aligned_in_smp89#endif /* CONFIG_SMP */90#endif91 92/*93 * The maximum alignment needed for some critical structures94 * These could be inter-node cacheline sizes/L3 cacheline95 * size etc. Define this in asm/cache.h for your arch96 */97#ifndef INTERNODE_CACHE_SHIFT98#define INTERNODE_CACHE_SHIFT L1_CACHE_SHIFT99#endif100 101#if !defined(____cacheline_internodealigned_in_smp)102#if defined(CONFIG_SMP)103#define ____cacheline_internodealigned_in_smp \104 __attribute__((__aligned__(1 << (INTERNODE_CACHE_SHIFT))))105#else106#define ____cacheline_internodealigned_in_smp107#endif108#endif109 110#ifndef CONFIG_ARCH_HAS_CACHE_LINE_SIZE111#define cache_line_size() L1_CACHE_BYTES112#endif113 114#ifndef __cacheline_group_begin115#define __cacheline_group_begin(GROUP) \116 __u8 __cacheline_group_begin__##GROUP[0]117#endif118 119#ifndef __cacheline_group_end120#define __cacheline_group_end(GROUP) \121 __u8 __cacheline_group_end__##GROUP[0]122#endif123 124/**125 * __cacheline_group_begin_aligned - declare an aligned group start126 * @GROUP: name of the group127 * @...: optional group alignment128 *129 * The following block inside a struct:130 *131 * __cacheline_group_begin_aligned(grp);132 * field a;133 * field b;134 * __cacheline_group_end_aligned(grp);135 *136 * will always be aligned to either the specified alignment or137 * ``SMP_CACHE_BYTES``.138 */139#define __cacheline_group_begin_aligned(GROUP, ...) \140 __cacheline_group_begin(GROUP) \141 __aligned((__VA_ARGS__ + 0) ? : SMP_CACHE_BYTES)142 143/**144 * __cacheline_group_end_aligned - declare an aligned group end145 * @GROUP: name of the group146 * @...: optional alignment (same as was in __cacheline_group_begin_aligned())147 *148 * Note that the end marker is aligned to sizeof(long) to allow more precise149 * size assertion. It also declares a padding at the end to avoid next field150 * falling into this cacheline.151 */152#define __cacheline_group_end_aligned(GROUP, ...) \153 __cacheline_group_end(GROUP) __aligned(sizeof(long)); \154 struct { } __cacheline_group_pad__##GROUP \155 __aligned((__VA_ARGS__ + 0) ? : SMP_CACHE_BYTES)156 157#ifndef CACHELINE_ASSERT_GROUP_MEMBER158#define CACHELINE_ASSERT_GROUP_MEMBER(TYPE, GROUP, MEMBER) \159 BUILD_BUG_ON(!(offsetof(TYPE, MEMBER) >= \160 offsetofend(TYPE, __cacheline_group_begin__##GROUP) && \161 offsetofend(TYPE, MEMBER) <= \162 offsetof(TYPE, __cacheline_group_end__##GROUP)))163#endif164 165#ifndef CACHELINE_ASSERT_GROUP_SIZE166#define CACHELINE_ASSERT_GROUP_SIZE(TYPE, GROUP, SIZE) \167 BUILD_BUG_ON(offsetof(TYPE, __cacheline_group_end__##GROUP) - \168 offsetofend(TYPE, __cacheline_group_begin__##GROUP) > \169 SIZE)170#endif171 172/*173 * Helper to add padding within a struct to ensure data fall into separate174 * cachelines.175 */176#if defined(CONFIG_SMP)177struct cacheline_padding {178 char x[0];179} ____cacheline_internodealigned_in_smp;180#define CACHELINE_PADDING(name) struct cacheline_padding name181#else182#define CACHELINE_PADDING(name)183#endif184 185#ifdef ARCH_DMA_MINALIGN186#define ARCH_HAS_DMA_MINALIGN187#else188#define ARCH_DMA_MINALIGN __alignof__(unsigned long long)189#endif190 191#endif /* __LINUX_CACHE_H */192