150 lines · plain
1#include <TargetConditionals.h>2#if TARGET_OS_OSX || TARGET_OS_IPHONE3#include <sys/sysctl.h>4 5#if __has_include(<arm/cpu_capabilities_public.h>)6#include <arm/cpu_capabilities_public.h>7#define HAS_CPU_CAPABILITIES_PUBLIC_H 18 9// FB13964283 - A few of these didn't make it into the public SDK yet.10#ifndef CAP_BIT_FEAT_SME11#define CAP_BIT_FEAT_SME 4012#endif13#ifndef CAP_BIT_FEAT_SME214#define CAP_BIT_FEAT_SME2 4115#endif16#ifndef CAP_BIT_FEAT_SME_F64F6417#define CAP_BIT_FEAT_SME_F64F64 4218#endif19#ifndef CAP_BIT_FEAT_SME_I16I6420#define CAP_BIT_FEAT_SME_I16I64 4321#endif22 23#endif24 25static bool isKnownAndSupported(const char *name) {26 int32_t val = 0;27 size_t size = sizeof(val);28 if (sysctlbyname(name, &val, &size, NULL, 0))29 return false;30 return val;31}32 33static uint64_t deriveImplicitFeatures(uint64_t features) {34 // FEAT_FP is always enabled35 features |= (1ULL << FEAT_FP);36 37 features |= (1ULL << FEAT_INIT);38 39 return features;40}41 42void __init_cpu_features_resolver(void) {43 // On Darwin platforms, this may be called concurrently by multiple threads44 // because the resolvers that use it are called lazily at runtime (unlike on45 // ELF platforms, where IFuncs are resolved serially at load time). This46 // function's effect on __aarch64_cpu_features must be idempotent.47 48 if (__atomic_load_n(&__aarch64_cpu_features.features, __ATOMIC_RELAXED))49 return;50 51 uint64_t features = 0;52 53#ifdef HAS_CPU_CAPABILITIES_PUBLIC_H54 uint8_t feats_bitvec[(CAP_BIT_NB + 7) / 8] = {0};55 size_t len = sizeof(feats_bitvec);56 // When hw.optional.arm.feats is available (macOS 15.0+, iOS 18.0+), use the57 // fast path to get all the feature bits, otherwise fall back to the slow58 // ~20-something sysctls path.59 if (!sysctlbyname("hw.optional.arm.caps", &feats_bitvec, &len, 0, 0)) {60 61#define CHECK_BIT(FROM, TO) \62 do { \63 if (feats_bitvec[FROM / 8] & (1u << ((FROM) & 7))) { \64 features |= (1ULL << TO); \65 } \66 } while (0)67 68 CHECK_BIT(CAP_BIT_FEAT_FlagM, FEAT_FLAGM);69 CHECK_BIT(CAP_BIT_FEAT_FlagM2, FEAT_FLAGM2);70 CHECK_BIT(CAP_BIT_FEAT_FHM, FEAT_FP16FML);71 CHECK_BIT(CAP_BIT_FEAT_DotProd, FEAT_DOTPROD);72 CHECK_BIT(CAP_BIT_FEAT_SHA3, FEAT_SHA3);73 CHECK_BIT(CAP_BIT_FEAT_RDM, FEAT_RDM);74 CHECK_BIT(CAP_BIT_FEAT_LSE, FEAT_LSE);75 CHECK_BIT(CAP_BIT_FEAT_SHA256, FEAT_SHA2);76 CHECK_BIT(CAP_BIT_FEAT_PMULL, FEAT_PMULL);77 CHECK_BIT(CAP_BIT_FEAT_SB, FEAT_SB);78 CHECK_BIT(CAP_BIT_FEAT_FRINTTS, FEAT_FRINTTS);79 CHECK_BIT(CAP_BIT_FEAT_LRCPC, FEAT_RCPC);80 CHECK_BIT(CAP_BIT_FEAT_LRCPC2, FEAT_RCPC2);81 CHECK_BIT(CAP_BIT_FEAT_FCMA, FEAT_FCMA);82 CHECK_BIT(CAP_BIT_FEAT_JSCVT, FEAT_JSCVT);83 CHECK_BIT(CAP_BIT_FEAT_DPB, FEAT_DPB);84 CHECK_BIT(CAP_BIT_FEAT_DPB2, FEAT_DPB2);85 CHECK_BIT(CAP_BIT_FEAT_BF16, FEAT_BF16);86 CHECK_BIT(CAP_BIT_FEAT_I8MM, FEAT_I8MM);87 CHECK_BIT(CAP_BIT_FEAT_DIT, FEAT_DIT);88 CHECK_BIT(CAP_BIT_FEAT_FP16, FEAT_FP16);89 CHECK_BIT(CAP_BIT_FEAT_SSBS, FEAT_SSBS2);90 CHECK_BIT(CAP_BIT_FEAT_BTI, FEAT_BTI);91 CHECK_BIT(CAP_BIT_AdvSIMD, FEAT_SIMD);92 CHECK_BIT(CAP_BIT_CRC32, FEAT_CRC);93 CHECK_BIT(CAP_BIT_FEAT_SME, FEAT_SME);94 CHECK_BIT(CAP_BIT_FEAT_SME2, FEAT_SME2);95 CHECK_BIT(CAP_BIT_FEAT_SME_F64F64, FEAT_SME_F64);96 CHECK_BIT(CAP_BIT_FEAT_SME_I16I64, FEAT_SME_I64);97 98 features = deriveImplicitFeatures(features);99 100 __atomic_store(&__aarch64_cpu_features.features, &features,101 __ATOMIC_RELAXED);102 return;103 }104#endif105 106 // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics107 static const struct {108 const char *sysctl_name;109 enum CPUFeatures feature;110 } feature_checks[] = {111 {"hw.optional.arm.FEAT_FlagM", FEAT_FLAGM},112 {"hw.optional.arm.FEAT_FlagM2", FEAT_FLAGM2},113 {"hw.optional.arm.FEAT_FHM", FEAT_FP16FML},114 {"hw.optional.arm.FEAT_DotProd", FEAT_DOTPROD},115 {"hw.optional.arm.FEAT_RDM", FEAT_RDM},116 {"hw.optional.arm.FEAT_LSE", FEAT_LSE},117 {"hw.optional.AdvSIMD", FEAT_SIMD},118 {"hw.optional.armv8_crc32", FEAT_CRC},119 {"hw.optional.arm.FEAT_SHA256", FEAT_SHA2},120 {"hw.optional.arm.FEAT_SHA3", FEAT_SHA3},121 {"hw.optional.arm.FEAT_PMULL", FEAT_PMULL},122 {"hw.optional.arm.FEAT_FP16", FEAT_FP16},123 {"hw.optional.arm.FEAT_DIT", FEAT_DIT},124 {"hw.optional.arm.FEAT_DPB", FEAT_DPB},125 {"hw.optional.arm.FEAT_DPB2", FEAT_DPB2},126 {"hw.optional.arm.FEAT_JSCVT", FEAT_JSCVT},127 {"hw.optional.arm.FEAT_FCMA", FEAT_FCMA},128 {"hw.optional.arm.FEAT_LRCPC", FEAT_RCPC},129 {"hw.optional.arm.FEAT_LRCPC2", FEAT_RCPC2},130 {"hw.optional.arm.FEAT_FRINTTS", FEAT_FRINTTS},131 {"hw.optional.arm.FEAT_I8MM", FEAT_I8MM},132 {"hw.optional.arm.FEAT_BF16", FEAT_BF16},133 {"hw.optional.arm.FEAT_SB", FEAT_SB},134 {"hw.optional.arm.FEAT_SSBS", FEAT_SSBS2},135 {"hw.optional.arm.FEAT_BTI", FEAT_BTI},136 };137 138 for (size_t I = 0, E = sizeof(feature_checks) / sizeof(feature_checks[0]);139 I != E; ++I)140 if (isKnownAndSupported(feature_checks[I].sysctl_name))141 features |= (1ULL << feature_checks[I].feature);142 143 features = deriveImplicitFeatures(features);144 145 __atomic_store(&__aarch64_cpu_features.features, &features,146 __ATOMIC_RELAXED);147}148 149#endif // TARGET_OS_OSX || TARGET_OS_IPHONE150