596 lines · c
1/*===---- immintrin.h - Intel intrinsics -----------------------------------===2 *3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4 * See https://llvm.org/LICENSE.txt for license information.5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6 *7 *===-----------------------------------------------------------------------===8 */9 10#ifndef __IMMINTRIN_H11#define __IMMINTRIN_H12 13#if !defined(__i386__) && !defined(__x86_64__)14#error "This header is only meant to be used on x86 and x64 architecture"15#endif16 17#include <x86gprintrin.h>18 19#include <mmintrin.h>20 21#include <xmmintrin.h>22 23#include <emmintrin.h>24 25#include <pmmintrin.h>26 27#include <tmmintrin.h>28 29#include <smmintrin.h>30 31#include <wmmintrin.h>32 33#include <clflushoptintrin.h>34 35#include <clwbintrin.h>36 37#include <avxintrin.h>38 39#include <avx2intrin.h>40 41#include <f16cintrin.h>42 43#include <bmiintrin.h>44 45#include <bmi2intrin.h>46 47#include <lzcntintrin.h>48 49#include <popcntintrin.h>50 51#include <fmaintrin.h>52 53#include <avx512fintrin.h>54 55#include <avx512vlintrin.h>56 57#include <avx512bwintrin.h>58 59#include <avx512bitalgintrin.h>60 61#include <avx512cdintrin.h>62 63#include <avx512vpopcntdqintrin.h>64 65#include <avx512vpopcntdqvlintrin.h>66 67#include <avx512vnniintrin.h>68 69#include <avx512vlvnniintrin.h>70 71#include <avxvnniintrin.h>72 73#include <avx512dqintrin.h>74 75#include <avx512vlbitalgintrin.h>76 77#include <avx512vlbwintrin.h>78 79#include <avx512vlcdintrin.h>80 81#include <avx512vldqintrin.h>82 83#include <avx512ifmaintrin.h>84 85#include <avx512ifmavlintrin.h>86 87#include <avxifmaintrin.h>88 89#include <avx512vbmiintrin.h>90 91#include <avx512vbmivlintrin.h>92 93#include <avx512vbmi2intrin.h>94 95#include <avx512vlvbmi2intrin.h>96 97#include <avx512fp16intrin.h>98 99#include <avx512vlfp16intrin.h>100 101#include <avx512bf16intrin.h>102 103#include <avx512vlbf16intrin.h>104 105#include <pkuintrin.h>106 107#include <vpclmulqdqintrin.h>108 109#include <vaesintrin.h>110 111#include <gfniintrin.h>112 113#include <avxvnniint8intrin.h>114 115#include <avxneconvertintrin.h>116 117#include <sha512intrin.h>118 119#include <sm3intrin.h>120 121#include <sm4intrin.h>122 123#include <avxvnniint16intrin.h>124 125/// Reads the value of the IA32_TSC_AUX MSR (0xc0000103).126///127/// \headerfile <immintrin.h>128///129/// This intrinsic corresponds to the <c> RDPID </c> instruction.130///131/// \returns The 32-bit contents of the MSR.132static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid")))133_rdpid_u32(void) {134 return __builtin_ia32_rdpid();135}136 137/// Returns a 16-bit hardware-generated random value.138///139/// \headerfile <immintrin.h>140///141/// This intrinsic corresponds to the <c> RDRAND </c> instruction.142///143/// \param __p144/// A pointer to a 16-bit memory location to place the random value.145/// \returns 1 if the value was successfully generated, 0 otherwise.146static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))147_rdrand16_step(unsigned short *__p)148{149 return (int)__builtin_ia32_rdrand16_step(__p);150}151 152/// Returns a 32-bit hardware-generated random value.153///154/// \headerfile <immintrin.h>155///156/// This intrinsic corresponds to the <c> RDRAND </c> instruction.157///158/// \param __p159/// A pointer to a 32-bit memory location to place the random value.160/// \returns 1 if the value was successfully generated, 0 otherwise.161static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))162_rdrand32_step(unsigned int *__p)163{164 return (int)__builtin_ia32_rdrand32_step(__p);165}166 167/// Returns a 64-bit hardware-generated random value.168///169/// \headerfile <immintrin.h>170///171/// This intrinsic corresponds to the <c> RDRAND </c> instruction.172///173/// \param __p174/// A pointer to a 64-bit memory location to place the random value.175/// \returns 1 if the value was successfully generated, 0 otherwise.176static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))177_rdrand64_step(unsigned long long *__p)178{179#ifdef __x86_64__180 return (int)__builtin_ia32_rdrand64_step(__p);181#else182 // We need to emulate the functionality of 64-bit rdrand with 2 32-bit183 // rdrand instructions.184 unsigned int __lo, __hi;185 unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo);186 unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi);187 if (__res_lo && __res_hi) {188 *__p = ((unsigned long long)__hi << 32) | (unsigned long long)__lo;189 return 1;190 } else {191 *__p = 0;192 return 0;193 }194#endif195}196 197#ifdef __x86_64__198/// Reads the FS base register.199///200/// \headerfile <immintrin.h>201///202/// This intrinsic corresponds to the <c> RDFSBASE </c> instruction.203///204/// \returns The lower 32 bits of the FS base register.205static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))206_readfsbase_u32(void)207{208 return __builtin_ia32_rdfsbase32();209}210 211/// Reads the FS base register.212///213/// \headerfile <immintrin.h>214///215/// This intrinsic corresponds to the <c> RDFSBASE </c> instruction.216///217/// \returns The contents of the FS base register.218static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))219_readfsbase_u64(void)220{221 return __builtin_ia32_rdfsbase64();222}223 224/// Reads the GS base register.225///226/// \headerfile <immintrin.h>227///228/// This intrinsic corresponds to the <c> RDGSBASE </c> instruction.229///230/// \returns The lower 32 bits of the GS base register.231static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))232_readgsbase_u32(void)233{234 return __builtin_ia32_rdgsbase32();235}236 237/// Reads the GS base register.238///239/// \headerfile <immintrin.h>240///241/// This intrinsic corresponds to the <c> RDGSBASE </c> instruction.242///243/// \returns The contents of the GS base register.244static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))245_readgsbase_u64(void)246{247 return __builtin_ia32_rdgsbase64();248}249 250/// Modifies the FS base register.251///252/// \headerfile <immintrin.h>253///254/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.255///256/// \param __V257/// Value to use for the lower 32 bits of the FS base register.258static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))259_writefsbase_u32(unsigned int __V)260{261 __builtin_ia32_wrfsbase32(__V);262}263 264/// Modifies the FS base register.265///266/// \headerfile <immintrin.h>267///268/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.269///270/// \param __V271/// Value to use for the FS base register.272static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))273_writefsbase_u64(unsigned long long __V)274{275 __builtin_ia32_wrfsbase64(__V);276}277 278/// Modifies the GS base register.279///280/// \headerfile <immintrin.h>281///282/// This intrinsic corresponds to the <c> WRGSBASE </c> instruction.283///284/// \param __V285/// Value to use for the lower 32 bits of the GS base register.286static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))287_writegsbase_u32(unsigned int __V)288{289 __builtin_ia32_wrgsbase32(__V);290}291 292/// Modifies the GS base register.293///294/// \headerfile <immintrin.h>295///296/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.297///298/// \param __V299/// Value to use for GS base register.300static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))301_writegsbase_u64(unsigned long long __V)302{303 __builtin_ia32_wrgsbase64(__V);304}305 306#endif307 308/* The structs used below are to force the load/store to be unaligned. This309 * is accomplished with the __packed__ attribute. The __may_alias__ prevents310 * tbaa metadata from being generated based on the struct and the type of the311 * field inside of it.312 */313 314/// Load a 16-bit value from memory and swap its bytes.315///316/// \headerfile <x86intrin.h>317///318/// This intrinsic corresponds to the MOVBE instruction.319///320/// \param __P321/// A pointer to the 16-bit value to load.322/// \returns The byte-swapped value.323static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe")))324_loadbe_i16(void const * __P) {325 struct __loadu_i16 {326 unsigned short __v;327 } __attribute__((__packed__, __may_alias__));328 return (short)__builtin_bswap16(((const struct __loadu_i16*)__P)->__v);329}330 331/// Swap the bytes of a 16-bit value and store it to memory.332///333/// \headerfile <x86intrin.h>334///335/// This intrinsic corresponds to the MOVBE instruction.336///337/// \param __P338/// A pointer to the memory for storing the swapped value.339/// \param __D340/// The 16-bit value to be byte-swapped.341static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))342_storebe_i16(void * __P, short __D) {343 struct __storeu_i16 {344 unsigned short __v;345 } __attribute__((__packed__, __may_alias__));346 ((struct __storeu_i16*)__P)->__v = __builtin_bswap16((unsigned short)__D);347}348 349/// Load a 32-bit value from memory and swap its bytes.350///351/// \headerfile <x86intrin.h>352///353/// This intrinsic corresponds to the MOVBE instruction.354///355/// \param __P356/// A pointer to the 32-bit value to load.357/// \returns The byte-swapped value.358static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe")))359_loadbe_i32(void const * __P) {360 struct __loadu_i32 {361 unsigned int __v;362 } __attribute__((__packed__, __may_alias__));363 return (int)__builtin_bswap32(((const struct __loadu_i32*)__P)->__v);364}365 366/// Swap the bytes of a 32-bit value and store it to memory.367///368/// \headerfile <x86intrin.h>369///370/// This intrinsic corresponds to the MOVBE instruction.371///372/// \param __P373/// A pointer to the memory for storing the swapped value.374/// \param __D375/// The 32-bit value to be byte-swapped.376static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))377_storebe_i32(void * __P, int __D) {378 struct __storeu_i32 {379 unsigned int __v;380 } __attribute__((__packed__, __may_alias__));381 ((struct __storeu_i32*)__P)->__v = __builtin_bswap32((unsigned int)__D);382}383 384#ifdef __x86_64__385/// Load a 64-bit value from memory and swap its bytes.386///387/// \headerfile <x86intrin.h>388///389/// This intrinsic corresponds to the MOVBE instruction.390///391/// \param __P392/// A pointer to the 64-bit value to load.393/// \returns The byte-swapped value.394static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe")))395_loadbe_i64(void const * __P) {396 struct __loadu_i64 {397 unsigned long long __v;398 } __attribute__((__packed__, __may_alias__));399 return (long long)__builtin_bswap64(((const struct __loadu_i64*)__P)->__v);400}401 402/// Swap the bytes of a 64-bit value and store it to memory.403///404/// \headerfile <x86intrin.h>405///406/// This intrinsic corresponds to the MOVBE instruction.407///408/// \param __P409/// A pointer to the memory for storing the swapped value.410/// \param __D411/// The 64-bit value to be byte-swapped.412static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))413_storebe_i64(void * __P, long long __D) {414 struct __storeu_i64 {415 unsigned long long __v;416 } __attribute__((__packed__, __may_alias__));417 ((struct __storeu_i64*)__P)->__v = __builtin_bswap64((unsigned long long)__D);418}419#endif420 421#include <rtmintrin.h>422#include <xtestintrin.h>423 424#include <shaintrin.h>425 426#include <fxsrintrin.h>427 428/* No feature check desired due to internal MSC_VER checks */429#include <xsaveintrin.h>430 431#include <xsaveoptintrin.h>432 433#include <xsavecintrin.h>434 435#include <xsavesintrin.h>436 437#include <cetintrin.h>438 439/* Intrinsics inside adcintrin.h are available at all times. */440#include <adcintrin.h>441 442#include <adxintrin.h>443 444#include <rdseedintrin.h>445 446#include <wbnoinvdintrin.h>447 448#include <cldemoteintrin.h>449 450#include <waitpkgintrin.h>451 452#include <movdirintrin.h>453 454#include <movrsintrin.h>455 456#include <movrs_avx10_2intrin.h>457 458#include <movrs_avx10_2_512intrin.h>459 460#include <pconfigintrin.h>461 462#include <sgxintrin.h>463 464#include <ptwriteintrin.h>465 466#include <invpcidintrin.h>467 468#include <keylockerintrin.h>469 470#include <amxintrin.h>471 472#include <amxfp16intrin.h>473 474#include <amxcomplexintrin.h>475 476#include <amxfp8intrin.h>477 478#include <amxmovrsintrin.h>479 480#include <amxavx512intrin.h>481 482#include <amxtf32intrin.h>483 484#include <avx512vp2intersectintrin.h>485 486#include <avx512vlvp2intersectintrin.h>487 488#include <avx10_2bf16intrin.h>489#include <avx10_2convertintrin.h>490#include <avx10_2copyintrin.h>491#include <avx10_2minmaxintrin.h>492#include <avx10_2niintrin.h>493#include <avx10_2satcvtdsintrin.h>494#include <avx10_2satcvtintrin.h>495 496#include <avx10_2_512bf16intrin.h>497#include <avx10_2_512convertintrin.h>498#include <avx10_2_512minmaxintrin.h>499#include <avx10_2_512niintrin.h>500#include <avx10_2_512satcvtdsintrin.h>501#include <avx10_2_512satcvtintrin.h>502 503#include <sm4evexintrin.h>504 505#include <enqcmdintrin.h>506 507#include <serializeintrin.h>508 509#include <tsxldtrkintrin.h>510 511#if defined(_MSC_VER) && __has_extension(gnu_asm)512/* Define the default attributes for these intrinsics */513#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))514#ifdef __cplusplus515extern "C" {516#endif517/*----------------------------------------------------------------------------*\518|* Interlocked Exchange HLE519\*----------------------------------------------------------------------------*/520#if defined(__i386__) || defined(__x86_64__)521static __inline__ long __DEFAULT_FN_ATTRS522_InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) {523 __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}"524 : "+r" (_Value), "+m" (*_Target) :: "memory");525 return _Value;526}527static __inline__ long __DEFAULT_FN_ATTRS528_InterlockedExchange_HLERelease(long volatile *_Target, long _Value) {529 __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}"530 : "+r" (_Value), "+m" (*_Target) :: "memory");531 return _Value;532}533#endif534#if defined(__x86_64__)535static __inline__ __int64 __DEFAULT_FN_ATTRS536_InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) {537 __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}"538 : "+r" (_Value), "+m" (*_Target) :: "memory");539 return _Value;540}541static __inline__ __int64 __DEFAULT_FN_ATTRS542_InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) {543 __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}"544 : "+r" (_Value), "+m" (*_Target) :: "memory");545 return _Value;546}547#endif548/*----------------------------------------------------------------------------*\549|* Interlocked Compare Exchange HLE550\*----------------------------------------------------------------------------*/551#if defined(__i386__) || defined(__x86_64__)552static __inline__ long __DEFAULT_FN_ATTRS553_InterlockedCompareExchange_HLEAcquire(long volatile *_Destination,554 long _Exchange, long _Comparand) {555 __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}"556 : "+a" (_Comparand), "+m" (*_Destination)557 : "r" (_Exchange) : "memory");558 return _Comparand;559}560static __inline__ long __DEFAULT_FN_ATTRS561_InterlockedCompareExchange_HLERelease(long volatile *_Destination,562 long _Exchange, long _Comparand) {563 __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}"564 : "+a" (_Comparand), "+m" (*_Destination)565 : "r" (_Exchange) : "memory");566 return _Comparand;567}568#endif569#if defined(__x86_64__)570static __inline__ __int64 __DEFAULT_FN_ATTRS571_InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination,572 __int64 _Exchange, __int64 _Comparand) {573 __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}"574 : "+a" (_Comparand), "+m" (*_Destination)575 : "r" (_Exchange) : "memory");576 return _Comparand;577}578static __inline__ __int64 __DEFAULT_FN_ATTRS579_InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,580 __int64 _Exchange, __int64 _Comparand) {581 __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}"582 : "+a" (_Comparand), "+m" (*_Destination)583 : "r" (_Exchange) : "memory");584 return _Comparand;585}586#endif587#ifdef __cplusplus588}589#endif590 591#undef __DEFAULT_FN_ATTRS592 593#endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */594 595#endif /* __IMMINTRIN_H */596