605 lines · c
1/*===---- bmiintrin.h - BMI intrinsics -------------------------------------===2 *3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4 * See https://llvm.org/LICENSE.txt for license information.5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6 *7 *===-----------------------------------------------------------------------===8 */9 10#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H11#error "Never use <bmiintrin.h> directly; include <x86intrin.h> instead."12#endif13 14#ifndef __BMIINTRIN_H15#define __BMIINTRIN_H16 17/* Allow using the tzcnt intrinsics even for non-BMI targets. Since the TZCNT18 instruction behaves as BSF on non-BMI targets, there is code that expects19 to use it as a potentially faster version of BSF. */20#if defined(__cplusplus) && (__cplusplus >= 201103L)21#define __RELAXED_FN_ATTRS \22 __attribute__((__always_inline__, __nodebug__)) constexpr23#else24#define __RELAXED_FN_ATTRS __attribute__((__always_inline__, __nodebug__))25#endif26 27/// Counts the number of trailing zero bits in the operand.28///29/// \headerfile <x86intrin.h>30///31/// This intrinsic corresponds to the \c TZCNT instruction.32///33/// \param __X34/// An unsigned 16-bit integer whose trailing zeros are to be counted.35/// \returns An unsigned 16-bit integer containing the number of trailing zero36/// bits in the operand.37/// \see _tzcnt_u1638static __inline__ unsigned short __RELAXED_FN_ATTRS39__tzcnt_u16(unsigned short __X) {40 return __builtin_ia32_tzcnt_u16(__X);41}42 43/// Counts the number of trailing zero bits in the operand.44///45/// \headerfile <x86intrin.h>46///47/// \code48/// unsigned short _tzcnt_u16(unsigned short __X);49/// \endcode50///51/// This intrinsic corresponds to the \c TZCNT instruction.52///53/// \param __X54/// An unsigned 16-bit integer whose trailing zeros are to be counted.55/// \returns An unsigned 16-bit integer containing the number of trailing zero56/// bits in the operand.57/// \see __tzcnt_u1658#define _tzcnt_u16 __tzcnt_u1659 60/// Counts the number of trailing zero bits in the operand.61///62/// \headerfile <x86intrin.h>63///64/// This intrinsic corresponds to the \c TZCNT instruction.65///66/// \param __X67/// An unsigned 32-bit integer whose trailing zeros are to be counted.68/// \returns An unsigned 32-bit integer containing the number of trailing zero69/// bits in the operand.70/// \see { _mm_tzcnt_32 _tzcnt_u32 }71static __inline__ unsigned int __RELAXED_FN_ATTRS72__tzcnt_u32(unsigned int __X) {73 return __builtin_ia32_tzcnt_u32(__X);74}75 76/// Counts the number of trailing zero bits in the operand.77///78/// \headerfile <x86intrin.h>79///80/// This intrinsic corresponds to the \c TZCNT instruction.81///82/// \param __X83/// An unsigned 32-bit integer whose trailing zeros are to be counted.84/// \returns A 32-bit integer containing the number of trailing zero bits in85/// the operand.86/// \see { __tzcnt_u32 _tzcnt_u32 }87static __inline__ int __RELAXED_FN_ATTRS88_mm_tzcnt_32(unsigned int __X) {89 return (int)__builtin_ia32_tzcnt_u32(__X);90}91 92/// Counts the number of trailing zero bits in the operand.93///94/// \headerfile <x86intrin.h>95///96/// \code97/// unsigned int _tzcnt_u32(unsigned int __X);98/// \endcode99///100/// This intrinsic corresponds to the \c TZCNT instruction.101///102/// \param __X103/// An unsigned 32-bit integer whose trailing zeros are to be counted.104/// \returns An unsigned 32-bit integer containing the number of trailing zero105/// bits in the operand.106/// \see { _mm_tzcnt_32 __tzcnt_u32 }107#define _tzcnt_u32 __tzcnt_u32108 109#ifdef __x86_64__110 111/// Counts the number of trailing zero bits in the operand.112///113/// \headerfile <x86intrin.h>114///115/// This intrinsic corresponds to the \c TZCNT instruction.116///117/// \param __X118/// An unsigned 64-bit integer whose trailing zeros are to be counted.119/// \returns An unsigned 64-bit integer containing the number of trailing zero120/// bits in the operand.121/// \see { _mm_tzcnt_64 _tzcnt_u64 }122static __inline__ unsigned long long __RELAXED_FN_ATTRS123__tzcnt_u64(unsigned long long __X) {124 return __builtin_ia32_tzcnt_u64(__X);125}126 127/// Counts the number of trailing zero bits in the operand.128///129/// \headerfile <x86intrin.h>130///131/// This intrinsic corresponds to the \c TZCNT instruction.132///133/// \param __X134/// An unsigned 64-bit integer whose trailing zeros are to be counted.135/// \returns An 64-bit integer containing the number of trailing zero bits in136/// the operand.137/// \see { __tzcnt_u64 _tzcnt_u64 }138static __inline__ long long __RELAXED_FN_ATTRS139_mm_tzcnt_64(unsigned long long __X) {140 return (long long)__builtin_ia32_tzcnt_u64(__X);141}142 143/// Counts the number of trailing zero bits in the operand.144///145/// \headerfile <x86intrin.h>146///147/// \code148/// unsigned long long _tzcnt_u64(unsigned long long __X);149/// \endcode150///151/// This intrinsic corresponds to the \c TZCNT instruction.152///153/// \param __X154/// An unsigned 64-bit integer whose trailing zeros are to be counted.155/// \returns An unsigned 64-bit integer containing the number of trailing zero156/// bits in the operand.157/// \see { _mm_tzcnt_64 __tzcnt_u64158#define _tzcnt_u64 __tzcnt_u64159 160#endif /* __x86_64__ */161 162#undef __RELAXED_FN_ATTRS163 164/* Define the default attributes for the functions in this file. */165#if defined(__cplusplus) && (__cplusplus >= 201103L)166#define __DEFAULT_FN_ATTRS \167 __attribute__((__always_inline__, __nodebug__, __target__("bmi"))) constexpr168#else169#define __DEFAULT_FN_ATTRS \170 __attribute__((__always_inline__, __nodebug__, __target__("bmi")))171#endif172 173/// Performs a bitwise AND of the second operand with the one's174/// complement of the first operand.175///176/// \headerfile <x86intrin.h>177///178/// This intrinsic corresponds to the \c ANDN instruction.179///180/// \param __X181/// An unsigned integer containing one of the operands.182/// \param __Y183/// An unsigned integer containing one of the operands.184/// \returns An unsigned integer containing the bitwise AND of the second185/// operand with the one's complement of the first operand.186/// \see _andn_u32187static __inline__ unsigned int __DEFAULT_FN_ATTRS188__andn_u32(unsigned int __X, unsigned int __Y) {189 return ~__X & __Y;190}191 192/// Performs a bitwise AND of the second operand with the one's193/// complement of the first operand.194///195/// \headerfile <x86intrin.h>196///197/// \code198/// unsigned int _andn_u32(unsigned int __X, unsigned int __Y);199/// \endcode200///201/// This intrinsic corresponds to the \c ANDN instruction.202///203/// \param __X204/// An unsigned integer containing one of the operands.205/// \param __Y206/// An unsigned integer containing one of the operands.207/// \returns An unsigned integer containing the bitwise AND of the second208/// operand with the one's complement of the first operand.209/// \see __andn_u32210#define _andn_u32 __andn_u32211 212/* AMD-specified, double-leading-underscore version of BEXTR */213/// Extracts the specified bits from the first operand and returns them214/// in the least significant bits of the result.215///216/// \headerfile <x86intrin.h>217///218/// This intrinsic corresponds to the \c BEXTR instruction.219///220/// \param __X221/// An unsigned integer whose bits are to be extracted.222/// \param __Y223/// An unsigned integer used to specify which bits are extracted. Bits [7:0]224/// specify the index of the least significant bit. Bits [15:8] specify the225/// number of bits to be extracted.226/// \returns An unsigned integer whose least significant bits contain the227/// extracted bits.228/// \see _bextr_u32229static __inline__ unsigned int __DEFAULT_FN_ATTRS230__bextr_u32(unsigned int __X, unsigned int __Y) {231 return __builtin_ia32_bextr_u32(__X, __Y);232}233 234/* Intel-specified, single-leading-underscore version of BEXTR */235/// Extracts the specified bits from the first operand and returns them236/// in the least significant bits of the result.237///238/// \headerfile <x86intrin.h>239///240/// This intrinsic corresponds to the \c BEXTR instruction.241///242/// \param __X243/// An unsigned integer whose bits are to be extracted.244/// \param __Y245/// An unsigned integer used to specify the index of the least significant246/// bit for the bits to be extracted. Bits [7:0] specify the index.247/// \param __Z248/// An unsigned integer used to specify the number of bits to be extracted.249/// Bits [7:0] specify the number of bits.250/// \returns An unsigned integer whose least significant bits contain the251/// extracted bits.252/// \see __bextr_u32253static __inline__ unsigned int __DEFAULT_FN_ATTRS254_bextr_u32(unsigned int __X, unsigned int __Y, unsigned int __Z) {255 return __builtin_ia32_bextr_u32(__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));256}257 258/* Intel-specified, single-leading-underscore version of BEXTR2 */259/// Extracts the specified bits from the first operand and returns them260/// in the least significant bits of the result.261///262/// \headerfile <x86intrin.h>263///264/// This intrinsic corresponds to the \c BEXTR instruction.265///266/// \param __X267/// An unsigned integer whose bits are to be extracted.268/// \param __Y269/// An unsigned integer used to specify which bits are extracted. Bits [7:0]270/// specify the index of the least significant bit. Bits [15:8] specify the271/// number of bits to be extracted.272/// \returns An unsigned integer whose least significant bits contain the273/// extracted bits.274/// \see __bextr_u32275static __inline__ unsigned int __DEFAULT_FN_ATTRS276_bextr2_u32(unsigned int __X, unsigned int __Y) {277 return __builtin_ia32_bextr_u32(__X, __Y);278}279 280/// Clears all bits in the source except for the least significant bit281/// containing a value of 1 and returns the result.282///283/// \headerfile <x86intrin.h>284///285/// This intrinsic corresponds to the \c BLSI instruction.286///287/// \param __X288/// An unsigned integer whose bits are to be cleared.289/// \returns An unsigned integer containing the result of clearing the bits from290/// the source operand.291/// \see _blsi_u32292static __inline__ unsigned int __DEFAULT_FN_ATTRS293__blsi_u32(unsigned int __X) {294 return __X & -__X;295}296 297/// Clears all bits in the source except for the least significant bit298/// containing a value of 1 and returns the result.299///300/// \headerfile <x86intrin.h>301///302/// \code303/// unsigned int _blsi_u32(unsigned int __X);304/// \endcode305///306/// This intrinsic corresponds to the \c BLSI instruction.307///308/// \param __X309/// An unsigned integer whose bits are to be cleared.310/// \returns An unsigned integer containing the result of clearing the bits from311/// the source operand.312/// \see __blsi_u32313#define _blsi_u32 __blsi_u32314 315/// Creates a mask whose bits are set to 1, using bit 0 up to and316/// including the least significant bit that is set to 1 in the source317/// operand and returns the result.318///319/// \headerfile <x86intrin.h>320///321/// This intrinsic corresponds to the \c BLSMSK instruction.322///323/// \param __X324/// An unsigned integer used to create the mask.325/// \returns An unsigned integer containing the newly created mask.326/// \see _blsmsk_u32327static __inline__ unsigned int __DEFAULT_FN_ATTRS328__blsmsk_u32(unsigned int __X) {329 return __X ^ (__X - 1);330}331 332/// Creates a mask whose bits are set to 1, using bit 0 up to and333/// including the least significant bit that is set to 1 in the source334/// operand and returns the result.335///336/// \headerfile <x86intrin.h>337///338/// \code339/// unsigned int _blsmsk_u32(unsigned int __X);340/// \endcode341///342/// This intrinsic corresponds to the \c BLSMSK instruction.343///344/// \param __X345/// An unsigned integer used to create the mask.346/// \returns An unsigned integer containing the newly created mask.347/// \see __blsmsk_u32348#define _blsmsk_u32 __blsmsk_u32349 350/// Clears the least significant bit that is set to 1 in the source351/// operand and returns the result.352///353/// \headerfile <x86intrin.h>354///355/// This intrinsic corresponds to the \c BLSR instruction.356///357/// \param __X358/// An unsigned integer containing the operand to be cleared.359/// \returns An unsigned integer containing the result of clearing the source360/// operand.361/// \see _blsr_u32362static __inline__ unsigned int __DEFAULT_FN_ATTRS363__blsr_u32(unsigned int __X) {364 return __X & (__X - 1);365}366 367/// Clears the least significant bit that is set to 1 in the source368/// operand and returns the result.369///370/// \headerfile <x86intrin.h>371///372/// \code373/// unsigned int _bls4_u32(unsigned int __X);374/// \endcode375///376/// This intrinsic corresponds to the \c BLSR instruction.377///378/// \param __X379/// An unsigned integer containing the operand to be cleared.380/// \returns An unsigned integer containing the result of clearing the source381/// operand.382/// \see __blsr_u32383#define _blsr_u32 __blsr_u32384 385#ifdef __x86_64__386 387/// Performs a bitwise AND of the second operand with the one's388/// complement of the first operand.389///390/// \headerfile <x86intrin.h>391///392/// This intrinsic corresponds to the \c ANDN instruction.393///394/// \param __X395/// An unsigned 64-bit integer containing one of the operands.396/// \param __Y397/// An unsigned 64-bit integer containing one of the operands.398/// \returns An unsigned 64-bit integer containing the bitwise AND of the second399/// operand with the one's complement of the first operand.400/// \see _andn_u64401static __inline__ unsigned long long __DEFAULT_FN_ATTRS402__andn_u64 (unsigned long long __X, unsigned long long __Y) {403 return ~__X & __Y;404}405 406/// Performs a bitwise AND of the second operand with the one's407/// complement of the first operand.408///409/// \headerfile <x86intrin.h>410///411/// \code412/// unsigned long long _andn_u64(unsigned long long __X,413/// unsigned long long __Y);414/// \endcode415///416/// This intrinsic corresponds to the \c ANDN instruction.417///418/// \param __X419/// An unsigned 64-bit integer containing one of the operands.420/// \param __Y421/// An unsigned 64-bit integer containing one of the operands.422/// \returns An unsigned 64-bit integer containing the bitwise AND of the second423/// operand with the one's complement of the first operand.424/// \see __andn_u64425#define _andn_u64 __andn_u64426 427/* AMD-specified, double-leading-underscore version of BEXTR */428/// Extracts the specified bits from the first operand and returns them429/// in the least significant bits of the result.430///431/// \headerfile <x86intrin.h>432///433/// This intrinsic corresponds to the \c BEXTR instruction.434///435/// \param __X436/// An unsigned 64-bit integer whose bits are to be extracted.437/// \param __Y438/// An unsigned 64-bit integer used to specify which bits are extracted. Bits439/// [7:0] specify the index of the least significant bit. Bits [15:8] specify440/// the number of bits to be extracted.441/// \returns An unsigned 64-bit integer whose least significant bits contain the442/// extracted bits.443/// \see _bextr_u64444static __inline__ unsigned long long __DEFAULT_FN_ATTRS445__bextr_u64(unsigned long long __X, unsigned long long __Y) {446 return __builtin_ia32_bextr_u64(__X, __Y);447}448 449/* Intel-specified, single-leading-underscore version of BEXTR */450/// Extracts the specified bits from the first operand and returns them451/// in the least significant bits of the result.452///453/// \headerfile <x86intrin.h>454///455/// This intrinsic corresponds to the \c BEXTR instruction.456///457/// \param __X458/// An unsigned 64-bit integer whose bits are to be extracted.459/// \param __Y460/// An unsigned integer used to specify the index of the least significant461/// bit for the bits to be extracted. Bits [7:0] specify the index.462/// \param __Z463/// An unsigned integer used to specify the number of bits to be extracted.464/// Bits [7:0] specify the number of bits.465/// \returns An unsigned 64-bit integer whose least significant bits contain the466/// extracted bits.467/// \see __bextr_u64468static __inline__ unsigned long long __DEFAULT_FN_ATTRS469_bextr_u64(unsigned long long __X, unsigned int __Y, unsigned int __Z) {470 return __builtin_ia32_bextr_u64(__X, ((__Y & 0xff) | ((__Z & 0xff) << 8)));471}472 473/* Intel-specified, single-leading-underscore version of BEXTR2 */474/// Extracts the specified bits from the first operand and returns them475/// in the least significant bits of the result.476///477/// \headerfile <x86intrin.h>478///479/// This intrinsic corresponds to the \c BEXTR instruction.480///481/// \param __X482/// An unsigned 64-bit integer whose bits are to be extracted.483/// \param __Y484/// An unsigned 64-bit integer used to specify which bits are extracted. Bits485/// [7:0] specify the index of the least significant bit. Bits [15:8] specify486/// the number of bits to be extracted.487/// \returns An unsigned 64-bit integer whose least significant bits contain the488/// extracted bits.489/// \see __bextr_u64490static __inline__ unsigned long long __DEFAULT_FN_ATTRS491_bextr2_u64(unsigned long long __X, unsigned long long __Y) {492 return __builtin_ia32_bextr_u64(__X, __Y);493}494 495/// Clears all bits in the source except for the least significant bit496/// containing a value of 1 and returns the result.497///498/// \headerfile <x86intrin.h>499///500/// This intrinsic corresponds to the \c BLSI instruction.501///502/// \param __X503/// An unsigned 64-bit integer whose bits are to be cleared.504/// \returns An unsigned 64-bit integer containing the result of clearing the505/// bits from the source operand.506/// \see _blsi_u64507static __inline__ unsigned long long __DEFAULT_FN_ATTRS508__blsi_u64(unsigned long long __X) {509 return __X & -__X;510}511 512/// Clears all bits in the source except for the least significant bit513/// containing a value of 1 and returns the result.514///515/// \headerfile <x86intrin.h>516///517/// \code518/// unsigned long long _blsi_u64(unsigned long long __X);519/// \endcode520///521/// This intrinsic corresponds to the \c BLSI instruction.522///523/// \param __X524/// An unsigned 64-bit integer whose bits are to be cleared.525/// \returns An unsigned 64-bit integer containing the result of clearing the526/// bits from the source operand.527/// \see __blsi_u64528#define _blsi_u64 __blsi_u64529 530/// Creates a mask whose bits are set to 1, using bit 0 up to and531/// including the least significant bit that is set to 1 in the source532/// operand and returns the result.533///534/// \headerfile <x86intrin.h>535///536/// This intrinsic corresponds to the \c BLSMSK instruction.537///538/// \param __X539/// An unsigned 64-bit integer used to create the mask.540/// \returns An unsigned 64-bit integer containing the newly created mask.541/// \see _blsmsk_u64542static __inline__ unsigned long long __DEFAULT_FN_ATTRS543__blsmsk_u64(unsigned long long __X) {544 return __X ^ (__X - 1);545}546 547/// Creates a mask whose bits are set to 1, using bit 0 up to and548/// including the least significant bit that is set to 1 in the source549/// operand and returns the result.550///551/// \headerfile <x86intrin.h>552///553/// \code554/// unsigned long long _blsmsk_u64(unsigned long long __X);555/// \endcode556///557/// This intrinsic corresponds to the \c BLSMSK instruction.558///559/// \param __X560/// An unsigned 64-bit integer used to create the mask.561/// \returns An unsigned 64-bit integer containing the newly created mask.562/// \see __blsmsk_u64563#define _blsmsk_u64 __blsmsk_u64564 565/// Clears the least significant bit that is set to 1 in the source566/// operand and returns the result.567///568/// \headerfile <x86intrin.h>569///570/// This intrinsic corresponds to the \c BLSR instruction.571///572/// \param __X573/// An unsigned 64-bit integer containing the operand to be cleared.574/// \returns An unsigned 64-bit integer containing the result of clearing the575/// source operand.576/// \see _blsr_u64577static __inline__ unsigned long long __DEFAULT_FN_ATTRS578__blsr_u64(unsigned long long __X) {579 return __X & (__X - 1);580}581 582/// Clears the least significant bit that is set to 1 in the source583/// operand and returns the result.584///585/// \headerfile <x86intrin.h>586///587/// \code588/// unsigned long long _blsr_u64(unsigned long long __X);589/// \endcode590///591/// This intrinsic corresponds to the \c BLSR instruction.592///593/// \param __X594/// An unsigned 64-bit integer containing the operand to be cleared.595/// \returns An unsigned 64-bit integer containing the result of clearing the596/// source operand.597/// \see __blsr_u64598#define _blsr_u64 __blsr_u64599 600#endif /* __x86_64__ */601 602#undef __DEFAULT_FN_ATTRS603 604#endif /* __BMIINTRIN_H */605