2270 lines · c
1/*===---- emmintrin.h - Implementation of SSE2 intrinsics on PowerPC -------===2 *3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4 * See https://llvm.org/LICENSE.txt for license information.5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6 *7 *===-----------------------------------------------------------------------===8 */9 10/* Implemented from the specification included in the Intel C++ Compiler11 User Guide and Reference, version 9.0. */12 13#ifndef NO_WARN_X86_INTRINSICS14/* This header file is to help porting code using Intel intrinsics15 explicitly from x86_64 to powerpc64/powerpc64le.16 17 Since X86 SSE2 intrinsics mainly handles __m128i and __m128d type,18 PowerPC VMX/VSX ISA is a good match for vector float SIMD operations.19 However scalar float operations in vector (XMM) registers require20 the POWER8 VSX ISA (2.07) level. There are differences for data21 format and placement of float scalars in the vector register, which22 require extra steps to match SSE2 scalar float semantics on POWER.23 24 It should be noted that there's much difference between X86_64's25 MXSCR and PowerISA's FPSCR/VSCR registers. It's recommended to use26 portable <fenv.h> instead of access MXSCR directly.27 28 Most SSE2 scalar float intrinsic operations can be performed more29 efficiently as C language float scalar operations or optimized to30 use vector SIMD operations. We recommend this for new applications.31*/32#error \33 "Please read comment above. Use -DNO_WARN_X86_INTRINSICS to disable this error."34#endif35 36#ifndef EMMINTRIN_H_37#define EMMINTRIN_H_38 39#if defined(__powerpc64__) && \40 (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX))41 42#include <altivec.h>43 44/* We need definitions from the SSE header files. */45#include <xmmintrin.h>46 47/* SSE2 */48typedef __vector double __v2df;49typedef __vector float __v4f;50typedef __vector long long __v2di;51typedef __vector unsigned long long __v2du;52typedef __vector int __v4si;53typedef __vector unsigned int __v4su;54typedef __vector short __v8hi;55typedef __vector unsigned short __v8hu;56typedef __vector signed char __v16qi;57typedef __vector unsigned char __v16qu;58 59/* The Intel API is flexible enough that we must allow aliasing with other60 vector types, and their scalar components. */61typedef long long __m128i __attribute__((__vector_size__(16), __may_alias__));62typedef double __m128d __attribute__((__vector_size__(16), __may_alias__));63 64/* Unaligned version of the same types. */65typedef long long __m128i_u66 __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));67typedef double __m128d_u68 __attribute__((__vector_size__(16), __may_alias__, __aligned__(1)));69 70/* Define two value permute mask. */71#define _MM_SHUFFLE2(x, y) (((x) << 1) | (y))72 73/* Create a vector with element 0 as F and the rest zero. */74extern __inline __m128d75 __attribute__((__gnu_inline__, __always_inline__, __artificial__))76 _mm_set_sd(double __F) {77 return __extension__(__m128d){__F, 0.0};78}79 80/* Create a vector with both elements equal to F. */81extern __inline __m128d82 __attribute__((__gnu_inline__, __always_inline__, __artificial__))83 _mm_set1_pd(double __F) {84 return __extension__(__m128d){__F, __F};85}86 87extern __inline __m128d88 __attribute__((__gnu_inline__, __always_inline__, __artificial__))89 _mm_set_pd1(double __F) {90 return _mm_set1_pd(__F);91}92 93/* Create a vector with the lower value X and upper value W. */94extern __inline __m128d95 __attribute__((__gnu_inline__, __always_inline__, __artificial__))96 _mm_set_pd(double __W, double __X) {97 return __extension__(__m128d){__X, __W};98}99 100/* Create a vector with the lower value W and upper value X. */101extern __inline __m128d102 __attribute__((__gnu_inline__, __always_inline__, __artificial__))103 _mm_setr_pd(double __W, double __X) {104 return __extension__(__m128d){__W, __X};105}106 107/* Create an undefined vector. */108extern __inline __m128d109 __attribute__((__gnu_inline__, __always_inline__, __artificial__))110 _mm_undefined_pd(void) {111 __m128d __Y = __Y;112 return __Y;113}114 115/* Create a vector of zeros. */116extern __inline __m128d117 __attribute__((__gnu_inline__, __always_inline__, __artificial__))118 _mm_setzero_pd(void) {119 return (__m128d)vec_splats(0);120}121 122/* Sets the low DPFP value of A from the low value of B. */123extern __inline __m128d124 __attribute__((__gnu_inline__, __always_inline__, __artificial__))125 _mm_move_sd(__m128d __A, __m128d __B) {126 __v2df __result = (__v2df)__A;127 __result[0] = ((__v2df)__B)[0];128 return (__m128d)__result;129}130 131/* Load two DPFP values from P. The address must be 16-byte aligned. */132extern __inline __m128d133 __attribute__((__gnu_inline__, __always_inline__, __artificial__))134 _mm_load_pd(double const *__P) {135 return ((__m128d)vec_ld(0, (__v16qu *)__P));136}137 138/* Load two DPFP values from P. The address need not be 16-byte aligned. */139extern __inline __m128d140 __attribute__((__gnu_inline__, __always_inline__, __artificial__))141 _mm_loadu_pd(double const *__P) {142 return (vec_vsx_ld(0, __P));143}144 145/* Create a vector with all two elements equal to *P. */146extern __inline __m128d147 __attribute__((__gnu_inline__, __always_inline__, __artificial__))148 _mm_load1_pd(double const *__P) {149 return (vec_splats(*__P));150}151 152/* Create a vector with element 0 as *P and the rest zero. */153extern __inline __m128d154 __attribute__((__gnu_inline__, __always_inline__, __artificial__))155 _mm_load_sd(double const *__P) {156 return _mm_set_sd(*__P);157}158 159extern __inline __m128d160 __attribute__((__gnu_inline__, __always_inline__, __artificial__))161 _mm_load_pd1(double const *__P) {162 return _mm_load1_pd(__P);163}164 165/* Load two DPFP values in reverse order. The address must be aligned. */166extern __inline __m128d167 __attribute__((__gnu_inline__, __always_inline__, __artificial__))168 _mm_loadr_pd(double const *__P) {169 __v2df __tmp = _mm_load_pd(__P);170 return (__m128d)vec_xxpermdi(__tmp, __tmp, 2);171}172 173/* Store two DPFP values. The address must be 16-byte aligned. */174extern __inline void175 __attribute__((__gnu_inline__, __always_inline__, __artificial__))176 _mm_store_pd(double *__P, __m128d __A) {177 vec_st((__v16qu)__A, 0, (__v16qu *)__P);178}179 180/* Store two DPFP values. The address need not be 16-byte aligned. */181extern __inline void182 __attribute__((__gnu_inline__, __always_inline__, __artificial__))183 _mm_storeu_pd(double *__P, __m128d __A) {184 *(__m128d_u *)__P = __A;185}186 187/* Stores the lower DPFP value. */188extern __inline void189 __attribute__((__gnu_inline__, __always_inline__, __artificial__))190 _mm_store_sd(double *__P, __m128d __A) {191 *__P = ((__v2df)__A)[0];192}193 194extern __inline double195 __attribute__((__gnu_inline__, __always_inline__, __artificial__))196 _mm_cvtsd_f64(__m128d __A) {197 return ((__v2df)__A)[0];198}199 200extern __inline void201 __attribute__((__gnu_inline__, __always_inline__, __artificial__))202 _mm_storel_pd(double *__P, __m128d __A) {203 _mm_store_sd(__P, __A);204}205 206/* Stores the upper DPFP value. */207extern __inline void208 __attribute__((__gnu_inline__, __always_inline__, __artificial__))209 _mm_storeh_pd(double *__P, __m128d __A) {210 *__P = ((__v2df)__A)[1];211}212/* Store the lower DPFP value across two words.213 The address must be 16-byte aligned. */214extern __inline void215 __attribute__((__gnu_inline__, __always_inline__, __artificial__))216 _mm_store1_pd(double *__P, __m128d __A) {217 _mm_store_pd(__P, vec_splat(__A, 0));218}219 220extern __inline void221 __attribute__((__gnu_inline__, __always_inline__, __artificial__))222 _mm_store_pd1(double *__P, __m128d __A) {223 _mm_store1_pd(__P, __A);224}225 226/* Store two DPFP values in reverse order. The address must be aligned. */227extern __inline void228 __attribute__((__gnu_inline__, __always_inline__, __artificial__))229 _mm_storer_pd(double *__P, __m128d __A) {230 _mm_store_pd(__P, vec_xxpermdi(__A, __A, 2));231}232 233/* Intel intrinsic. */234extern __inline long long235 __attribute__((__gnu_inline__, __always_inline__, __artificial__))236 _mm_cvtsi128_si64(__m128i __A) {237 return ((__v2di)__A)[0];238}239 240/* Microsoft intrinsic. */241extern __inline long long242 __attribute__((__gnu_inline__, __always_inline__, __artificial__))243 _mm_cvtsi128_si64x(__m128i __A) {244 return ((__v2di)__A)[0];245}246 247extern __inline __m128d248 __attribute__((__gnu_inline__, __always_inline__, __artificial__))249 _mm_add_pd(__m128d __A, __m128d __B) {250 return (__m128d)((__v2df)__A + (__v2df)__B);251}252 253/* Add the lower double-precision (64-bit) floating-point element in254 a and b, store the result in the lower element of dst, and copy255 the upper element from a to the upper element of dst. */256extern __inline __m128d257 __attribute__((__gnu_inline__, __always_inline__, __artificial__))258 _mm_add_sd(__m128d __A, __m128d __B) {259 __A[0] = __A[0] + __B[0];260 return (__A);261}262 263extern __inline __m128d264 __attribute__((__gnu_inline__, __always_inline__, __artificial__))265 _mm_sub_pd(__m128d __A, __m128d __B) {266 return (__m128d)((__v2df)__A - (__v2df)__B);267}268 269extern __inline __m128d270 __attribute__((__gnu_inline__, __always_inline__, __artificial__))271 _mm_sub_sd(__m128d __A, __m128d __B) {272 __A[0] = __A[0] - __B[0];273 return (__A);274}275 276extern __inline __m128d277 __attribute__((__gnu_inline__, __always_inline__, __artificial__))278 _mm_mul_pd(__m128d __A, __m128d __B) {279 return (__m128d)((__v2df)__A * (__v2df)__B);280}281 282extern __inline __m128d283 __attribute__((__gnu_inline__, __always_inline__, __artificial__))284 _mm_mul_sd(__m128d __A, __m128d __B) {285 __A[0] = __A[0] * __B[0];286 return (__A);287}288 289extern __inline __m128d290 __attribute__((__gnu_inline__, __always_inline__, __artificial__))291 _mm_div_pd(__m128d __A, __m128d __B) {292 return (__m128d)((__v2df)__A / (__v2df)__B);293}294 295extern __inline __m128d296 __attribute__((__gnu_inline__, __always_inline__, __artificial__))297 _mm_div_sd(__m128d __A, __m128d __B) {298 __A[0] = __A[0] / __B[0];299 return (__A);300}301 302extern __inline __m128d303 __attribute__((__gnu_inline__, __always_inline__, __artificial__))304 _mm_sqrt_pd(__m128d __A) {305 return (vec_sqrt(__A));306}307 308/* Return pair {sqrt (B[0]), A[1]}. */309extern __inline __m128d310 __attribute__((__gnu_inline__, __always_inline__, __artificial__))311 _mm_sqrt_sd(__m128d __A, __m128d __B) {312 __v2df __c;313 __c = vec_sqrt((__v2df)_mm_set1_pd(__B[0]));314 return (__m128d)_mm_setr_pd(__c[0], __A[1]);315}316 317extern __inline __m128d318 __attribute__((__gnu_inline__, __always_inline__, __artificial__))319 _mm_min_pd(__m128d __A, __m128d __B) {320 return (vec_min(__A, __B));321}322 323extern __inline __m128d324 __attribute__((__gnu_inline__, __always_inline__, __artificial__))325 _mm_min_sd(__m128d __A, __m128d __B) {326 __v2df __a, __b, __c;327 __a = vec_splats(__A[0]);328 __b = vec_splats(__B[0]);329 __c = vec_min(__a, __b);330 return (__m128d)_mm_setr_pd(__c[0], __A[1]);331}332 333extern __inline __m128d334 __attribute__((__gnu_inline__, __always_inline__, __artificial__))335 _mm_max_pd(__m128d __A, __m128d __B) {336 return (vec_max(__A, __B));337}338 339extern __inline __m128d340 __attribute__((__gnu_inline__, __always_inline__, __artificial__))341 _mm_max_sd(__m128d __A, __m128d __B) {342 __v2df __a, __b, __c;343 __a = vec_splats(__A[0]);344 __b = vec_splats(__B[0]);345 __c = vec_max(__a, __b);346 return (__m128d)_mm_setr_pd(__c[0], __A[1]);347}348 349extern __inline __m128d350 __attribute__((__gnu_inline__, __always_inline__, __artificial__))351 _mm_cmpeq_pd(__m128d __A, __m128d __B) {352 return ((__m128d)vec_cmpeq((__v2df)__A, (__v2df)__B));353}354 355extern __inline __m128d356 __attribute__((__gnu_inline__, __always_inline__, __artificial__))357 _mm_cmplt_pd(__m128d __A, __m128d __B) {358 return ((__m128d)vec_cmplt((__v2df)__A, (__v2df)__B));359}360 361extern __inline __m128d362 __attribute__((__gnu_inline__, __always_inline__, __artificial__))363 _mm_cmple_pd(__m128d __A, __m128d __B) {364 return ((__m128d)vec_cmple((__v2df)__A, (__v2df)__B));365}366 367extern __inline __m128d368 __attribute__((__gnu_inline__, __always_inline__, __artificial__))369 _mm_cmpgt_pd(__m128d __A, __m128d __B) {370 return ((__m128d)vec_cmpgt((__v2df)__A, (__v2df)__B));371}372 373extern __inline __m128d374 __attribute__((__gnu_inline__, __always_inline__, __artificial__))375 _mm_cmpge_pd(__m128d __A, __m128d __B) {376 return ((__m128d)vec_cmpge((__v2df)__A, (__v2df)__B));377}378 379extern __inline __m128d380 __attribute__((__gnu_inline__, __always_inline__, __artificial__))381 _mm_cmpneq_pd(__m128d __A, __m128d __B) {382 __v2df __temp = (__v2df)vec_cmpeq((__v2df)__A, (__v2df)__B);383 return ((__m128d)vec_nor(__temp, __temp));384}385 386extern __inline __m128d387 __attribute__((__gnu_inline__, __always_inline__, __artificial__))388 _mm_cmpnlt_pd(__m128d __A, __m128d __B) {389 return ((__m128d)vec_cmpge((__v2df)__A, (__v2df)__B));390}391 392extern __inline __m128d393 __attribute__((__gnu_inline__, __always_inline__, __artificial__))394 _mm_cmpnle_pd(__m128d __A, __m128d __B) {395 return ((__m128d)vec_cmpgt((__v2df)__A, (__v2df)__B));396}397 398extern __inline __m128d399 __attribute__((__gnu_inline__, __always_inline__, __artificial__))400 _mm_cmpngt_pd(__m128d __A, __m128d __B) {401 return ((__m128d)vec_cmple((__v2df)__A, (__v2df)__B));402}403 404extern __inline __m128d405 __attribute__((__gnu_inline__, __always_inline__, __artificial__))406 _mm_cmpnge_pd(__m128d __A, __m128d __B) {407 return ((__m128d)vec_cmplt((__v2df)__A, (__v2df)__B));408}409 410extern __inline __m128d411 __attribute__((__gnu_inline__, __always_inline__, __artificial__))412 _mm_cmpord_pd(__m128d __A, __m128d __B) {413 __v2du __c, __d;414 /* Compare against self will return false (0's) if NAN. */415 __c = (__v2du)vec_cmpeq(__A, __A);416 __d = (__v2du)vec_cmpeq(__B, __B);417 /* A != NAN and B != NAN. */418 return ((__m128d)vec_and(__c, __d));419}420 421extern __inline __m128d422 __attribute__((__gnu_inline__, __always_inline__, __artificial__))423 _mm_cmpunord_pd(__m128d __A, __m128d __B) {424#if _ARCH_PWR8425 __v2du __c, __d;426 /* Compare against self will return false (0's) if NAN. */427 __c = (__v2du)vec_cmpeq((__v2df)__A, (__v2df)__A);428 __d = (__v2du)vec_cmpeq((__v2df)__B, (__v2df)__B);429 /* A == NAN OR B == NAN converts too:430 NOT(A != NAN) OR NOT(B != NAN). */431 __c = vec_nor(__c, __c);432 return ((__m128d)vec_orc(__c, __d));433#else434 __v2du __c, __d;435 /* Compare against self will return false (0's) if NAN. */436 __c = (__v2du)vec_cmpeq((__v2df)__A, (__v2df)__A);437 __d = (__v2du)vec_cmpeq((__v2df)__B, (__v2df)__B);438 /* Convert the true ('1's) is NAN. */439 __c = vec_nor(__c, __c);440 __d = vec_nor(__d, __d);441 return ((__m128d)vec_or(__c, __d));442#endif443}444 445extern __inline __m128d446 __attribute__((__gnu_inline__, __always_inline__, __artificial__))447 _mm_cmpeq_sd(__m128d __A, __m128d __B) {448 __v2df __a, __b, __c;449 /* PowerISA VSX does not allow partial (for just lower double)450 results. So to insure we don't generate spurious exceptions451 (from the upper double values) we splat the lower double452 before we do the operation. */453 __a = vec_splats(__A[0]);454 __b = vec_splats(__B[0]);455 __c = (__v2df)vec_cmpeq(__a, __b);456 /* Then we merge the lower double result with the original upper457 double from __A. */458 return (__m128d)_mm_setr_pd(__c[0], __A[1]);459}460 461extern __inline __m128d462 __attribute__((__gnu_inline__, __always_inline__, __artificial__))463 _mm_cmplt_sd(__m128d __A, __m128d __B) {464 __v2df __a, __b, __c;465 __a = vec_splats(__A[0]);466 __b = vec_splats(__B[0]);467 __c = (__v2df)vec_cmplt(__a, __b);468 return (__m128d)_mm_setr_pd(__c[0], __A[1]);469}470 471extern __inline __m128d472 __attribute__((__gnu_inline__, __always_inline__, __artificial__))473 _mm_cmple_sd(__m128d __A, __m128d __B) {474 __v2df __a, __b, __c;475 __a = vec_splats(__A[0]);476 __b = vec_splats(__B[0]);477 __c = (__v2df)vec_cmple(__a, __b);478 return (__m128d)_mm_setr_pd(__c[0], __A[1]);479}480 481extern __inline __m128d482 __attribute__((__gnu_inline__, __always_inline__, __artificial__))483 _mm_cmpgt_sd(__m128d __A, __m128d __B) {484 __v2df __a, __b, __c;485 __a = vec_splats(__A[0]);486 __b = vec_splats(__B[0]);487 __c = (__v2df)vec_cmpgt(__a, __b);488 return (__m128d)_mm_setr_pd(__c[0], __A[1]);489}490 491extern __inline __m128d492 __attribute__((__gnu_inline__, __always_inline__, __artificial__))493 _mm_cmpge_sd(__m128d __A, __m128d __B) {494 __v2df __a, __b, __c;495 __a = vec_splats(__A[0]);496 __b = vec_splats(__B[0]);497 __c = (__v2df)vec_cmpge(__a, __b);498 return (__m128d)_mm_setr_pd(__c[0], __A[1]);499}500 501extern __inline __m128d502 __attribute__((__gnu_inline__, __always_inline__, __artificial__))503 _mm_cmpneq_sd(__m128d __A, __m128d __B) {504 __v2df __a, __b, __c;505 __a = vec_splats(__A[0]);506 __b = vec_splats(__B[0]);507 __c = (__v2df)vec_cmpeq(__a, __b);508 __c = vec_nor(__c, __c);509 return (__m128d)_mm_setr_pd(__c[0], __A[1]);510}511 512extern __inline __m128d513 __attribute__((__gnu_inline__, __always_inline__, __artificial__))514 _mm_cmpnlt_sd(__m128d __A, __m128d __B) {515 __v2df __a, __b, __c;516 __a = vec_splats(__A[0]);517 __b = vec_splats(__B[0]);518 /* Not less than is just greater than or equal. */519 __c = (__v2df)vec_cmpge(__a, __b);520 return (__m128d)_mm_setr_pd(__c[0], __A[1]);521}522 523extern __inline __m128d524 __attribute__((__gnu_inline__, __always_inline__, __artificial__))525 _mm_cmpnle_sd(__m128d __A, __m128d __B) {526 __v2df __a, __b, __c;527 __a = vec_splats(__A[0]);528 __b = vec_splats(__B[0]);529 /* Not less than or equal is just greater than. */530 __c = (__v2df)vec_cmpge(__a, __b);531 return (__m128d)_mm_setr_pd(__c[0], __A[1]);532}533 534extern __inline __m128d535 __attribute__((__gnu_inline__, __always_inline__, __artificial__))536 _mm_cmpngt_sd(__m128d __A, __m128d __B) {537 __v2df __a, __b, __c;538 __a = vec_splats(__A[0]);539 __b = vec_splats(__B[0]);540 /* Not greater than is just less than or equal. */541 __c = (__v2df)vec_cmple(__a, __b);542 return (__m128d)_mm_setr_pd(__c[0], __A[1]);543}544 545extern __inline __m128d546 __attribute__((__gnu_inline__, __always_inline__, __artificial__))547 _mm_cmpnge_sd(__m128d __A, __m128d __B) {548 __v2df __a, __b, __c;549 __a = vec_splats(__A[0]);550 __b = vec_splats(__B[0]);551 /* Not greater than or equal is just less than. */552 __c = (__v2df)vec_cmplt(__a, __b);553 return (__m128d)_mm_setr_pd(__c[0], __A[1]);554}555 556extern __inline __m128d557 __attribute__((__gnu_inline__, __always_inline__, __artificial__))558 _mm_cmpord_sd(__m128d __A, __m128d __B) {559 __v2df __r;560 __r = (__v2df)_mm_cmpord_pd(vec_splats(__A[0]), vec_splats(__B[0]));561 return (__m128d)_mm_setr_pd(__r[0], ((__v2df)__A)[1]);562}563 564extern __inline __m128d565 __attribute__((__gnu_inline__, __always_inline__, __artificial__))566 _mm_cmpunord_sd(__m128d __A, __m128d __B) {567 __v2df __r;568 __r = _mm_cmpunord_pd(vec_splats(__A[0]), vec_splats(__B[0]));569 return (__m128d)_mm_setr_pd(__r[0], __A[1]);570}571 572/* FIXME573 The __mm_comi??_sd and __mm_ucomi??_sd implementations below are574 exactly the same because GCC for PowerPC only generates unordered575 compares (scalar and vector).576 Technically __mm_comieq_sp et all should be using the ordered577 compare and signal for QNaNs. The __mm_ucomieq_sd et all should578 be OK. */579extern __inline int580 __attribute__((__gnu_inline__, __always_inline__, __artificial__))581 _mm_comieq_sd(__m128d __A, __m128d __B) {582 return (__A[0] == __B[0]);583}584 585extern __inline int586 __attribute__((__gnu_inline__, __always_inline__, __artificial__))587 _mm_comilt_sd(__m128d __A, __m128d __B) {588 return (__A[0] < __B[0]);589}590 591extern __inline int592 __attribute__((__gnu_inline__, __always_inline__, __artificial__))593 _mm_comile_sd(__m128d __A, __m128d __B) {594 return (__A[0] <= __B[0]);595}596 597extern __inline int598 __attribute__((__gnu_inline__, __always_inline__, __artificial__))599 _mm_comigt_sd(__m128d __A, __m128d __B) {600 return (__A[0] > __B[0]);601}602 603extern __inline int604 __attribute__((__gnu_inline__, __always_inline__, __artificial__))605 _mm_comige_sd(__m128d __A, __m128d __B) {606 return (__A[0] >= __B[0]);607}608 609extern __inline int610 __attribute__((__gnu_inline__, __always_inline__, __artificial__))611 _mm_comineq_sd(__m128d __A, __m128d __B) {612 return (__A[0] != __B[0]);613}614 615extern __inline int616 __attribute__((__gnu_inline__, __always_inline__, __artificial__))617 _mm_ucomieq_sd(__m128d __A, __m128d __B) {618 return (__A[0] == __B[0]);619}620 621extern __inline int622 __attribute__((__gnu_inline__, __always_inline__, __artificial__))623 _mm_ucomilt_sd(__m128d __A, __m128d __B) {624 return (__A[0] < __B[0]);625}626 627extern __inline int628 __attribute__((__gnu_inline__, __always_inline__, __artificial__))629 _mm_ucomile_sd(__m128d __A, __m128d __B) {630 return (__A[0] <= __B[0]);631}632 633extern __inline int634 __attribute__((__gnu_inline__, __always_inline__, __artificial__))635 _mm_ucomigt_sd(__m128d __A, __m128d __B) {636 return (__A[0] > __B[0]);637}638 639extern __inline int640 __attribute__((__gnu_inline__, __always_inline__, __artificial__))641 _mm_ucomige_sd(__m128d __A, __m128d __B) {642 return (__A[0] >= __B[0]);643}644 645extern __inline int646 __attribute__((__gnu_inline__, __always_inline__, __artificial__))647 _mm_ucomineq_sd(__m128d __A, __m128d __B) {648 return (__A[0] != __B[0]);649}650 651/* Create a vector of Qi, where i is the element number. */652extern __inline __m128i653 __attribute__((__gnu_inline__, __always_inline__, __artificial__))654 _mm_set_epi64x(long long __q1, long long __q0) {655 return __extension__(__m128i)(__v2di){__q0, __q1};656}657 658extern __inline __m128i659 __attribute__((__gnu_inline__, __always_inline__, __artificial__))660 _mm_set_epi64(__m64 __q1, __m64 __q0) {661 return _mm_set_epi64x((long long)__q1, (long long)__q0);662}663 664extern __inline __m128i665 __attribute__((__gnu_inline__, __always_inline__, __artificial__))666 _mm_set_epi32(int __q3, int __q2, int __q1, int __q0) {667 return __extension__(__m128i)(__v4si){__q0, __q1, __q2, __q3};668}669 670extern __inline __m128i671 __attribute__((__gnu_inline__, __always_inline__, __artificial__))672 _mm_set_epi16(short __q7, short __q6, short __q5, short __q4, short __q3,673 short __q2, short __q1, short __q0) {674 return __extension__(__m128i)(__v8hi){__q0, __q1, __q2, __q3,675 __q4, __q5, __q6, __q7};676}677 678extern __inline __m128i679 __attribute__((__gnu_inline__, __always_inline__, __artificial__))680 _mm_set_epi8(char __q15, char __q14, char __q13, char __q12, char __q11,681 char __q10, char __q09, char __q08, char __q07, char __q06,682 char __q05, char __q04, char __q03, char __q02, char __q01,683 char __q00) {684 return __extension__(__m128i)(__v16qi){685 __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07,686 __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15};687}688 689/* Set all of the elements of the vector to A. */690extern __inline __m128i691 __attribute__((__gnu_inline__, __always_inline__, __artificial__))692 _mm_set1_epi64x(long long __A) {693 return _mm_set_epi64x(__A, __A);694}695 696extern __inline __m128i697 __attribute__((__gnu_inline__, __always_inline__, __artificial__))698 _mm_set1_epi64(__m64 __A) {699 return _mm_set_epi64(__A, __A);700}701 702extern __inline __m128i703 __attribute__((__gnu_inline__, __always_inline__, __artificial__))704 _mm_set1_epi32(int __A) {705 return _mm_set_epi32(__A, __A, __A, __A);706}707 708extern __inline __m128i709 __attribute__((__gnu_inline__, __always_inline__, __artificial__))710 _mm_set1_epi16(short __A) {711 return _mm_set_epi16(__A, __A, __A, __A, __A, __A, __A, __A);712}713 714extern __inline __m128i715 __attribute__((__gnu_inline__, __always_inline__, __artificial__))716 _mm_set1_epi8(char __A) {717 return _mm_set_epi8(__A, __A, __A, __A, __A, __A, __A, __A, __A, __A, __A,718 __A, __A, __A, __A, __A);719}720 721/* Create a vector of Qi, where i is the element number.722 The parameter order is reversed from the _mm_set_epi* functions. */723extern __inline __m128i724 __attribute__((__gnu_inline__, __always_inline__, __artificial__))725 _mm_setr_epi64(__m64 __q0, __m64 __q1) {726 return _mm_set_epi64(__q1, __q0);727}728 729extern __inline __m128i730 __attribute__((__gnu_inline__, __always_inline__, __artificial__))731 _mm_setr_epi32(int __q0, int __q1, int __q2, int __q3) {732 return _mm_set_epi32(__q3, __q2, __q1, __q0);733}734 735extern __inline __m128i736 __attribute__((__gnu_inline__, __always_inline__, __artificial__))737 _mm_setr_epi16(short __q0, short __q1, short __q2, short __q3, short __q4,738 short __q5, short __q6, short __q7) {739 return _mm_set_epi16(__q7, __q6, __q5, __q4, __q3, __q2, __q1, __q0);740}741 742extern __inline __m128i743 __attribute__((__gnu_inline__, __always_inline__, __artificial__))744 _mm_setr_epi8(char __q00, char __q01, char __q02, char __q03, char __q04,745 char __q05, char __q06, char __q07, char __q08, char __q09,746 char __q10, char __q11, char __q12, char __q13, char __q14,747 char __q15) {748 return _mm_set_epi8(__q15, __q14, __q13, __q12, __q11, __q10, __q09, __q08,749 __q07, __q06, __q05, __q04, __q03, __q02, __q01, __q00);750}751 752/* Create a vector with element 0 as *P and the rest zero. */753extern __inline __m128i754 __attribute__((__gnu_inline__, __always_inline__, __artificial__))755 _mm_load_si128(__m128i const *__P) {756 return *__P;757}758 759extern __inline __m128i760 __attribute__((__gnu_inline__, __always_inline__, __artificial__))761 _mm_loadu_si128(__m128i_u const *__P) {762 return (__m128i)(vec_vsx_ld(0, (signed int const *)__P));763}764 765extern __inline __m128i766 __attribute__((__gnu_inline__, __always_inline__, __artificial__))767 _mm_loadl_epi64(__m128i_u const *__P) {768 return _mm_set_epi64((__m64)0LL, *(__m64 *)__P);769}770 771extern __inline void772 __attribute__((__gnu_inline__, __always_inline__, __artificial__))773 _mm_store_si128(__m128i *__P, __m128i __B) {774 vec_st((__v16qu)__B, 0, (__v16qu *)__P);775}776 777extern __inline void778 __attribute__((__gnu_inline__, __always_inline__, __artificial__))779 _mm_storeu_si128(__m128i_u *__P, __m128i __B) {780 *__P = __B;781}782 783extern __inline void784 __attribute__((__gnu_inline__, __always_inline__, __artificial__))785 _mm_storel_epi64(__m128i_u *__P, __m128i __B) {786 *(long long *)__P = ((__v2di)__B)[0];787}788 789extern __inline __m64790 __attribute__((__gnu_inline__, __always_inline__, __artificial__))791 _mm_movepi64_pi64(__m128i_u __B) {792 return (__m64)((__v2di)__B)[0];793}794 795extern __inline __m128i796 __attribute__((__gnu_inline__, __always_inline__, __artificial__))797 _mm_movpi64_epi64(__m64 __A) {798 return _mm_set_epi64((__m64)0LL, __A);799}800 801extern __inline __m128i802 __attribute__((__gnu_inline__, __always_inline__, __artificial__))803 _mm_move_epi64(__m128i __A) {804 return _mm_set_epi64((__m64)0LL, (__m64)__A[0]);805}806 807/* Create an undefined vector. */808extern __inline __m128i809 __attribute__((__gnu_inline__, __always_inline__, __artificial__))810 _mm_undefined_si128(void) {811 __m128i __Y = __Y;812 return __Y;813}814 815/* Create a vector of zeros. */816extern __inline __m128i817 __attribute__((__gnu_inline__, __always_inline__, __artificial__))818 _mm_setzero_si128(void) {819 return __extension__(__m128i)(__v4si){0, 0, 0, 0};820}821 822#ifdef _ARCH_PWR8823extern __inline __m128d824 __attribute__((__gnu_inline__, __always_inline__, __artificial__))825 _mm_cvtepi32_pd(__m128i __A) {826 __v2di __val;827 /* For LE need to generate Vector Unpack Low Signed Word.828 Which is generated from unpackh. */829 __val = (__v2di)vec_unpackh((__v4si)__A);830 831 return (__m128d)vec_ctf(__val, 0);832}833#endif834 835extern __inline __m128836 __attribute__((__gnu_inline__, __always_inline__, __artificial__))837 _mm_cvtepi32_ps(__m128i __A) {838 return ((__m128)vec_ctf((__v4si)__A, 0));839}840 841extern __inline __m128i842 __attribute__((__gnu_inline__, __always_inline__, __artificial__))843 _mm_cvtpd_epi32(__m128d __A) {844 __v2df __rounded = vec_rint(__A);845 __v4si __result, __temp;846 const __v4si __vzero = {0, 0, 0, 0};847 848 /* VSX Vector truncate Double-Precision to integer and Convert to849 Signed Integer Word format with Saturate. */850 __asm__("xvcvdpsxws %x0,%x1" : "=wa"(__temp) : "wa"(__rounded) :);851 852#ifdef _ARCH_PWR8853#ifdef __LITTLE_ENDIAN__854 __temp = vec_mergeo(__temp, __temp);855#else856 __temp = vec_mergee(__temp, __temp);857#endif858 __result = (__v4si)vec_vpkudum((__vector long long)__temp,859 (__vector long long)__vzero);860#else861 {862 const __v16qu __pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,863 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f};864 __result = (__v4si)vec_perm((__v16qu)__temp, (__v16qu)__vzero, __pkperm);865 }866#endif867 return (__m128i)__result;868}869 870extern __inline __m64871 __attribute__((__gnu_inline__, __always_inline__, __artificial__))872 _mm_cvtpd_pi32(__m128d __A) {873 __m128i __result = _mm_cvtpd_epi32(__A);874 875 return (__m64)__result[0];876}877 878extern __inline __m128879 __attribute__((__gnu_inline__, __always_inline__, __artificial__))880 _mm_cvtpd_ps(__m128d __A) {881 __v4sf __result;882 __v4si __temp;883 const __v4si __vzero = {0, 0, 0, 0};884 885 __asm__("xvcvdpsp %x0,%x1" : "=wa"(__temp) : "wa"(__A) :);886 887#ifdef _ARCH_PWR8888#ifdef __LITTLE_ENDIAN__889 __temp = vec_mergeo(__temp, __temp);890#else891 __temp = vec_mergee(__temp, __temp);892#endif893 __result = (__v4sf)vec_vpkudum((__vector long long)__temp,894 (__vector long long)__vzero);895#else896 {897 const __v16qu __pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,898 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f};899 __result = (__v4sf)vec_perm((__v16qu)__temp, (__v16qu)__vzero, __pkperm);900 }901#endif902 return ((__m128)__result);903}904 905extern __inline __m128i906 __attribute__((__gnu_inline__, __always_inline__, __artificial__))907 _mm_cvttpd_epi32(__m128d __A) {908 __v4si __result;909 __v4si __temp;910 const __v4si __vzero = {0, 0, 0, 0};911 912 /* VSX Vector truncate Double-Precision to integer and Convert to913 Signed Integer Word format with Saturate. */914 __asm__("xvcvdpsxws %x0,%x1" : "=wa"(__temp) : "wa"(__A) :);915 916#ifdef _ARCH_PWR8917#ifdef __LITTLE_ENDIAN__918 __temp = vec_mergeo(__temp, __temp);919#else920 __temp = vec_mergee(__temp, __temp);921#endif922 __result = (__v4si)vec_vpkudum((__vector long long)__temp,923 (__vector long long)__vzero);924#else925 {926 const __v16qu __pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b,927 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f};928 __result = (__v4si)vec_perm((__v16qu)__temp, (__v16qu)__vzero, __pkperm);929 }930#endif931 932 return ((__m128i)__result);933}934 935extern __inline __m64936 __attribute__((__gnu_inline__, __always_inline__, __artificial__))937 _mm_cvttpd_pi32(__m128d __A) {938 __m128i __result = _mm_cvttpd_epi32(__A);939 940 return (__m64)__result[0];941}942 943extern __inline int944 __attribute__((__gnu_inline__, __always_inline__, __artificial__))945 _mm_cvtsi128_si32(__m128i __A) {946 return ((__v4si)__A)[0];947}948 949#ifdef _ARCH_PWR8950extern __inline __m128d951 __attribute__((__gnu_inline__, __always_inline__, __artificial__))952 _mm_cvtpi32_pd(__m64 __A) {953 __v4si __temp;954 __v2di __tmp2;955 __v4f __result;956 957 __temp = (__v4si)vec_splats(__A);958 __tmp2 = (__v2di)vec_unpackl(__temp);959 __result = vec_ctf((__vector signed long long)__tmp2, 0);960 return (__m128d)__result;961}962#endif963 964extern __inline __m128i965 __attribute__((__gnu_inline__, __always_inline__, __artificial__))966 _mm_cvtps_epi32(__m128 __A) {967 __v4sf __rounded;968 __v4si __result;969 970 __rounded = vec_rint((__v4sf)__A);971 __result = vec_cts(__rounded, 0);972 return (__m128i)__result;973}974 975extern __inline __m128i976 __attribute__((__gnu_inline__, __always_inline__, __artificial__))977 _mm_cvttps_epi32(__m128 __A) {978 __v4si __result;979 980 __result = vec_cts((__v4sf)__A, 0);981 return (__m128i)__result;982}983 984extern __inline __m128d985 __attribute__((__gnu_inline__, __always_inline__, __artificial__))986 _mm_cvtps_pd(__m128 __A) {987 /* Check if vec_doubleh is defined by <altivec.h>. If so use that. */988#ifdef vec_doubleh989 return (__m128d)vec_doubleh((__v4sf)__A);990#else991 /* Otherwise the compiler is not current and so need to generate the992 equivalent code. */993 __v4sf __a = (__v4sf)__A;994 __v4sf __temp;995 __v2df __result;996#ifdef __LITTLE_ENDIAN__997 /* The input float values are in elements {[0], [1]} but the convert998 instruction needs them in elements {[1], [3]}, So we use two999 shift left double vector word immediates to get the elements1000 lined up. */1001 __temp = __builtin_vsx_xxsldwi(__a, __a, 3);1002 __temp = __builtin_vsx_xxsldwi(__a, __temp, 2);1003#else1004 /* The input float values are in elements {[0], [1]} but the convert1005 instruction needs them in elements {[0], [2]}, So we use two1006 shift left double vector word immediates to get the elements1007 lined up. */1008 __temp = vec_vmrghw(__a, __a);1009#endif1010 __asm__(" xvcvspdp %x0,%x1" : "=wa"(__result) : "wa"(__temp) :);1011 return (__m128d)__result;1012#endif1013}1014 1015extern __inline int1016 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1017 _mm_cvtsd_si32(__m128d __A) {1018 __v2df __rounded = vec_rint((__v2df)__A);1019 int __result = ((__v2df)__rounded)[0];1020 1021 return __result;1022}1023/* Intel intrinsic. */1024extern __inline long long1025 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1026 _mm_cvtsd_si64(__m128d __A) {1027 __v2df __rounded = vec_rint((__v2df)__A);1028 long long __result = ((__v2df)__rounded)[0];1029 1030 return __result;1031}1032 1033/* Microsoft intrinsic. */1034extern __inline long long1035 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1036 _mm_cvtsd_si64x(__m128d __A) {1037 return _mm_cvtsd_si64((__v2df)__A);1038}1039 1040extern __inline int1041 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1042 _mm_cvttsd_si32(__m128d __A) {1043 int __result = ((__v2df)__A)[0];1044 1045 return __result;1046}1047 1048/* Intel intrinsic. */1049extern __inline long long1050 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1051 _mm_cvttsd_si64(__m128d __A) {1052 long long __result = ((__v2df)__A)[0];1053 1054 return __result;1055}1056 1057/* Microsoft intrinsic. */1058extern __inline long long1059 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1060 _mm_cvttsd_si64x(__m128d __A) {1061 return _mm_cvttsd_si64(__A);1062}1063 1064extern __inline __m1281065 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1066 _mm_cvtsd_ss(__m128 __A, __m128d __B) {1067 __v4sf __result = (__v4sf)__A;1068 1069#ifdef __LITTLE_ENDIAN__1070 __v4sf __temp_s;1071 /* Copy double element[0] to element [1] for conversion. */1072 __v2df __temp_b = vec_splat((__v2df)__B, 0);1073 1074 /* Pre-rotate __A left 3 (logically right 1) elements. */1075 __result = __builtin_vsx_xxsldwi(__result, __result, 3);1076 /* Convert double to single float scalar in a vector. */1077 __asm__("xscvdpsp %x0,%x1" : "=wa"(__temp_s) : "wa"(__temp_b) :);1078 /* Shift the resulting scalar into vector element [0]. */1079 __result = __builtin_vsx_xxsldwi(__result, __temp_s, 1);1080#else1081 __result[0] = ((__v2df)__B)[0];1082#endif1083 return (__m128)__result;1084}1085 1086extern __inline __m128d1087 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1088 _mm_cvtsi32_sd(__m128d __A, int __B) {1089 __v2df __result = (__v2df)__A;1090 double __db = __B;1091 __result[0] = __db;1092 return (__m128d)__result;1093}1094 1095/* Intel intrinsic. */1096extern __inline __m128d1097 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1098 _mm_cvtsi64_sd(__m128d __A, long long __B) {1099 __v2df __result = (__v2df)__A;1100 double __db = __B;1101 __result[0] = __db;1102 return (__m128d)__result;1103}1104 1105/* Microsoft intrinsic. */1106extern __inline __m128d1107 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1108 _mm_cvtsi64x_sd(__m128d __A, long long __B) {1109 return _mm_cvtsi64_sd(__A, __B);1110}1111 1112extern __inline __m128d1113 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1114 _mm_cvtss_sd(__m128d __A, __m128 __B) {1115#ifdef __LITTLE_ENDIAN__1116 /* Use splat to move element [0] into position for the convert. */1117 __v4sf __temp = vec_splat((__v4sf)__B, 0);1118 __v2df __res;1119 /* Convert single float scalar to double in a vector. */1120 __asm__("xscvspdp %x0,%x1" : "=wa"(__res) : "wa"(__temp) :);1121 return (__m128d)vec_mergel(__res, (__v2df)__A);1122#else1123 __v2df __res = (__v2df)__A;1124 __res[0] = ((__v4sf)__B)[0];1125 return (__m128d)__res;1126#endif1127}1128 1129extern __inline __m128d1130 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1131 _mm_shuffle_pd(__m128d __A, __m128d __B, const int __mask) {1132 __vector double __result;1133 const int __litmsk = __mask & 0x3;1134 1135 if (__litmsk == 0)1136 __result = vec_mergeh(__A, __B);1137#if __GNUC__ < 61138 else if (__litmsk == 1)1139 __result = vec_xxpermdi(__B, __A, 2);1140 else if (__litmsk == 2)1141 __result = vec_xxpermdi(__B, __A, 1);1142#else1143 else if (__litmsk == 1)1144 __result = vec_xxpermdi(__A, __B, 2);1145 else if (__litmsk == 2)1146 __result = vec_xxpermdi(__A, __B, 1);1147#endif1148 else1149 __result = vec_mergel(__A, __B);1150 1151 return __result;1152}1153 1154extern __inline __m128d1155 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1156 _mm_unpackhi_pd(__m128d __A, __m128d __B) {1157 return (__m128d)vec_mergel((__v2df)__A, (__v2df)__B);1158}1159 1160extern __inline __m128d1161 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1162 _mm_unpacklo_pd(__m128d __A, __m128d __B) {1163 return (__m128d)vec_mergeh((__v2df)__A, (__v2df)__B);1164}1165 1166extern __inline __m128d1167 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1168 _mm_loadh_pd(__m128d __A, double const *__B) {1169 __v2df __result = (__v2df)__A;1170 __result[1] = *__B;1171 return (__m128d)__result;1172}1173 1174extern __inline __m128d1175 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1176 _mm_loadl_pd(__m128d __A, double const *__B) {1177 __v2df __result = (__v2df)__A;1178 __result[0] = *__B;1179 return (__m128d)__result;1180}1181 1182#ifdef _ARCH_PWR81183/* Intrinsic functions that require PowerISA 2.07 minimum. */1184 1185/* Creates a 2-bit mask from the most significant bits of the DPFP values. */1186extern __inline int1187 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1188 _mm_movemask_pd(__m128d __A) {1189#ifdef _ARCH_PWR101190 return vec_extractm((__v2du)__A);1191#else1192 __vector unsigned long long __result;1193 static const __vector unsigned int __perm_mask = {1194#ifdef __LITTLE_ENDIAN__1195 0x80800040, 0x80808080, 0x80808080, 0x808080801196#else1197 0x80808080, 0x80808080, 0x80808080, 0x808040001198#endif1199 };1200 1201 __result = ((__vector unsigned long long)vec_vbpermq(1202 (__vector unsigned char)__A, (__vector unsigned char)__perm_mask));1203 1204#ifdef __LITTLE_ENDIAN__1205 return __result[1];1206#else1207 return __result[0];1208#endif1209#endif /* !_ARCH_PWR10 */1210}1211#endif /* _ARCH_PWR8 */1212 1213extern __inline __m128i1214 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1215 _mm_packs_epi16(__m128i __A, __m128i __B) {1216 return (__m128i)vec_packs((__v8hi)__A, (__v8hi)__B);1217}1218 1219extern __inline __m128i1220 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1221 _mm_packs_epi32(__m128i __A, __m128i __B) {1222 return (__m128i)vec_packs((__v4si)__A, (__v4si)__B);1223}1224 1225extern __inline __m128i1226 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1227 _mm_packus_epi16(__m128i __A, __m128i __B) {1228 return (__m128i)vec_packsu((__v8hi)__A, (__v8hi)__B);1229}1230 1231extern __inline __m128i1232 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1233 _mm_unpackhi_epi8(__m128i __A, __m128i __B) {1234 return (__m128i)vec_mergel((__v16qu)__A, (__v16qu)__B);1235}1236 1237extern __inline __m128i1238 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1239 _mm_unpackhi_epi16(__m128i __A, __m128i __B) {1240 return (__m128i)vec_mergel((__v8hu)__A, (__v8hu)__B);1241}1242 1243extern __inline __m128i1244 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1245 _mm_unpackhi_epi32(__m128i __A, __m128i __B) {1246 return (__m128i)vec_mergel((__v4su)__A, (__v4su)__B);1247}1248 1249extern __inline __m128i1250 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1251 _mm_unpackhi_epi64(__m128i __A, __m128i __B) {1252 return (__m128i)vec_mergel((__vector long long)__A, (__vector long long)__B);1253}1254 1255extern __inline __m128i1256 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1257 _mm_unpacklo_epi8(__m128i __A, __m128i __B) {1258 return (__m128i)vec_mergeh((__v16qu)__A, (__v16qu)__B);1259}1260 1261extern __inline __m128i1262 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1263 _mm_unpacklo_epi16(__m128i __A, __m128i __B) {1264 return (__m128i)vec_mergeh((__v8hi)__A, (__v8hi)__B);1265}1266 1267extern __inline __m128i1268 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1269 _mm_unpacklo_epi32(__m128i __A, __m128i __B) {1270 return (__m128i)vec_mergeh((__v4si)__A, (__v4si)__B);1271}1272 1273extern __inline __m128i1274 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1275 _mm_unpacklo_epi64(__m128i __A, __m128i __B) {1276 return (__m128i)vec_mergeh((__vector long long)__A, (__vector long long)__B);1277}1278 1279extern __inline __m128i1280 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1281 _mm_add_epi8(__m128i __A, __m128i __B) {1282 return (__m128i)((__v16qu)__A + (__v16qu)__B);1283}1284 1285extern __inline __m128i1286 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1287 _mm_add_epi16(__m128i __A, __m128i __B) {1288 return (__m128i)((__v8hu)__A + (__v8hu)__B);1289}1290 1291extern __inline __m128i1292 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1293 _mm_add_epi32(__m128i __A, __m128i __B) {1294 return (__m128i)((__v4su)__A + (__v4su)__B);1295}1296 1297extern __inline __m128i1298 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1299 _mm_add_epi64(__m128i __A, __m128i __B) {1300 return (__m128i)((__v2du)__A + (__v2du)__B);1301}1302 1303extern __inline __m128i1304 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1305 _mm_adds_epi8(__m128i __A, __m128i __B) {1306 return (__m128i)vec_adds((__v16qi)__A, (__v16qi)__B);1307}1308 1309extern __inline __m128i1310 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1311 _mm_adds_epi16(__m128i __A, __m128i __B) {1312 return (__m128i)vec_adds((__v8hi)__A, (__v8hi)__B);1313}1314 1315extern __inline __m128i1316 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1317 _mm_adds_epu8(__m128i __A, __m128i __B) {1318 return (__m128i)vec_adds((__v16qu)__A, (__v16qu)__B);1319}1320 1321extern __inline __m128i1322 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1323 _mm_adds_epu16(__m128i __A, __m128i __B) {1324 return (__m128i)vec_adds((__v8hu)__A, (__v8hu)__B);1325}1326 1327extern __inline __m128i1328 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1329 _mm_sub_epi8(__m128i __A, __m128i __B) {1330 return (__m128i)((__v16qu)__A - (__v16qu)__B);1331}1332 1333extern __inline __m128i1334 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1335 _mm_sub_epi16(__m128i __A, __m128i __B) {1336 return (__m128i)((__v8hu)__A - (__v8hu)__B);1337}1338 1339extern __inline __m128i1340 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1341 _mm_sub_epi32(__m128i __A, __m128i __B) {1342 return (__m128i)((__v4su)__A - (__v4su)__B);1343}1344 1345extern __inline __m128i1346 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1347 _mm_sub_epi64(__m128i __A, __m128i __B) {1348 return (__m128i)((__v2du)__A - (__v2du)__B);1349}1350 1351extern __inline __m128i1352 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1353 _mm_subs_epi8(__m128i __A, __m128i __B) {1354 return (__m128i)vec_subs((__v16qi)__A, (__v16qi)__B);1355}1356 1357extern __inline __m128i1358 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1359 _mm_subs_epi16(__m128i __A, __m128i __B) {1360 return (__m128i)vec_subs((__v8hi)__A, (__v8hi)__B);1361}1362 1363extern __inline __m128i1364 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1365 _mm_subs_epu8(__m128i __A, __m128i __B) {1366 return (__m128i)vec_subs((__v16qu)__A, (__v16qu)__B);1367}1368 1369extern __inline __m128i1370 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1371 _mm_subs_epu16(__m128i __A, __m128i __B) {1372 return (__m128i)vec_subs((__v8hu)__A, (__v8hu)__B);1373}1374 1375extern __inline __m128i1376 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1377 _mm_madd_epi16(__m128i __A, __m128i __B) {1378 __vector signed int __zero = {0, 0, 0, 0};1379 1380 return (__m128i)vec_vmsumshm((__v8hi)__A, (__v8hi)__B, __zero);1381}1382 1383extern __inline __m128i1384 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1385 _mm_mulhi_epi16(__m128i __A, __m128i __B) {1386 __vector signed int __w0, __w1;1387 1388 __vector unsigned char __xform1 = {1389#ifdef __LITTLE_ENDIAN__1390 0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17, 0x0A,1391 0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F1392#else1393 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15, 0x08,1394 0x09, 0x18, 0x19, 0x0C, 0x0D, 0x1C, 0x1D1395#endif1396 };1397 1398 __w0 = vec_vmulesh((__v8hi)__A, (__v8hi)__B);1399 __w1 = vec_vmulosh((__v8hi)__A, (__v8hi)__B);1400 return (__m128i)vec_perm(__w0, __w1, __xform1);1401}1402 1403extern __inline __m128i1404 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1405 _mm_mullo_epi16(__m128i __A, __m128i __B) {1406 return (__m128i)((__v8hi)__A * (__v8hi)__B);1407}1408 1409extern __inline __m641410 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1411 _mm_mul_su32(__m64 __A, __m64 __B) {1412 unsigned int __a = __A;1413 unsigned int __b = __B;1414 1415 return ((__m64)__a * (__m64)__b);1416}1417 1418#ifdef _ARCH_PWR81419extern __inline __m128i1420 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1421 _mm_mul_epu32(__m128i __A, __m128i __B) {1422#if __GNUC__ < 81423 __v2du __result;1424 1425#ifdef __LITTLE_ENDIAN__1426 /* VMX Vector Multiply Odd Unsigned Word. */1427 __asm__("vmulouw %0,%1,%2" : "=v"(__result) : "v"(__A), "v"(__B) :);1428#else1429 /* VMX Vector Multiply Even Unsigned Word. */1430 __asm__("vmuleuw %0,%1,%2" : "=v"(__result) : "v"(__A), "v"(__B) :);1431#endif1432 return (__m128i)__result;1433#else1434 return (__m128i)vec_mule((__v4su)__A, (__v4su)__B);1435#endif1436}1437#endif1438 1439extern __inline __m128i1440 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1441 _mm_slli_epi16(__m128i __A, int __B) {1442 __v8hu __lshift;1443 __v8hi __result = {0, 0, 0, 0, 0, 0, 0, 0};1444 1445 if (__B >= 0 && __B < 16) {1446 if (__builtin_constant_p(__B))1447 __lshift = (__v8hu)vec_splat_s16(__B);1448 else1449 __lshift = vec_splats((unsigned short)__B);1450 1451 __result = vec_sl((__v8hi)__A, __lshift);1452 }1453 1454 return (__m128i)__result;1455}1456 1457extern __inline __m128i1458 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1459 _mm_slli_epi32(__m128i __A, int __B) {1460 __v4su __lshift;1461 __v4si __result = {0, 0, 0, 0};1462 1463 if (__B >= 0 && __B < 32) {1464 if (__builtin_constant_p(__B) && __B < 16)1465 __lshift = (__v4su)vec_splat_s32(__B);1466 else1467 __lshift = vec_splats((unsigned int)__B);1468 1469 __result = vec_sl((__v4si)__A, __lshift);1470 }1471 1472 return (__m128i)__result;1473}1474 1475#ifdef _ARCH_PWR81476extern __inline __m128i1477 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1478 _mm_slli_epi64(__m128i __A, int __B) {1479 __v2du __lshift;1480 __v2di __result = {0, 0};1481 1482 if (__B >= 0 && __B < 64) {1483 if (__builtin_constant_p(__B) && __B < 16)1484 __lshift = (__v2du)vec_splat_s32(__B);1485 else1486 __lshift = (__v2du)vec_splats((unsigned int)__B);1487 1488 __result = vec_sl((__v2di)__A, __lshift);1489 }1490 1491 return (__m128i)__result;1492}1493#endif1494 1495extern __inline __m128i1496 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1497 _mm_srai_epi16(__m128i __A, int __B) {1498 __v8hu __rshift = {15, 15, 15, 15, 15, 15, 15, 15};1499 __v8hi __result;1500 1501 if (__B < 16) {1502 if (__builtin_constant_p(__B))1503 __rshift = (__v8hu)vec_splat_s16(__B);1504 else1505 __rshift = vec_splats((unsigned short)__B);1506 }1507 __result = vec_sra((__v8hi)__A, __rshift);1508 1509 return (__m128i)__result;1510}1511 1512extern __inline __m128i1513 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1514 _mm_srai_epi32(__m128i __A, int __B) {1515 __v4su __rshift = {31, 31, 31, 31};1516 __v4si __result;1517 1518 if (__B < 32) {1519 if (__builtin_constant_p(__B)) {1520 if (__B < 16)1521 __rshift = (__v4su)vec_splat_s32(__B);1522 else1523 __rshift = (__v4su)vec_splats((unsigned int)__B);1524 } else1525 __rshift = vec_splats((unsigned int)__B);1526 }1527 __result = vec_sra((__v4si)__A, __rshift);1528 1529 return (__m128i)__result;1530}1531 1532extern __inline __m128i1533 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1534 _mm_bslli_si128(__m128i __A, const int __N) {1535 __v16qu __result;1536 const __v16qu __zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};1537 1538 if (__N < 16)1539 __result = vec_sld((__v16qu)__A, __zeros, __N);1540 else1541 __result = __zeros;1542 1543 return (__m128i)__result;1544}1545 1546extern __inline __m128i1547 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1548 _mm_bsrli_si128(__m128i __A, const int __N) {1549 __v16qu __result;1550 const __v16qu __zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};1551 1552 if (__N < 16)1553#ifdef __LITTLE_ENDIAN__1554 if (__builtin_constant_p(__N))1555 /* Would like to use Vector Shift Left Double by Octet1556 Immediate here to use the immediate form and avoid1557 load of __N * 8 value into a separate VR. */1558 __result = vec_sld(__zeros, (__v16qu)__A, (16 - __N));1559 else1560#endif1561 {1562 __v16qu __shift = vec_splats((unsigned char)(__N * 8));1563#ifdef __LITTLE_ENDIAN__1564 __result = vec_sro((__v16qu)__A, __shift);1565#else1566 __result = vec_slo((__v16qu)__A, __shift);1567#endif1568 }1569 else1570 __result = __zeros;1571 1572 return (__m128i)__result;1573}1574 1575extern __inline __m128i1576 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1577 _mm_srli_si128(__m128i __A, const int __N) {1578 return _mm_bsrli_si128(__A, __N);1579}1580 1581extern __inline __m128i1582 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1583 _mm_slli_si128(__m128i __A, const int _imm5) {1584 __v16qu __result;1585 const __v16qu __zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};1586 1587 if (_imm5 < 16)1588#ifdef __LITTLE_ENDIAN__1589 __result = vec_sld((__v16qu)__A, __zeros, _imm5);1590#else1591 __result = vec_sld(__zeros, (__v16qu)__A, (16 - _imm5));1592#endif1593 else1594 __result = __zeros;1595 1596 return (__m128i)__result;1597}1598 1599extern __inline __m128i1600 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1601 1602 _mm_srli_epi16(__m128i __A, int __B) {1603 __v8hu __rshift;1604 __v8hi __result = {0, 0, 0, 0, 0, 0, 0, 0};1605 1606 if (__B < 16) {1607 if (__builtin_constant_p(__B))1608 __rshift = (__v8hu)vec_splat_s16(__B);1609 else1610 __rshift = vec_splats((unsigned short)__B);1611 1612 __result = vec_sr((__v8hi)__A, __rshift);1613 }1614 1615 return (__m128i)__result;1616}1617 1618extern __inline __m128i1619 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1620 _mm_srli_epi32(__m128i __A, int __B) {1621 __v4su __rshift;1622 __v4si __result = {0, 0, 0, 0};1623 1624 if (__B < 32) {1625 if (__builtin_constant_p(__B)) {1626 if (__B < 16)1627 __rshift = (__v4su)vec_splat_s32(__B);1628 else1629 __rshift = (__v4su)vec_splats((unsigned int)__B);1630 } else1631 __rshift = vec_splats((unsigned int)__B);1632 1633 __result = vec_sr((__v4si)__A, __rshift);1634 }1635 1636 return (__m128i)__result;1637}1638 1639#ifdef _ARCH_PWR81640extern __inline __m128i1641 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1642 _mm_srli_epi64(__m128i __A, int __B) {1643 __v2du __rshift;1644 __v2di __result = {0, 0};1645 1646 if (__B < 64) {1647 if (__builtin_constant_p(__B)) {1648 if (__B < 16)1649 __rshift = (__v2du)vec_splat_s32(__B);1650 else1651 __rshift = (__v2du)vec_splats((unsigned long long)__B);1652 } else1653 __rshift = (__v2du)vec_splats((unsigned int)__B);1654 1655 __result = vec_sr((__v2di)__A, __rshift);1656 }1657 1658 return (__m128i)__result;1659}1660#endif1661 1662extern __inline __m128i1663 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1664 _mm_sll_epi16(__m128i __A, __m128i __B) {1665 __v8hu __lshift;1666 __vector __bool short __shmask;1667 const __v8hu __shmax = {15, 15, 15, 15, 15, 15, 15, 15};1668 __v8hu __result;1669 1670#ifdef __LITTLE_ENDIAN__1671 __lshift = vec_splat((__v8hu)__B, 0);1672#else1673 __lshift = vec_splat((__v8hu)__B, 3);1674#endif1675 __shmask = vec_cmple(__lshift, __shmax);1676 __result = vec_sl((__v8hu)__A, __lshift);1677 __result = vec_sel((__v8hu)__shmask, __result, __shmask);1678 1679 return (__m128i)__result;1680}1681 1682extern __inline __m128i1683 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1684 _mm_sll_epi32(__m128i __A, __m128i __B) {1685 __v4su __lshift;1686 __vector __bool int __shmask;1687 const __v4su __shmax = {32, 32, 32, 32};1688 __v4su __result;1689#ifdef __LITTLE_ENDIAN__1690 __lshift = vec_splat((__v4su)__B, 0);1691#else1692 __lshift = vec_splat((__v4su)__B, 1);1693#endif1694 __shmask = vec_cmplt(__lshift, __shmax);1695 __result = vec_sl((__v4su)__A, __lshift);1696 __result = vec_sel((__v4su)__shmask, __result, __shmask);1697 1698 return (__m128i)__result;1699}1700 1701#ifdef _ARCH_PWR81702extern __inline __m128i1703 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1704 _mm_sll_epi64(__m128i __A, __m128i __B) {1705 __v2du __lshift;1706 __vector __bool long long __shmask;1707 const __v2du __shmax = {64, 64};1708 __v2du __result;1709 1710 __lshift = vec_splat((__v2du)__B, 0);1711 __shmask = vec_cmplt(__lshift, __shmax);1712 __result = vec_sl((__v2du)__A, __lshift);1713 __result = vec_sel((__v2du)__shmask, __result, __shmask);1714 1715 return (__m128i)__result;1716}1717#endif1718 1719extern __inline __m128i1720 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1721 _mm_sra_epi16(__m128i __A, __m128i __B) {1722 const __v8hu __rshmax = {15, 15, 15, 15, 15, 15, 15, 15};1723 __v8hu __rshift;1724 __v8hi __result;1725 1726#ifdef __LITTLE_ENDIAN__1727 __rshift = vec_splat((__v8hu)__B, 0);1728#else1729 __rshift = vec_splat((__v8hu)__B, 3);1730#endif1731 __rshift = vec_min(__rshift, __rshmax);1732 __result = vec_sra((__v8hi)__A, __rshift);1733 1734 return (__m128i)__result;1735}1736 1737extern __inline __m128i1738 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1739 _mm_sra_epi32(__m128i __A, __m128i __B) {1740 const __v4su __rshmax = {31, 31, 31, 31};1741 __v4su __rshift;1742 __v4si __result;1743 1744#ifdef __LITTLE_ENDIAN__1745 __rshift = vec_splat((__v4su)__B, 0);1746#else1747 __rshift = vec_splat((__v4su)__B, 1);1748#endif1749 __rshift = vec_min(__rshift, __rshmax);1750 __result = vec_sra((__v4si)__A, __rshift);1751 1752 return (__m128i)__result;1753}1754 1755extern __inline __m128i1756 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1757 _mm_srl_epi16(__m128i __A, __m128i __B) {1758 __v8hu __rshift;1759 __vector __bool short __shmask;1760 const __v8hu __shmax = {15, 15, 15, 15, 15, 15, 15, 15};1761 __v8hu __result;1762 1763#ifdef __LITTLE_ENDIAN__1764 __rshift = vec_splat((__v8hu)__B, 0);1765#else1766 __rshift = vec_splat((__v8hu)__B, 3);1767#endif1768 __shmask = vec_cmple(__rshift, __shmax);1769 __result = vec_sr((__v8hu)__A, __rshift);1770 __result = vec_sel((__v8hu)__shmask, __result, __shmask);1771 1772 return (__m128i)__result;1773}1774 1775extern __inline __m128i1776 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1777 _mm_srl_epi32(__m128i __A, __m128i __B) {1778 __v4su __rshift;1779 __vector __bool int __shmask;1780 const __v4su __shmax = {32, 32, 32, 32};1781 __v4su __result;1782 1783#ifdef __LITTLE_ENDIAN__1784 __rshift = vec_splat((__v4su)__B, 0);1785#else1786 __rshift = vec_splat((__v4su)__B, 1);1787#endif1788 __shmask = vec_cmplt(__rshift, __shmax);1789 __result = vec_sr((__v4su)__A, __rshift);1790 __result = vec_sel((__v4su)__shmask, __result, __shmask);1791 1792 return (__m128i)__result;1793}1794 1795#ifdef _ARCH_PWR81796extern __inline __m128i1797 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1798 _mm_srl_epi64(__m128i __A, __m128i __B) {1799 __v2du __rshift;1800 __vector __bool long long __shmask;1801 const __v2du __shmax = {64, 64};1802 __v2du __result;1803 1804 __rshift = vec_splat((__v2du)__B, 0);1805 __shmask = vec_cmplt(__rshift, __shmax);1806 __result = vec_sr((__v2du)__A, __rshift);1807 __result = vec_sel((__v2du)__shmask, __result, __shmask);1808 1809 return (__m128i)__result;1810}1811#endif1812 1813extern __inline __m128d1814 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1815 _mm_and_pd(__m128d __A, __m128d __B) {1816 return (vec_and((__v2df)__A, (__v2df)__B));1817}1818 1819extern __inline __m128d1820 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1821 _mm_andnot_pd(__m128d __A, __m128d __B) {1822 return (vec_andc((__v2df)__B, (__v2df)__A));1823}1824 1825extern __inline __m128d1826 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1827 _mm_or_pd(__m128d __A, __m128d __B) {1828 return (vec_or((__v2df)__A, (__v2df)__B));1829}1830 1831extern __inline __m128d1832 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1833 _mm_xor_pd(__m128d __A, __m128d __B) {1834 return (vec_xor((__v2df)__A, (__v2df)__B));1835}1836 1837extern __inline __m128i1838 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1839 _mm_and_si128(__m128i __A, __m128i __B) {1840 return (__m128i)vec_and((__v2di)__A, (__v2di)__B);1841}1842 1843extern __inline __m128i1844 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1845 _mm_andnot_si128(__m128i __A, __m128i __B) {1846 return (__m128i)vec_andc((__v2di)__B, (__v2di)__A);1847}1848 1849extern __inline __m128i1850 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1851 _mm_or_si128(__m128i __A, __m128i __B) {1852 return (__m128i)vec_or((__v2di)__A, (__v2di)__B);1853}1854 1855extern __inline __m128i1856 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1857 _mm_xor_si128(__m128i __A, __m128i __B) {1858 return (__m128i)vec_xor((__v2di)__A, (__v2di)__B);1859}1860 1861extern __inline __m128i1862 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1863 _mm_cmpeq_epi8(__m128i __A, __m128i __B) {1864 return (__m128i)vec_cmpeq((__v16qi)__A, (__v16qi)__B);1865}1866 1867extern __inline __m128i1868 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1869 _mm_cmpeq_epi16(__m128i __A, __m128i __B) {1870 return (__m128i)vec_cmpeq((__v8hi)__A, (__v8hi)__B);1871}1872 1873extern __inline __m128i1874 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1875 _mm_cmpeq_epi32(__m128i __A, __m128i __B) {1876 return (__m128i)vec_cmpeq((__v4si)__A, (__v4si)__B);1877}1878 1879extern __inline __m128i1880 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1881 _mm_cmplt_epi8(__m128i __A, __m128i __B) {1882 return (__m128i)vec_cmplt((__v16qi)__A, (__v16qi)__B);1883}1884 1885extern __inline __m128i1886 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1887 _mm_cmplt_epi16(__m128i __A, __m128i __B) {1888 return (__m128i)vec_cmplt((__v8hi)__A, (__v8hi)__B);1889}1890 1891extern __inline __m128i1892 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1893 _mm_cmplt_epi32(__m128i __A, __m128i __B) {1894 return (__m128i)vec_cmplt((__v4si)__A, (__v4si)__B);1895}1896 1897extern __inline __m128i1898 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1899 _mm_cmpgt_epi8(__m128i __A, __m128i __B) {1900 return (__m128i)vec_cmpgt((__v16qi)__A, (__v16qi)__B);1901}1902 1903extern __inline __m128i1904 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1905 _mm_cmpgt_epi16(__m128i __A, __m128i __B) {1906 return (__m128i)vec_cmpgt((__v8hi)__A, (__v8hi)__B);1907}1908 1909extern __inline __m128i1910 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1911 _mm_cmpgt_epi32(__m128i __A, __m128i __B) {1912 return (__m128i)vec_cmpgt((__v4si)__A, (__v4si)__B);1913}1914 1915extern __inline int1916 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1917 _mm_extract_epi16(__m128i const __A, int const __N) {1918 return (unsigned short)((__v8hi)__A)[__N & 7];1919}1920 1921extern __inline __m128i1922 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1923 _mm_insert_epi16(__m128i const __A, int const __D, int const __N) {1924 __v8hi __result = (__v8hi)__A;1925 1926 __result[(__N & 7)] = __D;1927 1928 return (__m128i)__result;1929}1930 1931extern __inline __m128i1932 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1933 _mm_max_epi16(__m128i __A, __m128i __B) {1934 return (__m128i)vec_max((__v8hi)__A, (__v8hi)__B);1935}1936 1937extern __inline __m128i1938 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1939 _mm_max_epu8(__m128i __A, __m128i __B) {1940 return (__m128i)vec_max((__v16qu)__A, (__v16qu)__B);1941}1942 1943extern __inline __m128i1944 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1945 _mm_min_epi16(__m128i __A, __m128i __B) {1946 return (__m128i)vec_min((__v8hi)__A, (__v8hi)__B);1947}1948 1949extern __inline __m128i1950 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1951 _mm_min_epu8(__m128i __A, __m128i __B) {1952 return (__m128i)vec_min((__v16qu)__A, (__v16qu)__B);1953}1954 1955#ifdef _ARCH_PWR81956/* Intrinsic functions that require PowerISA 2.07 minimum. */1957 1958/* Return a mask created from the most significant bit of each 8-bit1959 element in A. */1960extern __inline int1961 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1962 _mm_movemask_epi8(__m128i __A) {1963#ifdef _ARCH_PWR101964 return vec_extractm((__v16qu)__A);1965#else1966 __vector unsigned long long __result;1967 static const __vector unsigned char __perm_mask = {1968 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40,1969 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00};1970 1971 __result = ((__vector unsigned long long)vec_vbpermq(1972 (__vector unsigned char)__A, (__vector unsigned char)__perm_mask));1973 1974#ifdef __LITTLE_ENDIAN__1975 return __result[1];1976#else1977 return __result[0];1978#endif1979#endif /* !_ARCH_PWR10 */1980}1981#endif /* _ARCH_PWR8 */1982 1983extern __inline __m128i1984 __attribute__((__gnu_inline__, __always_inline__, __artificial__))1985 _mm_mulhi_epu16(__m128i __A, __m128i __B) {1986 __v4su __w0, __w1;1987 __v16qu __xform1 = {1988#ifdef __LITTLE_ENDIAN__1989 0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17, 0x0A,1990 0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F1991#else1992 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15, 0x08,1993 0x09, 0x18, 0x19, 0x0C, 0x0D, 0x1C, 0x1D1994#endif1995 };1996 1997 __w0 = vec_vmuleuh((__v8hu)__A, (__v8hu)__B);1998 __w1 = vec_vmulouh((__v8hu)__A, (__v8hu)__B);1999 return (__m128i)vec_perm(__w0, __w1, __xform1);2000}2001 2002extern __inline __m128i2003 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2004 _mm_shufflehi_epi16(__m128i __A, const int __mask) {2005 unsigned long __element_selector_98 = __mask & 0x03;2006 unsigned long __element_selector_BA = (__mask >> 2) & 0x03;2007 unsigned long __element_selector_DC = (__mask >> 4) & 0x03;2008 unsigned long __element_selector_FE = (__mask >> 6) & 0x03;2009 static const unsigned short __permute_selectors[4] = {2010#ifdef __LITTLE_ENDIAN__2011 0x0908, 0x0B0A, 0x0D0C, 0x0F0E2012#else2013 0x0809, 0x0A0B, 0x0C0D, 0x0E0F2014#endif2015 };2016 __v2du __pmask =2017#ifdef __LITTLE_ENDIAN__2018 {0x1716151413121110UL, 0UL};2019#else2020 {0x1011121314151617UL, 0UL};2021#endif2022 __m64_union __t;2023 __v2du __a, __r;2024 2025 __t.as_short[0] = __permute_selectors[__element_selector_98];2026 __t.as_short[1] = __permute_selectors[__element_selector_BA];2027 __t.as_short[2] = __permute_selectors[__element_selector_DC];2028 __t.as_short[3] = __permute_selectors[__element_selector_FE];2029 __pmask[1] = __t.as_m64;2030 __a = (__v2du)__A;2031 __r = vec_perm(__a, __a, (__vector unsigned char)__pmask);2032 return (__m128i)__r;2033}2034 2035extern __inline __m128i2036 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2037 _mm_shufflelo_epi16(__m128i __A, const int __mask) {2038 unsigned long __element_selector_10 = __mask & 0x03;2039 unsigned long __element_selector_32 = (__mask >> 2) & 0x03;2040 unsigned long __element_selector_54 = (__mask >> 4) & 0x03;2041 unsigned long __element_selector_76 = (__mask >> 6) & 0x03;2042 static const unsigned short __permute_selectors[4] = {2043#ifdef __LITTLE_ENDIAN__2044 0x0100, 0x0302, 0x0504, 0x07062045#else2046 0x0001, 0x0203, 0x0405, 0x06072047#endif2048 };2049 __v2du __pmask =2050#ifdef __LITTLE_ENDIAN__2051 {0UL, 0x1f1e1d1c1b1a1918UL};2052#else2053 {0UL, 0x18191a1b1c1d1e1fUL};2054#endif2055 __m64_union __t;2056 __v2du __a, __r;2057 __t.as_short[0] = __permute_selectors[__element_selector_10];2058 __t.as_short[1] = __permute_selectors[__element_selector_32];2059 __t.as_short[2] = __permute_selectors[__element_selector_54];2060 __t.as_short[3] = __permute_selectors[__element_selector_76];2061 __pmask[0] = __t.as_m64;2062 __a = (__v2du)__A;2063 __r = vec_perm(__a, __a, (__vector unsigned char)__pmask);2064 return (__m128i)__r;2065}2066 2067extern __inline __m128i2068 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2069 _mm_shuffle_epi32(__m128i __A, const int __mask) {2070 unsigned long __element_selector_10 = __mask & 0x03;2071 unsigned long __element_selector_32 = (__mask >> 2) & 0x03;2072 unsigned long __element_selector_54 = (__mask >> 4) & 0x03;2073 unsigned long __element_selector_76 = (__mask >> 6) & 0x03;2074 static const unsigned int __permute_selectors[4] = {2075#ifdef __LITTLE_ENDIAN__2076 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C2077#else2078 0x00010203, 0x04050607, 0x08090A0B, 0x0C0D0E0F2079#endif2080 };2081 __v4su __t;2082 2083 __t[0] = __permute_selectors[__element_selector_10];2084 __t[1] = __permute_selectors[__element_selector_32];2085 __t[2] = __permute_selectors[__element_selector_54] + 0x10101010;2086 __t[3] = __permute_selectors[__element_selector_76] + 0x10101010;2087 return (__m128i)vec_perm((__v4si)__A, (__v4si)__A,2088 (__vector unsigned char)__t);2089}2090 2091extern __inline void2092 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2093 _mm_maskmoveu_si128(__m128i __A, __m128i __B, char *__C) {2094 __v2du __hibit = {0x7f7f7f7f7f7f7f7fUL, 0x7f7f7f7f7f7f7f7fUL};2095 __v16qu __mask, __tmp;2096 __m128i_u *__p = (__m128i_u *)__C;2097 2098 __tmp = (__v16qu)_mm_loadu_si128(__p);2099 __mask = (__v16qu)vec_cmpgt((__v16qu)__B, (__v16qu)__hibit);2100 __tmp = vec_sel(__tmp, (__v16qu)__A, __mask);2101 _mm_storeu_si128(__p, (__m128i)__tmp);2102}2103 2104extern __inline __m128i2105 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2106 _mm_avg_epu8(__m128i __A, __m128i __B) {2107 return (__m128i)vec_avg((__v16qu)__A, (__v16qu)__B);2108}2109 2110extern __inline __m128i2111 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2112 _mm_avg_epu16(__m128i __A, __m128i __B) {2113 return (__m128i)vec_avg((__v8hu)__A, (__v8hu)__B);2114}2115 2116extern __inline __m128i2117 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2118 _mm_sad_epu8(__m128i __A, __m128i __B) {2119 __v16qu __a, __b;2120 __v16qu __vabsdiff;2121 __v4si __vsum;2122 const __v4su __zero = {0, 0, 0, 0};2123 __v4si __result;2124 2125 __a = (__v16qu)__A;2126 __b = (__v16qu)__B;2127#ifndef _ARCH_PWR92128 __v16qu __vmin = vec_min(__a, __b);2129 __v16qu __vmax = vec_max(__a, __b);2130 __vabsdiff = vec_sub(__vmax, __vmin);2131#else2132 __vabsdiff = vec_absd(__a, __b);2133#endif2134 /* Sum four groups of bytes into integers. */2135 __vsum = (__vector signed int)vec_sum4s(__vabsdiff, __zero);2136#ifdef __LITTLE_ENDIAN__2137 /* Sum across four integers with two integer results. */2138 __asm__("vsum2sws %0,%1,%2" : "=v"(__result) : "v"(__vsum), "v"(__zero));2139 /* Note: vec_sum2s could be used here, but on little-endian, vector2140 shifts are added that are not needed for this use-case.2141 A vector shift to correctly position the 32-bit integer results2142 (currently at [0] and [2]) to [1] and [3] would then need to be2143 swapped back again since the desired results are two 64-bit2144 integers ([1]|[0] and [3]|[2]). Thus, no shift is performed. */2145#else2146 /* Sum across four integers with two integer results. */2147 __result = vec_sum2s(__vsum, (__vector signed int)__zero);2148 /* Rotate the sums into the correct position. */2149 __result = vec_sld(__result, __result, 6);2150#endif2151 return (__m128i)__result;2152}2153 2154extern __inline void2155 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2156 _mm_stream_si32(int *__A, int __B) {2157 /* Use the data cache block touch for store transient. */2158 __asm__("dcbtstt 0,%0" : : "b"(__A) : "memory");2159 *__A = __B;2160}2161 2162extern __inline void2163 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2164 _mm_stream_si64(long long int *__A, long long int __B) {2165 /* Use the data cache block touch for store transient. */2166 __asm__(" dcbtstt 0,%0" : : "b"(__A) : "memory");2167 *__A = __B;2168}2169 2170extern __inline void2171 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2172 _mm_stream_si128(__m128i *__A, __m128i __B) {2173 /* Use the data cache block touch for store transient. */2174 __asm__("dcbtstt 0,%0" : : "b"(__A) : "memory");2175 *__A = __B;2176}2177 2178extern __inline void2179 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2180 _mm_stream_pd(double *__A, __m128d __B) {2181 /* Use the data cache block touch for store transient. */2182 __asm__("dcbtstt 0,%0" : : "b"(__A) : "memory");2183 *(__m128d *)__A = __B;2184}2185 2186extern __inline void2187 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2188 _mm_clflush(void const *__A) {2189 /* Use the data cache block flush. */2190 __asm__("dcbf 0,%0" : : "b"(__A) : "memory");2191}2192 2193extern __inline void2194 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2195 _mm_lfence(void) {2196 /* Use light weight sync for load to load ordering. */2197 __atomic_thread_fence(__ATOMIC_RELEASE);2198}2199 2200extern __inline void2201 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2202 _mm_mfence(void) {2203 /* Use heavy weight sync for any to any ordering. */2204 __atomic_thread_fence(__ATOMIC_SEQ_CST);2205}2206 2207extern __inline __m128i2208 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2209 _mm_cvtsi32_si128(int __A) {2210 return _mm_set_epi32(0, 0, 0, __A);2211}2212 2213extern __inline __m128i2214 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2215 _mm_cvtsi64_si128(long long __A) {2216 return __extension__(__m128i)(__v2di){__A, 0LL};2217}2218 2219/* Microsoft intrinsic. */2220extern __inline __m128i2221 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2222 _mm_cvtsi64x_si128(long long __A) {2223 return __extension__(__m128i)(__v2di){__A, 0LL};2224}2225 2226/* Casts between various SP, DP, INT vector types. Note that these do no2227 conversion of values, they just change the type. */2228extern __inline __m1282229 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2230 _mm_castpd_ps(__m128d __A) {2231 return (__m128)__A;2232}2233 2234extern __inline __m128i2235 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2236 _mm_castpd_si128(__m128d __A) {2237 return (__m128i)__A;2238}2239 2240extern __inline __m128d2241 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2242 _mm_castps_pd(__m128 __A) {2243 return (__m128d)__A;2244}2245 2246extern __inline __m128i2247 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2248 _mm_castps_si128(__m128 __A) {2249 return (__m128i)__A;2250}2251 2252extern __inline __m1282253 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2254 _mm_castsi128_ps(__m128i __A) {2255 return (__m128)__A;2256}2257 2258extern __inline __m128d2259 __attribute__((__gnu_inline__, __always_inline__, __artificial__))2260 _mm_castsi128_pd(__m128i __A) {2261 return (__m128d)__A;2262}2263 2264#else2265#include_next <emmintrin.h>2266#endif /* defined(__powerpc64__) && \2267 * (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX)) */2268 2269#endif /* EMMINTRIN_H_ */2270