436 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefixes=ALL,AVX512F3; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw | FileCheck %s --check-prefixes=ALL,AVX512BW4 5define <16 x i32> @_inreg16xi32(i32 %a) {6; ALL-LABEL: _inreg16xi32:7; ALL: # %bb.0:8; ALL-NEXT: vpbroadcastd %edi, %zmm09; ALL-NEXT: retq10 %b = insertelement <16 x i32> undef, i32 %a, i32 011 %c = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer12 ret <16 x i32> %c13}14 15define <8 x i64> @_inreg8xi64(i64 %a) {16; ALL-LABEL: _inreg8xi64:17; ALL: # %bb.0:18; ALL-NEXT: vpbroadcastq %rdi, %zmm019; ALL-NEXT: retq20 %b = insertelement <8 x i64> undef, i64 %a, i32 021 %c = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer22 ret <8 x i64> %c23}24 25define <16 x float> @_ss16xfloat_v4(<4 x float> %a) {26; ALL-LABEL: _ss16xfloat_v4:27; ALL: # %bb.0:28; ALL-NEXT: vbroadcastss %xmm0, %zmm029; ALL-NEXT: retq30 %b = shufflevector <4 x float> %a, <4 x float> undef, <16 x i32> zeroinitializer31 ret <16 x float> %b32}33 34define <16 x float> @_inreg16xfloat(float %a) {35; ALL-LABEL: _inreg16xfloat:36; ALL: # %bb.0:37; ALL-NEXT: vbroadcastss %xmm0, %zmm038; ALL-NEXT: retq39 %b = insertelement <16 x float> undef, float %a, i32 040 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer41 ret <16 x float> %c42}43 44define <16 x float> @_ss16xfloat_mask(float %a, <16 x float> %i, <16 x i32> %mask1) {45; ALL-LABEL: _ss16xfloat_mask:46; ALL: # %bb.0:47; ALL-NEXT: vptestmd %zmm2, %zmm2, %k148; ALL-NEXT: vbroadcastss %xmm0, %zmm1 {%k1}49; ALL-NEXT: vmovaps %zmm1, %zmm050; ALL-NEXT: retq51 %mask = icmp ne <16 x i32> %mask1, zeroinitializer52 %b = insertelement <16 x float> undef, float %a, i32 053 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer54 %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> %i55 ret <16 x float> %r56}57 58define <16 x float> @_ss16xfloat_maskz(float %a, <16 x i32> %mask1) {59; ALL-LABEL: _ss16xfloat_maskz:60; ALL: # %bb.0:61; ALL-NEXT: vptestmd %zmm1, %zmm1, %k162; ALL-NEXT: vbroadcastss %xmm0, %zmm0 {%k1} {z}63; ALL-NEXT: retq64 %mask = icmp ne <16 x i32> %mask1, zeroinitializer65 %b = insertelement <16 x float> undef, float %a, i32 066 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer67 %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> zeroinitializer68 ret <16 x float> %r69}70 71define <16 x float> @_ss16xfloat_load(ptr %a.ptr) {72; ALL-LABEL: _ss16xfloat_load:73; ALL: # %bb.0:74; ALL-NEXT: vbroadcastss (%rdi), %zmm075; ALL-NEXT: retq76 %a = load float, ptr %a.ptr77 %b = insertelement <16 x float> undef, float %a, i32 078 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer79 ret <16 x float> %c80}81 82define <16 x float> @_ss16xfloat_mask_load(ptr %a.ptr, <16 x float> %i, <16 x i32> %mask1) {83; ALL-LABEL: _ss16xfloat_mask_load:84; ALL: # %bb.0:85; ALL-NEXT: vptestmd %zmm1, %zmm1, %k186; ALL-NEXT: vbroadcastss (%rdi), %zmm0 {%k1}87; ALL-NEXT: retq88 %a = load float, ptr %a.ptr89 %mask = icmp ne <16 x i32> %mask1, zeroinitializer90 %b = insertelement <16 x float> undef, float %a, i32 091 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer92 %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> %i93 ret <16 x float> %r94}95 96define <16 x float> @_ss16xfloat_maskz_load(ptr %a.ptr, <16 x i32> %mask1) {97; ALL-LABEL: _ss16xfloat_maskz_load:98; ALL: # %bb.0:99; ALL-NEXT: vptestmd %zmm0, %zmm0, %k1100; ALL-NEXT: vbroadcastss (%rdi), %zmm0 {%k1} {z}101; ALL-NEXT: retq102 %a = load float, ptr %a.ptr103 %mask = icmp ne <16 x i32> %mask1, zeroinitializer104 %b = insertelement <16 x float> undef, float %a, i32 0105 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer106 %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> zeroinitializer107 ret <16 x float> %r108}109 110define <8 x double> @_inreg8xdouble(double %a) {111; ALL-LABEL: _inreg8xdouble:112; ALL: # %bb.0:113; ALL-NEXT: vbroadcastsd %xmm0, %zmm0114; ALL-NEXT: retq115 %b = insertelement <8 x double> undef, double %a, i32 0116 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer117 ret <8 x double> %c118}119 120define <8 x double> @_sd8xdouble_mask(double %a, <8 x double> %i, <8 x i32> %mask1) {121; ALL-LABEL: _sd8xdouble_mask:122; ALL: # %bb.0:123; ALL-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2124; ALL-NEXT: vptestmd %zmm2, %zmm2, %k1125; ALL-NEXT: vbroadcastsd %xmm0, %zmm1 {%k1}126; ALL-NEXT: vmovapd %zmm1, %zmm0127; ALL-NEXT: retq128 %mask = icmp ne <8 x i32> %mask1, zeroinitializer129 %b = insertelement <8 x double> undef, double %a, i32 0130 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer131 %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> %i132 ret <8 x double> %r133}134 135define <8 x double> @_sd8xdouble_maskz(double %a, <8 x i32> %mask1) {136; ALL-LABEL: _sd8xdouble_maskz:137; ALL: # %bb.0:138; ALL-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1139; ALL-NEXT: vptestmd %zmm1, %zmm1, %k1140; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 {%k1} {z}141; ALL-NEXT: retq142 %mask = icmp ne <8 x i32> %mask1, zeroinitializer143 %b = insertelement <8 x double> undef, double %a, i32 0144 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer145 %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> zeroinitializer146 ret <8 x double> %r147}148 149define <8 x double> @_sd8xdouble_load(ptr %a.ptr) {150; ALL-LABEL: _sd8xdouble_load:151; ALL: # %bb.0:152; ALL-NEXT: vbroadcastsd (%rdi), %zmm0153; ALL-NEXT: retq154 %a = load double, ptr %a.ptr155 %b = insertelement <8 x double> undef, double %a, i32 0156 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer157 ret <8 x double> %c158}159 160define <8 x double> @_sd8xdouble_mask_load(ptr %a.ptr, <8 x double> %i, <8 x i32> %mask1) {161; ALL-LABEL: _sd8xdouble_mask_load:162; ALL: # %bb.0:163; ALL-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1164; ALL-NEXT: vptestmd %zmm1, %zmm1, %k1165; ALL-NEXT: vbroadcastsd (%rdi), %zmm0 {%k1}166; ALL-NEXT: retq167 %a = load double, ptr %a.ptr168 %mask = icmp ne <8 x i32> %mask1, zeroinitializer169 %b = insertelement <8 x double> undef, double %a, i32 0170 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer171 %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> %i172 ret <8 x double> %r173}174 175define <8 x double> @_sd8xdouble_maskz_load(ptr %a.ptr, <8 x i32> %mask1) {176; ALL-LABEL: _sd8xdouble_maskz_load:177; ALL: # %bb.0:178; ALL-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0179; ALL-NEXT: vptestmd %zmm0, %zmm0, %k1180; ALL-NEXT: vbroadcastsd (%rdi), %zmm0 {%k1} {z}181; ALL-NEXT: retq182 %a = load double, ptr %a.ptr183 %mask = icmp ne <8 x i32> %mask1, zeroinitializer184 %b = insertelement <8 x double> undef, double %a, i32 0185 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer186 %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> zeroinitializer187 ret <8 x double> %r188}189 190define <16 x i32> @_xmm16xi32(<16 x i32> %a) {191; ALL-LABEL: _xmm16xi32:192; ALL: # %bb.0:193; ALL-NEXT: vbroadcastss %xmm0, %zmm0194; ALL-NEXT: retq195 %b = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> zeroinitializer196 ret <16 x i32> %b197}198 199define <16 x float> @_xmm16xfloat(<16 x float> %a) {200; ALL-LABEL: _xmm16xfloat:201; ALL: # %bb.0:202; ALL-NEXT: vbroadcastss %xmm0, %zmm0203; ALL-NEXT: retq204 %b = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> zeroinitializer205 ret <16 x float> %b206}207 208define <16 x i32> @test_vbroadcast(<16 x float> %a0) {209; ALL-LABEL: test_vbroadcast:210; ALL: # %bb.0: # %entry211; ALL-NEXT: vxorps %xmm1, %xmm1, %xmm1212; ALL-NEXT: vcmpunordps %zmm1, %zmm0, %k1213; ALL-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1214; ALL-NEXT: knotw %k1, %k1215; ALL-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}216; ALL-NEXT: retq217entry:218 %0 = sext <16 x i1> zeroinitializer to <16 x i32>219 %1 = fcmp uno <16 x float> %a0, zeroinitializer220 %2 = sext <16 x i1> %1 to <16 x i32>221 %3 = select <16 x i1> %1, <16 x i32> %0, <16 x i32> %2222 ret <16 x i32> %3223}224 225; We implement the set1 intrinsics with vector initializers. Verify that the226; IR generated will produce broadcasts at the end.227define <8 x double> @test_set1_pd(double %d) #2 {228; ALL-LABEL: test_set1_pd:229; ALL: # %bb.0: # %entry230; ALL-NEXT: vbroadcastsd %xmm0, %zmm0231; ALL-NEXT: retq232entry:233 %vecinit.i = insertelement <8 x double> undef, double %d, i32 0234 %vecinit1.i = insertelement <8 x double> %vecinit.i, double %d, i32 1235 %vecinit2.i = insertelement <8 x double> %vecinit1.i, double %d, i32 2236 %vecinit3.i = insertelement <8 x double> %vecinit2.i, double %d, i32 3237 %vecinit4.i = insertelement <8 x double> %vecinit3.i, double %d, i32 4238 %vecinit5.i = insertelement <8 x double> %vecinit4.i, double %d, i32 5239 %vecinit6.i = insertelement <8 x double> %vecinit5.i, double %d, i32 6240 %vecinit7.i = insertelement <8 x double> %vecinit6.i, double %d, i32 7241 ret <8 x double> %vecinit7.i242}243 244define <8 x i64> @test_set1_epi64(i64 %d) #2 {245; ALL-LABEL: test_set1_epi64:246; ALL: # %bb.0: # %entry247; ALL-NEXT: vpbroadcastq %rdi, %zmm0248; ALL-NEXT: retq249entry:250 %vecinit.i = insertelement <8 x i64> undef, i64 %d, i32 0251 %vecinit1.i = insertelement <8 x i64> %vecinit.i, i64 %d, i32 1252 %vecinit2.i = insertelement <8 x i64> %vecinit1.i, i64 %d, i32 2253 %vecinit3.i = insertelement <8 x i64> %vecinit2.i, i64 %d, i32 3254 %vecinit4.i = insertelement <8 x i64> %vecinit3.i, i64 %d, i32 4255 %vecinit5.i = insertelement <8 x i64> %vecinit4.i, i64 %d, i32 5256 %vecinit6.i = insertelement <8 x i64> %vecinit5.i, i64 %d, i32 6257 %vecinit7.i = insertelement <8 x i64> %vecinit6.i, i64 %d, i32 7258 ret <8 x i64> %vecinit7.i259}260 261define <16 x float> @test_set1_ps(float %f) #2 {262; ALL-LABEL: test_set1_ps:263; ALL: # %bb.0: # %entry264; ALL-NEXT: vbroadcastss %xmm0, %zmm0265; ALL-NEXT: retq266entry:267 %vecinit.i = insertelement <16 x float> undef, float %f, i32 0268 %vecinit1.i = insertelement <16 x float> %vecinit.i, float %f, i32 1269 %vecinit2.i = insertelement <16 x float> %vecinit1.i, float %f, i32 2270 %vecinit3.i = insertelement <16 x float> %vecinit2.i, float %f, i32 3271 %vecinit4.i = insertelement <16 x float> %vecinit3.i, float %f, i32 4272 %vecinit5.i = insertelement <16 x float> %vecinit4.i, float %f, i32 5273 %vecinit6.i = insertelement <16 x float> %vecinit5.i, float %f, i32 6274 %vecinit7.i = insertelement <16 x float> %vecinit6.i, float %f, i32 7275 %vecinit8.i = insertelement <16 x float> %vecinit7.i, float %f, i32 8276 %vecinit9.i = insertelement <16 x float> %vecinit8.i, float %f, i32 9277 %vecinit10.i = insertelement <16 x float> %vecinit9.i, float %f, i32 10278 %vecinit11.i = insertelement <16 x float> %vecinit10.i, float %f, i32 11279 %vecinit12.i = insertelement <16 x float> %vecinit11.i, float %f, i32 12280 %vecinit13.i = insertelement <16 x float> %vecinit12.i, float %f, i32 13281 %vecinit14.i = insertelement <16 x float> %vecinit13.i, float %f, i32 14282 %vecinit15.i = insertelement <16 x float> %vecinit14.i, float %f, i32 15283 ret <16 x float> %vecinit15.i284}285 286define <16 x i32> @test_set1_epi32(i32 %f) #2 {287; ALL-LABEL: test_set1_epi32:288; ALL: # %bb.0: # %entry289; ALL-NEXT: vpbroadcastd %edi, %zmm0290; ALL-NEXT: retq291entry:292 %vecinit.i = insertelement <16 x i32> undef, i32 %f, i32 0293 %vecinit1.i = insertelement <16 x i32> %vecinit.i, i32 %f, i32 1294 %vecinit2.i = insertelement <16 x i32> %vecinit1.i, i32 %f, i32 2295 %vecinit3.i = insertelement <16 x i32> %vecinit2.i, i32 %f, i32 3296 %vecinit4.i = insertelement <16 x i32> %vecinit3.i, i32 %f, i32 4297 %vecinit5.i = insertelement <16 x i32> %vecinit4.i, i32 %f, i32 5298 %vecinit6.i = insertelement <16 x i32> %vecinit5.i, i32 %f, i32 6299 %vecinit7.i = insertelement <16 x i32> %vecinit6.i, i32 %f, i32 7300 %vecinit8.i = insertelement <16 x i32> %vecinit7.i, i32 %f, i32 8301 %vecinit9.i = insertelement <16 x i32> %vecinit8.i, i32 %f, i32 9302 %vecinit10.i = insertelement <16 x i32> %vecinit9.i, i32 %f, i32 10303 %vecinit11.i = insertelement <16 x i32> %vecinit10.i, i32 %f, i32 11304 %vecinit12.i = insertelement <16 x i32> %vecinit11.i, i32 %f, i32 12305 %vecinit13.i = insertelement <16 x i32> %vecinit12.i, i32 %f, i32 13306 %vecinit14.i = insertelement <16 x i32> %vecinit13.i, i32 %f, i32 14307 %vecinit15.i = insertelement <16 x i32> %vecinit14.i, i32 %f, i32 15308 ret <16 x i32> %vecinit15.i309}310 311; We implement the scalar broadcast intrinsics with vector initializers.312; Verify that the IR generated will produce the broadcast at the end.313define <8 x double> @test_mm512_broadcastsd_pd(<2 x double> %a) {314; ALL-LABEL: test_mm512_broadcastsd_pd:315; ALL: # %bb.0: # %entry316; ALL-NEXT: vbroadcastsd %xmm0, %zmm0317; ALL-NEXT: retq318entry:319 %0 = extractelement <2 x double> %a, i32 0320 %vecinit.i = insertelement <8 x double> undef, double %0, i32 0321 %vecinit1.i = insertelement <8 x double> %vecinit.i, double %0, i32 1322 %vecinit2.i = insertelement <8 x double> %vecinit1.i, double %0, i32 2323 %vecinit3.i = insertelement <8 x double> %vecinit2.i, double %0, i32 3324 %vecinit4.i = insertelement <8 x double> %vecinit3.i, double %0, i32 4325 %vecinit5.i = insertelement <8 x double> %vecinit4.i, double %0, i32 5326 %vecinit6.i = insertelement <8 x double> %vecinit5.i, double %0, i32 6327 %vecinit7.i = insertelement <8 x double> %vecinit6.i, double %0, i32 7328 ret <8 x double> %vecinit7.i329}330 331define <16 x float> @test1(<8 x float>%a) {332; ALL-LABEL: test1:333; ALL: # %bb.0:334; ALL-NEXT: vbroadcastss %xmm0, %zmm0335; ALL-NEXT: retq336 %res = shufflevector <8 x float> %a, <8 x float> undef, <16 x i32> zeroinitializer337 ret <16 x float>%res338}339 340define <8 x double> @test2(<4 x double>%a) {341; ALL-LABEL: test2:342; ALL: # %bb.0:343; ALL-NEXT: vbroadcastsd %xmm0, %zmm0344; ALL-NEXT: retq345 %res = shufflevector <4 x double> %a, <4 x double> undef, <8 x i32> zeroinitializer346 ret <8 x double>%res347}348 349define <64 x i8> @_invec32xi8(<32 x i8>%a) {350; AVX512F-LABEL: _invec32xi8:351; AVX512F: # %bb.0:352; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm0353; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0354; AVX512F-NEXT: retq355;356; AVX512BW-LABEL: _invec32xi8:357; AVX512BW: # %bb.0:358; AVX512BW-NEXT: vpbroadcastb %xmm0, %zmm0359; AVX512BW-NEXT: retq360 %res = shufflevector <32 x i8> %a, <32 x i8> undef, <64 x i32> zeroinitializer361 ret <64 x i8>%res362}363 364define <32 x i16> @_invec16xi16(<16 x i16>%a) {365; AVX512F-LABEL: _invec16xi16:366; AVX512F: # %bb.0:367; AVX512F-NEXT: vpbroadcastw %xmm0, %ymm0368; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0369; AVX512F-NEXT: retq370;371; AVX512BW-LABEL: _invec16xi16:372; AVX512BW: # %bb.0:373; AVX512BW-NEXT: vpbroadcastw %xmm0, %zmm0374; AVX512BW-NEXT: retq375 %res = shufflevector <16 x i16> %a, <16 x i16> undef, <32 x i32> zeroinitializer376 ret <32 x i16>%res377}378 379define <16 x i32> @_invec8xi32(<8 x i32>%a) {380; ALL-LABEL: _invec8xi32:381; ALL: # %bb.0:382; ALL-NEXT: vbroadcastss %xmm0, %zmm0383; ALL-NEXT: retq384 %res = shufflevector <8 x i32> %a, <8 x i32> undef, <16 x i32> zeroinitializer385 ret <16 x i32>%res386}387 388define <8 x i64> @_invec4xi64(<4 x i64>%a) {389; ALL-LABEL: _invec4xi64:390; ALL: # %bb.0:391; ALL-NEXT: vbroadcastsd %xmm0, %zmm0392; ALL-NEXT: retq393 %res = shufflevector <4 x i64> %a, <4 x i64> undef, <8 x i32> zeroinitializer394 ret <8 x i64>%res395}396 397declare void @func_f32(float)398define <16 x float> @broadcast_ss_spill(float %x) {399; ALL-LABEL: broadcast_ss_spill:400; ALL: # %bb.0:401; ALL-NEXT: subq $24, %rsp402; ALL-NEXT: .cfi_def_cfa_offset 32403; ALL-NEXT: vaddss %xmm0, %xmm0, %xmm0404; ALL-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill405; ALL-NEXT: callq func_f32@PLT406; ALL-NEXT: vbroadcastss (%rsp), %zmm0 # 16-byte Folded Reload407; ALL-NEXT: addq $24, %rsp408; ALL-NEXT: .cfi_def_cfa_offset 8409; ALL-NEXT: retq410 %a = fadd float %x, %x411 call void @func_f32(float %a)412 %b = insertelement <16 x float> undef, float %a, i32 0413 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer414 ret <16 x float> %c415}416 417declare void @func_f64(double)418define <8 x double> @broadcast_sd_spill(double %x) {419; ALL-LABEL: broadcast_sd_spill:420; ALL: # %bb.0:421; ALL-NEXT: subq $24, %rsp422; ALL-NEXT: .cfi_def_cfa_offset 32423; ALL-NEXT: vaddsd %xmm0, %xmm0, %xmm0424; ALL-NEXT: vmovapd %xmm0, (%rsp) # 16-byte Spill425; ALL-NEXT: callq func_f64@PLT426; ALL-NEXT: vbroadcastsd (%rsp), %zmm0 # 16-byte Folded Reload427; ALL-NEXT: addq $24, %rsp428; ALL-NEXT: .cfi_def_cfa_offset 8429; ALL-NEXT: retq430 %a = fadd double %x, %x431 call void @func_f64(double %a)432 %b = insertelement <8 x double> undef, double %a, i32 0433 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer434 ret <8 x double> %c435}436