2492 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=CHECK,X64,X64VL3; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512fp16,+avx512vl | FileCheck %s --check-prefixes=CHECK,X864; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=CHECK,X64,X64-NOVL5 6define <8 x half> @broadcastph128(ptr %x) {7; X64-LABEL: broadcastph128:8; X64: # %bb.0:9; X64-NEXT: vpbroadcastw (%rdi), %xmm010; X64-NEXT: retq11;12; X86-LABEL: broadcastph128:13; X86: # %bb.0:14; X86-NEXT: movl {{[0-9]+}}(%esp), %eax15; X86-NEXT: vpbroadcastw (%eax), %xmm016; X86-NEXT: retl17 %l1 = load half, ptr %x, align 218 %vec = insertelement <8 x half> undef, half %l1, i32 019 %res = shufflevector <8 x half> %vec, <8 x half> undef, <8 x i32> zeroinitializer20 ret <8 x half> %res21}22 23define <16 x half> @broadcastph256(ptr %x) {24; X64-LABEL: broadcastph256:25; X64: # %bb.0:26; X64-NEXT: vpbroadcastw (%rdi), %ymm027; X64-NEXT: retq28;29; X86-LABEL: broadcastph256:30; X86: # %bb.0:31; X86-NEXT: movl {{[0-9]+}}(%esp), %eax32; X86-NEXT: vpbroadcastw (%eax), %ymm033; X86-NEXT: retl34 %l1 = load half, ptr %x, align 235 %vec = insertelement <16 x half> undef, half %l1, i32 036 %res = shufflevector <16 x half> %vec, <16 x half> undef, <16 x i32> zeroinitializer37 ret <16 x half> %res38}39 40define <32 x half> @broadcastph512(ptr %x) {41; X64-LABEL: broadcastph512:42; X64: # %bb.0:43; X64-NEXT: vpbroadcastw (%rdi), %zmm044; X64-NEXT: retq45;46; X86-LABEL: broadcastph512:47; X86: # %bb.0:48; X86-NEXT: movl {{[0-9]+}}(%esp), %eax49; X86-NEXT: vpbroadcastw (%eax), %zmm050; X86-NEXT: retl51 %l1 = load half, ptr %x, align 252 %vec = insertelement <32 x half> undef, half %l1, i32 053 %res = shufflevector <32 x half> %vec, <32 x half> undef, <32 x i32> zeroinitializer54 ret <32 x half> %res55}56 57define <8 x half> @broadcastph128_scalar(half %x) {58; X64-LABEL: broadcastph128_scalar:59; X64: # %bb.0:60; X64-NEXT: vpbroadcastw %xmm0, %xmm061; X64-NEXT: retq62;63; X86-LABEL: broadcastph128_scalar:64; X86: # %bb.0:65; X86-NEXT: vpbroadcastw {{[0-9]+}}(%esp), %xmm066; X86-NEXT: retl67 %vec = insertelement <8 x half> undef, half %x, i32 068 %res = shufflevector <8 x half> %vec, <8 x half> undef, <8 x i32> zeroinitializer69 ret <8 x half> %res70}71 72define <16 x half> @broadcastph256_scalar(half %x) {73; X64-LABEL: broadcastph256_scalar:74; X64: # %bb.0:75; X64-NEXT: vpbroadcastw %xmm0, %ymm076; X64-NEXT: retq77;78; X86-LABEL: broadcastph256_scalar:79; X86: # %bb.0:80; X86-NEXT: vpbroadcastw {{[0-9]+}}(%esp), %ymm081; X86-NEXT: retl82 %vec = insertelement <16 x half> undef, half %x, i32 083 %res = shufflevector <16 x half> %vec, <16 x half> undef, <16 x i32> zeroinitializer84 ret <16 x half> %res85}86 87define <32 x half> @broadcastph512_scalar(half %x) {88; X64-LABEL: broadcastph512_scalar:89; X64: # %bb.0:90; X64-NEXT: vpbroadcastw %xmm0, %zmm091; X64-NEXT: retq92;93; X86-LABEL: broadcastph512_scalar:94; X86: # %bb.0:95; X86-NEXT: vpbroadcastw {{[0-9]+}}(%esp), %zmm096; X86-NEXT: retl97 %vec = insertelement <32 x half> undef, half %x, i32 098 %res = shufflevector <32 x half> %vec, <32 x half> undef, <32 x i32> zeroinitializer99 ret <32 x half> %res100}101 102define <8 x half> @broadcastph128_reg(<8 x half> %x) {103; CHECK-LABEL: broadcastph128_reg:104; CHECK: # %bb.0:105; CHECK-NEXT: vpbroadcastw %xmm0, %xmm0106; CHECK-NEXT: ret{{[l|q]}}107 %res = shufflevector <8 x half> %x, <8 x half> undef, <8 x i32> zeroinitializer108 ret <8 x half> %res109}110 111define <16 x half> @broadcastph256_reg(<16 x half> %x) {112; CHECK-LABEL: broadcastph256_reg:113; CHECK: # %bb.0:114; CHECK-NEXT: vpbroadcastw %xmm0, %ymm0115; CHECK-NEXT: ret{{[l|q]}}116 %res = shufflevector <16 x half> %x, <16 x half> undef, <16 x i32> zeroinitializer117 ret <16 x half> %res118}119 120define <32 x half> @broadcastph512_reg(<32 x half> %x) {121; CHECK-LABEL: broadcastph512_reg:122; CHECK: # %bb.0:123; CHECK-NEXT: vpbroadcastw %xmm0, %zmm0124; CHECK-NEXT: ret{{[l|q]}}125 %res = shufflevector <32 x half> %x, <32 x half> undef, <32 x i32> zeroinitializer126 ret <32 x half> %res127}128 129define i16 @test1(half %x) {130; X64-LABEL: test1:131; X64: # %bb.0:132; X64-NEXT: vmovw %xmm0, %eax133; X64-NEXT: # kill: def $ax killed $ax killed $eax134; X64-NEXT: retq135;136; X86-LABEL: test1:137; X86: # %bb.0:138; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax139; X86-NEXT: retl140 %res = bitcast half %x to i16141 ret i16 %res142}143 144define <8 x i16> @test2(i16 %x) {145; X64-LABEL: test2:146; X64: # %bb.0:147; X64-NEXT: vmovw %edi, %xmm0148; X64-NEXT: retq149;150; X86-LABEL: test2:151; X86: # %bb.0:152; X86-NEXT: vpbroadcastw {{[0-9]+}}(%esp), %xmm0153; X86-NEXT: retl154 %res = insertelement <8 x i16>undef, i16 %x, i32 0155 ret <8 x i16>%res156}157 158define <8 x i16> @test4(ptr %x) {159; X64-LABEL: test4:160; X64: # %bb.0:161; X64-NEXT: vpbroadcastw (%rdi), %xmm0162; X64-NEXT: retq163;164; X86-LABEL: test4:165; X86: # %bb.0:166; X86-NEXT: movl {{[0-9]+}}(%esp), %eax167; X86-NEXT: vpbroadcastw (%eax), %xmm0168; X86-NEXT: retl169 %y = load i16, ptr %x170 %res = insertelement <8 x i16>undef, i16 %y, i32 0171 ret <8 x i16>%res172}173 174define void @test5(half %x, ptr %y) {175; X64-LABEL: test5:176; X64: # %bb.0:177; X64-NEXT: vmovsh %xmm0, (%rdi)178; X64-NEXT: retq179;180; X86-LABEL: test5:181; X86: # %bb.0:182; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero183; X86-NEXT: movl {{[0-9]+}}(%esp), %eax184; X86-NEXT: vmovsh %xmm0, (%eax)185; X86-NEXT: retl186 store half %x, ptr %y, align 2187 ret void188}189 190define half @test7(ptr %x) {191; X64-LABEL: test7:192; X64: # %bb.0:193; X64-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero194; X64-NEXT: retq195;196; X86-LABEL: test7:197; X86: # %bb.0:198; X86-NEXT: movl {{[0-9]+}}(%esp), %eax199; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero200; X86-NEXT: retl201 %y = load i16, ptr %x202 %res = bitcast i16 %y to half203 ret half %res204}205 206define <8 x i16> @test10(ptr %x) {207; X64-LABEL: test10:208; X64: # %bb.0:209; X64-NEXT: vmovw (%rdi), %xmm0210; X64-NEXT: retq211;212; X86-LABEL: test10:213; X86: # %bb.0:214; X86-NEXT: movl {{[0-9]+}}(%esp), %eax215; X86-NEXT: vmovw (%eax), %xmm0216; X86-NEXT: retl217 %y = load i16, ptr %x, align 2218 %res = insertelement <8 x i16>zeroinitializer, i16 %y, i32 0219 ret <8 x i16>%res220}221 222define <16 x i16> @test10b(ptr %x) {223; X64-LABEL: test10b:224; X64: # %bb.0:225; X64-NEXT: vmovw (%rdi), %xmm0226; X64-NEXT: retq227;228; X86-LABEL: test10b:229; X86: # %bb.0:230; X86-NEXT: movl {{[0-9]+}}(%esp), %eax231; X86-NEXT: vmovw (%eax), %xmm0232; X86-NEXT: retl233 %y = load i16, ptr %x, align 2234 %res = insertelement <16 x i16>zeroinitializer, i16 %y, i32 0235 ret <16 x i16>%res236}237 238define <32 x i16> @test10c(ptr %x) {239; X64-LABEL: test10c:240; X64: # %bb.0:241; X64-NEXT: vmovw (%rdi), %xmm0242; X64-NEXT: retq243;244; X86-LABEL: test10c:245; X86: # %bb.0:246; X86-NEXT: movl {{[0-9]+}}(%esp), %eax247; X86-NEXT: vmovw (%eax), %xmm0248; X86-NEXT: retl249 %y = load i16, ptr %x, align 2250 %res = insertelement <32 x i16>zeroinitializer, i16 %y, i32 0251 ret <32 x i16>%res252}253 254define <8 x half> @test11(ptr %x) {255; X64-LABEL: test11:256; X64: # %bb.0:257; X64-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero258; X64-NEXT: retq259;260; X86-LABEL: test11:261; X86: # %bb.0:262; X86-NEXT: movl {{[0-9]+}}(%esp), %eax263; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero264; X86-NEXT: retl265 %y = load half, ptr %x, align 2266 %res = insertelement <8 x half>zeroinitializer, half %y, i32 0267 ret <8 x half>%res268}269 270define <16 x half> @test11b(ptr %x) {271; X64-LABEL: test11b:272; X64: # %bb.0:273; X64-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero274; X64-NEXT: retq275;276; X86-LABEL: test11b:277; X86: # %bb.0:278; X86-NEXT: movl {{[0-9]+}}(%esp), %eax279; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero280; X86-NEXT: retl281 %y = load half, ptr %x, align 2282 %res = insertelement <16 x half>zeroinitializer, half %y, i32 0283 ret <16 x half>%res284}285 286define <32 x half> @test11c(ptr %x) {287; X64-LABEL: test11c:288; X64: # %bb.0:289; X64-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero290; X64-NEXT: retq291;292; X86-LABEL: test11c:293; X86: # %bb.0:294; X86-NEXT: movl {{[0-9]+}}(%esp), %eax295; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero296; X86-NEXT: retl297 %y = load half, ptr %x, align 2298 %res = insertelement <32 x half>zeroinitializer, half %y, i32 0299 ret <32 x half>%res300}301 302define <8 x half> @test14(half %x) {303; X64-LABEL: test14:304; X64: # %bb.0:305; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1306; X64-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]307; X64-NEXT: retq308;309; X86-LABEL: test14:310; X86: # %bb.0:311; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero312; X86-NEXT: retl313 %res = insertelement <8 x half>zeroinitializer, half %x, i32 0314 ret <8 x half>%res315}316 317define <16 x half> @test14b(half %x) {318; X64VL-LABEL: test14b:319; X64VL: # %bb.0:320; X64VL-NEXT: vxorps %xmm1, %xmm1, %xmm1321; X64VL-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]322; X64VL-NEXT: retq323;324; X86-LABEL: test14b:325; X86: # %bb.0:326; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero327; X86-NEXT: retl328;329; X64-NOVL-LABEL: test14b:330; X64-NOVL: # %bb.0:331; X64-NOVL-NEXT: vxorps %xmm1, %xmm1, %xmm1332; X64-NOVL-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]333; X64-NOVL-NEXT: vxorps %xmm1, %xmm1, %xmm1334; X64-NOVL-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]335; X64-NOVL-NEXT: retq336 %res = insertelement <16 x half>zeroinitializer, half %x, i32 0337 ret <16 x half>%res338}339 340define <32 x half> @test14c(half %x) {341; X64VL-LABEL: test14c:342; X64VL: # %bb.0:343; X64VL-NEXT: vxorps %xmm1, %xmm1, %xmm1344; X64VL-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]345; X64VL-NEXT: retq346;347; X86-LABEL: test14c:348; X86: # %bb.0:349; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero350; X86-NEXT: retl351;352; X64-NOVL-LABEL: test14c:353; X64-NOVL: # %bb.0:354; X64-NOVL-NEXT: vxorps %xmm1, %xmm1, %xmm1355; X64-NOVL-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]356; X64-NOVL-NEXT: vxorps %xmm1, %xmm1, %xmm1357; X64-NOVL-NEXT: vinsertf32x4 $0, %xmm0, %zmm1, %zmm0358; X64-NOVL-NEXT: retq359 %res = insertelement <32 x half>zeroinitializer, half %x, i32 0360 ret <32 x half>%res361}362 363define <8 x i16> @test15(i16 %x) {364; X64-LABEL: test15:365; X64: # %bb.0:366; X64-NEXT: vmovw %edi, %xmm0367; X64-NEXT: retq368;369; X86-LABEL: test15:370; X86: # %bb.0:371; X86-NEXT: vmovw {{[0-9]+}}(%esp), %xmm0372; X86-NEXT: retl373 %res = insertelement <8 x i16>zeroinitializer, i16 %x, i32 0374 ret <8 x i16>%res375}376 377define <16 x i16> @test16(i16 %x) {378; X64-LABEL: test16:379; X64: # %bb.0:380; X64-NEXT: vmovw %edi, %xmm0381; X64-NEXT: retq382;383; X86-LABEL: test16:384; X86: # %bb.0:385; X86-NEXT: vmovw {{[0-9]+}}(%esp), %xmm0386; X86-NEXT: retl387 %res = insertelement <16 x i16>zeroinitializer, i16 %x, i32 0388 ret <16 x i16>%res389}390 391define <32 x i16> @test17(i16 %x) {392; X64-LABEL: test17:393; X64: # %bb.0:394; X64-NEXT: vmovw %edi, %xmm0395; X64-NEXT: retq396;397; X86-LABEL: test17:398; X86: # %bb.0:399; X86-NEXT: vmovw {{[0-9]+}}(%esp), %xmm0400; X86-NEXT: retl401 %res = insertelement <32 x i16>zeroinitializer, i16 %x, i32 0402 ret <32 x i16>%res403}404 405define <8 x i16> @test18(i16 %x) {406; X64-LABEL: test18:407; X64: # %bb.0:408; X64-NEXT: vmovw %edi, %xmm0409; X64-NEXT: retq410;411; X86-LABEL: test18:412; X86: # %bb.0:413; X86-NEXT: vpbroadcastw {{[0-9]+}}(%esp), %xmm0414; X86-NEXT: retl415 %res = insertelement <8 x i16> undef, i16 %x, i32 0416 ret <8 x i16>%res417}418 419define <16 x i16> @test19(i16 %x) {420; X64-LABEL: test19:421; X64: # %bb.0:422; X64-NEXT: vmovw %edi, %xmm0423; X64-NEXT: retq424;425; X86-LABEL: test19:426; X86: # %bb.0:427; X86-NEXT: vpbroadcastw {{[0-9]+}}(%esp), %ymm0428; X86-NEXT: retl429 %res = insertelement <16 x i16> undef, i16 %x, i32 0430 ret <16 x i16>%res431}432 433define <32 x i16> @test20(i16 %x) {434; X64-LABEL: test20:435; X64: # %bb.0:436; X64-NEXT: vmovw %edi, %xmm0437; X64-NEXT: retq438;439; X86-LABEL: test20:440; X86: # %bb.0:441; X86-NEXT: vpbroadcastw {{[0-9]+}}(%esp), %zmm0442; X86-NEXT: retl443 %res = insertelement <32 x i16> undef, i16 %x, i32 0444 ret <32 x i16>%res445}446 447@g8f16 = external global <8 x half>448@g8f16u = external global <8 x half>, align 8449@g16f16 = external global <16 x half>450@g16f16u = external global <16 x half>, align 8451@g32f16 = external global <32 x half>452@g32f16u = external global <32 x half>, align 8453 454define <32 x half> @load32f16(ptr %a) {455; X64-LABEL: load32f16:456; X64: # %bb.0:457; X64-NEXT: vmovaps (%rdi), %zmm0458; X64-NEXT: retq459;460; X86-LABEL: load32f16:461; X86: # %bb.0:462; X86-NEXT: movl {{[0-9]+}}(%esp), %eax463; X86-NEXT: vmovaps (%eax), %zmm0464; X86-NEXT: retl465 %res = load <32 x half>, ptr %a466 ret <32 x half> %res467}468 469define <32 x half> @load32f16mask(ptr %a, <32 x half> %b, i32 %c) {470; X64-LABEL: load32f16mask:471; X64: # %bb.0:472; X64-NEXT: kmovd %esi, %k1473; X64-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1}474; X64-NEXT: retq475;476; X86-LABEL: load32f16mask:477; X86: # %bb.0:478; X86-NEXT: movl {{[0-9]+}}(%esp), %eax479; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1480; X86-NEXT: vmovdqu16 (%eax), %zmm0 {%k1}481; X86-NEXT: retl482 %msk = bitcast i32 %c to <32 x i1>483 %res0 = load <32 x half>, ptr %a484 %res = select <32 x i1> %msk, <32 x half> %res0, <32 x half> %b485 ret <32 x half> %res486}487 488define <32 x half> @load32f16maskz(ptr %a, i32 %c) {489; X64-LABEL: load32f16maskz:490; X64: # %bb.0:491; X64-NEXT: kmovd %esi, %k1492; X64-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}493; X64-NEXT: retq494;495; X86-LABEL: load32f16maskz:496; X86: # %bb.0:497; X86-NEXT: movl {{[0-9]+}}(%esp), %eax498; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1499; X86-NEXT: vmovdqu16 (%eax), %zmm0 {%k1} {z}500; X86-NEXT: retl501 %msk = bitcast i32 %c to <32 x i1>502 %res0 = load <32 x half>, ptr %a503 %res = select <32 x i1> %msk, <32 x half> %res0, <32 x half> zeroinitializer504 ret <32 x half> %res505}506 507define <32 x half> @loadu32f16(ptr %a) {508; X64-LABEL: loadu32f16:509; X64: # %bb.0:510; X64-NEXT: vmovups (%rdi), %zmm0511; X64-NEXT: retq512;513; X86-LABEL: loadu32f16:514; X86: # %bb.0:515; X86-NEXT: movl {{[0-9]+}}(%esp), %eax516; X86-NEXT: vmovups (%eax), %zmm0517; X86-NEXT: retl518 %res = load <32 x half>, ptr %a, align 8519 ret <32 x half> %res520}521 522define <32 x half> @loadu32f16mask(ptr %a, <32 x half> %b, i32 %c) {523; X64-LABEL: loadu32f16mask:524; X64: # %bb.0:525; X64-NEXT: kmovd %esi, %k1526; X64-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1}527; X64-NEXT: retq528;529; X86-LABEL: loadu32f16mask:530; X86: # %bb.0:531; X86-NEXT: movl {{[0-9]+}}(%esp), %eax532; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1533; X86-NEXT: vmovdqu16 (%eax), %zmm0 {%k1}534; X86-NEXT: retl535 %msk = bitcast i32 %c to <32 x i1>536 %res0 = load <32 x half>, ptr %a, align 8537 %res = select <32 x i1> %msk, <32 x half> %res0, <32 x half> %b538 ret <32 x half> %res539}540 541define <32 x half> @loadu32f16maskz(ptr %a, i32 %c) {542; X64-LABEL: loadu32f16maskz:543; X64: # %bb.0:544; X64-NEXT: kmovd %esi, %k1545; X64-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}546; X64-NEXT: retq547;548; X86-LABEL: loadu32f16maskz:549; X86: # %bb.0:550; X86-NEXT: movl {{[0-9]+}}(%esp), %eax551; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1552; X86-NEXT: vmovdqu16 (%eax), %zmm0 {%k1} {z}553; X86-NEXT: retl554 %msk = bitcast i32 %c to <32 x i1>555 %res0 = load <32 x half>, ptr %a, align 8556 %res = select <32 x i1> %msk, <32 x half> %res0, <32 x half> zeroinitializer557 ret <32 x half> %res558}559 560define void @store32f16(<32 x half> %a) {561; X64-LABEL: store32f16:562; X64: # %bb.0:563; X64-NEXT: movq g32f16@GOTPCREL(%rip), %rax564; X64-NEXT: vmovaps %zmm0, (%rax)565; X64-NEXT: vzeroupper566; X64-NEXT: retq567;568; X86-LABEL: store32f16:569; X86: # %bb.0:570; X86-NEXT: vmovaps %zmm0, g32f16571; X86-NEXT: vzeroupper572; X86-NEXT: retl573 store <32 x half> %a, ptr @g32f16574 ret void575}576 577define void @storeu32f16(<32 x half> %a) {578; X64-LABEL: storeu32f16:579; X64: # %bb.0:580; X64-NEXT: movq g32f16u@GOTPCREL(%rip), %rax581; X64-NEXT: vmovups %zmm0, (%rax)582; X64-NEXT: vzeroupper583; X64-NEXT: retq584;585; X86-LABEL: storeu32f16:586; X86: # %bb.0:587; X86-NEXT: vmovups %zmm0, g32f16u588; X86-NEXT: vzeroupper589; X86-NEXT: retl590 store <32 x half> %a, ptr @g32f16u, align 8591 ret void592}593 594declare void @llvm.masked.store.v32f16.p0(<32 x half>, ptr, i32, <32 x i1>)595declare <32 x half> @llvm.masked.load.v32f16.p0(ptr, i32, <32 x i1>, <32 x half>)596 597define void @storeu32f16mask(<32 x i1> %mask, ptr %addr, <32 x half> %val) {598; X64VL-LABEL: storeu32f16mask:599; X64VL: # %bb.0:600; X64VL-NEXT: vpsllw $7, %ymm0, %ymm0601; X64VL-NEXT: vpmovb2m %ymm0, %k1602; X64VL-NEXT: vmovdqu16 %zmm1, (%rdi) {%k1}603; X64VL-NEXT: vzeroupper604; X64VL-NEXT: retq605;606; X86-LABEL: storeu32f16mask:607; X86: # %bb.0:608; X86-NEXT: vpsllw $7, %ymm0, %ymm0609; X86-NEXT: vpmovb2m %ymm0, %k1610; X86-NEXT: movl {{[0-9]+}}(%esp), %eax611; X86-NEXT: vmovdqu16 %zmm1, (%eax) {%k1}612; X86-NEXT: vzeroupper613; X86-NEXT: retl614;615; X64-NOVL-LABEL: storeu32f16mask:616; X64-NOVL: # %bb.0:617; X64-NOVL-NEXT: vpsllw $7, %ymm0, %ymm0618; X64-NOVL-NEXT: vpmovb2m %zmm0, %k1619; X64-NOVL-NEXT: vmovdqu16 %zmm1, (%rdi) {%k1}620; X64-NOVL-NEXT: vzeroupper621; X64-NOVL-NEXT: retq622 call void @llvm.masked.store.v32f16.p0(<32 x half> %val, ptr %addr, i32 4, <32 x i1>%mask)623 ret void624}625 626define <32 x half> @maskloadu32f16(ptr %addr, <32 x half> %val, <32 x i1> %mask) {627; X64VL-LABEL: maskloadu32f16:628; X64VL: # %bb.0:629; X64VL-NEXT: vpsllw $7, %ymm1, %ymm1630; X64VL-NEXT: vpmovb2m %ymm1, %k1631; X64VL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1}632; X64VL-NEXT: retq633;634; X86-LABEL: maskloadu32f16:635; X86: # %bb.0:636; X86-NEXT: vpsllw $7, %ymm1, %ymm1637; X86-NEXT: vpmovb2m %ymm1, %k1638; X86-NEXT: movl {{[0-9]+}}(%esp), %eax639; X86-NEXT: vmovdqu16 (%eax), %zmm0 {%k1}640; X86-NEXT: retl641;642; X64-NOVL-LABEL: maskloadu32f16:643; X64-NOVL: # %bb.0:644; X64-NOVL-NEXT: vpsllw $7, %ymm1, %ymm1645; X64-NOVL-NEXT: vpmovb2m %zmm1, %k1646; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1}647; X64-NOVL-NEXT: retq648 %res = call <32 x half> @llvm.masked.load.v32f16.p0(ptr %addr, i32 4, <32 x i1> %mask, <32 x half> %val)649 ret <32 x half> %res650}651 652define <32 x half> @maskuloadu32f16(ptr %addr, <32 x i1> %mask) {653; X64VL-LABEL: maskuloadu32f16:654; X64VL: # %bb.0:655; X64VL-NEXT: vpsllw $7, %ymm0, %ymm0656; X64VL-NEXT: vpmovb2m %ymm0, %k1657; X64VL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}658; X64VL-NEXT: retq659;660; X86-LABEL: maskuloadu32f16:661; X86: # %bb.0:662; X86-NEXT: vpsllw $7, %ymm0, %ymm0663; X86-NEXT: vpmovb2m %ymm0, %k1664; X86-NEXT: movl {{[0-9]+}}(%esp), %eax665; X86-NEXT: vmovdqu16 (%eax), %zmm0 {%k1} {z}666; X86-NEXT: retl667;668; X64-NOVL-LABEL: maskuloadu32f16:669; X64-NOVL: # %bb.0:670; X64-NOVL-NEXT: vpsllw $7, %ymm0, %ymm0671; X64-NOVL-NEXT: vpmovb2m %zmm0, %k1672; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}673; X64-NOVL-NEXT: retq674 %res = call <32 x half> @llvm.masked.load.v32f16.p0(ptr %addr, i32 4, <32 x i1> %mask, <32 x half> undef)675 ret <32 x half> %res676}677 678define <32 x half> @maskzloadu32f16(ptr %addr, <32 x i1> %mask) {679; X64VL-LABEL: maskzloadu32f16:680; X64VL: # %bb.0:681; X64VL-NEXT: vpsllw $7, %ymm0, %ymm0682; X64VL-NEXT: vpmovb2m %ymm0, %k1683; X64VL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}684; X64VL-NEXT: retq685;686; X86-LABEL: maskzloadu32f16:687; X86: # %bb.0:688; X86-NEXT: vpsllw $7, %ymm0, %ymm0689; X86-NEXT: vpmovb2m %ymm0, %k1690; X86-NEXT: movl {{[0-9]+}}(%esp), %eax691; X86-NEXT: vmovdqu16 (%eax), %zmm0 {%k1} {z}692; X86-NEXT: retl693;694; X64-NOVL-LABEL: maskzloadu32f16:695; X64-NOVL: # %bb.0:696; X64-NOVL-NEXT: vpsllw $7, %ymm0, %ymm0697; X64-NOVL-NEXT: vpmovb2m %zmm0, %k1698; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}699; X64-NOVL-NEXT: retq700 %res = call <32 x half> @llvm.masked.load.v32f16.p0(ptr %addr, i32 4, <32 x i1> %mask, <32 x half> zeroinitializer)701 ret <32 x half> %res702}703 704define <32 x half> @movrr32f16(<32 x half> %a, <32 x half> %b) {705; CHECK-LABEL: movrr32f16:706; CHECK: # %bb.0:707; CHECK-NEXT: vmovaps %zmm1, %zmm0708; CHECK-NEXT: ret{{[l|q]}}709 ret <32 x half> %b710}711 712define <32 x half> @movrrk32f16(<32 x half> %a, <32 x half> %b, i32 %msk) {713; X64-LABEL: movrrk32f16:714; X64: # %bb.0:715; X64-NEXT: kmovd %edi, %k1716; X64-NEXT: vpblendmw %zmm0, %zmm1, %zmm0 {%k1}717; X64-NEXT: retq718;719; X86-LABEL: movrrk32f16:720; X86: # %bb.0:721; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1722; X86-NEXT: vpblendmw %zmm0, %zmm1, %zmm0 {%k1}723; X86-NEXT: retl724 %mask = bitcast i32 %msk to <32 x i1>725 %res = select <32 x i1> %mask, <32 x half> %a, <32 x half> %b726 ret <32 x half> %res727}728 729define <32 x half> @movrrkz32f16(<32 x half> %a, i32 %msk) {730; X64-LABEL: movrrkz32f16:731; X64: # %bb.0:732; X64-NEXT: kmovd %edi, %k1733; X64-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}734; X64-NEXT: retq735;736; X86-LABEL: movrrkz32f16:737; X86: # %bb.0:738; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1739; X86-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}740; X86-NEXT: retl741 %mask = bitcast i32 %msk to <32 x i1>742 %res = select <32 x i1> %mask, <32 x half> %a, <32 x half> zeroinitializer743 ret <32 x half> %res744}745 746define <16 x half> @load16f16(ptr %a) {747; X64-LABEL: load16f16:748; X64: # %bb.0:749; X64-NEXT: vmovaps (%rdi), %ymm0750; X64-NEXT: retq751;752; X86-LABEL: load16f16:753; X86: # %bb.0:754; X86-NEXT: movl {{[0-9]+}}(%esp), %eax755; X86-NEXT: vmovaps (%eax), %ymm0756; X86-NEXT: retl757 %res = load <16 x half>, ptr %a758 ret <16 x half> %res759}760 761define <16 x half> @load16f16mask(ptr %a, <16 x half> %b, i16 %c) {762; X64VL-LABEL: load16f16mask:763; X64VL: # %bb.0:764; X64VL-NEXT: kmovd %esi, %k1765; X64VL-NEXT: vmovdqu16 (%rdi), %ymm0 {%k1}766; X64VL-NEXT: retq767;768; X86-LABEL: load16f16mask:769; X86: # %bb.0:770; X86-NEXT: movl {{[0-9]+}}(%esp), %eax771; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1772; X86-NEXT: vmovdqu16 (%eax), %ymm0 {%k1}773; X86-NEXT: retl774;775; X64-NOVL-LABEL: load16f16mask:776; X64-NOVL: # %bb.0:777; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0778; X64-NOVL-NEXT: kmovd %esi, %k1779; X64-NOVL-NEXT: vmovdqa (%rdi), %ymm1780; X64-NOVL-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1}781; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0782; X64-NOVL-NEXT: retq783 %msk = bitcast i16 %c to <16 x i1>784 %res0 = load <16 x half>, ptr %a785 %res = select <16 x i1> %msk, <16 x half> %res0, <16 x half> %b786 ret <16 x half> %res787}788 789define <16 x half> @load16f16maskz(ptr %a, i16 %c) {790; X64VL-LABEL: load16f16maskz:791; X64VL: # %bb.0:792; X64VL-NEXT: kmovd %esi, %k1793; X64VL-NEXT: vmovdqu16 (%rdi), %ymm0 {%k1} {z}794; X64VL-NEXT: retq795;796; X86-LABEL: load16f16maskz:797; X86: # %bb.0:798; X86-NEXT: movl {{[0-9]+}}(%esp), %eax799; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1800; X86-NEXT: vmovdqu16 (%eax), %ymm0 {%k1} {z}801; X86-NEXT: retl802;803; X64-NOVL-LABEL: load16f16maskz:804; X64-NOVL: # %bb.0:805; X64-NOVL-NEXT: kmovd %esi, %k1806; X64-NOVL-NEXT: vmovdqa (%rdi), %ymm0807; X64-NOVL-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}808; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0809; X64-NOVL-NEXT: retq810 %msk = bitcast i16 %c to <16 x i1>811 %res0 = load <16 x half>, ptr %a812 %res = select <16 x i1> %msk, <16 x half> %res0, <16 x half> zeroinitializer813 ret <16 x half> %res814}815 816define <16 x half> @loadu16f16(ptr %a) {817; X64-LABEL: loadu16f16:818; X64: # %bb.0:819; X64-NEXT: vmovups (%rdi), %ymm0820; X64-NEXT: retq821;822; X86-LABEL: loadu16f16:823; X86: # %bb.0:824; X86-NEXT: movl {{[0-9]+}}(%esp), %eax825; X86-NEXT: vmovups (%eax), %ymm0826; X86-NEXT: retl827 %res = load <16 x half>, ptr %a, align 8828 ret <16 x half> %res829}830 831define <16 x half> @loadu16f16mask(ptr %a, <16 x half> %b, i16 %c) {832; X64VL-LABEL: loadu16f16mask:833; X64VL: # %bb.0:834; X64VL-NEXT: kmovd %esi, %k1835; X64VL-NEXT: vmovdqu16 (%rdi), %ymm0 {%k1}836; X64VL-NEXT: retq837;838; X86-LABEL: loadu16f16mask:839; X86: # %bb.0:840; X86-NEXT: movl {{[0-9]+}}(%esp), %eax841; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1842; X86-NEXT: vmovdqu16 (%eax), %ymm0 {%k1}843; X86-NEXT: retl844;845; X64-NOVL-LABEL: loadu16f16mask:846; X64-NOVL: # %bb.0:847; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0848; X64-NOVL-NEXT: kmovd %esi, %k1849; X64-NOVL-NEXT: vmovdqu (%rdi), %ymm1850; X64-NOVL-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1}851; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0852; X64-NOVL-NEXT: retq853 %msk = bitcast i16 %c to <16 x i1>854 %res0 = load <16 x half>, ptr %a, align 8855 %res = select <16 x i1> %msk, <16 x half> %res0, <16 x half> %b856 ret <16 x half> %res857}858 859define <16 x half> @loadu16f16maskz(ptr %a, i16 %c) {860; X64VL-LABEL: loadu16f16maskz:861; X64VL: # %bb.0:862; X64VL-NEXT: kmovd %esi, %k1863; X64VL-NEXT: vmovdqu16 (%rdi), %ymm0 {%k1} {z}864; X64VL-NEXT: retq865;866; X86-LABEL: loadu16f16maskz:867; X86: # %bb.0:868; X86-NEXT: movl {{[0-9]+}}(%esp), %eax869; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1870; X86-NEXT: vmovdqu16 (%eax), %ymm0 {%k1} {z}871; X86-NEXT: retl872;873; X64-NOVL-LABEL: loadu16f16maskz:874; X64-NOVL: # %bb.0:875; X64-NOVL-NEXT: kmovd %esi, %k1876; X64-NOVL-NEXT: vmovdqu (%rdi), %ymm0877; X64-NOVL-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}878; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0879; X64-NOVL-NEXT: retq880 %msk = bitcast i16 %c to <16 x i1>881 %res0 = load <16 x half>, ptr %a, align 8882 %res = select <16 x i1> %msk, <16 x half> %res0, <16 x half> zeroinitializer883 ret <16 x half> %res884}885 886define void @store16f16(<16 x half> %a) {887; X64-LABEL: store16f16:888; X64: # %bb.0:889; X64-NEXT: movq g16f16@GOTPCREL(%rip), %rax890; X64-NEXT: vmovaps %ymm0, (%rax)891; X64-NEXT: vzeroupper892; X64-NEXT: retq893;894; X86-LABEL: store16f16:895; X86: # %bb.0:896; X86-NEXT: vmovaps %ymm0, g16f16897; X86-NEXT: vzeroupper898; X86-NEXT: retl899 store <16 x half> %a, ptr @g16f16900 ret void901}902 903define void @storeu16f16(<16 x half> %a) {904; X64-LABEL: storeu16f16:905; X64: # %bb.0:906; X64-NEXT: movq g16f16u@GOTPCREL(%rip), %rax907; X64-NEXT: vmovups %ymm0, (%rax)908; X64-NEXT: vzeroupper909; X64-NEXT: retq910;911; X86-LABEL: storeu16f16:912; X86: # %bb.0:913; X86-NEXT: vmovups %ymm0, g16f16u914; X86-NEXT: vzeroupper915; X86-NEXT: retl916 store <16 x half> %a, ptr @g16f16u, align 8917 ret void918}919 920declare void @llvm.masked.store.v16f16.p0(<16 x half>, ptr, i32, <16 x i1>)921declare <16 x half> @llvm.masked.load.v16f16.p0(ptr, i32, <16 x i1>, <16 x half>)922 923define void @storeu16f16mask(<16 x i1> %mask, ptr %addr, <16 x half> %val) {924; X64VL-LABEL: storeu16f16mask:925; X64VL: # %bb.0:926; X64VL-NEXT: vpsllw $7, %xmm0, %xmm0927; X64VL-NEXT: vpmovb2m %xmm0, %k1928; X64VL-NEXT: vmovdqu16 %ymm1, (%rdi) {%k1}929; X64VL-NEXT: vzeroupper930; X64VL-NEXT: retq931;932; X86-LABEL: storeu16f16mask:933; X86: # %bb.0:934; X86-NEXT: vpsllw $7, %xmm0, %xmm0935; X86-NEXT: vpmovb2m %xmm0, %k1936; X86-NEXT: movl {{[0-9]+}}(%esp), %eax937; X86-NEXT: vmovdqu16 %ymm1, (%eax) {%k1}938; X86-NEXT: vzeroupper939; X86-NEXT: retl940;941; X64-NOVL-LABEL: storeu16f16mask:942; X64-NOVL: # %bb.0:943; X64-NOVL-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1944; X64-NOVL-NEXT: vpsllw $7, %xmm0, %xmm0945; X64-NOVL-NEXT: vpmovb2m %zmm0, %k0946; X64-NOVL-NEXT: kmovw %k0, %k1947; X64-NOVL-NEXT: vmovdqu16 %zmm1, (%rdi) {%k1}948; X64-NOVL-NEXT: vzeroupper949; X64-NOVL-NEXT: retq950 call void @llvm.masked.store.v16f16.p0(<16 x half> %val, ptr %addr, i32 4, <16 x i1>%mask)951 ret void952}953 954define <16 x half> @maskloadu16f16(ptr %addr, <16 x half> %val, <16 x i1> %mask) {955; X64VL-LABEL: maskloadu16f16:956; X64VL: # %bb.0:957; X64VL-NEXT: vpsllw $7, %xmm1, %xmm1958; X64VL-NEXT: vpmovb2m %xmm1, %k1959; X64VL-NEXT: vmovdqu16 (%rdi), %ymm0 {%k1}960; X64VL-NEXT: retq961;962; X86-LABEL: maskloadu16f16:963; X86: # %bb.0:964; X86-NEXT: vpsllw $7, %xmm1, %xmm1965; X86-NEXT: vpmovb2m %xmm1, %k1966; X86-NEXT: movl {{[0-9]+}}(%esp), %eax967; X86-NEXT: vmovdqu16 (%eax), %ymm0 {%k1}968; X86-NEXT: retl969;970; X64-NOVL-LABEL: maskloadu16f16:971; X64-NOVL: # %bb.0:972; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0973; X64-NOVL-NEXT: vpsllw $7, %xmm1, %xmm1974; X64-NOVL-NEXT: vpmovb2m %zmm1, %k0975; X64-NOVL-NEXT: kmovw %k0, %k1976; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1}977; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0978; X64-NOVL-NEXT: retq979 %res = call <16 x half> @llvm.masked.load.v16f16.p0(ptr %addr, i32 4, <16 x i1> %mask, <16 x half> %val)980 ret <16 x half> %res981}982 983define <16 x half> @maskuloadu16f16(ptr %addr, <16 x i1> %mask) {984; X64VL-LABEL: maskuloadu16f16:985; X64VL: # %bb.0:986; X64VL-NEXT: vpsllw $7, %xmm0, %xmm0987; X64VL-NEXT: vpmovb2m %xmm0, %k1988; X64VL-NEXT: vmovdqu16 (%rdi), %ymm0 {%k1} {z}989; X64VL-NEXT: retq990;991; X86-LABEL: maskuloadu16f16:992; X86: # %bb.0:993; X86-NEXT: vpsllw $7, %xmm0, %xmm0994; X86-NEXT: vpmovb2m %xmm0, %k1995; X86-NEXT: movl {{[0-9]+}}(%esp), %eax996; X86-NEXT: vmovdqu16 (%eax), %ymm0 {%k1} {z}997; X86-NEXT: retl998;999; X64-NOVL-LABEL: maskuloadu16f16:1000; X64-NOVL: # %bb.0:1001; X64-NOVL-NEXT: vpsllw $7, %xmm0, %xmm01002; X64-NOVL-NEXT: vpmovb2m %zmm0, %k01003; X64-NOVL-NEXT: kmovw %k0, %k11004; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}1005; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01006; X64-NOVL-NEXT: retq1007 %res = call <16 x half> @llvm.masked.load.v16f16.p0(ptr %addr, i32 4, <16 x i1> %mask, <16 x half> undef)1008 ret <16 x half> %res1009}1010 1011define <16 x half> @maskzloadu16f16(ptr %addr, <16 x i1> %mask) {1012; X64VL-LABEL: maskzloadu16f16:1013; X64VL: # %bb.0:1014; X64VL-NEXT: vpsllw $7, %xmm0, %xmm01015; X64VL-NEXT: vpmovb2m %xmm0, %k11016; X64VL-NEXT: vmovdqu16 (%rdi), %ymm0 {%k1} {z}1017; X64VL-NEXT: retq1018;1019; X86-LABEL: maskzloadu16f16:1020; X86: # %bb.0:1021; X86-NEXT: vpsllw $7, %xmm0, %xmm01022; X86-NEXT: vpmovb2m %xmm0, %k11023; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1024; X86-NEXT: vmovdqu16 (%eax), %ymm0 {%k1} {z}1025; X86-NEXT: retl1026;1027; X64-NOVL-LABEL: maskzloadu16f16:1028; X64-NOVL: # %bb.0:1029; X64-NOVL-NEXT: vpsllw $7, %xmm0, %xmm01030; X64-NOVL-NEXT: vpmovb2m %zmm0, %k01031; X64-NOVL-NEXT: kmovw %k0, %k11032; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}1033; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01034; X64-NOVL-NEXT: retq1035 %res = call <16 x half> @llvm.masked.load.v16f16.p0(ptr %addr, i32 4, <16 x i1> %mask, <16 x half> zeroinitializer)1036 ret <16 x half> %res1037}1038 1039define <16 x half> @movrr16f16(<16 x half> %a, <16 x half> %b) {1040; CHECK-LABEL: movrr16f16:1041; CHECK: # %bb.0:1042; CHECK-NEXT: vmovaps %ymm1, %ymm01043; CHECK-NEXT: ret{{[l|q]}}1044 ret <16 x half> %b1045}1046 1047define <16 x half> @movrrk16f16(<16 x half> %a, <16 x half> %b, i16 %msk) {1048; X64VL-LABEL: movrrk16f16:1049; X64VL: # %bb.0:1050; X64VL-NEXT: kmovd %edi, %k11051; X64VL-NEXT: vpblendmw %ymm0, %ymm1, %ymm0 {%k1}1052; X64VL-NEXT: retq1053;1054; X86-LABEL: movrrk16f16:1055; X86: # %bb.0:1056; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k11057; X86-NEXT: vpblendmw %ymm0, %ymm1, %ymm0 {%k1}1058; X86-NEXT: retl1059;1060; X64-NOVL-LABEL: movrrk16f16:1061; X64-NOVL: # %bb.0:1062; X64-NOVL-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm11063; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01064; X64-NOVL-NEXT: kmovd %edi, %k11065; X64-NOVL-NEXT: vpblendmw %zmm0, %zmm1, %zmm0 {%k1}1066; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01067; X64-NOVL-NEXT: retq1068 %mask = bitcast i16 %msk to <16 x i1>1069 %res = select <16 x i1> %mask, <16 x half> %a, <16 x half> %b1070 ret <16 x half> %res1071}1072 1073define <16 x half> @movrrkz16f16(<16 x half> %a, i16 %msk) {1074; X64VL-LABEL: movrrkz16f16:1075; X64VL: # %bb.0:1076; X64VL-NEXT: kmovd %edi, %k11077; X64VL-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}1078; X64VL-NEXT: retq1079;1080; X86-LABEL: movrrkz16f16:1081; X86: # %bb.0:1082; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k11083; X86-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}1084; X86-NEXT: retl1085;1086; X64-NOVL-LABEL: movrrkz16f16:1087; X64-NOVL: # %bb.0:1088; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01089; X64-NOVL-NEXT: kmovd %edi, %k11090; X64-NOVL-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}1091; X64-NOVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01092; X64-NOVL-NEXT: retq1093 %mask = bitcast i16 %msk to <16 x i1>1094 %res = select <16 x i1> %mask, <16 x half> %a, <16 x half> zeroinitializer1095 ret <16 x half> %res1096}1097 1098define <8 x half> @load8f16(ptr %a) {1099; X64-LABEL: load8f16:1100; X64: # %bb.0:1101; X64-NEXT: vmovaps (%rdi), %xmm01102; X64-NEXT: retq1103;1104; X86-LABEL: load8f16:1105; X86: # %bb.0:1106; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1107; X86-NEXT: vmovaps (%eax), %xmm01108; X86-NEXT: retl1109 %res = load <8 x half>, ptr %a1110 ret <8 x half> %res1111}1112 1113define <8 x half> @load8f16mask(ptr %a, <8 x half> %b, i8 %c) {1114; X64VL-LABEL: load8f16mask:1115; X64VL: # %bb.0:1116; X64VL-NEXT: kmovd %esi, %k11117; X64VL-NEXT: vmovdqu16 (%rdi), %xmm0 {%k1}1118; X64VL-NEXT: retq1119;1120; X86-LABEL: load8f16mask:1121; X86: # %bb.0:1122; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1123; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx1124; X86-NEXT: kmovd %ecx, %k11125; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1}1126; X86-NEXT: retl1127;1128; X64-NOVL-LABEL: load8f16mask:1129; X64-NOVL: # %bb.0:1130; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm01131; X64-NOVL-NEXT: kmovd %esi, %k11132; X64-NOVL-NEXT: vmovdqa (%rdi), %xmm11133; X64-NOVL-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1}1134; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01135; X64-NOVL-NEXT: vzeroupper1136; X64-NOVL-NEXT: retq1137 %msk = bitcast i8 %c to <8 x i1>1138 %res0 = load <8 x half>, ptr %a1139 %res = select <8 x i1> %msk, <8 x half> %res0, <8 x half> %b1140 ret <8 x half> %res1141}1142 1143define <8 x half> @load8f16maskz(ptr %a, i8 %c) {1144; X64VL-LABEL: load8f16maskz:1145; X64VL: # %bb.0:1146; X64VL-NEXT: kmovd %esi, %k11147; X64VL-NEXT: vmovdqu16 (%rdi), %xmm0 {%k1} {z}1148; X64VL-NEXT: retq1149;1150; X86-LABEL: load8f16maskz:1151; X86: # %bb.0:1152; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1153; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx1154; X86-NEXT: kmovd %ecx, %k11155; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1} {z}1156; X86-NEXT: retl1157;1158; X64-NOVL-LABEL: load8f16maskz:1159; X64-NOVL: # %bb.0:1160; X64-NOVL-NEXT: kmovd %esi, %k11161; X64-NOVL-NEXT: vmovdqa (%rdi), %xmm01162; X64-NOVL-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}1163; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01164; X64-NOVL-NEXT: vzeroupper1165; X64-NOVL-NEXT: retq1166 %msk = bitcast i8 %c to <8 x i1>1167 %res0 = load <8 x half>, ptr %a1168 %res = select <8 x i1> %msk, <8 x half> %res0, <8 x half> zeroinitializer1169 ret <8 x half> %res1170}1171 1172define <8 x half> @loadu8f16(ptr %a) {1173; X64-LABEL: loadu8f16:1174; X64: # %bb.0:1175; X64-NEXT: vmovups (%rdi), %xmm01176; X64-NEXT: retq1177;1178; X86-LABEL: loadu8f16:1179; X86: # %bb.0:1180; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1181; X86-NEXT: vmovups (%eax), %xmm01182; X86-NEXT: retl1183 %res = load <8 x half>, ptr %a, align 81184 ret <8 x half> %res1185}1186 1187define <8 x half> @loadu8f16mask(ptr %a, <8 x half> %b, i8 %c) {1188; X64VL-LABEL: loadu8f16mask:1189; X64VL: # %bb.0:1190; X64VL-NEXT: kmovd %esi, %k11191; X64VL-NEXT: vmovdqu16 (%rdi), %xmm0 {%k1}1192; X64VL-NEXT: retq1193;1194; X86-LABEL: loadu8f16mask:1195; X86: # %bb.0:1196; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1197; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx1198; X86-NEXT: kmovd %ecx, %k11199; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1}1200; X86-NEXT: retl1201;1202; X64-NOVL-LABEL: loadu8f16mask:1203; X64-NOVL: # %bb.0:1204; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm01205; X64-NOVL-NEXT: kmovd %esi, %k11206; X64-NOVL-NEXT: vmovdqu (%rdi), %xmm11207; X64-NOVL-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1}1208; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01209; X64-NOVL-NEXT: vzeroupper1210; X64-NOVL-NEXT: retq1211 %msk = bitcast i8 %c to <8 x i1>1212 %res0 = load <8 x half>, ptr %a, align 81213 %res = select <8 x i1> %msk, <8 x half> %res0, <8 x half> %b1214 ret <8 x half> %res1215}1216 1217define <8 x half> @loadu8f16maskz(ptr %a, i8 %c) {1218; X64VL-LABEL: loadu8f16maskz:1219; X64VL: # %bb.0:1220; X64VL-NEXT: kmovd %esi, %k11221; X64VL-NEXT: vmovdqu16 (%rdi), %xmm0 {%k1} {z}1222; X64VL-NEXT: retq1223;1224; X86-LABEL: loadu8f16maskz:1225; X86: # %bb.0:1226; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1227; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx1228; X86-NEXT: kmovd %ecx, %k11229; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1} {z}1230; X86-NEXT: retl1231;1232; X64-NOVL-LABEL: loadu8f16maskz:1233; X64-NOVL: # %bb.0:1234; X64-NOVL-NEXT: kmovd %esi, %k11235; X64-NOVL-NEXT: vmovdqu (%rdi), %xmm01236; X64-NOVL-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}1237; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01238; X64-NOVL-NEXT: vzeroupper1239; X64-NOVL-NEXT: retq1240 %msk = bitcast i8 %c to <8 x i1>1241 %res0 = load <8 x half>, ptr %a, align 81242 %res = select <8 x i1> %msk, <8 x half> %res0, <8 x half> zeroinitializer1243 ret <8 x half> %res1244}1245 1246define void @store8f16(<8 x half> %a) {1247; X64-LABEL: store8f16:1248; X64: # %bb.0:1249; X64-NEXT: movq g8f16@GOTPCREL(%rip), %rax1250; X64-NEXT: vmovaps %xmm0, (%rax)1251; X64-NEXT: retq1252;1253; X86-LABEL: store8f16:1254; X86: # %bb.0:1255; X86-NEXT: vmovaps %xmm0, g8f161256; X86-NEXT: retl1257 store <8 x half> %a, ptr @g8f161258 ret void1259}1260 1261define void @storeu8f16(<8 x half> %a) {1262; X64-LABEL: storeu8f16:1263; X64: # %bb.0:1264; X64-NEXT: movq g8f16u@GOTPCREL(%rip), %rax1265; X64-NEXT: vmovups %xmm0, (%rax)1266; X64-NEXT: retq1267;1268; X86-LABEL: storeu8f16:1269; X86: # %bb.0:1270; X86-NEXT: vmovups %xmm0, g8f16u1271; X86-NEXT: retl1272 store <8 x half> %a, ptr @g8f16u, align 81273 ret void1274}1275 1276declare void @llvm.masked.store.v8f16.p0(<8 x half>, ptr, i32, <8 x i1>)1277declare <8 x half> @llvm.masked.load.v8f16.p0(ptr, i32, <8 x i1>, <8 x half>)1278 1279define void @storeu8f16mask(<8 x i1> %mask, ptr %addr, <8 x half> %val) {1280; X64VL-LABEL: storeu8f16mask:1281; X64VL: # %bb.0:1282; X64VL-NEXT: vpsllw $15, %xmm0, %xmm01283; X64VL-NEXT: vpmovw2m %xmm0, %k11284; X64VL-NEXT: vmovdqu16 %xmm1, (%rdi) {%k1}1285; X64VL-NEXT: retq1286;1287; X86-LABEL: storeu8f16mask:1288; X86: # %bb.0:1289; X86-NEXT: vpsllw $15, %xmm0, %xmm01290; X86-NEXT: vpmovw2m %xmm0, %k11291; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1292; X86-NEXT: vmovdqu16 %xmm1, (%eax) {%k1}1293; X86-NEXT: retl1294;1295; X64-NOVL-LABEL: storeu8f16mask:1296; X64-NOVL: # %bb.0:1297; X64-NOVL-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm11298; X64-NOVL-NEXT: vpsllw $15, %xmm0, %xmm01299; X64-NOVL-NEXT: vpmovw2m %zmm0, %k01300; X64-NOVL-NEXT: kshiftld $24, %k0, %k01301; X64-NOVL-NEXT: kshiftrd $24, %k0, %k11302; X64-NOVL-NEXT: vmovdqu16 %zmm1, (%rdi) {%k1}1303; X64-NOVL-NEXT: vzeroupper1304; X64-NOVL-NEXT: retq1305 call void @llvm.masked.store.v8f16.p0(<8 x half> %val, ptr %addr, i32 4, <8 x i1>%mask)1306 ret void1307}1308 1309define <8 x half> @maskloadu8f16(ptr %addr, <8 x half> %val, <8 x i1> %mask) {1310; X64VL-LABEL: maskloadu8f16:1311; X64VL: # %bb.0:1312; X64VL-NEXT: vpsllw $15, %xmm1, %xmm11313; X64VL-NEXT: vpmovw2m %xmm1, %k11314; X64VL-NEXT: vmovdqu16 (%rdi), %xmm0 {%k1}1315; X64VL-NEXT: retq1316;1317; X86-LABEL: maskloadu8f16:1318; X86: # %bb.0:1319; X86-NEXT: vpsllw $15, %xmm1, %xmm11320; X86-NEXT: vpmovw2m %xmm1, %k11321; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1322; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1}1323; X86-NEXT: retl1324;1325; X64-NOVL-LABEL: maskloadu8f16:1326; X64-NOVL: # %bb.0:1327; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm01328; X64-NOVL-NEXT: vpsllw $15, %xmm1, %xmm11329; X64-NOVL-NEXT: vpmovw2m %zmm1, %k01330; X64-NOVL-NEXT: kshiftld $24, %k0, %k01331; X64-NOVL-NEXT: kshiftrd $24, %k0, %k11332; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1}1333; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01334; X64-NOVL-NEXT: vzeroupper1335; X64-NOVL-NEXT: retq1336 %res = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %addr, i32 4, <8 x i1> %mask, <8 x half> %val)1337 ret <8 x half> %res1338}1339 1340define <8 x half> @maskuloadu8f16(ptr %addr, <8 x i1> %mask) {1341; X64VL-LABEL: maskuloadu8f16:1342; X64VL: # %bb.0:1343; X64VL-NEXT: vpsllw $15, %xmm0, %xmm01344; X64VL-NEXT: vpmovw2m %xmm0, %k11345; X64VL-NEXT: vmovdqu16 (%rdi), %xmm0 {%k1} {z}1346; X64VL-NEXT: retq1347;1348; X86-LABEL: maskuloadu8f16:1349; X86: # %bb.0:1350; X86-NEXT: vpsllw $15, %xmm0, %xmm01351; X86-NEXT: vpmovw2m %xmm0, %k11352; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1353; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1} {z}1354; X86-NEXT: retl1355;1356; X64-NOVL-LABEL: maskuloadu8f16:1357; X64-NOVL: # %bb.0:1358; X64-NOVL-NEXT: vpsllw $15, %xmm0, %xmm01359; X64-NOVL-NEXT: vpmovw2m %zmm0, %k01360; X64-NOVL-NEXT: kshiftld $24, %k0, %k01361; X64-NOVL-NEXT: kshiftrd $24, %k0, %k11362; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}1363; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01364; X64-NOVL-NEXT: vzeroupper1365; X64-NOVL-NEXT: retq1366 %res = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %addr, i32 4, <8 x i1> %mask, <8 x half> undef)1367 ret <8 x half> %res1368}1369 1370define <8 x half> @maskzloadu8f16(ptr %addr, <8 x i1> %mask) {1371; X64VL-LABEL: maskzloadu8f16:1372; X64VL: # %bb.0:1373; X64VL-NEXT: vpsllw $15, %xmm0, %xmm01374; X64VL-NEXT: vpmovw2m %xmm0, %k11375; X64VL-NEXT: vmovdqu16 (%rdi), %xmm0 {%k1} {z}1376; X64VL-NEXT: retq1377;1378; X86-LABEL: maskzloadu8f16:1379; X86: # %bb.0:1380; X86-NEXT: vpsllw $15, %xmm0, %xmm01381; X86-NEXT: vpmovw2m %xmm0, %k11382; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1383; X86-NEXT: vmovdqu16 (%eax), %xmm0 {%k1} {z}1384; X86-NEXT: retl1385;1386; X64-NOVL-LABEL: maskzloadu8f16:1387; X64-NOVL: # %bb.0:1388; X64-NOVL-NEXT: vpsllw $15, %xmm0, %xmm01389; X64-NOVL-NEXT: vpmovw2m %zmm0, %k01390; X64-NOVL-NEXT: kshiftld $24, %k0, %k01391; X64-NOVL-NEXT: kshiftrd $24, %k0, %k11392; X64-NOVL-NEXT: vmovdqu16 (%rdi), %zmm0 {%k1} {z}1393; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01394; X64-NOVL-NEXT: vzeroupper1395; X64-NOVL-NEXT: retq1396 %res = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %addr, i32 4, <8 x i1> %mask, <8 x half> zeroinitializer)1397 ret <8 x half> %res1398}1399 1400define <8 x half> @movrr8f16(<8 x half> %a, <8 x half> %b) {1401; CHECK-LABEL: movrr8f16:1402; CHECK: # %bb.0:1403; CHECK-NEXT: vmovaps %xmm1, %xmm01404; CHECK-NEXT: ret{{[l|q]}}1405 ret <8 x half> %b1406}1407 1408define <8 x half> @movrrk8f16(<8 x half> %a, <8 x half> %b, i8 %msk) {1409; X64VL-LABEL: movrrk8f16:1410; X64VL: # %bb.0:1411; X64VL-NEXT: kmovd %edi, %k11412; X64VL-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}1413; X64VL-NEXT: retq1414;1415; X86-LABEL: movrrk8f16:1416; X86: # %bb.0:1417; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax1418; X86-NEXT: kmovd %eax, %k11419; X86-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}1420; X86-NEXT: retl1421;1422; X64-NOVL-LABEL: movrrk8f16:1423; X64-NOVL: # %bb.0:1424; X64-NOVL-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm11425; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm01426; X64-NOVL-NEXT: kmovd %edi, %k11427; X64-NOVL-NEXT: vpblendmw %zmm0, %zmm1, %zmm0 {%k1}1428; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01429; X64-NOVL-NEXT: vzeroupper1430; X64-NOVL-NEXT: retq1431 %mask = bitcast i8 %msk to <8 x i1>1432 %res = select <8 x i1> %mask, <8 x half> %a, <8 x half> %b1433 ret <8 x half> %res1434}1435 1436define <8 x half> @movrrkz8f16(<8 x half> %a, i8 %msk) {1437; X64VL-LABEL: movrrkz8f16:1438; X64VL: # %bb.0:1439; X64VL-NEXT: kmovd %edi, %k11440; X64VL-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}1441; X64VL-NEXT: retq1442;1443; X86-LABEL: movrrkz8f16:1444; X86: # %bb.0:1445; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax1446; X86-NEXT: kmovd %eax, %k11447; X86-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}1448; X86-NEXT: retl1449;1450; X64-NOVL-LABEL: movrrkz8f16:1451; X64-NOVL: # %bb.0:1452; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm01453; X64-NOVL-NEXT: kmovd %edi, %k11454; X64-NOVL-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}1455; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01456; X64-NOVL-NEXT: vzeroupper1457; X64-NOVL-NEXT: retq1458 %mask = bitcast i8 %msk to <8 x i1>1459 %res = select <8 x i1> %mask, <8 x half> %a, <8 x half> zeroinitializer1460 ret <8 x half> %res1461}1462 1463define <8 x half> @movsh(<8 x half> %a, <8 x half> %b) {1464; X64VL-LABEL: movsh:1465; X64VL: # %bb.0:1466; X64VL-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,0,1,2,3,4,5,6,7,14,15,10,11]1467; X64VL-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]1468; X64VL-NEXT: vaddph %xmm0, %xmm2, %xmm01469; X64VL-NEXT: retq1470;1471; X86-LABEL: movsh:1472; X86: # %bb.0:1473; X86-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,0,1,2,3,4,5,6,7,14,15,10,11]1474; X86-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]1475; X86-NEXT: vaddph %xmm0, %xmm2, %xmm01476; X86-NEXT: retl1477;1478; X64-NOVL-LABEL: movsh:1479; X64-NOVL: # %bb.0:1480; X64-NOVL-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,14,15,0,1,2,3,4,5,6,7,14,15,10,11]1481; X64-NOVL-NEXT: vmovsh {{.*#+}} xmm3 = xmm0[0],xmm1[1,2,3,4,5,6,7]1482; X64-NOVL-NEXT: vpsrldq {{.*#+}} xmm4 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1483; X64-NOVL-NEXT: vpsrldq {{.*#+}} xmm5 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1484; X64-NOVL-NEXT: vaddsh %xmm4, %xmm5, %xmm41485; X64-NOVL-NEXT: vshufps {{.*#+}} xmm5 = xmm3[3,3,3,3]1486; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[3,3,3,3]1487; X64-NOVL-NEXT: vaddsh %xmm5, %xmm6, %xmm51488; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]1489; X64-NOVL-NEXT: vpsrldq {{.*#+}} xmm5 = xmm3[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1490; X64-NOVL-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]1491; X64-NOVL-NEXT: vaddsh %xmm5, %xmm0, %xmm01492; X64-NOVL-NEXT: vshufpd {{.*#+}} xmm5 = xmm3[1,0]1493; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[2,3,0,1]1494; X64-NOVL-NEXT: vaddsh %xmm5, %xmm6, %xmm51495; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]1496; X64-NOVL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]1497; X64-NOVL-NEXT: vpsrlq $48, %xmm1, %xmm41498; X64-NOVL-NEXT: vpsrlq $48, %xmm2, %xmm51499; X64-NOVL-NEXT: vaddsh %xmm4, %xmm5, %xmm41500; X64-NOVL-NEXT: vmovshdup {{.*#+}} xmm5 = xmm3[1,1,3,3]1501; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]1502; X64-NOVL-NEXT: vaddsh %xmm5, %xmm6, %xmm51503; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]1504; X64-NOVL-NEXT: vaddsh %xmm3, %xmm2, %xmm31505; X64-NOVL-NEXT: vpsrld $16, %xmm1, %xmm11506; X64-NOVL-NEXT: vpsrld $16, %xmm2, %xmm21507; X64-NOVL-NEXT: vaddsh %xmm1, %xmm2, %xmm11508; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]1509; X64-NOVL-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]1510; X64-NOVL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]1511; X64-NOVL-NEXT: retq1512 %res1 = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 0, i32 7, i32 0, i32 1, i32 2, i32 3, i32 7, i32 5>1513 %res2 = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1514 %res = fadd <8 x half> %res1, %res21515 ret <8 x half> %res1516}1517 1518define i16 @test_movw(half %x) {1519; X64-LABEL: test_movw:1520; X64: # %bb.0:1521; X64-NEXT: vmovw %xmm0, %eax1522; X64-NEXT: # kill: def $ax killed $ax killed $eax1523; X64-NEXT: retq1524;1525; X86-LABEL: test_movw:1526; X86: # %bb.0:1527; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax1528; X86-NEXT: retl1529 %res = bitcast half %x to i161530 ret i16 %res1531}1532 1533define half @test_movw2(i16 %x) {1534; X64-LABEL: test_movw2:1535; X64: # %bb.0:1536; X64-NEXT: vmovw %edi, %xmm01537; X64-NEXT: retq1538;1539; X86-LABEL: test_movw2:1540; X86: # %bb.0:1541; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero1542; X86-NEXT: retl1543 %res = bitcast i16 %x to half1544 ret half %res1545}1546 1547; sext avoids having a truncate in front of the bitcast input due to calling1548; convention or i16 op promotion.1549define half @test_movw3(i8 %x) {1550; X64-LABEL: test_movw3:1551; X64: # %bb.0:1552; X64-NEXT: movsbl %dil, %eax1553; X64-NEXT: vmovw %eax, %xmm01554; X64-NEXT: retq1555;1556; X86-LABEL: test_movw3:1557; X86: # %bb.0:1558; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax1559; X86-NEXT: vmovw %eax, %xmm01560; X86-NEXT: retl1561 %z = sext i8 %x to i161562 %a = bitcast i16 %z to half1563 ret half %a1564}1565 1566define half @extract_f16_0(<8 x half> %x) {1567; CHECK-LABEL: extract_f16_0:1568; CHECK: # %bb.0:1569; CHECK-NEXT: ret{{[l|q]}}1570 %res = extractelement <8 x half> %x, i32 01571 ret half %res1572}1573 1574define half @extract_f16_1(<8 x half> %x) {1575; CHECK-LABEL: extract_f16_1:1576; CHECK: # %bb.0:1577; CHECK-NEXT: vpsrld $16, %xmm0, %xmm01578; CHECK-NEXT: ret{{[l|q]}}1579 %res = extractelement <8 x half> %x, i32 11580 ret half %res1581}1582 1583define half @extract_f16_2(<8 x half> %x) {1584; CHECK-LABEL: extract_f16_2:1585; CHECK: # %bb.0:1586; CHECK-NEXT: vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]1587; CHECK-NEXT: ret{{[l|q]}}1588 %res = extractelement <8 x half> %x, i32 21589 ret half %res1590}1591 1592define half @extract_f16_3(<8 x half> %x) {1593; CHECK-LABEL: extract_f16_3:1594; CHECK: # %bb.0:1595; CHECK-NEXT: vpsrlq $48, %xmm0, %xmm01596; CHECK-NEXT: ret{{[l|q]}}1597 %res = extractelement <8 x half> %x, i32 31598 ret half %res1599}1600 1601define half @extract_f16_4(<8 x half> %x) {1602; CHECK-LABEL: extract_f16_4:1603; CHECK: # %bb.0:1604; CHECK-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0]1605; CHECK-NEXT: ret{{[l|q]}}1606 %res = extractelement <8 x half> %x, i32 41607 ret half %res1608}1609 1610define half @extract_f16_5(<8 x half> %x) {1611; CHECK-LABEL: extract_f16_5:1612; CHECK: # %bb.0:1613; CHECK-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1614; CHECK-NEXT: ret{{[l|q]}}1615 %res = extractelement <8 x half> %x, i32 51616 ret half %res1617}1618 1619define half @extract_f16_6(<8 x half> %x) {1620; CHECK-LABEL: extract_f16_6:1621; CHECK: # %bb.0:1622; CHECK-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]1623; CHECK-NEXT: ret{{[l|q]}}1624 %res = extractelement <8 x half> %x, i32 61625 ret half %res1626}1627 1628define half @extract_f16_7(<8 x half> %x) {1629; CHECK-LABEL: extract_f16_7:1630; CHECK: # %bb.0:1631; CHECK-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1632; CHECK-NEXT: ret{{[l|q]}}1633 %res = extractelement <8 x half> %x, i32 71634 ret half %res1635}1636 1637define half @extract_f16_8(<32 x half> %x, i64 %idx) nounwind {1638; X64-LABEL: extract_f16_8:1639; X64: # %bb.0:1640; X64-NEXT: pushq %rbp1641; X64-NEXT: movq %rsp, %rbp1642; X64-NEXT: andq $-64, %rsp1643; X64-NEXT: subq $128, %rsp1644; X64-NEXT: andl $31, %edi1645; X64-NEXT: vmovaps %zmm0, (%rsp)1646; X64-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero1647; X64-NEXT: movq %rbp, %rsp1648; X64-NEXT: popq %rbp1649; X64-NEXT: vzeroupper1650; X64-NEXT: retq1651;1652; X86-LABEL: extract_f16_8:1653; X86: # %bb.0:1654; X86-NEXT: pushl %ebp1655; X86-NEXT: movl %esp, %ebp1656; X86-NEXT: andl $-64, %esp1657; X86-NEXT: subl $128, %esp1658; X86-NEXT: movl 8(%ebp), %eax1659; X86-NEXT: andl $31, %eax1660; X86-NEXT: vmovaps %zmm0, (%esp)1661; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero1662; X86-NEXT: movl %ebp, %esp1663; X86-NEXT: popl %ebp1664; X86-NEXT: vzeroupper1665; X86-NEXT: retl1666 %res = extractelement <32 x half> %x, i64 %idx1667 ret half %res1668}1669 1670define half @extract_f16_9(<64 x half> %x, i64 %idx) nounwind {1671; X64-LABEL: extract_f16_9:1672; X64: # %bb.0:1673; X64-NEXT: pushq %rbp1674; X64-NEXT: movq %rsp, %rbp1675; X64-NEXT: andq $-64, %rsp1676; X64-NEXT: subq $192, %rsp1677; X64-NEXT: andl $63, %edi1678; X64-NEXT: vmovaps %zmm1, {{[0-9]+}}(%rsp)1679; X64-NEXT: vmovaps %zmm0, (%rsp)1680; X64-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero1681; X64-NEXT: movq %rbp, %rsp1682; X64-NEXT: popq %rbp1683; X64-NEXT: vzeroupper1684; X64-NEXT: retq1685;1686; X86-LABEL: extract_f16_9:1687; X86: # %bb.0:1688; X86-NEXT: pushl %ebp1689; X86-NEXT: movl %esp, %ebp1690; X86-NEXT: andl $-64, %esp1691; X86-NEXT: subl $192, %esp1692; X86-NEXT: movl 8(%ebp), %eax1693; X86-NEXT: andl $63, %eax1694; X86-NEXT: vmovaps %zmm1, {{[0-9]+}}(%esp)1695; X86-NEXT: vmovaps %zmm0, (%esp)1696; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero1697; X86-NEXT: movl %ebp, %esp1698; X86-NEXT: popl %ebp1699; X86-NEXT: vzeroupper1700; X86-NEXT: retl1701 %res = extractelement <64 x half> %x, i64 %idx1702 ret half %res1703}1704 1705define i16 @extract_i16_0(<8 x i16> %x) {1706; CHECK-LABEL: extract_i16_0:1707; CHECK: # %bb.0:1708; CHECK-NEXT: vmovw %xmm0, %eax1709; CHECK-NEXT: # kill: def $ax killed $ax killed $eax1710; CHECK-NEXT: ret{{[l|q]}}1711 %res = extractelement <8 x i16> %x, i32 01712 ret i16 %res1713}1714 1715define i16 @extract_i16_1(<8 x i16> %x) {1716; CHECK-LABEL: extract_i16_1:1717; CHECK: # %bb.0:1718; CHECK-NEXT: vpextrw $1, %xmm0, %eax1719; CHECK-NEXT: # kill: def $ax killed $ax killed $eax1720; CHECK-NEXT: ret{{[l|q]}}1721 %res = extractelement <8 x i16> %x, i32 11722 ret i16 %res1723}1724 1725define i16 @extract_i16_2(<8 x i16> %x) {1726; CHECK-LABEL: extract_i16_2:1727; CHECK: # %bb.0:1728; CHECK-NEXT: vpextrw $2, %xmm0, %eax1729; CHECK-NEXT: # kill: def $ax killed $ax killed $eax1730; CHECK-NEXT: ret{{[l|q]}}1731 %res = extractelement <8 x i16> %x, i32 21732 ret i16 %res1733}1734 1735define i16 @extract_i16_3(<8 x i16> %x) {1736; CHECK-LABEL: extract_i16_3:1737; CHECK: # %bb.0:1738; CHECK-NEXT: vpextrw $3, %xmm0, %eax1739; CHECK-NEXT: # kill: def $ax killed $ax killed $eax1740; CHECK-NEXT: ret{{[l|q]}}1741 %res = extractelement <8 x i16> %x, i32 31742 ret i16 %res1743}1744 1745define i16 @extract_i16_4(<8 x i16> %x) {1746; CHECK-LABEL: extract_i16_4:1747; CHECK: # %bb.0:1748; CHECK-NEXT: vpextrw $4, %xmm0, %eax1749; CHECK-NEXT: # kill: def $ax killed $ax killed $eax1750; CHECK-NEXT: ret{{[l|q]}}1751 %res = extractelement <8 x i16> %x, i32 41752 ret i16 %res1753}1754 1755define i16 @extract_i16_5(<8 x i16> %x) {1756; CHECK-LABEL: extract_i16_5:1757; CHECK: # %bb.0:1758; CHECK-NEXT: vpextrw $5, %xmm0, %eax1759; CHECK-NEXT: # kill: def $ax killed $ax killed $eax1760; CHECK-NEXT: ret{{[l|q]}}1761 %res = extractelement <8 x i16> %x, i32 51762 ret i16 %res1763}1764 1765define i16 @extract_i16_6(<8 x i16> %x) {1766; CHECK-LABEL: extract_i16_6:1767; CHECK: # %bb.0:1768; CHECK-NEXT: vpextrw $6, %xmm0, %eax1769; CHECK-NEXT: # kill: def $ax killed $ax killed $eax1770; CHECK-NEXT: ret{{[l|q]}}1771 %res = extractelement <8 x i16> %x, i32 61772 ret i16 %res1773}1774 1775define i16 @extract_i16_7(<8 x i16> %x) {1776; CHECK-LABEL: extract_i16_7:1777; CHECK: # %bb.0:1778; CHECK-NEXT: vpextrw $7, %xmm0, %eax1779; CHECK-NEXT: # kill: def $ax killed $ax killed $eax1780; CHECK-NEXT: ret{{[l|q]}}1781 %res = extractelement <8 x i16> %x, i32 71782 ret i16 %res1783}1784 1785define void @extract_store_f16_0(<8 x half> %x, ptr %y) {1786; X64-LABEL: extract_store_f16_0:1787; X64: # %bb.0:1788; X64-NEXT: vmovsh %xmm0, (%rdi)1789; X64-NEXT: retq1790;1791; X86-LABEL: extract_store_f16_0:1792; X86: # %bb.0:1793; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1794; X86-NEXT: vmovsh %xmm0, (%eax)1795; X86-NEXT: retl1796 %res = extractelement <8 x half> %x, i32 01797 store half %res, ptr %y1798 ret void1799}1800 1801define void @extract_store_f16_1(<8 x half> %x, ptr %y) {1802; X64-LABEL: extract_store_f16_1:1803; X64: # %bb.0:1804; X64-NEXT: vpsrld $16, %xmm0, %xmm01805; X64-NEXT: vmovsh %xmm0, (%rdi)1806; X64-NEXT: retq1807;1808; X86-LABEL: extract_store_f16_1:1809; X86: # %bb.0:1810; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1811; X86-NEXT: vpsrld $16, %xmm0, %xmm01812; X86-NEXT: vmovsh %xmm0, (%eax)1813; X86-NEXT: retl1814 %res = extractelement <8 x half> %x, i32 11815 store half %res, ptr %y1816 ret void1817}1818 1819define void @extract_store_f16_2(<8 x half> %x, ptr %y) {1820; X64-LABEL: extract_store_f16_2:1821; X64: # %bb.0:1822; X64-NEXT: vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]1823; X64-NEXT: vmovsh %xmm0, (%rdi)1824; X64-NEXT: retq1825;1826; X86-LABEL: extract_store_f16_2:1827; X86: # %bb.0:1828; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1829; X86-NEXT: vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]1830; X86-NEXT: vmovsh %xmm0, (%eax)1831; X86-NEXT: retl1832 %res = extractelement <8 x half> %x, i32 21833 store half %res, ptr %y1834 ret void1835}1836 1837define void @extract_store_f16_3(<8 x half> %x, ptr %y) {1838; X64-LABEL: extract_store_f16_3:1839; X64: # %bb.0:1840; X64-NEXT: vpsrlq $48, %xmm0, %xmm01841; X64-NEXT: vmovsh %xmm0, (%rdi)1842; X64-NEXT: retq1843;1844; X86-LABEL: extract_store_f16_3:1845; X86: # %bb.0:1846; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1847; X86-NEXT: vpsrlq $48, %xmm0, %xmm01848; X86-NEXT: vmovsh %xmm0, (%eax)1849; X86-NEXT: retl1850 %res = extractelement <8 x half> %x, i32 31851 store half %res, ptr %y1852 ret void1853}1854 1855define void @extract_store_f16_4(<8 x half> %x, ptr %y) {1856; X64-LABEL: extract_store_f16_4:1857; X64: # %bb.0:1858; X64-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0]1859; X64-NEXT: vmovsh %xmm0, (%rdi)1860; X64-NEXT: retq1861;1862; X86-LABEL: extract_store_f16_4:1863; X86: # %bb.0:1864; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1865; X86-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0]1866; X86-NEXT: vmovsh %xmm0, (%eax)1867; X86-NEXT: retl1868 %res = extractelement <8 x half> %x, i32 41869 store half %res, ptr %y1870 ret void1871}1872 1873define void @extract_store_f16_5(<8 x half> %x, ptr %y) {1874; X64-LABEL: extract_store_f16_5:1875; X64: # %bb.0:1876; X64-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1877; X64-NEXT: vmovsh %xmm0, (%rdi)1878; X64-NEXT: retq1879;1880; X86-LABEL: extract_store_f16_5:1881; X86: # %bb.0:1882; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1883; X86-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1884; X86-NEXT: vmovsh %xmm0, (%eax)1885; X86-NEXT: retl1886 %res = extractelement <8 x half> %x, i32 51887 store half %res, ptr %y1888 ret void1889}1890 1891define void @extract_store_f16_6(<8 x half> %x, ptr %y) {1892; X64-LABEL: extract_store_f16_6:1893; X64: # %bb.0:1894; X64-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]1895; X64-NEXT: vmovsh %xmm0, (%rdi)1896; X64-NEXT: retq1897;1898; X86-LABEL: extract_store_f16_6:1899; X86: # %bb.0:1900; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1901; X86-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]1902; X86-NEXT: vmovsh %xmm0, (%eax)1903; X86-NEXT: retl1904 %res = extractelement <8 x half> %x, i32 61905 store half %res, ptr %y1906 ret void1907}1908 1909define void @extract_store_f16_7(<8 x half> %x, ptr %y) {1910; X64-LABEL: extract_store_f16_7:1911; X64: # %bb.0:1912; X64-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1913; X64-NEXT: vmovsh %xmm0, (%rdi)1914; X64-NEXT: retq1915;1916; X86-LABEL: extract_store_f16_7:1917; X86: # %bb.0:1918; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1919; X86-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1920; X86-NEXT: vmovsh %xmm0, (%eax)1921; X86-NEXT: retl1922 %res = extractelement <8 x half> %x, i32 71923 store half %res, ptr %y1924 ret void1925}1926 1927define void @extract_store_i16_0(<8 x i16> %x, ptr %y) {1928; X64-LABEL: extract_store_i16_0:1929; X64: # %bb.0:1930; X64-NEXT: vpextrw $0, %xmm0, (%rdi)1931; X64-NEXT: retq1932;1933; X86-LABEL: extract_store_i16_0:1934; X86: # %bb.0:1935; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1936; X86-NEXT: vpextrw $0, %xmm0, (%eax)1937; X86-NEXT: retl1938 %res = extractelement <8 x i16> %x, i32 01939 store i16 %res, ptr %y1940 ret void1941}1942 1943define void @extract_store_i16_1(<8 x i16> %x, ptr %y) {1944; X64-LABEL: extract_store_i16_1:1945; X64: # %bb.0:1946; X64-NEXT: vpextrw $1, %xmm0, (%rdi)1947; X64-NEXT: retq1948;1949; X86-LABEL: extract_store_i16_1:1950; X86: # %bb.0:1951; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1952; X86-NEXT: vpextrw $1, %xmm0, (%eax)1953; X86-NEXT: retl1954 %res = extractelement <8 x i16> %x, i32 11955 store i16 %res, ptr %y1956 ret void1957}1958 1959define void @extract_store_i16_2(<8 x i16> %x, ptr %y) {1960; X64-LABEL: extract_store_i16_2:1961; X64: # %bb.0:1962; X64-NEXT: vpextrw $2, %xmm0, (%rdi)1963; X64-NEXT: retq1964;1965; X86-LABEL: extract_store_i16_2:1966; X86: # %bb.0:1967; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1968; X86-NEXT: vpextrw $2, %xmm0, (%eax)1969; X86-NEXT: retl1970 %res = extractelement <8 x i16> %x, i32 21971 store i16 %res, ptr %y1972 ret void1973}1974 1975define void @extract_store_i16_3(<8 x i16> %x, ptr %y) {1976; X64-LABEL: extract_store_i16_3:1977; X64: # %bb.0:1978; X64-NEXT: vpextrw $3, %xmm0, (%rdi)1979; X64-NEXT: retq1980;1981; X86-LABEL: extract_store_i16_3:1982; X86: # %bb.0:1983; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1984; X86-NEXT: vpextrw $3, %xmm0, (%eax)1985; X86-NEXT: retl1986 %res = extractelement <8 x i16> %x, i32 31987 store i16 %res, ptr %y1988 ret void1989}1990 1991define void @extract_store_i16_4(<8 x i16> %x, ptr %y) {1992; X64-LABEL: extract_store_i16_4:1993; X64: # %bb.0:1994; X64-NEXT: vpextrw $4, %xmm0, (%rdi)1995; X64-NEXT: retq1996;1997; X86-LABEL: extract_store_i16_4:1998; X86: # %bb.0:1999; X86-NEXT: movl {{[0-9]+}}(%esp), %eax2000; X86-NEXT: vpextrw $4, %xmm0, (%eax)2001; X86-NEXT: retl2002 %res = extractelement <8 x i16> %x, i32 42003 store i16 %res, ptr %y2004 ret void2005}2006 2007define void @extract_store_i16_5(<8 x i16> %x, ptr %y) {2008; X64-LABEL: extract_store_i16_5:2009; X64: # %bb.0:2010; X64-NEXT: vpextrw $5, %xmm0, (%rdi)2011; X64-NEXT: retq2012;2013; X86-LABEL: extract_store_i16_5:2014; X86: # %bb.0:2015; X86-NEXT: movl {{[0-9]+}}(%esp), %eax2016; X86-NEXT: vpextrw $5, %xmm0, (%eax)2017; X86-NEXT: retl2018 %res = extractelement <8 x i16> %x, i32 52019 store i16 %res, ptr %y2020 ret void2021}2022 2023define void @extract_store_i16_6(<8 x i16> %x, ptr %y) {2024; X64-LABEL: extract_store_i16_6:2025; X64: # %bb.0:2026; X64-NEXT: vpextrw $6, %xmm0, (%rdi)2027; X64-NEXT: retq2028;2029; X86-LABEL: extract_store_i16_6:2030; X86: # %bb.0:2031; X86-NEXT: movl {{[0-9]+}}(%esp), %eax2032; X86-NEXT: vpextrw $6, %xmm0, (%eax)2033; X86-NEXT: retl2034 %res = extractelement <8 x i16> %x, i32 62035 store i16 %res, ptr %y2036 ret void2037}2038 2039define void @extract_store_i16_7(<8 x i16> %x, ptr %y) {2040; X64-LABEL: extract_store_i16_7:2041; X64: # %bb.0:2042; X64-NEXT: vpextrw $7, %xmm0, (%rdi)2043; X64-NEXT: retq2044;2045; X86-LABEL: extract_store_i16_7:2046; X86: # %bb.0:2047; X86-NEXT: movl {{[0-9]+}}(%esp), %eax2048; X86-NEXT: vpextrw $7, %xmm0, (%eax)2049; X86-NEXT: retl2050 %res = extractelement <8 x i16> %x, i32 72051 store i16 %res, ptr %y2052 ret void2053}2054 2055define i32 @extract_zext_i16_0(<8 x i16> %x) {2056; CHECK-LABEL: extract_zext_i16_0:2057; CHECK: # %bb.0:2058; CHECK-NEXT: vpextrw $0, %xmm0, %eax2059; CHECK-NEXT: ret{{[l|q]}}2060 %res = extractelement <8 x i16> %x, i32 02061 %res2 = zext i16 %res to i322062 ret i32 %res22063}2064 2065define i32 @extract_zext_i16_1(<8 x i16> %x) {2066; CHECK-LABEL: extract_zext_i16_1:2067; CHECK: # %bb.0:2068; CHECK-NEXT: vpextrw $1, %xmm0, %eax2069; CHECK-NEXT: ret{{[l|q]}}2070 %res = extractelement <8 x i16> %x, i32 12071 %res2 = zext i16 %res to i322072 ret i32 %res22073}2074 2075define <8 x half> @build_vector_xxxxuuuu(half %a0, half %a1, half %a2, half %a3) {2076; X64-LABEL: build_vector_xxxxuuuu:2077; X64: # %bb.0:2078; X64-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2079; X64-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2080; X64-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],zero,zero2081; X64-NEXT: retq2082;2083; X86-LABEL: build_vector_xxxxuuuu:2084; X86: # %bb.0:2085; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero2086; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2087; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2088; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2089; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero2090; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2091; X86-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero2092; X86-NEXT: retl2093 %a = insertelement <8 x half> undef, half %a0, i32 02094 %b = insertelement <8 x half> %a, half %a1, i32 12095 %c = insertelement <8 x half> %b, half %a2, i32 22096 %d = insertelement <8 x half> %c, half %a3, i32 32097 ret <8 x half> %d2098}2099 2100define <8 x half> @build_vector_uuuuxxxx(half %a0, half %a1, half %a2, half %a3) {2101; X64-LABEL: build_vector_uuuuxxxx:2102; X64: # %bb.0:2103; X64-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2104; X64-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2105; X64-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]2106; X64-NEXT: vpbroadcastq %xmm0, %xmm02107; X64-NEXT: retq2108;2109; X86-LABEL: build_vector_uuuuxxxx:2110; X86: # %bb.0:2111; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero2112; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2113; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2114; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2115; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero2116; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2117; X86-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]2118; X86-NEXT: vpbroadcastq %xmm0, %xmm02119; X86-NEXT: retl2120 %a = insertelement <8 x half> undef, half %a0, i32 42121 %b = insertelement <8 x half> %a, half %a1, i32 52122 %c = insertelement <8 x half> %b, half %a2, i32 62123 %d = insertelement <8 x half> %c, half %a3, i32 72124 ret <8 x half> %d2125}2126 2127define <8 x half> @build_vector_xxxxxxxx(half %a0, half %a1, half %a2, half %a3, half %a4, half %a5, half %a6, half %a7) {2128; X64-LABEL: build_vector_xxxxxxxx:2129; X64: # %bb.0:2130; X64-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]2131; X64-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]2132; X64-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]2133; X64-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2134; X64-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2135; X64-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]2136; X64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]2137; X64-NEXT: retq2138;2139; X86-LABEL: build_vector_xxxxxxxx:2140; X86: # %bb.0:2141; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero2142; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2143; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2144; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2145; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero2146; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2147; X86-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]2148; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2149; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero2150; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2151; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero2152; X86-NEXT: vmovsh {{.*#+}} xmm3 = mem[0],zero,zero,zero,zero,zero,zero,zero2153; X86-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2154; X86-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]2155; X86-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]2156; X86-NEXT: retl2157 %a = insertelement <8 x half> undef, half %a0, i32 02158 %b = insertelement <8 x half> %a, half %a1, i32 12159 %c = insertelement <8 x half> %b, half %a2, i32 22160 %d = insertelement <8 x half> %c, half %a3, i32 32161 %e = insertelement <8 x half> %d, half %a4, i32 42162 %f = insertelement <8 x half> %e, half %a5, i32 52163 %g = insertelement <8 x half> %f, half %a6, i32 62164 %h = insertelement <8 x half> %g, half %a7, i32 72165 ret <8 x half> %h2166}2167 2168define <16 x half> @build_vector_xxxxuuuuuuuuxxxx(half %a0, half %a1, half %a2, half %a3, half %a4, half %a5, half %a6, half %a7) {2169; X64-LABEL: build_vector_xxxxuuuuuuuuxxxx:2170; X64: # %bb.0:2171; X64-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2172; X64-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2173; X64-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],zero,zero2174; X64-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]2175; X64-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]2176; X64-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2177; X64-NEXT: vpbroadcastq %xmm1, %xmm12178; X64-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm02179; X64-NEXT: retq2180;2181; X86-LABEL: build_vector_xxxxuuuuuuuuxxxx:2182; X86: # %bb.0:2183; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero2184; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2185; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2186; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2187; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero2188; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2189; X86-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]2190; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2191; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero2192; X86-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2193; X86-NEXT: vmovsh {{.*#+}} xmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero2194; X86-NEXT: vmovsh {{.*#+}} xmm3 = mem[0],zero,zero,zero,zero,zero,zero,zero2195; X86-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2196; X86-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero2197; X86-NEXT: vpbroadcastq %xmm0, %xmm02198; X86-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm02199; X86-NEXT: retl2200 %a = insertelement <16 x half> undef, half %a0, i32 02201 %b = insertelement <16 x half> %a, half %a1, i32 12202 %c = insertelement <16 x half> %b, half %a2, i32 22203 %d = insertelement <16 x half> %c, half %a3, i32 32204 %e = insertelement <16 x half> %d, half %a4, i32 122205 %f = insertelement <16 x half> %e, half %a5, i32 132206 %g = insertelement <16 x half> %f, half %a6, i32 142207 %h = insertelement <16 x half> %g, half %a7, i32 152208 ret <16 x half> %h2209}2210 2211define <8 x half> @regression1(<8 x half> %a, <8 x half> %b) {2212; CHECK-LABEL: regression1:2213; CHECK: # %bb.0:2214; CHECK-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,14,15,0,1,2,3,4,5,6,7,14,15,10,11]2215; CHECK-NEXT: ret{{[l|q]}}2216 %res = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 0, i32 7, i32 0, i32 1, i32 2, i32 3, i32 7, i32 5>2217 ret <8 x half> %res2218}2219 2220define <4 x float> @regression2(ptr addrspace(1) %0, <4 x i32> %1, <4 x i32> %2, <4 x float> %3, ptr %4) {2221; X64VL-LABEL: regression2:2222; X64VL: # %bb.0:2223; X64VL-NEXT: vmovw (%rsi), %xmm02224; X64VL-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2225; X64VL-NEXT: vcvtdq2ps %xmm0, %xmm02226; X64VL-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]2227; X64VL-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm02228; X64VL-NEXT: retq2229;2230; X86-LABEL: regression2:2231; X86: # %bb.0:2232; X86-NEXT: movl {{[0-9]+}}(%esp), %eax2233; X86-NEXT: vmovw (%eax), %xmm02234; X86-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2235; X86-NEXT: vcvtdq2ps %xmm0, %xmm02236; X86-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]2237; X86-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm02238; X86-NEXT: retl2239;2240; X64-NOVL-LABEL: regression2:2241; X64-NOVL: # %bb.0:2242; X64-NOVL-NEXT: vmovw (%rsi), %xmm02243; X64-NOVL-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2244; X64-NOVL-NEXT: vcvtdq2ps %xmm0, %xmm02245; X64-NOVL-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]2246; X64-NOVL-NEXT: vbroadcastss {{.*#+}} xmm1 = [3.92156886E-3,3.92156886E-3,3.92156886E-3,3.92156886E-3]2247; X64-NOVL-NEXT: vmulps %xmm1, %xmm0, %xmm02248; X64-NOVL-NEXT: retq2249 %6 = load i8, ptr %4, align 12250 %7 = getelementptr i8, ptr %4, i64 12251 %8 = addrspacecast ptr %7 to ptr addrspace(4)2252 %9 = load i8, ptr addrspace(4) %8, align 12253 %10 = insertelement <2 x i8> poison, i8 %6, i32 02254 %11 = insertelement <2 x i8> %10, i8 %9, i32 12255 %12 = uitofp <2 x i8> %11 to <2 x float>2256 %13 = shufflevector <2 x float> %12, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>2257 %14 = shufflevector <4 x float> %13, <4 x float> <float poison, float poison, float 0.000000e+00, float 2.550000e+02>, <4 x i32> <i32 0, i32 1, i32 6, i32 7>2258 %15 = fmul contract <4 x float> %14, <float 0x3F70101020000000, float 0x3F70101020000000, float 0x3F70101020000000, float 0x3F70101020000000>2259 ret <4 x float> %152260}2261 2262; Make sure load/stores of v4f16 are handled well on 32-bit targets where2263; default widening legalization can't use i64.2264define void @load_store_v4f16(ptr %x, ptr %y, ptr %z) {2265; X64VL-LABEL: load_store_v4f16:2266; X64VL: # %bb.0:2267; X64VL-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero2268; X64VL-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero2269; X64VL-NEXT: vaddph %xmm1, %xmm0, %xmm02270; X64VL-NEXT: vmovlps %xmm0, (%rdx)2271; X64VL-NEXT: retq2272;2273; X86-LABEL: load_store_v4f16:2274; X86: # %bb.0:2275; X86-NEXT: movl {{[0-9]+}}(%esp), %eax2276; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx2277; X86-NEXT: movl {{[0-9]+}}(%esp), %edx2278; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero2279; X86-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero2280; X86-NEXT: vaddph %xmm1, %xmm0, %xmm02281; X86-NEXT: vmovlps %xmm0, (%eax)2282; X86-NEXT: retl2283;2284; X64-NOVL-LABEL: load_store_v4f16:2285; X64-NOVL: # %bb.0:2286; X64-NOVL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero2287; X64-NOVL-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero2288; X64-NOVL-NEXT: vpsrlq $48, %xmm1, %xmm22289; X64-NOVL-NEXT: vpsrlq $48, %xmm0, %xmm32290; X64-NOVL-NEXT: vaddsh %xmm2, %xmm3, %xmm22291; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]2292; X64-NOVL-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]2293; X64-NOVL-NEXT: vaddsh %xmm3, %xmm4, %xmm32294; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]2295; X64-NOVL-NEXT: vaddsh %xmm1, %xmm0, %xmm32296; X64-NOVL-NEXT: vpsrld $16, %xmm1, %xmm12297; X64-NOVL-NEXT: vpsrld $16, %xmm0, %xmm02298; X64-NOVL-NEXT: vaddsh %xmm1, %xmm0, %xmm02299; X64-NOVL-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]2300; X64-NOVL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]2301; X64-NOVL-NEXT: vmovq %xmm0, (%rdx)2302; X64-NOVL-NEXT: retq2303 %a = load <4 x half>, ptr %x2304 %b = load <4 x half>, ptr %y2305 %c = fadd <4 x half> %a, %b2306 store <4 x half> %c, ptr %z2307 ret void2308}2309 2310define <8 x half> @test21(half %a, half %b, half %c) nounwind {2311; X64-LABEL: test21:2312; X64: # %bb.0:2313; X64-NEXT: vxorps %xmm3, %xmm3, %xmm32314; X64-NEXT: vmovsh {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3,4,5,6,7]2315; X64-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2316; X64-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm2[0],zero,zero2317; X64-NEXT: retq2318;2319; X86-LABEL: test21:2320; X86: # %bb.0:2321; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero2322; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2323; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2324; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero2325; X86-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero2326; X86-NEXT: retl2327 %1 = insertelement <8 x half> <half poison, half poison, half poison, half 0xH0000, half 0xH0000, half 0xH0000, half 0xH0000, half 0xH0000>, half %a, i32 02328 %2 = insertelement <8 x half> %1, half %b, i32 12329 %3 = insertelement <8 x half> %2, half %c, i32 22330 ret <8 x half> %32331}2332 2333define <16 x i16> @test22(ptr %mem) nounwind {2334; X64-LABEL: test22:2335; X64: # %bb.0:2336; X64-NEXT: movzwl 0, %eax2337; X64-NEXT: andw (%rdi), %ax2338; X64-NEXT: vmovw %eax, %xmm02339; X64-NEXT: retq2340;2341; X86-LABEL: test22:2342; X86: # %bb.0:2343; X86-NEXT: movl {{[0-9]+}}(%esp), %eax2344; X86-NEXT: movzwl 0, %ecx2345; X86-NEXT: andw (%eax), %cx2346; X86-NEXT: vmovw %ecx, %xmm02347; X86-NEXT: retl2348 %1 = load i16, ptr null, align 22349 %2 = load i16, ptr %mem, align 22350 %3 = and i16 %1, %22351 %4 = insertelement <16 x i16> <i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, i16 %3, i32 02352 ret <16 x i16> %42353}2354 2355define void @pr52560(i8 %0, <2 x i16> %1, ptr %c) nounwind {2356; X64VL-LABEL: pr52560:2357; X64VL: # %bb.0: # %entry2358; X64VL-NEXT: movsbl %dil, %eax2359; X64VL-NEXT: vmovw %eax, %xmm12360; X64VL-NEXT: vpxor %xmm2, %xmm2, %xmm22361; X64VL-NEXT: vpcmpgtw %xmm2, %xmm1, %k12362; X64VL-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}2363; X64VL-NEXT: vmovw %xmm0, %eax2364; X64VL-NEXT: testw %ax, %ax2365; X64VL-NEXT: je .LBB123_22366; X64VL-NEXT: # %bb.1: # %for.body.preheader2367; X64VL-NEXT: movb $0, (%rsi)2368; X64VL-NEXT: .LBB123_2: # %for.end2369; X64VL-NEXT: retq2370;2371; X86-LABEL: pr52560:2372; X86: # %bb.0: # %entry2373; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax2374; X86-NEXT: vmovw %eax, %xmm12375; X86-NEXT: vpxor %xmm2, %xmm2, %xmm22376; X86-NEXT: vpcmpgtw %xmm2, %xmm1, %k12377; X86-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}2378; X86-NEXT: vmovw %xmm0, %eax2379; X86-NEXT: testw %ax, %ax2380; X86-NEXT: je .LBB123_22381; X86-NEXT: # %bb.1: # %for.body.preheader2382; X86-NEXT: movl {{[0-9]+}}(%esp), %eax2383; X86-NEXT: movb $0, (%eax)2384; X86-NEXT: .LBB123_2: # %for.end2385; X86-NEXT: retl2386;2387; X64-NOVL-LABEL: pr52560:2388; X64-NOVL: # %bb.0: # %entry2389; X64-NOVL-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm02390; X64-NOVL-NEXT: movsbl %dil, %eax2391; X64-NOVL-NEXT: vmovw %eax, %xmm12392; X64-NOVL-NEXT: vpxor %xmm2, %xmm2, %xmm22393; X64-NOVL-NEXT: vpcmpgtw %zmm2, %zmm1, %k12394; X64-NOVL-NEXT: vmovdqu16 %zmm0, %zmm0 {%k1} {z}2395; X64-NOVL-NEXT: vmovw %xmm0, %eax2396; X64-NOVL-NEXT: testw %ax, %ax2397; X64-NOVL-NEXT: je .LBB123_22398; X64-NOVL-NEXT: # %bb.1: # %for.body.preheader2399; X64-NOVL-NEXT: movb $0, (%rsi)2400; X64-NOVL-NEXT: .LBB123_2: # %for.end2401; X64-NOVL-NEXT: vzeroupper2402; X64-NOVL-NEXT: retq2403entry:2404 %conv = sext i8 %0 to i162405 %2 = insertelement <2 x i16> <i16 poison, i16 0>, i16 %conv, i32 02406 %3 = icmp sgt <2 x i16> %2, zeroinitializer2407 %4 = select <2 x i1> %3, <2 x i16> %1, <2 x i16> <i16 0, i16 poison>2408 %5 = extractelement <2 x i16> %4, i32 02409 %tobool.not14 = icmp eq i16 %5, 02410 br i1 %tobool.not14, label %for.end, label %for.body.preheader2411 2412for.body.preheader: ; preds = %entry2413 store i8 0, ptr %c, align 12414 br label %for.end2415 2416for.end: ; preds = %for.body.preheader, %entry2417 ret void2418}2419 2420define <16 x i32> @pr52561(<16 x i32> %a, <16 x i32> %b) "min-legal-vector-width"="256" "prefer-vector-width"="256" nounwind {2421; X64VL-LABEL: pr52561:2422; X64VL: # %bb.0:2423; X64VL-NEXT: vpaddd %ymm3, %ymm1, %ymm12424; X64VL-NEXT: vpaddd %ymm2, %ymm0, %ymm02425; X64VL-NEXT: vpbroadcastd {{.*#+}} ymm2 = [112,112,112,112,112,112,112,112]2426; X64VL-NEXT: vpaddd %ymm2, %ymm0, %ymm02427; X64VL-NEXT: vpaddd %ymm2, %ymm1, %ymm12428; X64VL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm12429; X64VL-NEXT: vpxor %xmm2, %xmm2, %xmm22430; X64VL-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]2431; X64VL-NEXT: retq2432;2433; X86-LABEL: pr52561:2434; X86: # %bb.0:2435; X86-NEXT: pushl %ebp2436; X86-NEXT: movl %esp, %ebp2437; X86-NEXT: andl $-32, %esp2438; X86-NEXT: subl $32, %esp2439; X86-NEXT: vpaddd %ymm2, %ymm0, %ymm02440; X86-NEXT: vpaddd 8(%ebp), %ymm1, %ymm12441; X86-NEXT: vpbroadcastd {{.*#+}} ymm2 = [112,112,112,112,112,112,112,112]2442; X86-NEXT: vpaddd %ymm2, %ymm0, %ymm02443; X86-NEXT: vpaddd %ymm2, %ymm1, %ymm12444; X86-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm1, %ymm12445; X86-NEXT: vpxor %xmm2, %xmm2, %xmm22446; X86-NEXT: vmovsh {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]2447; X86-NEXT: movl %ebp, %esp2448; X86-NEXT: popl %ebp2449; X86-NEXT: retl2450;2451; X64-NOVL-LABEL: pr52561:2452; X64-NOVL: # %bb.0:2453; X64-NOVL-NEXT: vpaddd %zmm1, %zmm0, %zmm02454; X64-NOVL-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm02455; X64-NOVL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm02456; X64-NOVL-NEXT: retq2457 %1 = add <16 x i32> %a, <i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>2458 %2 = add <16 x i32> %1, %b2459 %3 = and <16 x i32> %2, <i32 65535, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 65535>2460 ret <16 x i32> %32461}2462 2463define <8 x i16> @pr59628_xmm(i16 %arg) {2464; X64VL-LABEL: pr59628_xmm:2465; X64VL: # %bb.0:2466; X64VL-NEXT: vmovw %edi, %xmm02467; X64VL-NEXT: vpbroadcastw %edi, %xmm12468; X64VL-NEXT: vpcmpneqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %k12469; X64VL-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}2470; X64VL-NEXT: retq2471;2472; X86-LABEL: pr59628_xmm:2473; X86: # %bb.0:2474; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax2475; X86-NEXT: vxorps %xmm0, %xmm0, %xmm02476; X86-NEXT: vpbroadcastw %eax, %xmm12477; X86-NEXT: vmovsh {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3,4,5,6,7]2478; X86-NEXT: vpcmpneqw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %k12479; X86-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}2480; X86-NEXT: retl2481;2482; X64-NOVL-LABEL: pr59628_xmm:2483; X64-NOVL: # %bb.0:2484; X64-NOVL-NEXT: vmovw %edi, %xmm02485; X64-NOVL-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm12486; X64-NOVL-NEXT: vpandn %xmm0, %xmm1, %xmm02487; X64-NOVL-NEXT: retq2488 %I1 = insertelement <8 x i16> zeroinitializer, i16 %arg, i16 02489 %I2 = insertelement <8 x i16> %I1, i16 0, i16 %arg2490 ret <8 x i16> %I22491}2492