1284 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6target triple = "aarch64-unknown-linux-gnu"7 8;9; LD1B10;11 12define void @masked_gather_v2i8(ptr %a, ptr %b) vscale_range(2,0) #0 {13; CHECK-LABEL: masked_gather_v2i8:14; CHECK: // %bb.0:15; CHECK-NEXT: ldr h0, [x0]16; CHECK-NEXT: ptrue p0.d, vl217; CHECK-NEXT: ushll v0.8h, v0.8b, #018; CHECK-NEXT: ushll v0.4s, v0.4h, #019; CHECK-NEXT: cmeq v0.2s, v0.2s, #020; CHECK-NEXT: sshll v0.2d, v0.2s, #021; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #022; CHECK-NEXT: ldr q0, [x1]23; CHECK-NEXT: ld1b { z0.d }, p0/z, [z0.d]24; CHECK-NEXT: ptrue p0.s, vl225; CHECK-NEXT: xtn v0.2s, v0.2d26; CHECK-NEXT: st1b { z0.s }, p0, [x0]27; CHECK-NEXT: ret28 %cval = load <2 x i8>, ptr %a29 %ptrs = load <2 x ptr>, ptr %b30 %mask = icmp eq <2 x i8> %cval, zeroinitializer31 %vals = call <2 x i8> @llvm.masked.gather.v2i8(<2 x ptr> %ptrs, i32 8, <2 x i1> %mask, <2 x i8> poison)32 store <2 x i8> %vals, ptr %a33 ret void34}35 36define void @masked_gather_v4i8(ptr %a, ptr %b) vscale_range(2,0) #0 {37; CHECK-LABEL: masked_gather_v4i8:38; CHECK: // %bb.0:39; CHECK-NEXT: ldr s0, [x0]40; CHECK-NEXT: ptrue p0.d, vl441; CHECK-NEXT: ushll v0.8h, v0.8b, #042; CHECK-NEXT: cmeq v0.4h, v0.4h, #043; CHECK-NEXT: sunpklo z0.s, z0.h44; CHECK-NEXT: sunpklo z0.d, z0.s45; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #046; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]47; CHECK-NEXT: ld1b { z0.d }, p1/z, [z0.d]48; CHECK-NEXT: st1b { z0.d }, p0, [x0]49; CHECK-NEXT: ret50 %cval = load <4 x i8>, ptr %a51 %ptrs = load <4 x ptr>, ptr %b52 %mask = icmp eq <4 x i8> %cval, zeroinitializer53 %vals = call <4 x i8> @llvm.masked.gather.v4i8(<4 x ptr> %ptrs, i32 8, <4 x i1> %mask, <4 x i8> poison)54 store <4 x i8> %vals, ptr %a55 ret void56}57 58define void @masked_gather_v8i8(ptr %a, ptr %b) #0 {59; VBITS_GE_256-LABEL: masked_gather_v8i8:60; VBITS_GE_256: // %bb.0:61; VBITS_GE_256-NEXT: ldr d0, [x0]62; VBITS_GE_256-NEXT: ptrue p0.d, vl463; VBITS_GE_256-NEXT: mov x8, #4 // =0x464; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #065; VBITS_GE_256-NEXT: zip2 v1.8b, v0.8b, v0.8b66; VBITS_GE_256-NEXT: zip1 v0.8b, v0.8b, v0.8b67; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #868; VBITS_GE_256-NEXT: shl v0.4h, v0.4h, #869; VBITS_GE_256-NEXT: sshr v1.4h, v1.4h, #870; VBITS_GE_256-NEXT: sshr v0.4h, v0.4h, #871; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h72; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h73; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s74; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s75; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #076; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]77; VBITS_GE_256-NEXT: ld1b { z1.d }, p1/z, [z1.d]78; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #079; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1]80; VBITS_GE_256-NEXT: ld1b { z0.d }, p1/z, [z0.d]81; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s82; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h83; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s84; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h85; VBITS_GE_256-NEXT: uzp1 v0.8b, v0.8b, v1.8b86; VBITS_GE_256-NEXT: str d0, [x0]87; VBITS_GE_256-NEXT: ret88;89; VBITS_GE_512-LABEL: masked_gather_v8i8:90; VBITS_GE_512: // %bb.0:91; VBITS_GE_512-NEXT: ldr d0, [x0]92; VBITS_GE_512-NEXT: ptrue p0.d, vl893; VBITS_GE_512-NEXT: cmeq v0.8b, v0.8b, #094; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b95; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h96; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s97; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #098; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]99; VBITS_GE_512-NEXT: ld1b { z0.d }, p1/z, [z0.d]100; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s101; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h102; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b103; VBITS_GE_512-NEXT: str d0, [x0]104; VBITS_GE_512-NEXT: ret105 %cval = load <8 x i8>, ptr %a106 %ptrs = load <8 x ptr>, ptr %b107 %mask = icmp eq <8 x i8> %cval, zeroinitializer108 %vals = call <8 x i8> @llvm.masked.gather.v8i8(<8 x ptr> %ptrs, i32 8, <8 x i1> %mask, <8 x i8> poison)109 store <8 x i8> %vals, ptr %a110 ret void111}112 113define void @masked_gather_v16i8(ptr %a, ptr %b) vscale_range(8,0) #0 {114; CHECK-LABEL: masked_gather_v16i8:115; CHECK: // %bb.0:116; CHECK-NEXT: ldr q0, [x0]117; CHECK-NEXT: ptrue p0.d, vl16118; CHECK-NEXT: cmeq v0.16b, v0.16b, #0119; CHECK-NEXT: sunpklo z0.h, z0.b120; CHECK-NEXT: sunpklo z0.s, z0.h121; CHECK-NEXT: sunpklo z0.d, z0.s122; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0123; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]124; CHECK-NEXT: ld1b { z0.d }, p1/z, [z0.d]125; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s126; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h127; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b128; CHECK-NEXT: str q0, [x0]129; CHECK-NEXT: ret130 %cval = load <16 x i8>, ptr %a131 %ptrs = load <16 x ptr>, ptr %b132 %mask = icmp eq <16 x i8> %cval, zeroinitializer133 %vals = call <16 x i8> @llvm.masked.gather.v16i8(<16 x ptr> %ptrs, i32 8, <16 x i1> %mask, <16 x i8> poison)134 store <16 x i8> %vals, ptr %a135 ret void136}137 138define void @masked_gather_v32i8(ptr %a, ptr %b) vscale_range(16,0) #0 {139; CHECK-LABEL: masked_gather_v32i8:140; CHECK: // %bb.0:141; CHECK-NEXT: ptrue p0.b, vl32142; CHECK-NEXT: ptrue p1.d, vl32143; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]144; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0145; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]146; CHECK-NEXT: punpklo p0.h, p0.b147; CHECK-NEXT: punpklo p0.h, p0.b148; CHECK-NEXT: punpklo p0.h, p0.b149; CHECK-NEXT: ld1b { z0.d }, p0/z, [z0.d]150; CHECK-NEXT: st1b { z0.d }, p1, [x0]151; CHECK-NEXT: ret152 %cval = load <32 x i8>, ptr %a153 %ptrs = load <32 x ptr>, ptr %b154 %mask = icmp eq <32 x i8> %cval, zeroinitializer155 %vals = call <32 x i8> @llvm.masked.gather.v32i8(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x i8> poison)156 store <32 x i8> %vals, ptr %a157 ret void158}159 160;161; LD1H162;163 164define void @masked_gather_v2i16(ptr %a, ptr %b) vscale_range(2,0) #0 {165; CHECK-LABEL: masked_gather_v2i16:166; CHECK: // %bb.0:167; CHECK-NEXT: ldr s0, [x0]168; CHECK-NEXT: ptrue p0.d, vl2169; CHECK-NEXT: ushll v0.4s, v0.4h, #0170; CHECK-NEXT: cmeq v0.2s, v0.2s, #0171; CHECK-NEXT: sshll v0.2d, v0.2s, #0172; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0173; CHECK-NEXT: ldr q0, [x1]174; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]175; CHECK-NEXT: ptrue p0.s, vl2176; CHECK-NEXT: xtn v0.2s, v0.2d177; CHECK-NEXT: st1h { z0.s }, p0, [x0]178; CHECK-NEXT: ret179 %cval = load <2 x i16>, ptr %a180 %ptrs = load <2 x ptr>, ptr %b181 %mask = icmp eq <2 x i16> %cval, zeroinitializer182 %vals = call <2 x i16> @llvm.masked.gather.v2i16(<2 x ptr> %ptrs, i32 8, <2 x i1> %mask, <2 x i16> poison)183 store <2 x i16> %vals, ptr %a184 ret void185}186 187define void @masked_gather_v4i16(ptr %a, ptr %b) vscale_range(2,0) #0 {188; CHECK-LABEL: masked_gather_v4i16:189; CHECK: // %bb.0:190; CHECK-NEXT: ldr d0, [x0]191; CHECK-NEXT: ptrue p0.d, vl4192; CHECK-NEXT: cmeq v0.4h, v0.4h, #0193; CHECK-NEXT: sunpklo z0.s, z0.h194; CHECK-NEXT: sunpklo z0.d, z0.s195; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0196; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]197; CHECK-NEXT: ld1h { z0.d }, p1/z, [z0.d]198; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s199; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h200; CHECK-NEXT: str d0, [x0]201; CHECK-NEXT: ret202 %cval = load <4 x i16>, ptr %a203 %ptrs = load <4 x ptr>, ptr %b204 %mask = icmp eq <4 x i16> %cval, zeroinitializer205 %vals = call <4 x i16> @llvm.masked.gather.v4i16(<4 x ptr> %ptrs, i32 8, <4 x i1> %mask, <4 x i16> poison)206 store <4 x i16> %vals, ptr %a207 ret void208}209 210define void @masked_gather_v8i16(ptr %a, ptr %b) #0 {211; VBITS_GE_256-LABEL: masked_gather_v8i16:212; VBITS_GE_256: // %bb.0:213; VBITS_GE_256-NEXT: ldr q0, [x0]214; VBITS_GE_256-NEXT: ptrue p0.d, vl4215; VBITS_GE_256-NEXT: mov x8, #4 // =0x4216; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0217; VBITS_GE_256-NEXT: sunpklo z1.s, z0.h218; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8219; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s220; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h221; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0222; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s223; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]224; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z1.d]225; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0226; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]227; VBITS_GE_256-NEXT: ld1h { z0.d }, p1/z, [z0.d]228; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s229; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h230; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s231; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h232; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]233; VBITS_GE_256-NEXT: str q1, [x0]234; VBITS_GE_256-NEXT: ret235;236; VBITS_GE_512-LABEL: masked_gather_v8i16:237; VBITS_GE_512: // %bb.0:238; VBITS_GE_512-NEXT: ldr q0, [x0]239; VBITS_GE_512-NEXT: ptrue p0.d, vl8240; VBITS_GE_512-NEXT: cmeq v0.8h, v0.8h, #0241; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h242; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s243; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0244; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]245; VBITS_GE_512-NEXT: ld1h { z0.d }, p1/z, [z0.d]246; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s247; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h248; VBITS_GE_512-NEXT: str q0, [x0]249; VBITS_GE_512-NEXT: ret250 %cval = load <8 x i16>, ptr %a251 %ptrs = load <8 x ptr>, ptr %b252 %mask = icmp eq <8 x i16> %cval, zeroinitializer253 %vals = call <8 x i16> @llvm.masked.gather.v8i16(<8 x ptr> %ptrs, i32 8, <8 x i1> %mask, <8 x i16> poison)254 store <8 x i16> %vals, ptr %a255 ret void256}257 258define void @masked_gather_v16i16(ptr %a, ptr %b) vscale_range(8,0) #0 {259; CHECK-LABEL: masked_gather_v16i16:260; CHECK: // %bb.0:261; CHECK-NEXT: ptrue p0.h, vl16262; CHECK-NEXT: ptrue p1.d, vl16263; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]264; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0265; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]266; CHECK-NEXT: punpklo p0.h, p0.b267; CHECK-NEXT: punpklo p0.h, p0.b268; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]269; CHECK-NEXT: st1h { z0.d }, p1, [x0]270; CHECK-NEXT: ret271 %cval = load <16 x i16>, ptr %a272 %ptrs = load <16 x ptr>, ptr %b273 %mask = icmp eq <16 x i16> %cval, zeroinitializer274 %vals = call <16 x i16> @llvm.masked.gather.v16i16(<16 x ptr> %ptrs, i32 8, <16 x i1> %mask, <16 x i16> poison)275 store <16 x i16> %vals, ptr %a276 ret void277}278 279define void @masked_gather_v32i16(ptr %a, ptr %b) vscale_range(16,0) #0 {280; CHECK-LABEL: masked_gather_v32i16:281; CHECK: // %bb.0:282; CHECK-NEXT: ptrue p0.h, vl32283; CHECK-NEXT: ptrue p1.d, vl32284; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]285; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0286; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]287; CHECK-NEXT: punpklo p0.h, p0.b288; CHECK-NEXT: punpklo p0.h, p0.b289; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]290; CHECK-NEXT: st1h { z0.d }, p1, [x0]291; CHECK-NEXT: ret292 %cval = load <32 x i16>, ptr %a293 %ptrs = load <32 x ptr>, ptr %b294 %mask = icmp eq <32 x i16> %cval, zeroinitializer295 %vals = call <32 x i16> @llvm.masked.gather.v32i16(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x i16> poison)296 store <32 x i16> %vals, ptr %a297 ret void298}299 300;301; LD1W302;303 304define void @masked_gather_v2i32(ptr %a, ptr %b) vscale_range(2,0) #0 {305; CHECK-LABEL: masked_gather_v2i32:306; CHECK: // %bb.0:307; CHECK-NEXT: ldr d0, [x0]308; CHECK-NEXT: ptrue p0.d, vl2309; CHECK-NEXT: ushll v0.2d, v0.2s, #0310; CHECK-NEXT: cmeq v0.2d, v0.2d, #0311; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0312; CHECK-NEXT: ldr q0, [x1]313; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]314; CHECK-NEXT: xtn v0.2s, v0.2d315; CHECK-NEXT: str d0, [x0]316; CHECK-NEXT: ret317 %cval = load <2 x i32>, ptr %a318 %ptrs = load <2 x ptr>, ptr %b319 %mask = icmp eq <2 x i32> %cval, zeroinitializer320 %vals = call <2 x i32> @llvm.masked.gather.v2i32(<2 x ptr> %ptrs, i32 8, <2 x i1> %mask, <2 x i32> poison)321 store <2 x i32> %vals, ptr %a322 ret void323}324 325define void @masked_gather_v4i32(ptr %a, ptr %b) vscale_range(2,0) #0 {326; CHECK-LABEL: masked_gather_v4i32:327; CHECK: // %bb.0:328; CHECK-NEXT: ldr q0, [x0]329; CHECK-NEXT: ptrue p0.d, vl4330; CHECK-NEXT: cmeq v0.4s, v0.4s, #0331; CHECK-NEXT: sunpklo z0.d, z0.s332; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0333; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]334; CHECK-NEXT: ld1w { z0.d }, p1/z, [z0.d]335; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s336; CHECK-NEXT: str q0, [x0]337; CHECK-NEXT: ret338 %cval = load <4 x i32>, ptr %a339 %ptrs = load <4 x ptr>, ptr %b340 %mask = icmp eq <4 x i32> %cval, zeroinitializer341 %vals = call <4 x i32> @llvm.masked.gather.v4i32(<4 x ptr> %ptrs, i32 8, <4 x i1> %mask, <4 x i32> poison)342 store <4 x i32> %vals, ptr %a343 ret void344}345 346define void @masked_gather_v8i32(ptr %a, ptr %b) #0 {347; VBITS_GE_256-LABEL: masked_gather_v8i32:348; VBITS_GE_256: // %bb.0:349; VBITS_GE_256-NEXT: ptrue p0.s, vl8350; VBITS_GE_256-NEXT: mov x8, #4 // =0x4351; VBITS_GE_256-NEXT: ptrue p2.d, vl4352; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0]353; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1]354; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #0355; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff356; VBITS_GE_256-NEXT: punpklo p1.h, p1.b357; VBITS_GE_256-NEXT: and p1.b, p1/z, p1.b, p2.b358; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16359; VBITS_GE_256-NEXT: ld1w { z1.d }, p1/z, [z1.d]360; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s361; VBITS_GE_256-NEXT: cmpne p1.d, p2/z, z0.d, #0362; VBITS_GE_256-NEXT: ld1d { z0.d }, p2/z, [x1, x8, lsl #3]363; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s364; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z0.d]365; VBITS_GE_256-NEXT: ptrue p1.s, vl4366; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s367; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s368; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]369; VBITS_GE_256-NEXT: ret370;371; VBITS_GE_512-LABEL: masked_gather_v8i32:372; VBITS_GE_512: // %bb.0:373; VBITS_GE_512-NEXT: ptrue p0.s, vl8374; VBITS_GE_512-NEXT: ptrue p1.d, vl8375; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]376; VBITS_GE_512-NEXT: cmpeq p0.s, p0/z, z0.s, #0377; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x1]378; VBITS_GE_512-NEXT: punpklo p0.h, p0.b379; VBITS_GE_512-NEXT: ld1w { z0.d }, p0/z, [z0.d]380; VBITS_GE_512-NEXT: st1w { z0.d }, p1, [x0]381; VBITS_GE_512-NEXT: ret382 %cval = load <8 x i32>, ptr %a383 %ptrs = load <8 x ptr>, ptr %b384 %mask = icmp eq <8 x i32> %cval, zeroinitializer385 %vals = call <8 x i32> @llvm.masked.gather.v8i32(<8 x ptr> %ptrs, i32 8, <8 x i1> %mask, <8 x i32> poison)386 store <8 x i32> %vals, ptr %a387 ret void388}389 390define void @masked_gather_v16i32(ptr %a, ptr %b) vscale_range(8,0) #0 {391; CHECK-LABEL: masked_gather_v16i32:392; CHECK: // %bb.0:393; CHECK-NEXT: ptrue p0.s, vl16394; CHECK-NEXT: ptrue p1.d, vl16395; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]396; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0397; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]398; CHECK-NEXT: punpklo p0.h, p0.b399; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]400; CHECK-NEXT: st1w { z0.d }, p1, [x0]401; CHECK-NEXT: ret402 %cval = load <16 x i32>, ptr %a403 %ptrs = load <16 x ptr>, ptr %b404 %mask = icmp eq <16 x i32> %cval, zeroinitializer405 %vals = call <16 x i32> @llvm.masked.gather.v16i32(<16 x ptr> %ptrs, i32 8, <16 x i1> %mask, <16 x i32> poison)406 store <16 x i32> %vals, ptr %a407 ret void408}409 410define void @masked_gather_v32i32(ptr %a, ptr %b) vscale_range(16,0) #0 {411; CHECK-LABEL: masked_gather_v32i32:412; CHECK: // %bb.0:413; CHECK-NEXT: ptrue p0.s, vl32414; CHECK-NEXT: ptrue p1.d, vl32415; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]416; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0417; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]418; CHECK-NEXT: punpklo p0.h, p0.b419; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]420; CHECK-NEXT: st1w { z0.d }, p1, [x0]421; CHECK-NEXT: ret422 %cval = load <32 x i32>, ptr %a423 %ptrs = load <32 x ptr>, ptr %b424 %mask = icmp eq <32 x i32> %cval, zeroinitializer425 %vals = call <32 x i32> @llvm.masked.gather.v32i32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x i32> poison)426 store <32 x i32> %vals, ptr %a427 ret void428}429 430;431; LD1D432;433 434; Scalarize 1 x i64 gathers435define void @masked_gather_v1i64(ptr %a, ptr %b) vscale_range(2,0) #0 {436; CHECK-LABEL: masked_gather_v1i64:437; CHECK: // %bb.0:438; CHECK-NEXT: ldr x8, [x0]439; CHECK-NEXT: // implicit-def: $d0440; CHECK-NEXT: cbnz x8, .LBB15_2441; CHECK-NEXT: // %bb.1: // %cond.load442; CHECK-NEXT: ldr d0, [x1]443; CHECK-NEXT: fmov x8, d0444; CHECK-NEXT: ldr d0, [x8]445; CHECK-NEXT: .LBB15_2: // %else446; CHECK-NEXT: str d0, [x0]447; CHECK-NEXT: ret448 %cval = load <1 x i64>, ptr %a449 %ptrs = load <1 x ptr>, ptr %b450 %mask = icmp eq <1 x i64> %cval, zeroinitializer451 %vals = call <1 x i64> @llvm.masked.gather.v1i64(<1 x ptr> %ptrs, i32 8, <1 x i1> %mask, <1 x i64> poison)452 store <1 x i64> %vals, ptr %a453 ret void454}455 456define void @masked_gather_v2i64(ptr %a, ptr %b) vscale_range(2,0) #0 {457; CHECK-LABEL: masked_gather_v2i64:458; CHECK: // %bb.0:459; CHECK-NEXT: ptrue p0.d, vl2460; CHECK-NEXT: ldr q0, [x0]461; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, #0462; CHECK-NEXT: ldr q0, [x1]463; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]464; CHECK-NEXT: str q0, [x0]465; CHECK-NEXT: ret466 %cval = load <2 x i64>, ptr %a467 %ptrs = load <2 x ptr>, ptr %b468 %mask = icmp eq <2 x i64> %cval, zeroinitializer469 %vals = call <2 x i64> @llvm.masked.gather.v2i64(<2 x ptr> %ptrs, i32 8, <2 x i1> %mask, <2 x i64> poison)470 store <2 x i64> %vals, ptr %a471 ret void472}473 474define void @masked_gather_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {475; CHECK-LABEL: masked_gather_v4i64:476; CHECK: // %bb.0:477; CHECK-NEXT: ptrue p0.d, vl4478; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]479; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0480; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]481; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d]482; CHECK-NEXT: st1d { z0.d }, p0, [x0]483; CHECK-NEXT: ret484 %cval = load <4 x i64>, ptr %a485 %ptrs = load <4 x ptr>, ptr %b486 %mask = icmp eq <4 x i64> %cval, zeroinitializer487 %vals = call <4 x i64> @llvm.masked.gather.v4i64(<4 x ptr> %ptrs, i32 8, <4 x i1> %mask, <4 x i64> poison)488 store <4 x i64> %vals, ptr %a489 ret void490}491 492define void @masked_gather_v8i64(ptr %a, ptr %b) #0 {493; VBITS_GE_256-LABEL: masked_gather_v8i64:494; VBITS_GE_256: // %bb.0:495; VBITS_GE_256-NEXT: ptrue p0.d, vl4496; VBITS_GE_256-NEXT: mov x8, #4 // =0x4497; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]498; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]499; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0500; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]501; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [z0.d]502; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z1.d, #0503; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]504; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [z1.d]505; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]506; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]507; VBITS_GE_256-NEXT: ret508;509; VBITS_GE_512-LABEL: masked_gather_v8i64:510; VBITS_GE_512: // %bb.0:511; VBITS_GE_512-NEXT: ptrue p0.d, vl8512; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]513; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0514; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]515; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [z0.d]516; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]517; VBITS_GE_512-NEXT: ret518 %cval = load <8 x i64>, ptr %a519 %ptrs = load <8 x ptr>, ptr %b520 %mask = icmp eq <8 x i64> %cval, zeroinitializer521 %vals = call <8 x i64> @llvm.masked.gather.v8i64(<8 x ptr> %ptrs, i32 8, <8 x i1> %mask, <8 x i64> poison)522 store <8 x i64> %vals, ptr %a523 ret void524}525 526define void @masked_gather_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {527; CHECK-LABEL: masked_gather_v16i64:528; CHECK: // %bb.0:529; CHECK-NEXT: ptrue p0.d, vl16530; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]531; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0532; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]533; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d]534; CHECK-NEXT: st1d { z0.d }, p0, [x0]535; CHECK-NEXT: ret536 %cval = load <16 x i64>, ptr %a537 %ptrs = load <16 x ptr>, ptr %b538 %mask = icmp eq <16 x i64> %cval, zeroinitializer539 %vals = call <16 x i64> @llvm.masked.gather.v16i64(<16 x ptr> %ptrs, i32 8, <16 x i1> %mask, <16 x i64> poison)540 store <16 x i64> %vals, ptr %a541 ret void542}543 544define void @masked_gather_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {545; CHECK-LABEL: masked_gather_v32i64:546; CHECK: // %bb.0:547; CHECK-NEXT: ptrue p0.d, vl32548; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]549; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0550; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]551; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d]552; CHECK-NEXT: st1d { z0.d }, p0, [x0]553; CHECK-NEXT: ret554 %cval = load <32 x i64>, ptr %a555 %ptrs = load <32 x ptr>, ptr %b556 %mask = icmp eq <32 x i64> %cval, zeroinitializer557 %vals = call <32 x i64> @llvm.masked.gather.v32i64(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x i64> poison)558 store <32 x i64> %vals, ptr %a559 ret void560}561 562;563; LD1H (float)564;565 566define void @masked_gather_v2f16(ptr %a, ptr %b) vscale_range(2,0) #0 {567; CHECK-LABEL: masked_gather_v2f16:568; CHECK: // %bb.0:569; CHECK-NEXT: ldr s1, [x0]570; CHECK-NEXT: movi v0.2d, #0000000000000000571; CHECK-NEXT: ptrue p0.d, vl4572; CHECK-NEXT: fcmeq v1.4h, v1.4h, #0.0573; CHECK-NEXT: sshll v1.4s, v1.4h, #0574; CHECK-NEXT: mov v0.h[0], v1.h[0]575; CHECK-NEXT: mov w8, v1.s[1]576; CHECK-NEXT: mov v0.h[1], w8577; CHECK-NEXT: sunpklo z0.s, z0.h578; CHECK-NEXT: sunpklo z0.d, z0.s579; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0580; CHECK-NEXT: ldr q0, [x1]581; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]582; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s583; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h584; CHECK-NEXT: str s0, [x0]585; CHECK-NEXT: ret586 %cval = load <2 x half>, ptr %a587 %ptrs = load <2 x ptr>, ptr %b588 %mask = fcmp oeq <2 x half> %cval, zeroinitializer589 %vals = call <2 x half> @llvm.masked.gather.v2f16(<2 x ptr> %ptrs, i32 8, <2 x i1> %mask, <2 x half> poison)590 store <2 x half> %vals, ptr %a591 ret void592}593 594define void @masked_gather_v4f16(ptr %a, ptr %b) vscale_range(2,0) #0 {595; CHECK-LABEL: masked_gather_v4f16:596; CHECK: // %bb.0:597; CHECK-NEXT: ldr d0, [x0]598; CHECK-NEXT: ptrue p0.d, vl4599; CHECK-NEXT: fcmeq v0.4h, v0.4h, #0.0600; CHECK-NEXT: sunpklo z0.s, z0.h601; CHECK-NEXT: sunpklo z0.d, z0.s602; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0603; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]604; CHECK-NEXT: ld1h { z0.d }, p1/z, [z0.d]605; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s606; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h607; CHECK-NEXT: str d0, [x0]608; CHECK-NEXT: ret609 %cval = load <4 x half>, ptr %a610 %ptrs = load <4 x ptr>, ptr %b611 %mask = fcmp oeq <4 x half> %cval, zeroinitializer612 %vals = call <4 x half> @llvm.masked.gather.v4f16(<4 x ptr> %ptrs, i32 8, <4 x i1> %mask, <4 x half> poison)613 store <4 x half> %vals, ptr %a614 ret void615}616 617define void @masked_gather_v8f16(ptr %a, ptr %b) #0 {618; VBITS_GE_256-LABEL: masked_gather_v8f16:619; VBITS_GE_256: // %bb.0:620; VBITS_GE_256-NEXT: ldr q0, [x0]621; VBITS_GE_256-NEXT: ptrue p0.d, vl4622; VBITS_GE_256-NEXT: mov x8, #4 // =0x4623; VBITS_GE_256-NEXT: fcmeq v0.8h, v0.8h, #0.0624; VBITS_GE_256-NEXT: sunpklo z1.s, z0.h625; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8626; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s627; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h628; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0629; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s630; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]631; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z1.d]632; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0633; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]634; VBITS_GE_256-NEXT: ld1h { z0.d }, p1/z, [z0.d]635; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s636; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h637; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s638; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h639; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]640; VBITS_GE_256-NEXT: str q1, [x0]641; VBITS_GE_256-NEXT: ret642;643; VBITS_GE_512-LABEL: masked_gather_v8f16:644; VBITS_GE_512: // %bb.0:645; VBITS_GE_512-NEXT: ldr q0, [x0]646; VBITS_GE_512-NEXT: ptrue p0.d, vl8647; VBITS_GE_512-NEXT: fcmeq v0.8h, v0.8h, #0.0648; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h649; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s650; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0651; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]652; VBITS_GE_512-NEXT: ld1h { z0.d }, p1/z, [z0.d]653; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s654; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h655; VBITS_GE_512-NEXT: str q0, [x0]656; VBITS_GE_512-NEXT: ret657 %cval = load <8 x half>, ptr %a658 %ptrs = load <8 x ptr>, ptr %b659 %mask = fcmp oeq <8 x half> %cval, zeroinitializer660 %vals = call <8 x half> @llvm.masked.gather.v8f16(<8 x ptr> %ptrs, i32 8, <8 x i1> %mask, <8 x half> poison)661 store <8 x half> %vals, ptr %a662 ret void663}664 665define void @masked_gather_v16f16(ptr %a, ptr %b) vscale_range(8,0) #0 {666; CHECK-LABEL: masked_gather_v16f16:667; CHECK: // %bb.0:668; CHECK-NEXT: ptrue p0.h, vl16669; CHECK-NEXT: ptrue p1.d, vl16670; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]671; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0672; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]673; CHECK-NEXT: punpklo p0.h, p0.b674; CHECK-NEXT: punpklo p0.h, p0.b675; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]676; CHECK-NEXT: st1h { z0.d }, p1, [x0]677; CHECK-NEXT: ret678 %cval = load <16 x half>, ptr %a679 %ptrs = load <16 x ptr>, ptr %b680 %mask = fcmp oeq <16 x half> %cval, zeroinitializer681 %vals = call <16 x half> @llvm.masked.gather.v16f16(<16 x ptr> %ptrs, i32 8, <16 x i1> %mask, <16 x half> poison)682 store <16 x half> %vals, ptr %a683 ret void684}685 686define void @masked_gather_v32f16(ptr %a, ptr %b) vscale_range(16,0) #0 {687; CHECK-LABEL: masked_gather_v32f16:688; CHECK: // %bb.0:689; CHECK-NEXT: ptrue p0.h, vl32690; CHECK-NEXT: ptrue p1.d, vl32691; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]692; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0693; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]694; CHECK-NEXT: punpklo p0.h, p0.b695; CHECK-NEXT: punpklo p0.h, p0.b696; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]697; CHECK-NEXT: st1h { z0.d }, p1, [x0]698; CHECK-NEXT: ret699 %cval = load <32 x half>, ptr %a700 %ptrs = load <32 x ptr>, ptr %b701 %mask = fcmp oeq <32 x half> %cval, zeroinitializer702 %vals = call <32 x half> @llvm.masked.gather.v32f16(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x half> poison)703 store <32 x half> %vals, ptr %a704 ret void705}706 707;708; LD1W (float)709;710 711define void @masked_gather_v2f32(ptr %a, ptr %b) vscale_range(2,0) #0 {712; CHECK-LABEL: masked_gather_v2f32:713; CHECK: // %bb.0:714; CHECK-NEXT: ldr d0, [x0]715; CHECK-NEXT: ptrue p0.d, vl2716; CHECK-NEXT: fcmeq v0.2s, v0.2s, #0.0717; CHECK-NEXT: sshll v0.2d, v0.2s, #0718; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0719; CHECK-NEXT: ldr q0, [x1]720; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]721; CHECK-NEXT: xtn v0.2s, v0.2d722; CHECK-NEXT: str d0, [x0]723; CHECK-NEXT: ret724 %cval = load <2 x float>, ptr %a725 %ptrs = load <2 x ptr>, ptr %b726 %mask = fcmp oeq <2 x float> %cval, zeroinitializer727 %vals = call <2 x float> @llvm.masked.gather.v2f32(<2 x ptr> %ptrs, i32 8, <2 x i1> %mask, <2 x float> poison)728 store <2 x float> %vals, ptr %a729 ret void730}731 732define void @masked_gather_v4f32(ptr %a, ptr %b) vscale_range(2,0) #0 {733; CHECK-LABEL: masked_gather_v4f32:734; CHECK: // %bb.0:735; CHECK-NEXT: ldr q0, [x0]736; CHECK-NEXT: ptrue p0.d, vl4737; CHECK-NEXT: fcmeq v0.4s, v0.4s, #0.0738; CHECK-NEXT: sunpklo z0.d, z0.s739; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0740; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]741; CHECK-NEXT: ld1w { z0.d }, p1/z, [z0.d]742; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s743; CHECK-NEXT: str q0, [x0]744; CHECK-NEXT: ret745 %cval = load <4 x float>, ptr %a746 %ptrs = load <4 x ptr>, ptr %b747 %mask = fcmp oeq <4 x float> %cval, zeroinitializer748 %vals = call <4 x float> @llvm.masked.gather.v4f32(<4 x ptr> %ptrs, i32 8, <4 x i1> %mask, <4 x float> poison)749 store <4 x float> %vals, ptr %a750 ret void751}752 753define void @masked_gather_v8f32(ptr %a, ptr %b) #0 {754; VBITS_GE_256-LABEL: masked_gather_v8f32:755; VBITS_GE_256: // %bb.0:756; VBITS_GE_256-NEXT: ptrue p0.s, vl8757; VBITS_GE_256-NEXT: mov x8, #4 // =0x4758; VBITS_GE_256-NEXT: ptrue p2.d, vl4759; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0]760; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1]761; VBITS_GE_256-NEXT: fcmeq p1.s, p0/z, z0.s, #0.0762; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff763; VBITS_GE_256-NEXT: punpklo p1.h, p1.b764; VBITS_GE_256-NEXT: and p1.b, p1/z, p1.b, p2.b765; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16766; VBITS_GE_256-NEXT: ld1w { z1.d }, p1/z, [z1.d]767; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s768; VBITS_GE_256-NEXT: cmpne p1.d, p2/z, z0.d, #0769; VBITS_GE_256-NEXT: ld1d { z0.d }, p2/z, [x1, x8, lsl #3]770; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s771; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z0.d]772; VBITS_GE_256-NEXT: ptrue p1.s, vl4773; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s774; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s775; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]776; VBITS_GE_256-NEXT: ret777;778; VBITS_GE_512-LABEL: masked_gather_v8f32:779; VBITS_GE_512: // %bb.0:780; VBITS_GE_512-NEXT: ptrue p0.s, vl8781; VBITS_GE_512-NEXT: ptrue p1.d, vl8782; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]783; VBITS_GE_512-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0784; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x1]785; VBITS_GE_512-NEXT: punpklo p0.h, p0.b786; VBITS_GE_512-NEXT: ld1w { z0.d }, p0/z, [z0.d]787; VBITS_GE_512-NEXT: st1w { z0.d }, p1, [x0]788; VBITS_GE_512-NEXT: ret789 %cval = load <8 x float>, ptr %a790 %ptrs = load <8 x ptr>, ptr %b791 %mask = fcmp oeq <8 x float> %cval, zeroinitializer792 %vals = call <8 x float> @llvm.masked.gather.v8f32(<8 x ptr> %ptrs, i32 8, <8 x i1> %mask, <8 x float> poison)793 store <8 x float> %vals, ptr %a794 ret void795}796 797define void @masked_gather_v16f32(ptr %a, ptr %b) vscale_range(8,0) #0 {798; CHECK-LABEL: masked_gather_v16f32:799; CHECK: // %bb.0:800; CHECK-NEXT: ptrue p0.s, vl16801; CHECK-NEXT: ptrue p1.d, vl16802; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]803; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0804; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]805; CHECK-NEXT: punpklo p0.h, p0.b806; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]807; CHECK-NEXT: st1w { z0.d }, p1, [x0]808; CHECK-NEXT: ret809 %cval = load <16 x float>, ptr %a810 %ptrs = load <16 x ptr>, ptr %b811 %mask = fcmp oeq <16 x float> %cval, zeroinitializer812 %vals = call <16 x float> @llvm.masked.gather.v16f32(<16 x ptr> %ptrs, i32 8, <16 x i1> %mask, <16 x float> poison)813 store <16 x float> %vals, ptr %a814 ret void815}816 817define void @masked_gather_v32f32(ptr %a, ptr %b) vscale_range(16,0) #0 {818; CHECK-LABEL: masked_gather_v32f32:819; CHECK: // %bb.0:820; CHECK-NEXT: ptrue p0.s, vl32821; CHECK-NEXT: ptrue p1.d, vl32822; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]823; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0824; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]825; CHECK-NEXT: punpklo p0.h, p0.b826; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]827; CHECK-NEXT: st1w { z0.d }, p1, [x0]828; CHECK-NEXT: ret829 %cval = load <32 x float>, ptr %a830 %ptrs = load <32 x ptr>, ptr %b831 %mask = fcmp oeq <32 x float> %cval, zeroinitializer832 %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x float> poison)833 store <32 x float> %vals, ptr %a834 ret void835}836 837;838; LD1D (float)839;840 841; Scalarize 1 x double gathers842define void @masked_gather_v1f64(ptr %a, ptr %b) vscale_range(2,0) #0 {843; CHECK-LABEL: masked_gather_v1f64:844; CHECK: // %bb.0:845; CHECK-NEXT: ldr d0, [x0]846; CHECK-NEXT: fcmp d0, #0.0847; CHECK-NEXT: // implicit-def: $d0848; CHECK-NEXT: b.ne .LBB31_2849; CHECK-NEXT: // %bb.1: // %cond.load850; CHECK-NEXT: ldr d0, [x1]851; CHECK-NEXT: fmov x8, d0852; CHECK-NEXT: ldr d0, [x8]853; CHECK-NEXT: .LBB31_2: // %else854; CHECK-NEXT: str d0, [x0]855; CHECK-NEXT: ret856 %cval = load <1 x double>, ptr %a857 %ptrs = load <1 x ptr>, ptr %b858 %mask = fcmp oeq <1 x double> %cval, zeroinitializer859 %vals = call <1 x double> @llvm.masked.gather.v1f64(<1 x ptr> %ptrs, i32 8, <1 x i1> %mask, <1 x double> poison)860 store <1 x double> %vals, ptr %a861 ret void862}863 864define void @masked_gather_v2f64(ptr %a, ptr %b) vscale_range(2,0) #0 {865; CHECK-LABEL: masked_gather_v2f64:866; CHECK: // %bb.0:867; CHECK-NEXT: ldr q0, [x0]868; CHECK-NEXT: ptrue p0.d, vl2869; CHECK-NEXT: fcmeq v0.2d, v0.2d, #0.0870; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0871; CHECK-NEXT: ldr q0, [x1]872; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]873; CHECK-NEXT: str q0, [x0]874; CHECK-NEXT: ret875 %cval = load <2 x double>, ptr %a876 %ptrs = load <2 x ptr>, ptr %b877 %mask = fcmp oeq <2 x double> %cval, zeroinitializer878 %vals = call <2 x double> @llvm.masked.gather.v2f64(<2 x ptr> %ptrs, i32 8, <2 x i1> %mask, <2 x double> poison)879 store <2 x double> %vals, ptr %a880 ret void881}882 883define void @masked_gather_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {884; CHECK-LABEL: masked_gather_v4f64:885; CHECK: // %bb.0:886; CHECK-NEXT: ptrue p0.d, vl4887; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]888; CHECK-NEXT: fcmeq p1.d, p0/z, z0.d, #0.0889; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]890; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d]891; CHECK-NEXT: st1d { z0.d }, p0, [x0]892; CHECK-NEXT: ret893 %cval = load <4 x double>, ptr %a894 %ptrs = load <4 x ptr>, ptr %b895 %mask = fcmp oeq <4 x double> %cval, zeroinitializer896 %vals = call <4 x double> @llvm.masked.gather.v4f64(<4 x ptr> %ptrs, i32 8, <4 x i1> %mask, <4 x double> poison)897 store <4 x double> %vals, ptr %a898 ret void899}900 901define void @masked_gather_v8f64(ptr %a, ptr %b) #0 {902; VBITS_GE_256-LABEL: masked_gather_v8f64:903; VBITS_GE_256: // %bb.0:904; VBITS_GE_256-NEXT: ptrue p0.d, vl4905; VBITS_GE_256-NEXT: mov x8, #4 // =0x4906; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]907; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]908; VBITS_GE_256-NEXT: fcmeq p1.d, p0/z, z0.d, #0.0909; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]910; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [z0.d]911; VBITS_GE_256-NEXT: fcmeq p1.d, p0/z, z1.d, #0.0912; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]913; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [z1.d]914; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]915; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]916; VBITS_GE_256-NEXT: ret917;918; VBITS_GE_512-LABEL: masked_gather_v8f64:919; VBITS_GE_512: // %bb.0:920; VBITS_GE_512-NEXT: ptrue p0.d, vl8921; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]922; VBITS_GE_512-NEXT: fcmeq p1.d, p0/z, z0.d, #0.0923; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]924; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [z0.d]925; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]926; VBITS_GE_512-NEXT: ret927 %cval = load <8 x double>, ptr %a928 %ptrs = load <8 x ptr>, ptr %b929 %mask = fcmp oeq <8 x double> %cval, zeroinitializer930 %vals = call <8 x double> @llvm.masked.gather.v8f64(<8 x ptr> %ptrs, i32 8, <8 x i1> %mask, <8 x double> poison)931 store <8 x double> %vals, ptr %a932 ret void933}934 935define void @masked_gather_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 {936; CHECK-LABEL: masked_gather_v16f64:937; CHECK: // %bb.0:938; CHECK-NEXT: ptrue p0.d, vl16939; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]940; CHECK-NEXT: fcmeq p1.d, p0/z, z0.d, #0.0941; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]942; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d]943; CHECK-NEXT: st1d { z0.d }, p0, [x0]944; CHECK-NEXT: ret945 %cval = load <16 x double>, ptr %a946 %ptrs = load <16 x ptr>, ptr %b947 %mask = fcmp oeq <16 x double> %cval, zeroinitializer948 %vals = call <16 x double> @llvm.masked.gather.v16f64(<16 x ptr> %ptrs, i32 8, <16 x i1> %mask, <16 x double> poison)949 store <16 x double> %vals, ptr %a950 ret void951}952 953define void @masked_gather_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 {954; CHECK-LABEL: masked_gather_v32f64:955; CHECK: // %bb.0:956; CHECK-NEXT: ptrue p0.d, vl32957; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]958; CHECK-NEXT: fcmeq p1.d, p0/z, z0.d, #0.0959; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]960; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d]961; CHECK-NEXT: st1d { z0.d }, p0, [x0]962; CHECK-NEXT: ret963 %cval = load <32 x double>, ptr %a964 %ptrs = load <32 x ptr>, ptr %b965 %mask = fcmp oeq <32 x double> %cval, zeroinitializer966 %vals = call <32 x double> @llvm.masked.gather.v32f64(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x double> poison)967 store <32 x double> %vals, ptr %a968 ret void969}970 971; The above tests test the types, the below tests check that the addressing972; modes still function973 974define void @masked_gather_32b_scaled_sext_f16(ptr %a, ptr %b, ptr %base) vscale_range(8,0) #0 {975; CHECK-LABEL: masked_gather_32b_scaled_sext_f16:976; CHECK: // %bb.0:977; CHECK-NEXT: ptrue p0.h, vl32978; CHECK-NEXT: ptrue p1.s, vl32979; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]980; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0981; CHECK-NEXT: ld1w { z0.s }, p1/z, [x1]982; CHECK-NEXT: punpklo p0.h, p0.b983; CHECK-NEXT: ld1h { z0.s }, p0/z, [x2, z0.s, sxtw #1]984; CHECK-NEXT: st1h { z0.s }, p1, [x0]985; CHECK-NEXT: ret986 %cvals = load <32 x half>, ptr %a987 %idxs = load <32 x i32>, ptr %b988 %ext = sext <32 x i32> %idxs to <32 x i64>989 %ptrs = getelementptr half, ptr %base, <32 x i64> %ext990 %mask = fcmp oeq <32 x half> %cvals, zeroinitializer991 %vals = call <32 x half> @llvm.masked.gather.v32f16(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x half> poison)992 store <32 x half> %vals, ptr %a993 ret void994}995 996define void @masked_gather_32b_scaled_sext_f32(ptr %a, ptr %b, ptr %base) vscale_range(8,0) #0 {997; CHECK-LABEL: masked_gather_32b_scaled_sext_f32:998; CHECK: // %bb.0:999; CHECK-NEXT: ptrue p0.s, vl321000; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1001; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, #0.01002; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1]1003; CHECK-NEXT: ld1w { z0.s }, p1/z, [x2, z0.s, sxtw #2]1004; CHECK-NEXT: st1w { z0.s }, p0, [x0]1005; CHECK-NEXT: ret1006 %cvals = load <32 x float>, ptr %a1007 %idxs = load <32 x i32>, ptr %b1008 %ext = sext <32 x i32> %idxs to <32 x i64>1009 %ptrs = getelementptr float, ptr %base, <32 x i64> %ext1010 %mask = fcmp oeq <32 x float> %cvals, zeroinitializer1011 %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x float> poison)1012 store <32 x float> %vals, ptr %a1013 ret void1014}1015 1016define void @masked_gather_32b_scaled_sext_f64(ptr %a, ptr %b, ptr %base) vscale_range(16,0) #0 {1017; CHECK-LABEL: masked_gather_32b_scaled_sext_f64:1018; CHECK: // %bb.0:1019; CHECK-NEXT: ptrue p0.d, vl321020; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1021; CHECK-NEXT: fcmeq p1.d, p0/z, z0.d, #0.01022; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x1]1023; CHECK-NEXT: ld1d { z0.d }, p1/z, [x2, z0.d, lsl #3]1024; CHECK-NEXT: st1d { z0.d }, p0, [x0]1025; CHECK-NEXT: ret1026 %cvals = load <32 x double>, ptr %a1027 %idxs = load <32 x i32>, ptr %b1028 %ext = sext <32 x i32> %idxs to <32 x i64>1029 %ptrs = getelementptr double, ptr %base, <32 x i64> %ext1030 %mask = fcmp oeq <32 x double> %cvals, zeroinitializer1031 %vals = call <32 x double> @llvm.masked.gather.v32f64(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x double> poison)1032 store <32 x double> %vals, ptr %a1033 ret void1034}1035 1036define void @masked_gather_32b_scaled_zext(ptr %a, ptr %b, ptr %base) vscale_range(8,0) #0 {1037; CHECK-LABEL: masked_gather_32b_scaled_zext:1038; CHECK: // %bb.0:1039; CHECK-NEXT: ptrue p0.h, vl321040; CHECK-NEXT: ptrue p1.s, vl321041; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1042; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.01043; CHECK-NEXT: ld1w { z0.s }, p1/z, [x1]1044; CHECK-NEXT: punpklo p0.h, p0.b1045; CHECK-NEXT: ld1h { z0.s }, p0/z, [x2, z0.s, uxtw #1]1046; CHECK-NEXT: st1h { z0.s }, p1, [x0]1047; CHECK-NEXT: ret1048 %cvals = load <32 x half>, ptr %a1049 %idxs = load <32 x i32>, ptr %b1050 %ext = zext <32 x i32> %idxs to <32 x i64>1051 %ptrs = getelementptr half, ptr %base, <32 x i64> %ext1052 %mask = fcmp oeq <32 x half> %cvals, zeroinitializer1053 %vals = call <32 x half> @llvm.masked.gather.v32f16(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x half> poison)1054 store <32 x half> %vals, ptr %a1055 ret void1056}1057 1058define void @masked_gather_32b_unscaled_sext(ptr %a, ptr %b, ptr %base) vscale_range(8,0) #0 {1059; CHECK-LABEL: masked_gather_32b_unscaled_sext:1060; CHECK: // %bb.0:1061; CHECK-NEXT: ptrue p0.h, vl321062; CHECK-NEXT: ptrue p1.s, vl321063; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1064; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.01065; CHECK-NEXT: ld1w { z0.s }, p1/z, [x1]1066; CHECK-NEXT: punpklo p0.h, p0.b1067; CHECK-NEXT: ld1h { z0.s }, p0/z, [x2, z0.s, sxtw]1068; CHECK-NEXT: st1h { z0.s }, p1, [x0]1069; CHECK-NEXT: ret1070 %cvals = load <32 x half>, ptr %a1071 %idxs = load <32 x i32>, ptr %b1072 %ext = sext <32 x i32> %idxs to <32 x i64>1073 %byte_ptrs = getelementptr i8, ptr %base, <32 x i64> %ext1074 %ptrs = bitcast <32 x ptr> %byte_ptrs to <32 x ptr>1075 %mask = fcmp oeq <32 x half> %cvals, zeroinitializer1076 %vals = call <32 x half> @llvm.masked.gather.v32f16(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x half> poison)1077 store <32 x half> %vals, ptr %a1078 ret void1079}1080 1081define void @masked_gather_32b_unscaled_zext(ptr %a, ptr %b, ptr %base) vscale_range(8,0) #0 {1082; CHECK-LABEL: masked_gather_32b_unscaled_zext:1083; CHECK: // %bb.0:1084; CHECK-NEXT: ptrue p0.h, vl321085; CHECK-NEXT: ptrue p1.s, vl321086; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1087; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.01088; CHECK-NEXT: ld1w { z0.s }, p1/z, [x1]1089; CHECK-NEXT: punpklo p0.h, p0.b1090; CHECK-NEXT: ld1h { z0.s }, p0/z, [x2, z0.s, uxtw]1091; CHECK-NEXT: st1h { z0.s }, p1, [x0]1092; CHECK-NEXT: ret1093 %cvals = load <32 x half>, ptr %a1094 %idxs = load <32 x i32>, ptr %b1095 %ext = zext <32 x i32> %idxs to <32 x i64>1096 %byte_ptrs = getelementptr i8, ptr %base, <32 x i64> %ext1097 %ptrs = bitcast <32 x ptr> %byte_ptrs to <32 x ptr>1098 %mask = fcmp oeq <32 x half> %cvals, zeroinitializer1099 %vals = call <32 x half> @llvm.masked.gather.v32f16(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x half> poison)1100 store <32 x half> %vals, ptr %a1101 ret void1102}1103 1104define void @masked_gather_64b_scaled(ptr %a, ptr %b, ptr %base) vscale_range(16,0) #0 {1105; CHECK-LABEL: masked_gather_64b_scaled:1106; CHECK: // %bb.0:1107; CHECK-NEXT: ptrue p0.s, vl321108; CHECK-NEXT: ptrue p1.d, vl321109; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1110; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.01111; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]1112; CHECK-NEXT: punpklo p0.h, p0.b1113; CHECK-NEXT: ld1w { z0.d }, p0/z, [x2, z0.d, lsl #2]1114; CHECK-NEXT: st1w { z0.d }, p1, [x0]1115; CHECK-NEXT: ret1116 %cvals = load <32 x float>, ptr %a1117 %idxs = load <32 x i64>, ptr %b1118 %ptrs = getelementptr float, ptr %base, <32 x i64> %idxs1119 %mask = fcmp oeq <32 x float> %cvals, zeroinitializer1120 %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x float> poison)1121 store <32 x float> %vals, ptr %a1122 ret void1123}1124 1125define void @masked_gather_64b_unscaled(ptr %a, ptr %b, ptr %base) vscale_range(16,0) #0 {1126; CHECK-LABEL: masked_gather_64b_unscaled:1127; CHECK: // %bb.0:1128; CHECK-NEXT: ptrue p0.s, vl321129; CHECK-NEXT: ptrue p1.d, vl321130; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1131; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.01132; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]1133; CHECK-NEXT: punpklo p0.h, p0.b1134; CHECK-NEXT: ld1w { z0.d }, p0/z, [x2, z0.d]1135; CHECK-NEXT: st1w { z0.d }, p1, [x0]1136; CHECK-NEXT: ret1137 %cvals = load <32 x float>, ptr %a1138 %idxs = load <32 x i64>, ptr %b1139 %byte_ptrs = getelementptr i8, ptr %base, <32 x i64> %idxs1140 %ptrs = bitcast <32 x ptr> %byte_ptrs to <32 x ptr>1141 %mask = fcmp oeq <32 x float> %cvals, zeroinitializer1142 %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x float> poison)1143 store <32 x float> %vals, ptr %a1144 ret void1145}1146 1147define void @masked_gather_vec_plus_reg(ptr %a, ptr %b, i64 %off) vscale_range(16,0) #0 {1148; CHECK-LABEL: masked_gather_vec_plus_reg:1149; CHECK: // %bb.0:1150; CHECK-NEXT: ptrue p0.s, vl321151; CHECK-NEXT: ptrue p1.d, vl321152; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1153; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.01154; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]1155; CHECK-NEXT: punpklo p0.h, p0.b1156; CHECK-NEXT: ld1w { z0.d }, p0/z, [x2, z0.d]1157; CHECK-NEXT: st1w { z0.d }, p1, [x0]1158; CHECK-NEXT: ret1159 %cvals = load <32 x float>, ptr %a1160 %bases = load <32 x ptr>, ptr %b1161 %byte_ptrs = getelementptr i8, <32 x ptr> %bases, i64 %off1162 %ptrs = bitcast <32 x ptr> %byte_ptrs to <32 x ptr>1163 %mask = fcmp oeq <32 x float> %cvals, zeroinitializer1164 %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x float> poison)1165 store <32 x float> %vals, ptr %a1166 ret void1167}1168 1169define void @masked_gather_vec_plus_imm(ptr %a, ptr %b) vscale_range(16,0) #0 {1170; CHECK-LABEL: masked_gather_vec_plus_imm:1171; CHECK: // %bb.0:1172; CHECK-NEXT: ptrue p0.s, vl321173; CHECK-NEXT: ptrue p1.d, vl321174; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1175; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.01176; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]1177; CHECK-NEXT: punpklo p0.h, p0.b1178; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d, #4]1179; CHECK-NEXT: st1w { z0.d }, p1, [x0]1180; CHECK-NEXT: ret1181 %cvals = load <32 x float>, ptr %a1182 %bases = load <32 x ptr>, ptr %b1183 %byte_ptrs = getelementptr i8, <32 x ptr> %bases, i64 41184 %ptrs = bitcast <32 x ptr> %byte_ptrs to <32 x ptr>1185 %mask = fcmp oeq <32 x float> %cvals, zeroinitializer1186 %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x float> poison)1187 store <32 x float> %vals, ptr %a1188 ret void1189}1190 1191define void @masked_gather_passthru(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {1192; CHECK-LABEL: masked_gather_passthru:1193; CHECK: // %bb.0:1194; CHECK-NEXT: ptrue p0.s, vl321195; CHECK-NEXT: ptrue p2.d, vl321196; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1197; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, #0.01198; CHECK-NEXT: ld1d { z0.d }, p2/z, [x1]1199; CHECK-NEXT: punpklo p2.h, p1.b1200; CHECK-NEXT: mov z1.s, p1/z, #-1 // =0xffffffffffffffff1201; CHECK-NEXT: ptrue p1.s1202; CHECK-NEXT: ld1w { z0.d }, p2/z, [z0.d]1203; CHECK-NEXT: and z1.s, z1.s, #0x11204; CHECK-NEXT: cmpne p1.s, p1/z, z1.s, #01205; CHECK-NEXT: ld1w { z1.s }, p0/z, [x2]1206; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s1207; CHECK-NEXT: sel z0.s, p1, z0.s, z1.s1208; CHECK-NEXT: st1w { z0.s }, p0, [x0]1209; CHECK-NEXT: ret1210 %cvals = load <32 x float>, ptr %a1211 %ptrs = load <32 x ptr>, ptr %b1212 %passthru = load <32 x float>, ptr %c1213 %mask = fcmp oeq <32 x float> %cvals, zeroinitializer1214 %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x float> %passthru)1215 store <32 x float> %vals, ptr %a1216 ret void1217}1218 1219define void @masked_gather_passthru_0(ptr %a, ptr %b) vscale_range(16,0) #0 {1220; CHECK-LABEL: masked_gather_passthru_0:1221; CHECK: // %bb.0:1222; CHECK-NEXT: ptrue p0.s, vl321223; CHECK-NEXT: ptrue p1.d, vl321224; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1225; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.01226; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1]1227; CHECK-NEXT: punpklo p0.h, p0.b1228; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]1229; CHECK-NEXT: st1w { z0.d }, p1, [x0]1230; CHECK-NEXT: ret1231 %cvals = load <32 x float>, ptr %a1232 %ptrs = load <32 x ptr>, ptr %b1233 %mask = fcmp oeq <32 x float> %cvals, zeroinitializer1234 %vals = call <32 x float> @llvm.masked.gather.v32f32(<32 x ptr> %ptrs, i32 8, <32 x i1> %mask, <32 x float> zeroinitializer)1235 store <32 x float> %vals, ptr %a1236 ret void1237}1238 1239declare <2 x i8> @llvm.masked.gather.v2i8(<2 x ptr>, i32, <2 x i1>, <2 x i8>)1240declare <4 x i8> @llvm.masked.gather.v4i8(<4 x ptr>, i32, <4 x i1>, <4 x i8>)1241declare <8 x i8> @llvm.masked.gather.v8i8(<8 x ptr>, i32, <8 x i1>, <8 x i8>)1242declare <16 x i8> @llvm.masked.gather.v16i8(<16 x ptr>, i32, <16 x i1>, <16 x i8>)1243declare <32 x i8> @llvm.masked.gather.v32i8(<32 x ptr>, i32, <32 x i1>, <32 x i8>)1244 1245declare <2 x i16> @llvm.masked.gather.v2i16(<2 x ptr>, i32, <2 x i1>, <2 x i16>)1246declare <4 x i16> @llvm.masked.gather.v4i16(<4 x ptr>, i32, <4 x i1>, <4 x i16>)1247declare <8 x i16> @llvm.masked.gather.v8i16(<8 x ptr>, i32, <8 x i1>, <8 x i16>)1248declare <16 x i16> @llvm.masked.gather.v16i16(<16 x ptr>, i32, <16 x i1>, <16 x i16>)1249declare <32 x i16> @llvm.masked.gather.v32i16(<32 x ptr>, i32, <32 x i1>, <32 x i16>)1250 1251declare <2 x i32> @llvm.masked.gather.v2i32(<2 x ptr>, i32, <2 x i1>, <2 x i32>)1252declare <4 x i32> @llvm.masked.gather.v4i32(<4 x ptr>, i32, <4 x i1>, <4 x i32>)1253declare <8 x i32> @llvm.masked.gather.v8i32(<8 x ptr>, i32, <8 x i1>, <8 x i32>)1254declare <16 x i32> @llvm.masked.gather.v16i32(<16 x ptr>, i32, <16 x i1>, <16 x i32>)1255declare <32 x i32> @llvm.masked.gather.v32i32(<32 x ptr>, i32, <32 x i1>, <32 x i32>)1256 1257declare <1 x i64> @llvm.masked.gather.v1i64(<1 x ptr>, i32, <1 x i1>, <1 x i64>)1258declare <2 x i64> @llvm.masked.gather.v2i64(<2 x ptr>, i32, <2 x i1>, <2 x i64>)1259declare <4 x i64> @llvm.masked.gather.v4i64(<4 x ptr>, i32, <4 x i1>, <4 x i64>)1260declare <8 x i64> @llvm.masked.gather.v8i64(<8 x ptr>, i32, <8 x i1>, <8 x i64>)1261declare <16 x i64> @llvm.masked.gather.v16i64(<16 x ptr>, i32, <16 x i1>, <16 x i64>)1262declare <32 x i64> @llvm.masked.gather.v32i64(<32 x ptr>, i32, <32 x i1>, <32 x i64>)1263 1264declare <2 x half> @llvm.masked.gather.v2f16(<2 x ptr>, i32, <2 x i1>, <2 x half>)1265declare <4 x half> @llvm.masked.gather.v4f16(<4 x ptr>, i32, <4 x i1>, <4 x half>)1266declare <8 x half> @llvm.masked.gather.v8f16(<8 x ptr>, i32, <8 x i1>, <8 x half>)1267declare <16 x half> @llvm.masked.gather.v16f16(<16 x ptr>, i32, <16 x i1>, <16 x half>)1268declare <32 x half> @llvm.masked.gather.v32f16(<32 x ptr>, i32, <32 x i1>, <32 x half>)1269 1270declare <2 x float> @llvm.masked.gather.v2f32(<2 x ptr>, i32, <2 x i1>, <2 x float>)1271declare <4 x float> @llvm.masked.gather.v4f32(<4 x ptr>, i32, <4 x i1>, <4 x float>)1272declare <8 x float> @llvm.masked.gather.v8f32(<8 x ptr>, i32, <8 x i1>, <8 x float>)1273declare <16 x float> @llvm.masked.gather.v16f32(<16 x ptr>, i32, <16 x i1>, <16 x float>)1274declare <32 x float> @llvm.masked.gather.v32f32(<32 x ptr>, i32, <32 x i1>, <32 x float>)1275 1276declare <1 x double> @llvm.masked.gather.v1f64(<1 x ptr>, i32, <1 x i1>, <1 x double>)1277declare <2 x double> @llvm.masked.gather.v2f64(<2 x ptr>, i32, <2 x i1>, <2 x double>)1278declare <4 x double> @llvm.masked.gather.v4f64(<4 x ptr>, i32, <4 x i1>, <4 x double>)1279declare <8 x double> @llvm.masked.gather.v8f64(<8 x ptr>, i32, <8 x i1>, <8 x double>)1280declare <16 x double> @llvm.masked.gather.v16f64(<16 x ptr>, i32, <16 x i1>, <16 x double>)1281declare <32 x double> @llvm.masked.gather.v32f64(<32 x ptr>, i32, <32 x i1>, <32 x double>)1282 1283attributes #0 = { "target-features"="+sve" }1284