474 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=aarch64-linux-unknown | FileCheck %s3 4 5; Ensure we use a "vscale x 4" wide scatter for the maximum supported offset.6define void @scatter_i8_index_offset_maximum(ptr %base, i64 %offset, <vscale x 4 x i1> %pg, <vscale x 4 x i8> %data) #0 {7; CHECK-LABEL: scatter_i8_index_offset_maximum:8; CHECK: // %bb.0:9; CHECK-NEXT: mov w8, #33554431 // =0x1ffffff10; CHECK-NEXT: index z1.s, #0, w811; CHECK-NEXT: add x8, x0, x112; CHECK-NEXT: st1b { z0.s }, p0, [x8, z1.s, sxtw]13; CHECK-NEXT: ret14 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 015 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer16 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()17 %t4 = mul <vscale x 4 x i64> splat(i64 33554431), %step18 %t5 = add <vscale x 4 x i64> %t1, %t419 %t6 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t520 call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)21 ret void22}23 24; Ensure we use a "vscale x 4" wide scatter for the minimum supported offset.25define void @scatter_i16_index_offset_minimum(ptr %base, i64 %offset, <vscale x 4 x i1> %pg, <vscale x 4 x i16> %data) #0 {26; CHECK-LABEL: scatter_i16_index_offset_minimum:27; CHECK: // %bb.0:28; CHECK-NEXT: mov w8, #-33554432 // =0xfe00000029; CHECK-NEXT: index z1.s, #0, w830; CHECK-NEXT: add x8, x0, x1, lsl #131; CHECK-NEXT: st1h { z0.s }, p0, [x8, z1.s, sxtw #1]32; CHECK-NEXT: ret33 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 034 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer35 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()36 %t4 = mul <vscale x 4 x i64> splat(i64 -33554432), %step37 %t5 = add <vscale x 4 x i64> %t1, %t438 %t6 = getelementptr i16, ptr %base, <vscale x 4 x i64> %t539 call void @llvm.masked.scatter.nxv4i16(<vscale x 4 x i16> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)40 ret void41}42 43; Ensure we use a "vscale x 4" gather for an offset in the limits of 32 bits.44define <vscale x 4 x i8> @gather_i8_index_offset_8(ptr %base, i64 %offset, <vscale x 4 x i1> %pg) #0 {45; CHECK-LABEL: gather_i8_index_offset_8:46; CHECK: // %bb.0:47; CHECK-NEXT: index z0.s, #0, #148; CHECK-NEXT: add x8, x0, x149; CHECK-NEXT: ld1b { z0.s }, p0/z, [x8, z0.s, sxtw]50; CHECK-NEXT: ret51 %splat.insert0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 052 %splat0 = shufflevector <vscale x 4 x i64> %splat.insert0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer53 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()54 %t1 = mul <vscale x 4 x i64> splat(i64 1), %step55 %t2 = add <vscale x 4 x i64> %splat0, %t156 %t3 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t257 %load = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr> %t3, i32 4, <vscale x 4 x i1> %pg, <vscale x 4 x i8> poison)58 ret <vscale x 4 x i8> %load59}60 61;; Negative tests62 63; Ensure we don't use a "vscale x 4" scatter. Cannot prove that variable stride64; will not wrap when shrunk to be i32 based.65define void @scatter_f16_index_offset_var(ptr %base, i64 %offset, i64 %scale, <vscale x 4 x i1> %pg, <vscale x 4 x half> %data) #0 {66; CHECK-LABEL: scatter_f16_index_offset_var:67; CHECK: // %bb.0:68; CHECK-NEXT: index z1.d, #0, #169; CHECK-NEXT: mov z2.d, x170; CHECK-NEXT: ptrue p1.d71; CHECK-NEXT: uunpklo z3.d, z0.s72; CHECK-NEXT: uunpkhi z0.d, z0.s73; CHECK-NEXT: punpklo p2.h, p0.b74; CHECK-NEXT: punpkhi p0.h, p0.b75; CHECK-NEXT: movprfx z4, z276; CHECK-NEXT: mla z4.d, p1/m, z1.d, z2.d77; CHECK-NEXT: incd z1.d78; CHECK-NEXT: mad z1.d, p1/m, z2.d, z2.d79; CHECK-NEXT: st1h { z3.d }, p2, [x0, z4.d, lsl #1]80; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d, lsl #1]81; CHECK-NEXT: ret82 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 083 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer84 %t2 = insertelement <vscale x 4 x i64> poison, i64 %scale, i32 085 %t3 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer86 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()87 %t4 = mul <vscale x 4 x i64> %t3, %step88 %t5 = add <vscale x 4 x i64> %t1, %t489 %t6 = getelementptr half, ptr %base, <vscale x 4 x i64> %t590 call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)91 ret void92}93 94; Ensure we don't use a "vscale x 4" wide scatter when the offset is too big.95define void @scatter_i8_index_offset_maximum_plus_one(ptr %base, i64 %offset, <vscale x 4 x i1> %pg, <vscale x 4 x i8> %data) #0 {96; CHECK-LABEL: scatter_i8_index_offset_maximum_plus_one:97; CHECK: // %bb.0:98; CHECK-NEXT: mov w8, #33554432 // =0x200000099; CHECK-NEXT: uunpklo z2.d, z0.s100; CHECK-NEXT: rdvl x9, #1101; CHECK-NEXT: index z1.d, #0, x8102; CHECK-NEXT: punpklo p1.h, p0.b103; CHECK-NEXT: lsr x9, x9, #4104; CHECK-NEXT: add x8, x0, x1105; CHECK-NEXT: mov w10, #67108864 // =0x4000000106; CHECK-NEXT: uunpkhi z0.d, z0.s107; CHECK-NEXT: punpkhi p0.h, p0.b108; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d]109; CHECK-NEXT: madd x8, x9, x10, x8110; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d]111; CHECK-NEXT: ret112 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0113 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer114 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()115 %t4 = mul <vscale x 4 x i64> splat(i64 33554432), %step116 %t5 = add <vscale x 4 x i64> %t1, %t4117 %t6 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t5118 call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)119 ret void120}121 122; Ensure we don't use a "vscale x 4" wide scatter when the offset is too small.123define void @scatter_i8_index_offset_minimum_minus_one(ptr %base, i64 %offset, <vscale x 4 x i1> %pg, <vscale x 4 x i8> %data) #0 {124; CHECK-LABEL: scatter_i8_index_offset_minimum_minus_one:125; CHECK: // %bb.0:126; CHECK-NEXT: mov x8, #-33554433 // =0xfffffffffdffffff127; CHECK-NEXT: uunpklo z2.d, z0.s128; CHECK-NEXT: rdvl x9, #1129; CHECK-NEXT: index z1.d, #0, x8130; CHECK-NEXT: punpklo p1.h, p0.b131; CHECK-NEXT: lsr x9, x9, #4132; CHECK-NEXT: mov x10, #-2 // =0xfffffffffffffffe133; CHECK-NEXT: add x8, x0, x1134; CHECK-NEXT: uunpkhi z0.d, z0.s135; CHECK-NEXT: movk x10, #64511, lsl #16136; CHECK-NEXT: punpkhi p0.h, p0.b137; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d]138; CHECK-NEXT: madd x8, x9, x10, x8139; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d]140; CHECK-NEXT: ret141 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0142 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer143 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()144 %t4 = mul <vscale x 4 x i64> splat(i64 -33554433), %step145 %t5 = add <vscale x 4 x i64> %t1, %t4146 %t6 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t5147 call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)148 ret void149}150 151; Ensure we don't use a "vscale x 4" wide scatter when the stride is too big .152define void @scatter_i8_index_stride_too_big(ptr %base, i64 %offset, <vscale x 4 x i1> %pg, <vscale x 4 x i8> %data) #0 {153; CHECK-LABEL: scatter_i8_index_stride_too_big:154; CHECK: // %bb.0:155; CHECK-NEXT: mov x8, #4611686018427387904 // =0x4000000000000000156; CHECK-NEXT: uunpklo z2.d, z0.s157; CHECK-NEXT: rdvl x9, #1158; CHECK-NEXT: index z1.d, #0, x8159; CHECK-NEXT: punpklo p1.h, p0.b160; CHECK-NEXT: lsr x9, x9, #4161; CHECK-NEXT: add x8, x0, x1162; CHECK-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000163; CHECK-NEXT: uunpkhi z0.d, z0.s164; CHECK-NEXT: punpkhi p0.h, p0.b165; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d]166; CHECK-NEXT: madd x8, x9, x10, x8167; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d]168; CHECK-NEXT: ret169 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0170 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer171 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()172 %t4 = mul <vscale x 4 x i64> splat(i64 4611686018427387904), %step173 %t5 = add <vscale x 4 x i64> %t1, %t4174 %t6 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t5175 call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)176 ret void177}178 179; Ensure the resulting load is "vscale x 4" wide, despite the offset giving the180; impression the gather must be split due to it's <vscale x 4 x i64> offset.181; gather_f32(base, index(offset, 8 * sizeof(float))182define <vscale x 4 x i8> @gather_8i8_index_offset_8(ptr %base, i64 %offset, <vscale x 4 x i1> %pg) #0 {183; CHECK-LABEL: gather_8i8_index_offset_8:184; CHECK: // %bb.0:185; CHECK-NEXT: index z0.s, #0, #8186; CHECK-NEXT: add x8, x0, x1, lsl #3187; CHECK-NEXT: ld1b { z0.s }, p0/z, [x8, z0.s, sxtw]188; CHECK-NEXT: ret189 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0190 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer191 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()192 %t2 = add <vscale x 4 x i64> %t1, %step193 %t3 = getelementptr [8 x i8], ptr %base, <vscale x 4 x i64> %t2194 %t4 = bitcast <vscale x 4 x ptr> %t3 to <vscale x 4 x ptr>195 %load = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr> %t4, i32 4, <vscale x 4 x i1> %pg, <vscale x 4 x i8> poison)196 ret <vscale x 4 x i8> %load197}198 199; Ensure the resulting load is "vscale x 4" wide, despite the offset giving the200; impression the gather must be split due to it's <vscale x 4 x i64> offset.201; gather_f32(base, index(offset, 8 * sizeof(float))202define <vscale x 4 x float> @gather_f32_index_offset_8(ptr %base, i64 %offset, <vscale x 4 x i1> %pg) #0 {203; CHECK-LABEL: gather_f32_index_offset_8:204; CHECK: // %bb.0:205; CHECK-NEXT: mov w8, #32 // =0x20206; CHECK-NEXT: index z0.s, #0, w8207; CHECK-NEXT: add x8, x0, x1, lsl #5208; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8, z0.s, sxtw]209; CHECK-NEXT: ret210 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0211 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer212 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()213 %t2 = add <vscale x 4 x i64> %t1, %step214 %t3 = getelementptr [8 x float], ptr %base, <vscale x 4 x i64> %t2215 %t4 = bitcast <vscale x 4 x ptr> %t3 to <vscale x 4 x ptr>216 %load = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %t4, i32 4, <vscale x 4 x i1> %pg, <vscale x 4 x float> poison)217 ret <vscale x 4 x float> %load218}219 220; Ensure the resulting store is "vscale x 4" wide, despite the offset giving the221; impression the scatter must be split due to it's <vscale x 4 x i64> offset.222; scatter_f16(base, index(offset, 8 * sizeof(i8))223define void @scatter_i8_index_offset_8(ptr %base, i64 %offset, <vscale x 4 x i1> %pg, <vscale x 4 x i8> %data) #0 {224; CHECK-LABEL: scatter_i8_index_offset_8:225; CHECK: // %bb.0:226; CHECK-NEXT: index z1.s, #0, #8227; CHECK-NEXT: add x8, x0, x1, lsl #3228; CHECK-NEXT: st1b { z0.s }, p0, [x8, z1.s, sxtw]229; CHECK-NEXT: ret230 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0231 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer232 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()233 %t2 = add <vscale x 4 x i64> %t1, %step234 %t3 = getelementptr [8 x i8], ptr %base, <vscale x 4 x i64> %t2235 %t4 = bitcast <vscale x 4 x ptr> %t3 to <vscale x 4 x ptr>236 call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t4, i32 2, <vscale x 4 x i1> %pg)237 ret void238}239 240; Ensure the resulting store is "vscale x 4" wide, despite the offset giving the241; impression the scatter must be split due to it's <vscale x 4 x i64> offset.242; scatter_f16(base, index(offset, 8 * sizeof(half))243define void @scatter_f16_index_offset_8(ptr %base, i64 %offset, <vscale x 4 x i1> %pg, <vscale x 4 x half> %data) #0 {244; CHECK-LABEL: scatter_f16_index_offset_8:245; CHECK: // %bb.0:246; CHECK-NEXT: mov w8, #16 // =0x10247; CHECK-NEXT: index z1.s, #0, w8248; CHECK-NEXT: add x8, x0, x1, lsl #4249; CHECK-NEXT: st1h { z0.s }, p0, [x8, z1.s, sxtw]250; CHECK-NEXT: ret251 %t0 = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0252 %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer253 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()254 %t2 = add <vscale x 4 x i64> %t1, %step255 %t3 = getelementptr [8 x half], ptr %base, <vscale x 4 x i64> %t2256 %t4 = bitcast <vscale x 4 x ptr> %t3 to <vscale x 4 x ptr>257 call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x ptr> %t4, i32 2, <vscale x 4 x i1> %pg)258 ret void259}260 261; stepvector is hidden further behind GEP and two adds.262define void @scatter_f16_index_add_add(ptr %base, i64 %offset, i64 %offset2, <vscale x 4 x i1> %pg, <vscale x 4 x half> %data) #0 {263; CHECK-LABEL: scatter_f16_index_add_add:264; CHECK: // %bb.0:265; CHECK-NEXT: mov w8, #16 // =0x10266; CHECK-NEXT: add x9, x0, x2, lsl #4267; CHECK-NEXT: index z1.s, #0, w8268; CHECK-NEXT: add x8, x9, x1, lsl #4269; CHECK-NEXT: st1h { z0.s }, p0, [x8, z1.s, sxtw]270; CHECK-NEXT: ret271 %splat.offset.ins = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0272 %splat.offset = shufflevector <vscale x 4 x i64> %splat.offset.ins, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer273 %splat.offset2.ins = insertelement <vscale x 4 x i64> poison, i64 %offset2, i32 0274 %splat.offset2 = shufflevector <vscale x 4 x i64> %splat.offset2.ins, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer275 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()276 %add1 = add <vscale x 4 x i64> %splat.offset, %step277 %add2 = add <vscale x 4 x i64> %add1, %splat.offset2278 %gep = getelementptr [8 x half], ptr %base, <vscale x 4 x i64> %add2279 %gep.bc = bitcast <vscale x 4 x ptr> %gep to <vscale x 4 x ptr>280 call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x ptr> %gep.bc, i32 2, <vscale x 4 x i1> %pg)281 ret void282}283 284; stepvector is hidden further behind GEP two adds and a shift.285define void @scatter_f16_index_add_add_mul(ptr %base, i64 %offset, i64 %offset2, <vscale x 4 x i1> %pg, <vscale x 4 x half> %data) #0 {286; CHECK-LABEL: scatter_f16_index_add_add_mul:287; CHECK: // %bb.0:288; CHECK-NEXT: mov w8, #128 // =0x80289; CHECK-NEXT: add x9, x0, x2, lsl #7290; CHECK-NEXT: index z1.s, #0, w8291; CHECK-NEXT: add x8, x9, x1, lsl #7292; CHECK-NEXT: st1h { z0.s }, p0, [x8, z1.s, sxtw]293; CHECK-NEXT: ret294 %splat.offset.ins = insertelement <vscale x 4 x i64> poison, i64 %offset, i32 0295 %splat.offset = shufflevector <vscale x 4 x i64> %splat.offset.ins, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer296 %splat.offset2.ins = insertelement <vscale x 4 x i64> poison, i64 %offset2, i32 0297 %splat.offset2 = shufflevector <vscale x 4 x i64> %splat.offset2.ins, <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer298 %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()299 %add1 = add <vscale x 4 x i64> %splat.offset, %step300 %add2 = add <vscale x 4 x i64> %add1, %splat.offset2301 %mul = mul <vscale x 4 x i64> %add2, splat(i64 8)302 %gep = getelementptr [8 x half], ptr %base, <vscale x 4 x i64> %mul303 %gep.bc = bitcast <vscale x 4 x ptr> %gep to <vscale x 4 x ptr>304 call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x ptr> %gep.bc, i32 2, <vscale x 4 x i1> %pg)305 ret void306}307 308define <vscale x 2 x i64> @masked_gather_nxv2i64_const_with_vec_offsets(<vscale x 2 x i64> %vector_offsets, <vscale x 2 x i1> %pg) #0 {309; CHECK-LABEL: masked_gather_nxv2i64_const_with_vec_offsets:310; CHECK: // %bb.0:311; CHECK-NEXT: mov w8, #8 // =0x8312; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8, z0.d, lsl #3]313; CHECK-NEXT: ret314 %ptrs = getelementptr i64, ptr inttoptr (i64 8 to ptr), <vscale x 2 x i64> %vector_offsets315 %data = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %pg, <vscale x 2 x i64> poison)316 ret <vscale x 2 x i64> %data317}318 319define <vscale x 2 x i64> @masked_gather_nxv2i64_null_with_vec_plus_scalar_offsets(<vscale x 2 x i64> %vector_offsets, i64 %scalar_offset, <vscale x 2 x i1> %pg) #0 {320; CHECK-LABEL: masked_gather_nxv2i64_null_with_vec_plus_scalar_offsets:321; CHECK: // %bb.0:322; CHECK-NEXT: lsl x8, x0, #3323; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8, z0.d, lsl #3]324; CHECK-NEXT: ret325 %scalar_offset.ins = insertelement <vscale x 2 x i64> poison, i64 %scalar_offset, i64 0326 %scalar_offset.splat = shufflevector <vscale x 2 x i64> %scalar_offset.ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer327 %offsets = add <vscale x 2 x i64> %vector_offsets, %scalar_offset.splat328 %ptrs = getelementptr i64, ptr null, <vscale x 2 x i64> %offsets329 %data = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %pg, <vscale x 2 x i64> poison)330 ret <vscale x 2 x i64> %data331}332 333define <vscale x 2 x i64> @masked_gather_nxv2i64_null_with__vec_plus_imm_offsets(<vscale x 2 x i64> %vector_offsets, <vscale x 2 x i1> %pg) #0 {334; CHECK-LABEL: masked_gather_nxv2i64_null_with__vec_plus_imm_offsets:335; CHECK: // %bb.0:336; CHECK-NEXT: mov w8, #8 // =0x8337; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8, z0.d, lsl #3]338; CHECK-NEXT: ret339 %offsets = add <vscale x 2 x i64> %vector_offsets, splat(i64 1)340 %ptrs = getelementptr i64, ptr null, <vscale x 2 x i64> %offsets341 %data = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %pg, <vscale x 2 x i64> poison)342 ret <vscale x 2 x i64> %data343}344 345define <vscale x 4 x i32> @masked_gather_nxv4i32_s8_offsets(ptr %base, <vscale x 4 x i8> %offsets, <vscale x 4 x i1> %mask) #0 {346; CHECK-LABEL: masked_gather_nxv4i32_s8_offsets:347; CHECK: // %bb.0:348; CHECK-NEXT: ptrue p1.s349; CHECK-NEXT: sxtb z0.s, p1/m, z0.s350; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]351; CHECK-NEXT: ret352 %offsets.sext = sext <vscale x 4 x i8> %offsets to <vscale x 4 x i32>353 %ptrs = getelementptr i32, ptr %base, <vscale x 4 x i32> %offsets.sext354 %data = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> poison)355 ret <vscale x 4 x i32> %data356}357 358define <vscale x 4 x i32> @masked_gather_nxv4i32_u8_offsets(ptr %base, <vscale x 4 x i8> %offsets, <vscale x 4 x i1> %mask) #0 {359; CHECK-LABEL: masked_gather_nxv4i32_u8_offsets:360; CHECK: // %bb.0:361; CHECK-NEXT: and z0.s, z0.s, #0xff362; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]363; CHECK-NEXT: ret364 %offsets.zext = zext <vscale x 4 x i8> %offsets to <vscale x 4 x i32>365 %ptrs = getelementptr i32, ptr %base, <vscale x 4 x i32> %offsets.zext366 %data = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> poison)367 ret <vscale x 4 x i32> %data368}369 370define <vscale x 4 x i32> @masked_gather_nxv4i32_u32s8_offsets(ptr %base, <vscale x 4 x i8> %offsets, <vscale x 4 x i1> %mask) #0 {371; CHECK-LABEL: masked_gather_nxv4i32_u32s8_offsets:372; CHECK: // %bb.0:373; CHECK-NEXT: ptrue p1.s374; CHECK-NEXT: sxtb z0.s, p1/m, z0.s375; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]376; CHECK-NEXT: ret377 %offsets.sext = sext <vscale x 4 x i8> %offsets to <vscale x 4 x i32>378 %offsets.sext.zext = zext <vscale x 4 x i32> %offsets.sext to <vscale x 4 x i64>379 %ptrs = getelementptr i32, ptr %base, <vscale x 4 x i64> %offsets.sext.zext380 %data = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> poison)381 ret <vscale x 4 x i32> %data382}383 384define void @masked_scatter_nxv2i64_const_with_vec_offsets(<vscale x 2 x i64> %vector_offsets, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %data) #0 {385; CHECK-LABEL: masked_scatter_nxv2i64_const_with_vec_offsets:386; CHECK: // %bb.0:387; CHECK-NEXT: mov w8, #8 // =0x8388; CHECK-NEXT: st1d { z1.d }, p0, [x8, z0.d, lsl #3]389; CHECK-NEXT: ret390 %ptrs = getelementptr i64, ptr inttoptr (i64 8 to ptr), <vscale x 2 x i64> %vector_offsets391 call void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %pg)392 ret void393}394 395define void @masked_scatter_nxv2i64_null_with_vec_plus_scalar_offsets(<vscale x 2 x i64> %vector_offsets, i64 %scalar_offset, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %data) #0 {396; CHECK-LABEL: masked_scatter_nxv2i64_null_with_vec_plus_scalar_offsets:397; CHECK: // %bb.0:398; CHECK-NEXT: lsl x8, x0, #3399; CHECK-NEXT: st1d { z1.d }, p0, [x8, z0.d, lsl #3]400; CHECK-NEXT: ret401 %scalar_offset.ins = insertelement <vscale x 2 x i64> poison, i64 %scalar_offset, i64 0402 %scalar_offset.splat = shufflevector <vscale x 2 x i64> %scalar_offset.ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer403 %offsets = add <vscale x 2 x i64> %vector_offsets, %scalar_offset.splat404 %ptrs = getelementptr i64, ptr null, <vscale x 2 x i64> %offsets405 call void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %pg)406 ret void407}408 409define void @masked_scatter_nxv2i64_null_with__vec_plus_imm_offsets(<vscale x 2 x i64> %vector_offsets, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %data) #0 {410; CHECK-LABEL: masked_scatter_nxv2i64_null_with__vec_plus_imm_offsets:411; CHECK: // %bb.0:412; CHECK-NEXT: mov w8, #8 // =0x8413; CHECK-NEXT: st1d { z1.d }, p0, [x8, z0.d, lsl #3]414; CHECK-NEXT: ret415 %offsets = add <vscale x 2 x i64> %vector_offsets, splat(i64 1)416 %ptrs = getelementptr i64, ptr null, <vscale x 2 x i64> %offsets417 call void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %pg)418 ret void419}420 421define void @masked_scatter_nxv4i32_s8_offsets(ptr %base, <vscale x 4 x i8> %offsets, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %data) #0 {422; CHECK-LABEL: masked_scatter_nxv4i32_s8_offsets:423; CHECK: // %bb.0:424; CHECK-NEXT: ptrue p1.s425; CHECK-NEXT: sxtb z0.s, p1/m, z0.s426; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, sxtw #2]427; CHECK-NEXT: ret428 %offsets.sext = sext <vscale x 4 x i8> %offsets to <vscale x 4 x i32>429 %ptrs = getelementptr i32, ptr %base, <vscale x 4 x i32> %offsets.sext430 call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask)431 ret void432}433 434define void @masked_scatter_nxv4i32_u8_offsets(ptr %base, <vscale x 4 x i8> %offsets, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %data) #0 {435; CHECK-LABEL: masked_scatter_nxv4i32_u8_offsets:436; CHECK: // %bb.0:437; CHECK-NEXT: and z0.s, z0.s, #0xff438; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, uxtw #2]439; CHECK-NEXT: ret440 %offsets.zext = zext <vscale x 4 x i8> %offsets to <vscale x 4 x i32>441 %ptrs = getelementptr i32, ptr %base, <vscale x 4 x i32> %offsets.zext442 call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask)443 ret void444}445 446define void @masked_scatter_nxv4i32_u32s8_offsets(ptr %base, <vscale x 4 x i8> %offsets, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %data) #0 {447; CHECK-LABEL: masked_scatter_nxv4i32_u32s8_offsets:448; CHECK: // %bb.0:449; CHECK-NEXT: ptrue p1.s450; CHECK-NEXT: sxtb z0.s, p1/m, z0.s451; CHECK-NEXT: st1w { z1.s }, p0, [x0, z0.s, uxtw #2]452; CHECK-NEXT: ret453 %offsets.sext = sext <vscale x 4 x i8> %offsets to <vscale x 4 x i32>454 %offsets.sext.zext = zext <vscale x 4 x i32> %offsets.sext to <vscale x 4 x i64>455 %ptrs = getelementptr i32, ptr %base, <vscale x 4 x i64> %offsets.sext.zext456 call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask)457 ret void458}459 460attributes #0 = { "target-features"="+sve" vscale_range(1, 16) }461 462declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)463declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)464declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)465declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)466 467declare void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)468declare void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x ptr>, i32, <vscale x 4 x i1>)469declare void @llvm.masked.scatter.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x ptr>, i32, <vscale x 4 x i1>)470declare void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x ptr>, i32, <vscale x 4 x i1>)471declare void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half>, <vscale x 4 x ptr>, i32, <vscale x 4 x i1>)472 473declare <vscale x 4 x i64> @llvm.stepvector.nxv4i64()474