944 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S < %s -passes=gvn,dce | FileCheck --check-prefixes=CHECK,MDEP %s3; RUN: opt -S < %s -passes='gvn<memoryssa>',dce | FileCheck --check-prefixes=CHECK,MSSA %s4 5; Analyze Load from clobbering Load.6 7define <vscale x 4 x i32> @load_store_clobber_load(ptr %p) {8; MDEP-LABEL: @load_store_clobber_load(9; MDEP-NEXT: [[LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[P:%.*]], align 1610; MDEP-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr undef, align 1611; MDEP-NEXT: [[ADD:%.*]] = add <vscale x 4 x i32> [[LOAD1]], [[LOAD1]]12; MDEP-NEXT: ret <vscale x 4 x i32> [[ADD]]13;14; MSSA-LABEL: @load_store_clobber_load(15; MSSA-NEXT: [[LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[P:%.*]], align 1616; MSSA-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr undef, align 1617; MSSA-NEXT: [[LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 1618; MSSA-NEXT: [[ADD:%.*]] = add <vscale x 4 x i32> [[LOAD1]], [[LOAD2]]19; MSSA-NEXT: ret <vscale x 4 x i32> [[ADD]]20;21 %load1 = load <vscale x 4 x i32>, ptr %p22 store <vscale x 4 x i32> zeroinitializer, ptr undef23 %load2 = load <vscale x 4 x i32>, ptr %p ; <- load to be eliminated24 %add = add <vscale x 4 x i32> %load1, %load225 ret <vscale x 4 x i32> %add26}27 28define <vscale x 4 x i32> @load_store_clobber_load_mayalias(ptr %p, ptr %p2) {29; CHECK-LABEL: @load_store_clobber_load_mayalias(30; CHECK-NEXT: [[LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[P:%.*]], align 1631; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[P2:%.*]], align 1632; CHECK-NEXT: [[LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 1633; CHECK-NEXT: [[SUB:%.*]] = sub <vscale x 4 x i32> [[LOAD1]], [[LOAD2]]34; CHECK-NEXT: ret <vscale x 4 x i32> [[SUB]]35;36 %load1 = load <vscale x 4 x i32>, ptr %p37 store <vscale x 4 x i32> zeroinitializer, ptr %p238 %load2 = load <vscale x 4 x i32>, ptr %p39 %sub = sub <vscale x 4 x i32> %load1, %load240 ret <vscale x 4 x i32> %sub41}42 43define <vscale x 4 x i32> @load_store_clobber_load_noalias(ptr noalias %p, ptr noalias %p2) {44; MDEP-LABEL: @load_store_clobber_load_noalias(45; MDEP-NEXT: [[LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[P:%.*]], align 1646; MDEP-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[P2:%.*]], align 1647; MDEP-NEXT: [[ADD:%.*]] = add <vscale x 4 x i32> [[LOAD1]], [[LOAD1]]48; MDEP-NEXT: ret <vscale x 4 x i32> [[ADD]]49;50; MSSA-LABEL: @load_store_clobber_load_noalias(51; MSSA-NEXT: [[LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[P:%.*]], align 1652; MSSA-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[P2:%.*]], align 1653; MSSA-NEXT: [[LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 1654; MSSA-NEXT: [[ADD:%.*]] = add <vscale x 4 x i32> [[LOAD1]], [[LOAD2]]55; MSSA-NEXT: ret <vscale x 4 x i32> [[ADD]]56;57 %load1 = load <vscale x 4 x i32>, ptr %p58 store <vscale x 4 x i32> zeroinitializer, ptr %p259 %load2 = load <vscale x 4 x i32>, ptr %p ; <- load to be eliminated60 %add = add <vscale x 4 x i32> %load1, %load261 ret <vscale x 4 x i32> %add62}63 64; BasicAA return MayAlias for %gep1,%gep2, could improve as MustAlias.65define i32 @load_clobber_load_gep1(ptr %p) {66; MDEP-LABEL: @load_clobber_load_gep1(67; MDEP-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 0, i64 168; MDEP-NEXT: [[LOAD1:%.*]] = load i32, ptr [[GEP1]], align 469; MDEP-NEXT: [[ADD:%.*]] = add i32 [[LOAD1]], [[LOAD1]]70; MDEP-NEXT: ret i32 [[ADD]]71;72; MSSA-LABEL: @load_clobber_load_gep1(73; MSSA-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 0, i64 174; MSSA-NEXT: [[LOAD1:%.*]] = load i32, ptr [[GEP1]], align 475; MSSA-NEXT: [[LOAD2:%.*]] = load i32, ptr [[GEP1]], align 476; MSSA-NEXT: [[ADD:%.*]] = add i32 [[LOAD1]], [[LOAD2]]77; MSSA-NEXT: ret i32 [[ADD]]78;79 %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 0, i64 180 %load1 = load i32, ptr %gep181 %gep2 = getelementptr i32, ptr %p, i64 182 %load2 = load i32, ptr %gep2 ; <- load could be eliminated83 %add = add i32 %load1, %load284 ret i32 %add85}86 87define i32 @load_clobber_load_gep2(ptr %p) {88; CHECK-LABEL: @load_clobber_load_gep2(89; CHECK-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 1, i64 090; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr [[GEP1]], align 491; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i32, ptr [[P]], i64 492; CHECK-NEXT: [[LOAD2:%.*]] = load i32, ptr [[GEP2]], align 493; CHECK-NEXT: [[ADD:%.*]] = add i32 [[LOAD1]], [[LOAD2]]94; CHECK-NEXT: ret i32 [[ADD]]95;96 %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 097 %load1 = load i32, ptr %gep198 %gep2 = getelementptr i32, ptr %p, i64 499 %load2 = load i32, ptr %gep2 ; <- can not determine at compile-time if %load1 and %load2 are same addr100 %add = add i32 %load1, %load2101 ret i32 %add102}103 104; TODO: BasicAA return MayAlias for %gep1,%gep2, could improve as MustAlias.105define i32 @load_clobber_load_gep3(ptr %p) {106; CHECK-LABEL: @load_clobber_load_gep3(107; CHECK-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 1, i64 0108; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr [[GEP1]], align 4109; CHECK-NEXT: [[GEP2:%.*]] = getelementptr <vscale x 4 x float>, ptr [[P]], i64 1, i64 0110; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4111; CHECK-NEXT: [[CAST:%.*]] = bitcast float [[LOAD2]] to i32112; CHECK-NEXT: [[ADD:%.*]] = add i32 [[LOAD1]], [[CAST]]113; CHECK-NEXT: ret i32 [[ADD]]114;115 %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 0116 %load1 = load i32, ptr %gep1117 %gep2 = getelementptr <vscale x 4 x float>, ptr %p, i64 1, i64 0118 %load2 = load float, ptr %gep2 ; <- load could be eliminated119 %cast = bitcast float %load2 to i32120 %add = add i32 %load1, %cast121 ret i32 %add122}123 124define <vscale x 4 x i32> @load_clobber_load_fence(ptr %p) {125; CHECK-LABEL: @load_clobber_load_fence(126; CHECK-NEXT: [[LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[P:%.*]], align 16127; CHECK-NEXT: call void asm "", "~{memory}"()128; CHECK-NEXT: [[LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16129; CHECK-NEXT: [[SUB:%.*]] = sub <vscale x 4 x i32> [[LOAD1]], [[LOAD2]]130; CHECK-NEXT: ret <vscale x 4 x i32> [[SUB]]131;132 %load1 = load <vscale x 4 x i32>, ptr %p133 call void asm "", "~{memory}"()134 %load2 = load <vscale x 4 x i32>, ptr %p135 %sub = sub <vscale x 4 x i32> %load1, %load2136 ret <vscale x 4 x i32> %sub137}138 139define <vscale x 4 x i32> @load_clobber_load_sideeffect(ptr %p) {140; CHECK-LABEL: @load_clobber_load_sideeffect(141; CHECK-NEXT: [[LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[P:%.*]], align 16142; CHECK-NEXT: call void asm sideeffect "", ""()143; CHECK-NEXT: [[LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16144; CHECK-NEXT: [[ADD:%.*]] = add <vscale x 4 x i32> [[LOAD1]], [[LOAD2]]145; CHECK-NEXT: ret <vscale x 4 x i32> [[ADD]]146;147 %load1 = load <vscale x 4 x i32>, ptr %p148 call void asm sideeffect "", ""()149 %load2 = load <vscale x 4 x i32>, ptr %p150 %add = add <vscale x 4 x i32> %load1, %load2151 ret <vscale x 4 x i32> %add152}153 154; Analyze Load from clobbering Store.155 156define <vscale x 4 x i32> @store_forward_to_load(ptr %p) {157; MDEP-LABEL: @store_forward_to_load(158; MDEP-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[P:%.*]], align 16159; MDEP-NEXT: ret <vscale x 4 x i32> zeroinitializer160;161; MSSA-LABEL: @store_forward_to_load(162; MSSA-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[P:%.*]], align 16163; MSSA-NEXT: [[LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16164; MSSA-NEXT: ret <vscale x 4 x i32> [[LOAD]]165;166 store <vscale x 4 x i32> zeroinitializer, ptr %p167 %load = load <vscale x 4 x i32>, ptr %p168 ret <vscale x 4 x i32> %load169}170 171define <vscale x 4 x i32> @store_forward_to_load_sideeffect(ptr %p) {172; CHECK-LABEL: @store_forward_to_load_sideeffect(173; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[P:%.*]], align 16174; CHECK-NEXT: call void asm sideeffect "", ""()175; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16176; CHECK-NEXT: ret <vscale x 4 x i32> [[LOAD]]177;178 store <vscale x 4 x i32> zeroinitializer, ptr %p179 call void asm sideeffect "", ""()180 %load = load <vscale x 4 x i32>, ptr %p181 ret <vscale x 4 x i32> %load182}183 184define i32 @store_clobber_load() {185; CHECK-LABEL: @store_clobber_load(186; CHECK-NEXT: [[ALLOC:%.*]] = alloca <vscale x 4 x i32>, align 16187; CHECK-NEXT: store <vscale x 4 x i32> undef, ptr [[ALLOC]], align 16188; CHECK-NEXT: [[PTR:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[ALLOC]], i32 0, i32 1189; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[PTR]], align 4190; CHECK-NEXT: ret i32 [[LOAD]]191;192 %alloc = alloca <vscale x 4 x i32>193 store <vscale x 4 x i32> undef, ptr %alloc194 %ptr = getelementptr <vscale x 4 x i32>, ptr %alloc, i32 0, i32 1195 %load = load i32, ptr %ptr196 ret i32 %load197}198 199; Analyze Load from clobbering MemInst.200 201declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1)202 203define i32 @memset_clobber_load(ptr %p) {204; MDEP-LABEL: @memset_clobber_load(205; MDEP-NEXT: tail call void @llvm.memset.p0.i64(ptr [[P:%.*]], i8 1, i64 200, i1 false)206; MDEP-NEXT: ret i32 16843009207;208; MSSA-LABEL: @memset_clobber_load(209; MSSA-NEXT: tail call void @llvm.memset.p0.i64(ptr [[P:%.*]], i8 1, i64 200, i1 false)210; MSSA-NEXT: [[GEP:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 0, i64 5211; MSSA-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4212; MSSA-NEXT: ret i32 [[LOAD]]213;214 tail call void @llvm.memset.p0.i64(ptr %p, i8 1, i64 200, i1 false)215 %gep = getelementptr <vscale x 4 x i32>, ptr %p, i64 0, i64 5216 %load = load i32, ptr %gep217 ret i32 %load218}219 220define i32 @memset_clobber_load_vscaled_base(ptr %p) {221; CHECK-LABEL: @memset_clobber_load_vscaled_base(222; CHECK-NEXT: tail call void @llvm.memset.p0.i64(ptr [[P:%.*]], i8 1, i64 200, i1 false)223; CHECK-NEXT: [[GEP:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 1, i64 1224; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4225; CHECK-NEXT: ret i32 [[LOAD]]226;227 tail call void @llvm.memset.p0.i64(ptr %p, i8 1, i64 200, i1 false)228 %gep = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 1229 %load = load i32, ptr %gep230 ret i32 %load231}232 233define i32 @memset_clobber_load_nonconst_index(ptr %p, i64 %idx1, i64 %idx2) {234; CHECK-LABEL: @memset_clobber_load_nonconst_index(235; CHECK-NEXT: tail call void @llvm.memset.p0.i64(ptr [[P:%.*]], i8 1, i64 200, i1 false)236; CHECK-NEXT: [[GEP:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[IDX1:%.*]], i64 [[IDX2:%.*]]237; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4238; CHECK-NEXT: ret i32 [[LOAD]]239;240 tail call void @llvm.memset.p0.i64(ptr %p, i8 1, i64 200, i1 false)241 %gep = getelementptr <vscale x 4 x i32>, ptr %p, i64 %idx1, i64 %idx2242 %load = load i32, ptr %gep243 ret i32 %load244}245 246 247; Load elimination across BBs248 249define ptr @load_from_alloc_replaced_with_undef() {250; MDEP-LABEL: @load_from_alloc_replaced_with_undef(251; MDEP-NEXT: entry:252; MDEP-NEXT: [[A:%.*]] = alloca <vscale x 4 x i32>, align 16253; MDEP-NEXT: br i1 undef, label [[IF_END:%.*]], label [[IF_THEN:%.*]]254; MDEP: if.then:255; MDEP-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[A]], align 16256; MDEP-NEXT: br label [[IF_END]]257; MDEP: if.end:258; MDEP-NEXT: ret ptr [[A]]259;260; MSSA-LABEL: @load_from_alloc_replaced_with_undef(261; MSSA-NEXT: entry:262; MSSA-NEXT: [[A:%.*]] = alloca <vscale x 4 x i32>, align 16263; MSSA-NEXT: [[GEP:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[A]], i64 0, i64 1264; MSSA-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4265; MSSA-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[LOAD]], 0266; MSSA-NEXT: br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]267; MSSA: if.then:268; MSSA-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[A]], align 16269; MSSA-NEXT: br label [[IF_END]]270; MSSA: if.end:271; MSSA-NEXT: ret ptr [[A]]272;273entry:274 %a = alloca <vscale x 4 x i32>275 %gep = getelementptr <vscale x 4 x i32>, ptr %a, i64 0, i64 1276 %load = load i32, ptr %gep ; <- load to be eliminated277 %tobool = icmp eq i32 %load, 0 ; <- icmp to be eliminated278 br i1 %tobool, label %if.end, label %if.then279 280if.then:281 store <vscale x 4 x i32> zeroinitializer, ptr %a282 br label %if.end283 284if.end:285 ret ptr %a286}287 288define i32 @redundant_load_elimination_1(ptr %p) {289; MDEP-LABEL: @redundant_load_elimination_1(290; MDEP-NEXT: entry:291; MDEP-NEXT: [[GEP:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 1, i64 1292; MDEP-NEXT: [[LOAD1:%.*]] = load i32, ptr [[GEP]], align 4293; MDEP-NEXT: [[CMP:%.*]] = icmp eq i32 [[LOAD1]], 0294; MDEP-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]295; MDEP: if.then:296; MDEP-NEXT: br label [[IF_END]]297; MDEP: if.end:298; MDEP-NEXT: ret i32 [[LOAD1]]299;300; MSSA-LABEL: @redundant_load_elimination_1(301; MSSA-NEXT: entry:302; MSSA-NEXT: [[GEP:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 1, i64 1303; MSSA-NEXT: [[LOAD1:%.*]] = load i32, ptr [[GEP]], align 4304; MSSA-NEXT: [[CMP:%.*]] = icmp eq i32 [[LOAD1]], 0305; MSSA-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]306; MSSA: if.then:307; MSSA-NEXT: [[LOAD2:%.*]] = load i32, ptr [[GEP]], align 4308; MSSA-NEXT: br label [[IF_END]]309; MSSA: if.end:310; MSSA-NEXT: [[RESULT:%.*]] = phi i32 [ [[LOAD2]], [[IF_THEN]] ], [ [[LOAD1]], [[ENTRY:%.*]] ]311; MSSA-NEXT: ret i32 [[RESULT]]312;313entry:314 %gep = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 1315 %load1 = load i32, ptr %gep316 %cmp = icmp eq i32 %load1, 0317 br i1 %cmp, label %if.then, label %if.end318 319if.then:320 %load2 = load i32, ptr %gep ; <- load to be eliminated321 %add = add i32 %load1, %load2322 br label %if.end323 324if.end:325 %result = phi i32 [ %add, %if.then ], [ %load1, %entry ]326 ret i32 %result327}328 329; TODO: BasicAA return MayAlias for %gep1,%gep2, could improve as NoAlias.330define void @redundant_load_elimination_2(i1 %c, ptr %p, ptr %q) {331; CHECK-LABEL: @redundant_load_elimination_2(332; CHECK-NEXT: entry:333; CHECK-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 1, i64 1334; CHECK-NEXT: store i32 0, ptr [[GEP1]], align 4335; CHECK-NEXT: [[GEP2:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 1, i64 0336; CHECK-NEXT: store i32 1, ptr [[GEP2]], align 4337; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]338; CHECK: if.then:339; CHECK-NEXT: [[T:%.*]] = load i32, ptr [[GEP1]], align 4340; CHECK-NEXT: store i32 [[T]], ptr [[Q:%.*]], align 4341; CHECK-NEXT: ret void342; CHECK: if.else:343; CHECK-NEXT: ret void344;345entry:346 %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 1347 store i32 0, ptr %gep1348 %gep2 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1, i64 0349 store i32 1, ptr %gep2350 br i1 %c, label %if.else, label %if.then351 352if.then:353 %t = load i32, ptr %gep1 ; <- load could be eliminated354 store i32 %t, ptr %q355 ret void356 357if.else:358 ret void359}360 361define void @redundant_load_elimination_zero_index(i1 %c, ptr %p, ptr %q) {362; MDEP-LABEL: @redundant_load_elimination_zero_index(363; MDEP-NEXT: entry:364; MDEP-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 0, i64 1365; MDEP-NEXT: store i32 0, ptr [[GEP1]], align 4366; MDEP-NEXT: store i32 1, ptr [[P]], align 4367; MDEP-NEXT: br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]368; MDEP: if.then:369; MDEP-NEXT: store i32 0, ptr [[Q:%.*]], align 4370; MDEP-NEXT: ret void371; MDEP: if.else:372; MDEP-NEXT: ret void373;374; MSSA-LABEL: @redundant_load_elimination_zero_index(375; MSSA-NEXT: entry:376; MSSA-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 0, i64 1377; MSSA-NEXT: store i32 0, ptr [[GEP1]], align 4378; MSSA-NEXT: store i32 1, ptr [[P]], align 4379; MSSA-NEXT: br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]380; MSSA: if.then:381; MSSA-NEXT: [[T:%.*]] = load i32, ptr [[GEP1]], align 4382; MSSA-NEXT: store i32 [[T]], ptr [[Q:%.*]], align 4383; MSSA-NEXT: ret void384; MSSA: if.else:385; MSSA-NEXT: ret void386;387entry:388 %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 0, i64 1389 store i32 0, ptr %gep1390 store i32 1, ptr %p391 br i1 %c, label %if.else, label %if.then392 393if.then:394 %t = load i32, ptr %gep1 ; <- load could be eliminated395 store i32 %t, ptr %q396 ret void397 398if.else:399 ret void400}401 402define void @redundant_load_elimination_zero_index_1(i1 %c, ptr %p, ptr %q, i64 %i) {403; MDEP-LABEL: @redundant_load_elimination_zero_index_1(404; MDEP-NEXT: entry:405; MDEP-NEXT: [[J:%.*]] = add i64 [[I:%.*]], 1406; MDEP-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 0, i64 [[J]]407; MDEP-NEXT: store i32 0, ptr [[GEP1]], align 4408; MDEP-NEXT: [[GEP2:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 0, i64 [[I]]409; MDEP-NEXT: store i32 1, ptr [[GEP2]], align 4410; MDEP-NEXT: br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]411; MDEP: if.then:412; MDEP-NEXT: store i32 0, ptr [[Q:%.*]], align 4413; MDEP-NEXT: ret void414; MDEP: if.else:415; MDEP-NEXT: ret void416;417; MSSA-LABEL: @redundant_load_elimination_zero_index_1(418; MSSA-NEXT: entry:419; MSSA-NEXT: [[J:%.*]] = add i64 [[I:%.*]], 1420; MSSA-NEXT: [[GEP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P:%.*]], i64 0, i64 [[J]]421; MSSA-NEXT: store i32 0, ptr [[GEP1]], align 4422; MSSA-NEXT: [[GEP2:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 0, i64 [[I]]423; MSSA-NEXT: store i32 1, ptr [[GEP2]], align 4424; MSSA-NEXT: br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]425; MSSA: if.then:426; MSSA-NEXT: [[T:%.*]] = load i32, ptr [[GEP1]], align 4427; MSSA-NEXT: store i32 [[T]], ptr [[Q:%.*]], align 4428; MSSA-NEXT: ret void429; MSSA: if.else:430; MSSA-NEXT: ret void431;432entry:433 %j = add i64 %i, 1434 %gep1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 0, i64 %j435 store i32 0, ptr %gep1436 %gep2 = getelementptr <vscale x 4 x i32>, ptr %p, i64 0, i64 %i437 store i32 1, ptr %gep2438 br i1 %c, label %if.else, label %if.then439 440if.then:441 %t = load i32, ptr %gep1 ; <- load could be eliminated442 store i32 %t, ptr %q443 ret void444 445if.else:446 ret void447}448; TODO: load in if.then could have been eliminated449define void @missing_load_elimination(i1 %c, ptr %p, ptr %q, <vscale x 4 x i32> %v) {450; CHECK-LABEL: @missing_load_elimination(451; CHECK-NEXT: entry:452; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[P:%.*]], align 16453; CHECK-NEXT: [[P1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 1454; CHECK-NEXT: store <vscale x 4 x i32> [[V:%.*]], ptr [[P1]], align 16455; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]456; CHECK: if.then:457; CHECK-NEXT: [[T:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16458; CHECK-NEXT: store <vscale x 4 x i32> [[T]], ptr [[Q:%.*]], align 16459; CHECK-NEXT: ret void460; CHECK: if.else:461; CHECK-NEXT: ret void462;463entry:464 store <vscale x 4 x i32> zeroinitializer, ptr %p465 %p1 = getelementptr <vscale x 4 x i32>, ptr %p, i64 1466 store <vscale x 4 x i32> %v, ptr %p1467 br i1 %c, label %if.else, label %if.then468 469if.then:470 %t = load <vscale x 4 x i32>, ptr %p ; load could be eliminated471 store <vscale x 4 x i32> %t, ptr %q472 ret void473 474if.else:475 ret void476}477 478; Different sizes / types479 480define <vscale x 16 x i8> @load_v16i8_store_v4i32_forward_load(ptr %p, <vscale x 4 x i32> %x) {481; MDEP-LABEL: @load_v16i8_store_v4i32_forward_load(482; MDEP-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16483; MDEP-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 16 x i8>484; MDEP-NEXT: ret <vscale x 16 x i8> [[TMP1]]485;486; MSSA-LABEL: @load_v16i8_store_v4i32_forward_load(487; MSSA-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16488; MSSA-NEXT: [[LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[P]], align 16489; MSSA-NEXT: ret <vscale x 16 x i8> [[LOAD]]490;491 store <vscale x 4 x i32> %x, ptr %p492 %load = load <vscale x 16 x i8>, ptr %p493 ret <vscale x 16 x i8> %load494}495 496define <vscale x 4 x float> @load_v4f32_store_v4i32_forward_load(ptr %p, <vscale x 4 x i32> %x) {497; MDEP-LABEL: @load_v4f32_store_v4i32_forward_load(498; MDEP-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16499; MDEP-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 4 x float>500; MDEP-NEXT: ret <vscale x 4 x float> [[TMP1]]501;502; MSSA-LABEL: @load_v4f32_store_v4i32_forward_load(503; MSSA-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16504; MSSA-NEXT: [[LOAD:%.*]] = load <vscale x 4 x float>, ptr [[P]], align 16505; MSSA-NEXT: ret <vscale x 4 x float> [[LOAD]]506;507 store <vscale x 4 x i32> %x, ptr %p508 %load = load <vscale x 4 x float>, ptr %p509 ret <vscale x 4 x float> %load510}511 512define <vscale x 4 x float> @load_v4f32_store_v16i8_forward_load(ptr %p, <vscale x 16 x i8> %x) {513; MDEP-LABEL: @load_v4f32_store_v16i8_forward_load(514; MDEP-NEXT: store <vscale x 16 x i8> [[X:%.*]], ptr [[P:%.*]], align 16515; MDEP-NEXT: [[TMP1:%.*]] = bitcast <vscale x 16 x i8> [[X]] to <vscale x 4 x float>516; MDEP-NEXT: ret <vscale x 4 x float> [[TMP1]]517;518; MSSA-LABEL: @load_v4f32_store_v16i8_forward_load(519; MSSA-NEXT: store <vscale x 16 x i8> [[X:%.*]], ptr [[P:%.*]], align 16520; MSSA-NEXT: [[LOAD:%.*]] = load <vscale x 4 x float>, ptr [[P]], align 16521; MSSA-NEXT: ret <vscale x 4 x float> [[LOAD]]522;523 store <vscale x 16 x i8> %x, ptr %p524 %load = load <vscale x 4 x float>, ptr %p525 ret <vscale x 4 x float> %load526}527 528define <vscale x 4 x i32> @load_v4i32_store_v4f32_forward_load(ptr %p, <vscale x 4 x float> %x) {529; MDEP-LABEL: @load_v4i32_store_v4f32_forward_load(530; MDEP-NEXT: store <vscale x 4 x float> [[X:%.*]], ptr [[P:%.*]], align 16531; MDEP-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x float> [[X]] to <vscale x 4 x i32>532; MDEP-NEXT: ret <vscale x 4 x i32> [[TMP1]]533;534; MSSA-LABEL: @load_v4i32_store_v4f32_forward_load(535; MSSA-NEXT: store <vscale x 4 x float> [[X:%.*]], ptr [[P:%.*]], align 16536; MSSA-NEXT: [[LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16537; MSSA-NEXT: ret <vscale x 4 x i32> [[LOAD]]538;539 store <vscale x 4 x float> %x, ptr %p540 %load = load <vscale x 4 x i32>, ptr %p541 ret <vscale x 4 x i32> %load542}543 544define <vscale x 4 x i32> @load_v4i32_store_v4i64_forward_load(ptr %p, <vscale x 4 x i64> %x) {545; CHECK-LABEL: @load_v4i32_store_v4i64_forward_load(546; CHECK-NEXT: store <vscale x 4 x i64> [[X:%.*]], ptr [[P:%.*]], align 32547; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16548; CHECK-NEXT: ret <vscale x 4 x i32> [[LOAD]]549;550 store <vscale x 4 x i64> %x, ptr %p551 %load = load <vscale x 4 x i32>, ptr %p552 ret <vscale x 4 x i32> %load553}554 555define <vscale x 4 x i64> @load_v4i64_store_v4i32_forward_load(ptr %p, <vscale x 4 x i32> %x) {556; CHECK-LABEL: @load_v4i64_store_v4i32_forward_load(557; CHECK-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16558; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[P]], align 32559; CHECK-NEXT: ret <vscale x 4 x i64> [[LOAD]]560;561 store <vscale x 4 x i32> %x, ptr %p562 %load = load <vscale x 4 x i64>, ptr %p563 ret <vscale x 4 x i64> %load564}565 566define <vscale x 2 x i32> @load_v2i32_store_v4i32_forward_load(ptr %p, <vscale x 4 x i32> %x) {567; CHECK-LABEL: @load_v2i32_store_v4i32_forward_load(568; CHECK-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16569; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[P]], align 8570; CHECK-NEXT: ret <vscale x 2 x i32> [[LOAD]]571;572 store <vscale x 4 x i32> %x, ptr %p573 %load = load <vscale x 2 x i32>, ptr %p574 ret <vscale x 2 x i32> %load575}576 577define <vscale x 2 x i32> @load_v2i32_store_v4i32_forward_load_offsets(ptr %p, <vscale x 4 x i32> %x) {578; CHECK-LABEL: @load_v2i32_store_v4i32_forward_load_offsets(579; CHECK-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16580; CHECK-NEXT: [[Q:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[P]], i64 1581; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[Q]], align 8582; CHECK-NEXT: ret <vscale x 2 x i32> [[LOAD]]583;584 store <vscale x 4 x i32> %x, ptr %p585 %q = getelementptr <vscale x 2 x i32>, ptr %p, i64 1586 %load = load <vscale x 2 x i32>, ptr %q587 ret <vscale x 2 x i32> %load588}589 590define <vscale x 2 x i32> @load_v2i32_store_v4i32_forward_load_offsetc(ptr %p, <vscale x 4 x i32> %x) {591; CHECK-LABEL: @load_v2i32_store_v4i32_forward_load_offsetc(592; CHECK-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16593; CHECK-NEXT: [[Q:%.*]] = getelementptr <2 x i32>, ptr [[P]], i64 1594; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[Q]], align 8595; CHECK-NEXT: ret <vscale x 2 x i32> [[LOAD]]596;597 store <vscale x 4 x i32> %x, ptr %p598 %q = getelementptr <2 x i32>, ptr %p, i64 1599 %load = load <vscale x 2 x i32>, ptr %q600 ret <vscale x 2 x i32> %load601}602 603define <vscale x 2 x ptr> @load_v2p0_store_v4i32_forward_load(ptr %p, <vscale x 4 x i32> %x) {604; MDEP-LABEL: @load_v2p0_store_v4i32_forward_load(605; MDEP-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16606; MDEP-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 2 x i64>607; MDEP-NEXT: [[TMP2:%.*]] = inttoptr <vscale x 2 x i64> [[TMP1]] to <vscale x 2 x ptr>608; MDEP-NEXT: ret <vscale x 2 x ptr> [[TMP2]]609;610; MSSA-LABEL: @load_v2p0_store_v4i32_forward_load(611; MSSA-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16612; MSSA-NEXT: [[LOAD:%.*]] = load <vscale x 2 x ptr>, ptr [[P]], align 16613; MSSA-NEXT: ret <vscale x 2 x ptr> [[LOAD]]614;615 store <vscale x 4 x i32> %x, ptr %p616 %load = load <vscale x 2 x ptr>, ptr %p617 ret <vscale x 2 x ptr> %load618}619 620define <vscale x 2 x i64> @load_v2i64_store_v2p0_forward_load(ptr %p, <vscale x 2 x ptr> %x) {621; MDEP-LABEL: @load_v2i64_store_v2p0_forward_load(622; MDEP-NEXT: store <vscale x 2 x ptr> [[X:%.*]], ptr [[P:%.*]], align 16623; MDEP-NEXT: [[TMP1:%.*]] = ptrtoint <vscale x 2 x ptr> [[X]] to <vscale x 2 x i64>624; MDEP-NEXT: ret <vscale x 2 x i64> [[TMP1]]625;626; MSSA-LABEL: @load_v2i64_store_v2p0_forward_load(627; MSSA-NEXT: store <vscale x 2 x ptr> [[X:%.*]], ptr [[P:%.*]], align 16628; MSSA-NEXT: [[LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16629; MSSA-NEXT: ret <vscale x 2 x i64> [[LOAD]]630;631 store <vscale x 2 x ptr> %x, ptr %p632 %load = load <vscale x 2 x i64>, ptr %p633 ret <vscale x 2 x i64> %load634}635 636define <vscale x 16 x i8> @load_nxv16i8_store_v4i32_forward_load(ptr %p, <4 x i32> %x) {637; CHECK-LABEL: @load_nxv16i8_store_v4i32_forward_load(638; CHECK-NEXT: store <4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16639; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[P]], align 16640; CHECK-NEXT: ret <vscale x 16 x i8> [[LOAD]]641;642 store <4 x i32> %x, ptr %p643 %load = load <vscale x 16 x i8>, ptr %p644 ret <vscale x 16 x i8> %load645}646 647define <16 x i8> @load_v16i8_store_nxv4i32_forward_load(ptr %p, <vscale x 4 x i32> %x) {648; CHECK-LABEL: @load_v16i8_store_nxv4i32_forward_load(649; CHECK-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16650; CHECK-NEXT: [[LOAD:%.*]] = load <16 x i8>, ptr [[P]], align 16651; CHECK-NEXT: ret <16 x i8> [[LOAD]]652;653 store <vscale x 4 x i32> %x, ptr %p654 %load = load <16 x i8>, ptr %p655 ret <16 x i8> %load656}657 658define <vscale x 16 x i8> @load_v16i8_store_v4i32_forward_constant(ptr %p) {659; MDEP-LABEL: @load_v16i8_store_v4i32_forward_constant(660; MDEP-NEXT: store <vscale x 4 x i32> splat (i32 4), ptr [[P:%.*]], align 16661; MDEP-NEXT: ret <vscale x 16 x i8> bitcast (<vscale x 4 x i32> splat (i32 4) to <vscale x 16 x i8>)662;663; MSSA-LABEL: @load_v16i8_store_v4i32_forward_constant(664; MSSA-NEXT: store <vscale x 4 x i32> splat (i32 4), ptr [[P:%.*]], align 16665; MSSA-NEXT: [[LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[P]], align 16666; MSSA-NEXT: ret <vscale x 16 x i8> [[LOAD]]667;668 store <vscale x 4 x i32> splat (i32 4), ptr %p669 %load = load <vscale x 16 x i8>, ptr %p670 ret <vscale x 16 x i8> %load671}672 673define <vscale x 16 x i8> @load_v16i8_struct_store_v4i32_forward_load(ptr %p, { <vscale x 4 x i32> } %x) {674; CHECK-LABEL: @load_v16i8_struct_store_v4i32_forward_load(675; CHECK-NEXT: store { <vscale x 4 x i32> } [[X:%.*]], ptr [[P:%.*]], align 16676; CHECK-NEXT: [[LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[P]], align 16677; CHECK-NEXT: ret <vscale x 16 x i8> [[LOAD]]678;679 store { <vscale x 4 x i32> } %x, ptr %p680 %load = load <vscale x 16 x i8>, ptr %p681 ret <vscale x 16 x i8> %load682}683 684define {<vscale x 16 x i8>} @load_v16i8_store_v4i32_struct_forward_load(ptr %p, <vscale x 4 x i32> %x) {685; CHECK-LABEL: @load_v16i8_store_v4i32_struct_forward_load(686; CHECK-NEXT: store <vscale x 4 x i32> [[X:%.*]], ptr [[P:%.*]], align 16687; CHECK-NEXT: [[LOAD:%.*]] = load { <vscale x 16 x i8> }, ptr [[P]], align 16688; CHECK-NEXT: ret { <vscale x 16 x i8> } [[LOAD]]689;690 store <vscale x 4 x i32> %x, ptr %p691 %load = load { <vscale x 16 x i8> }, ptr %p692 ret { <vscale x 16 x i8> } %load693}694 695define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @bigexample({ <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %a) vscale_range(1,16) {696; MDEP-LABEL: @bigexample(697; MDEP-NEXT: entry:698; MDEP-NEXT: [[REF_TMP:%.*]] = alloca { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }, align 16699; MDEP-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[REF_TMP]])700; MDEP-NEXT: [[A_ELT:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A:%.*]], 0701; MDEP-NEXT: store <vscale x 4 x i32> [[A_ELT]], ptr [[REF_TMP]], align 16702; MDEP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()703; MDEP-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4704; MDEP-NEXT: [[REF_TMP_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[REF_TMP]], i64 [[TMP1]]705; MDEP-NEXT: [[A_ELT2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A]], 1706; MDEP-NEXT: store <vscale x 4 x i32> [[A_ELT2]], ptr [[REF_TMP_REPACK1]], align 16707; MDEP-NEXT: [[TMP2:%.*]] = shl i64 [[TMP0]], 5708; MDEP-NEXT: [[REF_TMP_REPACK3:%.*]] = getelementptr inbounds i8, ptr [[REF_TMP]], i64 [[TMP2]]709; MDEP-NEXT: [[A_ELT4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A]], 2710; MDEP-NEXT: store <vscale x 4 x i32> [[A_ELT4]], ptr [[REF_TMP_REPACK3]], align 16711; MDEP-NEXT: [[TMP3:%.*]] = mul i64 [[TMP0]], 48712; MDEP-NEXT: [[REF_TMP_REPACK5:%.*]] = getelementptr inbounds i8, ptr [[REF_TMP]], i64 [[TMP3]]713; MDEP-NEXT: [[A_ELT6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A]], 3714; MDEP-NEXT: store <vscale x 4 x i32> [[A_ELT6]], ptr [[REF_TMP_REPACK5]], align 16715; MDEP-NEXT: [[TMP4:%.*]] = bitcast <vscale x 4 x i32> [[A_ELT]] to <vscale x 16 x i8>716; MDEP-NEXT: [[TMP5:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> [[TMP4]], 0717; MDEP-NEXT: [[TMP6:%.*]] = bitcast <vscale x 4 x i32> [[A_ELT2]] to <vscale x 16 x i8>718; MDEP-NEXT: [[TMP7:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP5]], <vscale x 16 x i8> [[TMP6]], 1719; MDEP-NEXT: [[TMP8:%.*]] = bitcast <vscale x 4 x i32> [[A_ELT4]] to <vscale x 16 x i8>720; MDEP-NEXT: [[TMP9:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP7]], <vscale x 16 x i8> [[TMP8]], 2721; MDEP-NEXT: [[TMP10:%.*]] = bitcast <vscale x 4 x i32> [[A_ELT6]] to <vscale x 16 x i8>722; MDEP-NEXT: [[TMP11:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP9]], <vscale x 16 x i8> [[TMP10]], 3723; MDEP-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[REF_TMP]])724; MDEP-NEXT: ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP11]]725;726; MSSA-LABEL: @bigexample(727; MSSA-NEXT: entry:728; MSSA-NEXT: [[REF_TMP:%.*]] = alloca { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }, align 16729; MSSA-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[REF_TMP]])730; MSSA-NEXT: [[A_ELT:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A:%.*]], 0731; MSSA-NEXT: store <vscale x 4 x i32> [[A_ELT]], ptr [[REF_TMP]], align 16732; MSSA-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()733; MSSA-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 4734; MSSA-NEXT: [[REF_TMP_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[REF_TMP]], i64 [[TMP1]]735; MSSA-NEXT: [[A_ELT2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A]], 1736; MSSA-NEXT: store <vscale x 4 x i32> [[A_ELT2]], ptr [[REF_TMP_REPACK1]], align 16737; MSSA-NEXT: [[TMP2:%.*]] = shl i64 [[TMP0]], 5738; MSSA-NEXT: [[REF_TMP_REPACK3:%.*]] = getelementptr inbounds i8, ptr [[REF_TMP]], i64 [[TMP2]]739; MSSA-NEXT: [[A_ELT4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A]], 2740; MSSA-NEXT: store <vscale x 4 x i32> [[A_ELT4]], ptr [[REF_TMP_REPACK3]], align 16741; MSSA-NEXT: [[TMP3:%.*]] = mul i64 [[TMP0]], 48742; MSSA-NEXT: [[REF_TMP_REPACK5:%.*]] = getelementptr inbounds i8, ptr [[REF_TMP]], i64 [[TMP3]]743; MSSA-NEXT: [[A_ELT6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A]], 3744; MSSA-NEXT: store <vscale x 4 x i32> [[A_ELT6]], ptr [[REF_TMP_REPACK5]], align 16745; MSSA-NEXT: [[DOTUNPACK:%.*]] = load <vscale x 16 x i8>, ptr [[REF_TMP]], align 16746; MSSA-NEXT: [[TMP4:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> [[DOTUNPACK]], 0747; MSSA-NEXT: [[DOTUNPACK8:%.*]] = load <vscale x 16 x i8>, ptr [[REF_TMP_REPACK1]], align 16748; MSSA-NEXT: [[TMP5:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], <vscale x 16 x i8> [[DOTUNPACK8]], 1749; MSSA-NEXT: [[DOTUNPACK10:%.*]] = load <vscale x 16 x i8>, ptr [[REF_TMP_REPACK3]], align 16750; MSSA-NEXT: [[TMP6:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP5]], <vscale x 16 x i8> [[DOTUNPACK10]], 2751; MSSA-NEXT: [[DOTUNPACK12:%.*]] = load <vscale x 16 x i8>, ptr [[REF_TMP_REPACK5]], align 16752; MSSA-NEXT: [[TMP7:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP6]], <vscale x 16 x i8> [[DOTUNPACK12]], 3753; MSSA-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[REF_TMP]])754; MSSA-NEXT: ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP7]]755;756entry:757 %ref.tmp = alloca { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }, align 16758 call void @llvm.lifetime.start.p0(ptr nonnull %ref.tmp)759 %a.elt = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %a, 0760 store <vscale x 4 x i32> %a.elt, ptr %ref.tmp, align 16761 %0 = call i64 @llvm.vscale.i64()762 %1 = shl i64 %0, 4763 %ref.tmp.repack1 = getelementptr inbounds i8, ptr %ref.tmp, i64 %1764 %a.elt2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %a, 1765 store <vscale x 4 x i32> %a.elt2, ptr %ref.tmp.repack1, align 16766 %2 = call i64 @llvm.vscale.i64()767 %3 = shl i64 %2, 5768 %ref.tmp.repack3 = getelementptr inbounds i8, ptr %ref.tmp, i64 %3769 %a.elt4 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %a, 2770 store <vscale x 4 x i32> %a.elt4, ptr %ref.tmp.repack3, align 16771 %4 = call i64 @llvm.vscale.i64()772 %5 = mul i64 %4, 48773 %ref.tmp.repack5 = getelementptr inbounds i8, ptr %ref.tmp, i64 %5774 %a.elt6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %a, 3775 store <vscale x 4 x i32> %a.elt6, ptr %ref.tmp.repack5, align 16776 %.unpack = load <vscale x 16 x i8>, ptr %ref.tmp, align 16777 %6 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> %.unpack, 0778 %7 = call i64 @llvm.vscale.i64()779 %8 = shl i64 %7, 4780 %.elt7 = getelementptr inbounds i8, ptr %ref.tmp, i64 %8781 %.unpack8 = load <vscale x 16 x i8>, ptr %.elt7, align 16782 %9 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, <vscale x 16 x i8> %.unpack8, 1783 %10 = call i64 @llvm.vscale.i64()784 %11 = shl i64 %10, 5785 %.elt9 = getelementptr inbounds i8, ptr %ref.tmp, i64 %11786 %.unpack10 = load <vscale x 16 x i8>, ptr %.elt9, align 16787 %12 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %9, <vscale x 16 x i8> %.unpack10, 2788 %13 = call i64 @llvm.vscale.i64()789 %14 = mul i64 %13, 48790 %.elt11 = getelementptr inbounds i8, ptr %ref.tmp, i64 %14791 %.unpack12 = load <vscale x 16 x i8>, ptr %.elt11, align 16792 %15 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %12, <vscale x 16 x i8> %.unpack12, 3793 call void @llvm.lifetime.end.p0(ptr nonnull %ref.tmp)794 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %15795}796 797define <vscale x 4 x float> @scalable_store_to_fixed_load(<vscale x 4 x float> %.coerce) vscale_range(4,4) {798; MDEP-LABEL: @scalable_store_to_fixed_load(799; MDEP-NEXT: entry:800; MDEP-NEXT: [[RETVAL:%.*]] = alloca { <16 x float> }, align 64801; MDEP-NEXT: [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]802; MDEP-NEXT: store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16803; MDEP-NEXT: ret <vscale x 4 x float> [[TMP0]]804;805; MSSA-LABEL: @scalable_store_to_fixed_load(806; MSSA-NEXT: entry:807; MSSA-NEXT: [[RETVAL:%.*]] = alloca { <16 x float> }, align 64808; MSSA-NEXT: [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]809; MSSA-NEXT: store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16810; MSSA-NEXT: [[TMP1:%.*]] = load <16 x float>, ptr [[RETVAL]], align 64811; MSSA-NEXT: [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP1]], i64 0)812; MSSA-NEXT: ret <vscale x 4 x float> [[CAST_SCALABLE]]813;814entry:815 %retval = alloca { <16 x float> }816 %0 = fadd <vscale x 4 x float> %.coerce, %.coerce817 store <vscale x 4 x float> %0, ptr %retval818 %1 = load <16 x float>, ptr %retval819 %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)820 ret <vscale x 4 x float> %cast.scalable821}822 823; Here, only the lower bound for the vscale is known, but this is enough to allow a forward to a load to 16 elements.824define <vscale x 4 x float> @scalable_store_to_fixed_load_only_lower_bound(<vscale x 4 x float> %a) vscale_range(4) {825; MDEP-LABEL: @scalable_store_to_fixed_load_only_lower_bound(826; MDEP-NEXT: entry:827; MDEP-NEXT: [[RETVAL:%.*]] = alloca { <vscale x 4 x float> }, align 16828; MDEP-NEXT: store <vscale x 4 x float> [[A:%.*]], ptr [[RETVAL]], align 16829; MDEP-NEXT: ret <vscale x 4 x float> [[A]]830;831; MSSA-LABEL: @scalable_store_to_fixed_load_only_lower_bound(832; MSSA-NEXT: entry:833; MSSA-NEXT: [[RETVAL:%.*]] = alloca { <vscale x 4 x float> }, align 16834; MSSA-NEXT: store <vscale x 4 x float> [[A:%.*]], ptr [[RETVAL]], align 16835; MSSA-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[RETVAL]], align 64836; MSSA-NEXT: [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP0]], i64 0)837; MSSA-NEXT: ret <vscale x 4 x float> [[CAST_SCALABLE]]838;839entry:840 %retval = alloca { <vscale x 4 x float> }841 store <vscale x 4 x float> %a, ptr %retval842 %1 = load <16 x float>, ptr %retval843 %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)844 ret <vscale x 4 x float> %cast.scalable845}846 847define <vscale x 4 x float> @scalable_store_to_fixed_load_with_offset(<vscale x 4 x float> %a) vscale_range(4,4) {848; CHECK-LABEL: @scalable_store_to_fixed_load_with_offset(849; CHECK-NEXT: entry:850; CHECK-NEXT: [[PTR:%.*]] = alloca { <32 x float> }, align 128851; CHECK-NEXT: store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16852; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 8853; CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[GEP]], align 64854; CHECK-NEXT: [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP0]], i64 0)855; CHECK-NEXT: ret <vscale x 4 x float> [[CAST_SCALABLE]]856;857entry:858 %ptr = alloca { <32 x float> }859 store <vscale x 4 x float> %a, ptr %ptr860 %gep = getelementptr inbounds i8, ptr %ptr, i64 8861 %1 = load <16 x float>, ptr %gep862 %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)863 ret <vscale x 4 x float> %cast.scalable864}865 866define <vscale x 4 x float> @scalable_store_to_fixed_load_unknown_vscale(<vscale x 4 x float> %.coerce) {867; CHECK-LABEL: @scalable_store_to_fixed_load_unknown_vscale(868; CHECK-NEXT: entry:869; CHECK-NEXT: [[RETVAL:%.*]] = alloca { <16 x float> }, align 64870; CHECK-NEXT: [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]871; CHECK-NEXT: store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16872; CHECK-NEXT: [[TMP1:%.*]] = load <16 x float>, ptr [[RETVAL]], align 64873; CHECK-NEXT: [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> [[TMP1]], i64 0)874; CHECK-NEXT: ret <vscale x 4 x float> [[CAST_SCALABLE]]875;876entry:877 %retval = alloca { <16 x float> }878 %0 = fadd <vscale x 4 x float> %.coerce, %.coerce879 store <vscale x 4 x float> %0, ptr %retval880 %1 = load <16 x float>, ptr %retval881 %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v16f32(<vscale x 4 x float> poison, <16 x float> %1, i64 0)882 ret <vscale x 4 x float> %cast.scalable883}884 885define <vscale x 4 x float> @scalable_store_to_fixed_load_size_missmatch(<vscale x 4 x float> %.coerce) vscale_range(4,4) {886; CHECK-LABEL: @scalable_store_to_fixed_load_size_missmatch(887; CHECK-NEXT: entry:888; CHECK-NEXT: [[RETVAL:%.*]] = alloca { <32 x float> }, align 128889; CHECK-NEXT: [[TMP0:%.*]] = fadd <vscale x 4 x float> [[DOTCOERCE:%.*]], [[DOTCOERCE]]890; CHECK-NEXT: store <vscale x 4 x float> [[TMP0]], ptr [[RETVAL]], align 16891; CHECK-NEXT: [[TMP1:%.*]] = load <32 x float>, ptr [[RETVAL]], align 128892; CHECK-NEXT: [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v32f32(<vscale x 4 x float> poison, <32 x float> [[TMP1]], i64 0)893; CHECK-NEXT: ret <vscale x 4 x float> [[CAST_SCALABLE]]894;895entry:896 %retval = alloca { <32 x float> }897 %0 = fadd <vscale x 4 x float> %.coerce, %.coerce898 store <vscale x 4 x float> %0, ptr %retval899 %1 = load <32 x float>, ptr %retval900 %cast.scalable = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v32f32(<vscale x 4 x float> poison, <32 x float> %1, i64 0)901 ret <vscale x 4 x float> %cast.scalable902}903 904define <vscale x 4 x i32> @scalable_store_to_fixed_load_different_types(<vscale x 4 x float> %a) vscale_range(4,4) {905; CHECK-LABEL: @scalable_store_to_fixed_load_different_types(906; CHECK-NEXT: entry:907; CHECK-NEXT: [[PTR:%.*]] = alloca { <16 x float> }, align 64908; CHECK-NEXT: store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16909; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[PTR]], align 64910; CHECK-NEXT: [[CAST_SCALABLE:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> [[TMP0]], i64 0)911; CHECK-NEXT: ret <vscale x 4 x i32> [[CAST_SCALABLE]]912;913entry:914 %ptr = alloca { <16 x float> }915 store <vscale x 4 x float> %a, ptr %ptr916 %1 = load <16 x i32>, ptr %ptr917 %cast.scalable = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> poison, <16 x i32> %1, i64 0)918 ret <vscale x 4 x i32> %cast.scalable919}920 921; This function does not have a fixed vscale, but the loaded vector is still known922; to be smaller or equal in size compared to the stored vector.923define <4 x float> @scalable_store_to_small_fixed_load(<vscale x 4 x float> %a) {924; MDEP-LABEL: @scalable_store_to_small_fixed_load(925; MDEP-NEXT: entry:926; MDEP-NEXT: [[PTR:%.*]] = alloca <vscale x 4 x float>, align 16927; MDEP-NEXT: store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16928; MDEP-NEXT: [[TMP0:%.*]] = call <4 x float> @llvm.vector.extract.v4f32.nxv4f32(<vscale x 4 x float> [[A]], i64 0)929; MDEP-NEXT: ret <4 x float> [[TMP0]]930;931; MSSA-LABEL: @scalable_store_to_small_fixed_load(932; MSSA-NEXT: entry:933; MSSA-NEXT: [[PTR:%.*]] = alloca <vscale x 4 x float>, align 16934; MSSA-NEXT: store <vscale x 4 x float> [[A:%.*]], ptr [[PTR]], align 16935; MSSA-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[PTR]], align 16936; MSSA-NEXT: ret <4 x float> [[TMP0]]937;938entry:939 %ptr = alloca <vscale x 4 x float>940 store <vscale x 4 x float> %a, ptr %ptr941 %1 = load <4 x float>, ptr %ptr942 ret <4 x float> %1943}944