1107 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt %s -S -riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+v,+zvl256b | FileCheck %s --check-prefixes=CHECK,V3; RUN: opt %s -S -riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+f,+zve32f,+zvl256b | FileCheck %s --check-prefixes=CHECK,ZVE32F4 5%struct.foo = type { i32, i32, i32, i32 }6 7; void gather(signed char * __restrict A, signed char * __restrict B) {8; for (int i = 0; i != 1024; ++i)9; A[i] += B[i * 5];10; }11define void @gather(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {12; CHECK-LABEL: @gather(13; CHECK-NEXT: entry:14; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]15; CHECK: vector.body:16; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]17; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]18; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]]19; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 5, <32 x i1> splat (i1 true), i32 32)20; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]21; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I2]], align 122; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[TMP1]]23; CHECK-NEXT: store <32 x i8> [[I4]], ptr [[I2]], align 124; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 3225; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 16026; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 102427; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]28; CHECK: for.cond.cleanup:29; CHECK-NEXT: ret void30;31entry:32 br label %vector.body33 34vector.body: ; preds = %vector.body, %entry35 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]36 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]37 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)38 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i39 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)40 %i2 = getelementptr inbounds i8, ptr %A, i64 %index41 %wide.load = load <32 x i8>, ptr %i2, align 142 %i4 = add <32 x i8> %wide.load, %wide.masked.gather43 store <32 x i8> %i4, ptr %i2, align 144 %index.next = add nuw i64 %index, 3245 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)46 %i6 = icmp eq i64 %index.next, 102447 br i1 %i6, label %for.cond.cleanup, label %vector.body48 49for.cond.cleanup: ; preds = %vector.body50 ret void51}52 53define void @gather_masked(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, <32 x i8> %maskedoff) {54; CHECK-LABEL: @gather_masked(55; CHECK-NEXT: entry:56; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]57; CHECK: vector.body:58; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]59; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]60; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]]61; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 5, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, i32 32)62; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = select <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <32 x i8> [[TMP1]], <32 x i8> [[MASKEDOFF:%.*]]63; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]64; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I2]], align 165; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]66; CHECK-NEXT: store <32 x i8> [[I4]], ptr [[I2]], align 167; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 3268; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 16069; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 102470; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]71; CHECK: for.cond.cleanup:72; CHECK-NEXT: ret void73;74entry:75 br label %vector.body76 77vector.body: ; preds = %vector.body, %entry78 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]79 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]80 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)81 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i82 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <32 x i8> %maskedoff)83 %i2 = getelementptr inbounds i8, ptr %A, i64 %index84 %wide.load = load <32 x i8>, ptr %i2, align 185 %i4 = add <32 x i8> %wide.load, %wide.masked.gather86 store <32 x i8> %i4, ptr %i2, align 187 %index.next = add nuw i64 %index, 3288 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)89 %i6 = icmp eq i64 %index.next, 102490 br i1 %i6, label %for.cond.cleanup, label %vector.body91 92for.cond.cleanup: ; preds = %vector.body93 ret void94}95 96define void @gather_negative_stride(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {97; CHECK-LABEL: @gather_negative_stride(98; CHECK-NEXT: entry:99; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]100; CHECK: vector.body:101; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]102; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 155, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]103; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]]104; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 -5, <32 x i1> splat (i1 true), i32 32)105; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]106; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I2]], align 1107; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[TMP1]]108; CHECK-NEXT: store <32 x i8> [[I4]], ptr [[I2]], align 1109; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32110; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 160111; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024112; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]113; CHECK: for.cond.cleanup:114; CHECK-NEXT: ret void115;116entry:117 br label %vector.body118 119vector.body: ; preds = %vector.body, %entry120 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]121 %vec.ind = phi <32 x i64> [ <i64 31, i64 30, i64 29, i64 28, i64 27, i64 26, i64 25, i64 24, i64 23, i64 22, i64 21, i64 20, i64 19, i64 18, i64 17, i64 16, i64 15, i64 14, i64 13, i64 12, i64 11, i64 10, i64 9, i64 8, i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, %entry ], [ %vec.ind.next, %vector.body ]122 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)123 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i124 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)125 %i2 = getelementptr inbounds i8, ptr %A, i64 %index126 %wide.load = load <32 x i8>, ptr %i2, align 1127 %i4 = add <32 x i8> %wide.load, %wide.masked.gather128 store <32 x i8> %i4, ptr %i2, align 1129 %index.next = add nuw i64 %index, 32130 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)131 %i6 = icmp eq i64 %index.next, 1024132 br i1 %i6, label %for.cond.cleanup, label %vector.body133 134for.cond.cleanup: ; preds = %vector.body135 ret void136}137 138define void @gather_zero_stride(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {139; CHECK-LABEL: @gather_zero_stride(140; CHECK-NEXT: entry:141; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]142; CHECK: vector.body:143; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]144; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]145; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]]146; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 0, <32 x i1> splat (i1 true), i32 32)147; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]148; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I2]], align 1149; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[TMP1]]150; CHECK-NEXT: store <32 x i8> [[I4]], ptr [[I2]], align 1151; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32152; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 160153; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024154; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]155; CHECK: for.cond.cleanup:156; CHECK-NEXT: ret void157;158entry:159 br label %vector.body160 161vector.body: ; preds = %vector.body, %entry162 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]163 %vec.ind = phi <32 x i64> [ zeroinitializer, %entry ], [ %vec.ind.next, %vector.body ]164 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)165 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i166 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)167 %i2 = getelementptr inbounds i8, ptr %A, i64 %index168 %wide.load = load <32 x i8>, ptr %i2, align 1169 %i4 = add <32 x i8> %wide.load, %wide.masked.gather170 store <32 x i8> %i4, ptr %i2, align 1171 %index.next = add nuw i64 %index, 32172 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)173 %i6 = icmp eq i64 %index.next, 1024174 br i1 %i6, label %for.cond.cleanup, label %vector.body175 176for.cond.cleanup: ; preds = %vector.body177 ret void178}179 180;void scatter(signed char * __restrict A, signed char * __restrict B) {181; for (int i = 0; i < 1024; ++i)182; A[i * 5] += B[i];183;}184define void @scatter(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {185; CHECK-LABEL: @scatter(186; CHECK-NEXT: entry:187; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]188; CHECK: vector.body:189; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]190; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]191; CHECK-NEXT: [[I:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 [[INDEX]]192; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I]], align 1193; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]]194; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 5, <32 x i1> splat (i1 true), i32 32)195; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[TMP1]], [[WIDE_LOAD]]196; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v32i8.p0.i64(<32 x i8> [[I4]], ptr [[TMP0]], i64 5, <32 x i1> splat (i1 true), i32 32)197; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32198; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 160199; CHECK-NEXT: [[I5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024200; CHECK-NEXT: br i1 [[I5]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]201; CHECK: for.cond.cleanup:202; CHECK-NEXT: ret void203;204entry:205 br label %vector.body206 207vector.body: ; preds = %vector.body, %entry208 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]209 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]210 %i = getelementptr inbounds i8, ptr %B, i64 %index211 %wide.load = load <32 x i8>, ptr %i, align 1212 %i2 = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)213 %i3 = getelementptr inbounds i8, ptr %A, <32 x i64> %i2214 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i3, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)215 %i4 = add <32 x i8> %wide.masked.gather, %wide.load216 call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> %i4, <32 x ptr> %i3, i32 1, <32 x i1> splat (i1 true))217 %index.next = add nuw i64 %index, 32218 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)219 %i5 = icmp eq i64 %index.next, 1024220 br i1 %i5, label %for.cond.cleanup, label %vector.body221 222for.cond.cleanup: ; preds = %vector.body223 ret void224}225 226define void @scatter_masked(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, <32 x i8> %maskedoff) {227; CHECK-LABEL: @scatter_masked(228; CHECK-NEXT: entry:229; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]230; CHECK: vector.body:231; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]232; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]233; CHECK-NEXT: [[I:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 [[INDEX]]234; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I]], align 1235; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]]236; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 5, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, i32 32)237; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = select <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <32 x i8> [[TMP1]], <32 x i8> [[MASKEDOFF:%.*]]238; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]]239; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v32i8.p0.i64(<32 x i8> [[I4]], ptr [[TMP0]], i64 5, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, i32 32)240; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32241; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 160242; CHECK-NEXT: [[I5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024243; CHECK-NEXT: br i1 [[I5]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]244; CHECK: for.cond.cleanup:245; CHECK-NEXT: ret void246;247entry:248 br label %vector.body249 250vector.body: ; preds = %vector.body, %entry251 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]252 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]253 %i = getelementptr inbounds i8, ptr %B, i64 %index254 %wide.load = load <32 x i8>, ptr %i, align 1255 %i2 = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)256 %i3 = getelementptr inbounds i8, ptr %A, <32 x i64> %i2257 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i3, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <32 x i8> %maskedoff)258 %i4 = add <32 x i8> %wide.masked.gather, %wide.load259 call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> %i4, <32 x ptr> %i3, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)260 %index.next = add nuw i64 %index, 32261 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)262 %i5 = icmp eq i64 %index.next, 1024263 br i1 %i5, label %for.cond.cleanup, label %vector.body264 265for.cond.cleanup: ; preds = %vector.body266 ret void267}268 269; void gather_pow2(signed char * __restrict A, signed char * __restrict B) {270; for (int i = 0; i != 1024; ++i)271; A[i] += B[i * 4];272; }273define void @gather_pow2(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {274; CHECK-LABEL: @gather_pow2(275; CHECK-NEXT: entry:276; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]277; CHECK: vector.body:278; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]279; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]280; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]]281; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP0]], i64 16, <8 x i1> splat (i1 true), i32 8)282; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]283; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[I2]], align 1284; CHECK-NEXT: [[I4:%.*]] = add <8 x i32> [[WIDE_LOAD]], [[TMP1]]285; CHECK-NEXT: store <8 x i32> [[I4]], ptr [[I2]], align 1286; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8287; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 32288; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024289; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]290; CHECK: for.cond.cleanup:291; CHECK-NEXT: ret void292;293entry:294 br label %vector.body295 296vector.body: ; preds = %vector.body, %entry297 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]298 %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]299 %i = shl nsw <8 x i64> %vec.ind, splat (i64 2)300 %i1 = getelementptr inbounds i32, ptr %B, <8 x i64> %i301 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)302 %i2 = getelementptr inbounds i32, ptr %A, i64 %index303 %wide.load = load <8 x i32>, ptr %i2, align 1304 %i4 = add <8 x i32> %wide.load, %wide.masked.gather305 store <8 x i32> %i4, ptr %i2, align 1306 %index.next = add nuw i64 %index, 8307 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 8)308 %i6 = icmp eq i64 %index.next, 1024309 br i1 %i6, label %for.cond.cleanup, label %vector.body310 311for.cond.cleanup: ; preds = %vector.body312 ret void313}314 315define void @gather_unknown_pow2(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, i64 %shift) {316; CHECK-LABEL: @gather_unknown_pow2(317; CHECK-NEXT: entry:318; CHECK-NEXT: [[STRIDE:%.*]] = shl i64 1, [[SHIFT:%.*]]319; CHECK-NEXT: [[STEP:%.*]] = shl i64 8, [[SHIFT]]320; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[STRIDE]], 4321; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]322; CHECK: vector.body:323; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]324; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]325; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]]326; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP1]], i64 [[TMP0]], <8 x i1> splat (i1 true), i32 8)327; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]328; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[I2]], align 1329; CHECK-NEXT: [[I4:%.*]] = add <8 x i32> [[WIDE_LOAD]], [[TMP2]]330; CHECK-NEXT: store <8 x i32> [[I4]], ptr [[I2]], align 1331; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8332; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], [[STEP]]333; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024334; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]335; CHECK: for.cond.cleanup:336; CHECK-NEXT: ret void337;338entry:339 %.splatinsert = insertelement <8 x i64> poison, i64 %shift, i64 0340 %.splat = shufflevector <8 x i64> %.splatinsert, <8 x i64> poison, <8 x i32> zeroinitializer341 br label %vector.body342 343vector.body: ; preds = %vector.body, %entry344 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]345 %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]346 %i = shl nsw <8 x i64> %vec.ind, %.splat347 %i1 = getelementptr inbounds i32, ptr %B, <8 x i64> %i348 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)349 %i2 = getelementptr inbounds i32, ptr %A, i64 %index350 %wide.load = load <8 x i32>, ptr %i2, align 1351 %i4 = add <8 x i32> %wide.load, %wide.masked.gather352 store <8 x i32> %i4, ptr %i2, align 1353 %index.next = add nuw i64 %index, 8354 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 8)355 %i6 = icmp eq i64 %index.next, 1024356 br i1 %i6, label %for.cond.cleanup, label %vector.body357 358for.cond.cleanup: ; preds = %vector.body359 ret void360}361 362define void @negative_shl_non_commute(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, i64 %shift) {363; CHECK-LABEL: @negative_shl_non_commute(364; CHECK-NEXT: entry:365; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[SHIFT:%.*]], i64 0366; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i64> [[DOTSPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer367; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]368; CHECK: vector.body:369; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]370; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]371; CHECK-NEXT: [[I:%.*]] = shl nsw <8 x i64> [[DOTSPLAT]], [[VEC_IND]]372; CHECK-NEXT: [[I1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], <8 x i64> [[I]]373; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[I1]], <8 x i1> splat (i1 true), <8 x i32> poison)374; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]375; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[I2]], align 1376; CHECK-NEXT: [[I4:%.*]] = add <8 x i32> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]377; CHECK-NEXT: store <8 x i32> [[I4]], ptr [[I2]], align 1378; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8379; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)380; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024381; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]382; CHECK: for.cond.cleanup:383; CHECK-NEXT: ret void384;385entry:386 %.splatinsert = insertelement <8 x i64> poison, i64 %shift, i64 0387 %.splat = shufflevector <8 x i64> %.splatinsert, <8 x i64> poison, <8 x i32> zeroinitializer388 br label %vector.body389 390vector.body: ; preds = %vector.body, %entry391 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]392 %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]393 %i = shl nsw <8 x i64> %.splat, %vec.ind394 %i1 = getelementptr inbounds i32, ptr %B, <8 x i64> %i395 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)396 %i2 = getelementptr inbounds i32, ptr %A, i64 %index397 %wide.load = load <8 x i32>, ptr %i2, align 1398 %i4 = add <8 x i32> %wide.load, %wide.masked.gather399 store <8 x i32> %i4, ptr %i2, align 1400 %index.next = add nuw i64 %index, 8401 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 8)402 %i6 = icmp eq i64 %index.next, 1024403 br i1 %i6, label %for.cond.cleanup, label %vector.body404 405for.cond.cleanup: ; preds = %vector.body406 ret void407}408 409;void scatter_pow2(signed char * __restrict A, signed char * __restrict B) {410; for (int i = 0; i < 1024; ++i)411; A[i * 4] += B[i];412;}413define void @scatter_pow2(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {414; CHECK-LABEL: @scatter_pow2(415; CHECK-NEXT: entry:416; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]417; CHECK: vector.body:418; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]419; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]420; CHECK-NEXT: [[I:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]421; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[I]], align 1422; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]]423; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP0]], i64 16, <8 x i1> splat (i1 true), i32 8)424; CHECK-NEXT: [[I4:%.*]] = add <8 x i32> [[TMP1]], [[WIDE_LOAD]]425; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i32.p0.i64(<8 x i32> [[I4]], ptr [[TMP0]], i64 16, <8 x i1> splat (i1 true), i32 8)426; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8427; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 32428; CHECK-NEXT: [[I5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024429; CHECK-NEXT: br i1 [[I5]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]430; CHECK: for.cond.cleanup:431; CHECK-NEXT: ret void432;433entry:434 br label %vector.body435 436vector.body: ; preds = %vector.body, %entry437 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]438 %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]439 %i = getelementptr inbounds i32, ptr %B, i64 %index440 %wide.load = load <8 x i32>, ptr %i, align 1441 %i2 = shl nuw nsw <8 x i64> %vec.ind, splat (i64 2)442 %i3 = getelementptr inbounds i32, ptr %A, <8 x i64> %i2443 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i3, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)444 %i4 = add <8 x i32> %wide.masked.gather, %wide.load445 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i4, <8 x ptr> %i3, i32 4, <8 x i1> splat (i1 true))446 %index.next = add nuw i64 %index, 8447 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 8)448 %i5 = icmp eq i64 %index.next, 1024449 br i1 %i5, label %for.cond.cleanup, label %vector.body450 451for.cond.cleanup: ; preds = %vector.body452 ret void453}454 455;struct foo {456; int a, b, c, d;457;};458;459;void struct_gather(int * __restrict A, struct foo * __restrict B) {460; for (int i = 0; i < 1024; ++i)461; A[i] += B[i].b;462;}463define void @struct_gather(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {464; CHECK-LABEL: @struct_gather(465; CHECK-NEXT: entry:466; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]467; CHECK: vector.body:468; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]469; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]470; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 8, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR2:%.*]], [[VECTOR_BODY]] ]471; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]], i32 1472; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [[STRUCT_FOO]], ptr [[B]], i64 [[VEC_IND_SCALAR1]], i32 1473; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP0]], i64 16, <8 x i1> splat (i1 true), i32 8)474; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP1]], i64 16, <8 x i1> splat (i1 true), i32 8)475; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]476; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[I2]], align 4477; CHECK-NEXT: [[I4:%.*]] = getelementptr inbounds i32, ptr [[I2]], i64 8478; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i32>, ptr [[I4]], align 4479; CHECK-NEXT: [[I6:%.*]] = add nsw <8 x i32> [[WIDE_LOAD]], [[TMP2]]480; CHECK-NEXT: [[I7:%.*]] = add nsw <8 x i32> [[WIDE_LOAD10]], [[TMP3]]481; CHECK-NEXT: store <8 x i32> [[I6]], ptr [[I2]], align 4482; CHECK-NEXT: store <8 x i32> [[I7]], ptr [[I4]], align 4483; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16484; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 16485; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR2]] = add i64 [[VEC_IND_SCALAR1]], 16486; CHECK-NEXT: [[I10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024487; CHECK-NEXT: br i1 [[I10]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]488; CHECK: for.cond.cleanup:489; CHECK-NEXT: ret void490;491entry:492 br label %vector.body493 494vector.body: ; preds = %vector.body, %entry495 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]496 %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]497 %step.add = add <8 x i64> %vec.ind, splat (i64 8)498 %i = getelementptr inbounds %struct.foo, ptr %B, <8 x i64> %vec.ind, i32 1499 %i1 = getelementptr inbounds %struct.foo, ptr %B, <8 x i64> %step.add, i32 1500 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)501 %wide.masked.gather9 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)502 %i2 = getelementptr inbounds i32, ptr %A, i64 %index503 %wide.load = load <8 x i32>, ptr %i2, align 4504 %i4 = getelementptr inbounds i32, ptr %i2, i64 8505 %wide.load10 = load <8 x i32>, ptr %i4, align 4506 %i6 = add nsw <8 x i32> %wide.load, %wide.masked.gather507 %i7 = add nsw <8 x i32> %wide.load10, %wide.masked.gather9508 store <8 x i32> %i6, ptr %i2, align 4509 store <8 x i32> %i7, ptr %i4, align 4510 %index.next = add nuw i64 %index, 16511 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 16)512 %i10 = icmp eq i64 %index.next, 1024513 br i1 %i10, label %for.cond.cleanup, label %vector.body514 515for.cond.cleanup: ; preds = %vector.body516 ret void517}518 519;void gather_unroll(int * __restrict A, int * __restrict B) {520; for (int i = 0; i < 1024; i+= 4 ) {521; A[i] += B[i * 4];522; A[i+1] += B[(i+1) * 4];523; A[i+2] += B[(i+2) * 4];524; A[i+3] += B[(i+3) * 4];525; }526;}527define void @gather_unroll(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {528; CHECK-LABEL: @gather_unroll(529; CHECK-NEXT: entry:530; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]531; CHECK: vector.body:532; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]533; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]534; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR2:%.*]], [[VECTOR_BODY]] ]535; CHECK-NEXT: [[VEC_IND_SCALAR3:%.*]] = phi i64 [ 4, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR4:%.*]], [[VECTOR_BODY]] ]536; CHECK-NEXT: [[VEC_IND_SCALAR5:%.*]] = phi i64 [ 1, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR6:%.*]], [[VECTOR_BODY]] ]537; CHECK-NEXT: [[VEC_IND_SCALAR7:%.*]] = phi i64 [ 8, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR8:%.*]], [[VECTOR_BODY]] ]538; CHECK-NEXT: [[VEC_IND_SCALAR9:%.*]] = phi i64 [ 2, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR10:%.*]], [[VECTOR_BODY]] ]539; CHECK-NEXT: [[VEC_IND_SCALAR11:%.*]] = phi i64 [ 12, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR12:%.*]], [[VECTOR_BODY]] ]540; CHECK-NEXT: [[VEC_IND_SCALAR13:%.*]] = phi i64 [ 3, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR14:%.*]], [[VECTOR_BODY]] ]541; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]]542; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP0]], i64 64, <8 x i1> splat (i1 true), i32 8)543; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]]544; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP2]], i64 16, <8 x i1> splat (i1 true), i32 8)545; CHECK-NEXT: [[I3:%.*]] = add nsw <8 x i32> [[TMP3]], [[TMP1]]546; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i32.p0.i64(<8 x i32> [[I3]], ptr [[TMP2]], i64 16, <8 x i1> splat (i1 true), i32 8)547; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[B]], i64 [[VEC_IND_SCALAR3]]548; CHECK-NEXT: [[TMP5:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP4]], i64 64, <8 x i1> splat (i1 true), i32 8)549; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[VEC_IND_SCALAR5]]550; CHECK-NEXT: [[TMP7:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP6]], i64 16, <8 x i1> splat (i1 true), i32 8)551; CHECK-NEXT: [[I8:%.*]] = add nsw <8 x i32> [[TMP7]], [[TMP5]]552; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i32.p0.i64(<8 x i32> [[I8]], ptr [[TMP6]], i64 16, <8 x i1> splat (i1 true), i32 8)553; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[B]], i64 [[VEC_IND_SCALAR7]]554; CHECK-NEXT: [[TMP9:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP8]], i64 64, <8 x i1> splat (i1 true), i32 8)555; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A]], i64 [[VEC_IND_SCALAR9]]556; CHECK-NEXT: [[TMP11:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP10]], i64 16, <8 x i1> splat (i1 true), i32 8)557; CHECK-NEXT: [[I13:%.*]] = add nsw <8 x i32> [[TMP11]], [[TMP9]]558; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i32.p0.i64(<8 x i32> [[I13]], ptr [[TMP10]], i64 16, <8 x i1> splat (i1 true), i32 8)559; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[B]], i64 [[VEC_IND_SCALAR11]]560; CHECK-NEXT: [[TMP13:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP12]], i64 64, <8 x i1> splat (i1 true), i32 8)561; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[A]], i64 [[VEC_IND_SCALAR13]]562; CHECK-NEXT: [[TMP15:%.*]] = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i64(ptr [[TMP14]], i64 16, <8 x i1> splat (i1 true), i32 8)563; CHECK-NEXT: [[I18:%.*]] = add nsw <8 x i32> [[TMP15]], [[TMP13]]564; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i32.p0.i64(<8 x i32> [[I18]], ptr [[TMP14]], i64 16, <8 x i1> splat (i1 true), i32 8)565; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8566; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 128567; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR2]] = add i64 [[VEC_IND_SCALAR1]], 32568; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR4]] = add i64 [[VEC_IND_SCALAR3]], 128569; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR6]] = add i64 [[VEC_IND_SCALAR5]], 32570; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR8]] = add i64 [[VEC_IND_SCALAR7]], 128571; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR10]] = add i64 [[VEC_IND_SCALAR9]], 32572; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR12]] = add i64 [[VEC_IND_SCALAR11]], 128573; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR14]] = add i64 [[VEC_IND_SCALAR13]], 32574; CHECK-NEXT: [[I19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256575; CHECK-NEXT: br i1 [[I19]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]576; CHECK: for.cond.cleanup:577; CHECK-NEXT: ret void578;579entry:580 br label %vector.body581 582vector.body: ; preds = %vector.body, %entry583 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]584 %vec.ind = phi <8 x i64> [ <i64 0, i64 4, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>, %entry ], [ %vec.ind.next, %vector.body ]585 %i = shl nuw nsw <8 x i64> %vec.ind, splat (i64 2)586 %i1 = getelementptr inbounds i32, ptr %B, <8 x i64> %i587 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)588 %i2 = getelementptr inbounds i32, ptr %A, <8 x i64> %vec.ind589 %wide.masked.gather52 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i2, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)590 %i3 = add nsw <8 x i32> %wide.masked.gather52, %wide.masked.gather591 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i3, <8 x ptr> %i2, i32 4, <8 x i1> splat (i1 true))592 %i4 = or disjoint <8 x i64> %vec.ind, splat (i64 1)593 %i5 = shl nsw <8 x i64> %i4, splat (i64 2)594 %i6 = getelementptr inbounds i32, ptr %B, <8 x i64> %i5595 %wide.masked.gather53 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i6, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)596 %i7 = getelementptr inbounds i32, ptr %A, <8 x i64> %i4597 %wide.masked.gather54 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i7, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)598 %i8 = add nsw <8 x i32> %wide.masked.gather54, %wide.masked.gather53599 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i8, <8 x ptr> %i7, i32 4, <8 x i1> splat (i1 true))600 %i9 = or disjoint <8 x i64> %vec.ind, splat (i64 2)601 %i10 = shl nsw <8 x i64> %i9, splat (i64 2)602 %i11 = getelementptr inbounds i32, ptr %B, <8 x i64> %i10603 %wide.masked.gather55 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i11, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)604 %i12 = getelementptr inbounds i32, ptr %A, <8 x i64> %i9605 %wide.masked.gather56 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i12, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)606 %i13 = add nsw <8 x i32> %wide.masked.gather56, %wide.masked.gather55607 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i13, <8 x ptr> %i12, i32 4, <8 x i1> splat (i1 true))608 %i14 = or disjoint <8 x i64> %vec.ind, splat (i64 3)609 %i15 = shl nsw <8 x i64> %i14, splat (i64 2)610 %i16 = getelementptr inbounds i32, ptr %B, <8 x i64> %i15611 %wide.masked.gather57 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i16, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)612 %i17 = getelementptr inbounds i32, ptr %A, <8 x i64> %i14613 %wide.masked.gather58 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i17, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)614 %i18 = add nsw <8 x i32> %wide.masked.gather58, %wide.masked.gather57615 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i18, <8 x ptr> %i17, i32 4, <8 x i1> splat (i1 true))616 %index.next = add nuw i64 %index, 8617 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 32)618 %i19 = icmp eq i64 %index.next, 256619 br i1 %i19, label %for.cond.cleanup, label %vector.body620 621for.cond.cleanup: ; preds = %vector.body622 ret void623}624 625; Make sure we don't crash in getTgtMemIntrinsic for a vector of pointers.626define void @gather_of_pointers(ptr noalias nocapture %arg, ptr noalias nocapture readonly %arg1) {627; V-LABEL: @gather_of_pointers(628; V-NEXT: bb:629; V-NEXT: br label [[BB2:%.*]]630; V: bb2:631; V-NEXT: [[I:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ [[I15:%.*]], [[BB2]] ]632; V-NEXT: [[I3_SCALAR:%.*]] = phi i64 [ 0, [[BB]] ], [ [[I16_SCALAR:%.*]], [[BB2]] ]633; V-NEXT: [[I3_SCALAR1:%.*]] = phi i64 [ 10, [[BB]] ], [ [[I16_SCALAR2:%.*]], [[BB2]] ]634; V-NEXT: [[TMP0:%.*]] = getelementptr ptr, ptr [[ARG1:%.*]], i64 [[I3_SCALAR]]635; V-NEXT: [[TMP1:%.*]] = getelementptr ptr, ptr [[ARG1]], i64 [[I3_SCALAR1]]636; V-NEXT: [[TMP2:%.*]] = call <2 x ptr> @llvm.experimental.vp.strided.load.v2p0.p0.i64(ptr [[TMP0]], i64 40, <2 x i1> splat (i1 true), i32 2)637; V-NEXT: [[TMP3:%.*]] = call <2 x ptr> @llvm.experimental.vp.strided.load.v2p0.p0.i64(ptr [[TMP1]], i64 40, <2 x i1> splat (i1 true), i32 2)638; V-NEXT: [[I11:%.*]] = getelementptr inbounds ptr, ptr [[ARG:%.*]], i64 [[I]]639; V-NEXT: store <2 x ptr> [[TMP2]], ptr [[I11]], align 8640; V-NEXT: [[I13:%.*]] = getelementptr inbounds ptr, ptr [[I11]], i64 2641; V-NEXT: store <2 x ptr> [[TMP3]], ptr [[I13]], align 8642; V-NEXT: [[I15]] = add nuw i64 [[I]], 4643; V-NEXT: [[I16_SCALAR]] = add i64 [[I3_SCALAR]], 20644; V-NEXT: [[I16_SCALAR2]] = add i64 [[I3_SCALAR1]], 20645; V-NEXT: [[I17:%.*]] = icmp eq i64 [[I15]], 1024646; V-NEXT: br i1 [[I17]], label [[BB18:%.*]], label [[BB2]]647; V: bb18:648; V-NEXT: ret void649;650; ZVE32F-LABEL: @gather_of_pointers(651; ZVE32F-NEXT: bb:652; ZVE32F-NEXT: br label [[BB2:%.*]]653; ZVE32F: bb2:654; ZVE32F-NEXT: [[I:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ [[I15:%.*]], [[BB2]] ]655; ZVE32F-NEXT: [[I3:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[BB]] ], [ [[I16:%.*]], [[BB2]] ]656; ZVE32F-NEXT: [[I4:%.*]] = mul nuw nsw <2 x i64> [[I3]], splat (i64 5)657; ZVE32F-NEXT: [[I5:%.*]] = mul <2 x i64> [[I3]], splat (i64 5)658; ZVE32F-NEXT: [[I6:%.*]] = add <2 x i64> [[I5]], splat (i64 10)659; ZVE32F-NEXT: [[I7:%.*]] = getelementptr inbounds ptr, ptr [[ARG1:%.*]], <2 x i64> [[I4]]660; ZVE32F-NEXT: [[I8:%.*]] = getelementptr inbounds ptr, ptr [[ARG1]], <2 x i64> [[I6]]661; ZVE32F-NEXT: [[I9:%.*]] = call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> align 8 [[I7]], <2 x i1> splat (i1 true), <2 x ptr> poison)662; ZVE32F-NEXT: [[I10:%.*]] = call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> align 8 [[I8]], <2 x i1> splat (i1 true), <2 x ptr> poison)663; ZVE32F-NEXT: [[I11:%.*]] = getelementptr inbounds ptr, ptr [[ARG:%.*]], i64 [[I]]664; ZVE32F-NEXT: store <2 x ptr> [[I9]], ptr [[I11]], align 8665; ZVE32F-NEXT: [[I13:%.*]] = getelementptr inbounds ptr, ptr [[I11]], i64 2666; ZVE32F-NEXT: store <2 x ptr> [[I10]], ptr [[I13]], align 8667; ZVE32F-NEXT: [[I15]] = add nuw i64 [[I]], 4668; ZVE32F-NEXT: [[I16]] = add <2 x i64> [[I3]], splat (i64 4)669; ZVE32F-NEXT: [[I17:%.*]] = icmp eq i64 [[I15]], 1024670; ZVE32F-NEXT: br i1 [[I17]], label [[BB18:%.*]], label [[BB2]]671; ZVE32F: bb18:672; ZVE32F-NEXT: ret void673;674bb:675 br label %bb2676 677bb2: ; preds = %bb2, %bb678 %i = phi i64 [ 0, %bb ], [ %i15, %bb2 ]679 %i3 = phi <2 x i64> [ <i64 0, i64 1>, %bb ], [ %i16, %bb2 ]680 %i4 = mul nuw nsw <2 x i64> %i3, splat (i64 5)681 %i5 = mul <2 x i64> %i3, splat (i64 5)682 %i6 = add <2 x i64> %i5, <i64 10, i64 10>683 %i7 = getelementptr inbounds ptr, ptr %arg1, <2 x i64> %i4684 %i8 = getelementptr inbounds ptr, ptr %arg1, <2 x i64> %i6685 %i9 = call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> %i7, i32 8, <2 x i1> splat (i1 true), <2 x ptr> poison)686 %i10 = call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> %i8, i32 8, <2 x i1> splat (i1 true), <2 x ptr> poison)687 %i11 = getelementptr inbounds ptr, ptr %arg, i64 %i688 store <2 x ptr> %i9, ptr %i11, align 8689 %i13 = getelementptr inbounds ptr, ptr %i11, i64 2690 store <2 x ptr> %i10, ptr %i13, align 8691 %i15 = add nuw i64 %i, 4692 %i16 = add <2 x i64> %i3, <i64 4, i64 4>693 %i17 = icmp eq i64 %i15, 1024694 br i1 %i17, label %bb18, label %bb2695 696bb18: ; preds = %bb2697 ret void698}699 700; Make sure we don't crash in getTgtMemIntrinsic for a vector of pointers.701define void @scatter_of_pointers(ptr noalias nocapture %arg, ptr noalias nocapture readonly %arg1) {702; V-LABEL: @scatter_of_pointers(703; V-NEXT: bb:704; V-NEXT: br label [[BB2:%.*]]705; V: bb2:706; V-NEXT: [[I:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ [[I15:%.*]], [[BB2]] ]707; V-NEXT: [[I3_SCALAR:%.*]] = phi i64 [ 0, [[BB]] ], [ [[I16_SCALAR:%.*]], [[BB2]] ]708; V-NEXT: [[I3_SCALAR1:%.*]] = phi i64 [ 10, [[BB]] ], [ [[I16_SCALAR2:%.*]], [[BB2]] ]709; V-NEXT: [[I4:%.*]] = getelementptr inbounds ptr, ptr [[ARG1:%.*]], i64 [[I]]710; V-NEXT: [[I6:%.*]] = load <2 x ptr>, ptr [[I4]], align 8711; V-NEXT: [[I7:%.*]] = getelementptr inbounds ptr, ptr [[I4]], i64 2712; V-NEXT: [[I9:%.*]] = load <2 x ptr>, ptr [[I7]], align 8713; V-NEXT: [[TMP0:%.*]] = getelementptr ptr, ptr [[ARG:%.*]], i64 [[I3_SCALAR]]714; V-NEXT: [[TMP1:%.*]] = getelementptr ptr, ptr [[ARG]], i64 [[I3_SCALAR1]]715; V-NEXT: call void @llvm.experimental.vp.strided.store.v2p0.p0.i64(<2 x ptr> [[I6]], ptr [[TMP0]], i64 40, <2 x i1> splat (i1 true), i32 2)716; V-NEXT: call void @llvm.experimental.vp.strided.store.v2p0.p0.i64(<2 x ptr> [[I9]], ptr [[TMP1]], i64 40, <2 x i1> splat (i1 true), i32 2)717; V-NEXT: [[I15]] = add nuw i64 [[I]], 4718; V-NEXT: [[I16_SCALAR]] = add i64 [[I3_SCALAR]], 20719; V-NEXT: [[I16_SCALAR2]] = add i64 [[I3_SCALAR1]], 20720; V-NEXT: [[I17:%.*]] = icmp eq i64 [[I15]], 1024721; V-NEXT: br i1 [[I17]], label [[BB18:%.*]], label [[BB2]]722; V: bb18:723; V-NEXT: ret void724;725; ZVE32F-LABEL: @scatter_of_pointers(726; ZVE32F-NEXT: bb:727; ZVE32F-NEXT: br label [[BB2:%.*]]728; ZVE32F: bb2:729; ZVE32F-NEXT: [[I:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ [[I15:%.*]], [[BB2]] ]730; ZVE32F-NEXT: [[I3:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[BB]] ], [ [[I16:%.*]], [[BB2]] ]731; ZVE32F-NEXT: [[I4:%.*]] = getelementptr inbounds ptr, ptr [[ARG1:%.*]], i64 [[I]]732; ZVE32F-NEXT: [[I6:%.*]] = load <2 x ptr>, ptr [[I4]], align 8733; ZVE32F-NEXT: [[I7:%.*]] = getelementptr inbounds ptr, ptr [[I4]], i64 2734; ZVE32F-NEXT: [[I9:%.*]] = load <2 x ptr>, ptr [[I7]], align 8735; ZVE32F-NEXT: [[I10:%.*]] = mul nuw nsw <2 x i64> [[I3]], splat (i64 5)736; ZVE32F-NEXT: [[I11:%.*]] = mul <2 x i64> [[I3]], splat (i64 5)737; ZVE32F-NEXT: [[I12:%.*]] = add <2 x i64> [[I11]], splat (i64 10)738; ZVE32F-NEXT: [[I13:%.*]] = getelementptr inbounds ptr, ptr [[ARG:%.*]], <2 x i64> [[I10]]739; ZVE32F-NEXT: [[I14:%.*]] = getelementptr inbounds ptr, ptr [[ARG]], <2 x i64> [[I12]]740; ZVE32F-NEXT: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> [[I6]], <2 x ptr> align 8 [[I13]], <2 x i1> splat (i1 true))741; ZVE32F-NEXT: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> [[I9]], <2 x ptr> align 8 [[I14]], <2 x i1> splat (i1 true))742; ZVE32F-NEXT: [[I15]] = add nuw i64 [[I]], 4743; ZVE32F-NEXT: [[I16]] = add <2 x i64> [[I3]], splat (i64 4)744; ZVE32F-NEXT: [[I17:%.*]] = icmp eq i64 [[I15]], 1024745; ZVE32F-NEXT: br i1 [[I17]], label [[BB18:%.*]], label [[BB2]]746; ZVE32F: bb18:747; ZVE32F-NEXT: ret void748;749bb:750 br label %bb2751 752bb2: ; preds = %bb2, %bb753 %i = phi i64 [ 0, %bb ], [ %i15, %bb2 ]754 %i3 = phi <2 x i64> [ <i64 0, i64 1>, %bb ], [ %i16, %bb2 ]755 %i4 = getelementptr inbounds ptr, ptr %arg1, i64 %i756 %i6 = load <2 x ptr>, ptr %i4, align 8757 %i7 = getelementptr inbounds ptr, ptr %i4, i64 2758 %i9 = load <2 x ptr>, ptr %i7, align 8759 %i10 = mul nuw nsw <2 x i64> %i3, splat (i64 5)760 %i11 = mul <2 x i64> %i3, splat (i64 5)761 %i12 = add <2 x i64> %i11, <i64 10, i64 10>762 %i13 = getelementptr inbounds ptr, ptr %arg, <2 x i64> %i10763 %i14 = getelementptr inbounds ptr, ptr %arg, <2 x i64> %i12764 call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> %i6, <2 x ptr> %i13, i32 8, <2 x i1> splat (i1 true))765 call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> %i9, <2 x ptr> %i14, i32 8, <2 x i1> splat (i1 true))766 %i15 = add nuw i64 %i, 4767 %i16 = add <2 x i64> %i3, <i64 4, i64 4>768 %i17 = icmp eq i64 %i15, 1024769 br i1 %i17, label %bb18, label %bb2770 771bb18: ; preds = %bb2772 ret void773}774 775define void @strided_load_startval_add_with_splat(ptr noalias nocapture %arg, ptr noalias nocapture readonly %arg1, i32 signext %arg2) {776; CHECK-LABEL: @strided_load_startval_add_with_splat(777; CHECK-NEXT: bb:778; CHECK-NEXT: [[I:%.*]] = icmp eq i32 [[ARG2:%.*]], 1024779; CHECK-NEXT: br i1 [[I]], label [[BB34:%.*]], label [[BB3:%.*]]780; CHECK: bb3:781; CHECK-NEXT: [[I4:%.*]] = sext i32 [[ARG2]] to i64782; CHECK-NEXT: [[I5:%.*]] = sub i32 1023, [[ARG2]]783; CHECK-NEXT: [[I6:%.*]] = zext i32 [[I5]] to i64784; CHECK-NEXT: [[I7:%.*]] = add nuw nsw i64 [[I6]], 1785; CHECK-NEXT: [[I8:%.*]] = icmp ult i32 [[I5]], 31786; CHECK-NEXT: br i1 [[I8]], label [[BB32:%.*]], label [[BB9:%.*]]787; CHECK: bb9:788; CHECK-NEXT: [[I10:%.*]] = and i64 [[I7]], 8589934560789; CHECK-NEXT: [[I11:%.*]] = add nsw i64 [[I10]], [[I4]]790; CHECK-NEXT: [[START:%.*]] = mul i64 [[I4]], 5791; CHECK-NEXT: br label [[BB15:%.*]]792; CHECK: bb15:793; CHECK-NEXT: [[I16:%.*]] = phi i64 [ 0, [[BB9]] ], [ [[I27:%.*]], [[BB15]] ]794; CHECK-NEXT: [[I17_SCALAR:%.*]] = phi i64 [ [[START]], [[BB9]] ], [ [[I28_SCALAR:%.*]], [[BB15]] ]795; CHECK-NEXT: [[I18:%.*]] = add i64 [[I16]], [[I4]]796; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[ARG1:%.*]], i64 [[I17_SCALAR]]797; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 5, <32 x i1> splat (i1 true), i32 32)798; CHECK-NEXT: [[I22:%.*]] = getelementptr inbounds i8, ptr [[ARG:%.*]], i64 [[I18]]799; CHECK-NEXT: [[I24:%.*]] = load <32 x i8>, ptr [[I22]], align 1800; CHECK-NEXT: [[I25:%.*]] = add <32 x i8> [[I24]], [[TMP1]]801; CHECK-NEXT: store <32 x i8> [[I25]], ptr [[I22]], align 1802; CHECK-NEXT: [[I27]] = add nuw i64 [[I16]], 32803; CHECK-NEXT: [[I28_SCALAR]] = add i64 [[I17_SCALAR]], 160804; CHECK-NEXT: [[I29:%.*]] = icmp eq i64 [[I27]], [[I10]]805; CHECK-NEXT: br i1 [[I29]], label [[BB30:%.*]], label [[BB15]]806; CHECK: bb30:807; CHECK-NEXT: [[I31:%.*]] = icmp eq i64 [[I7]], [[I10]]808; CHECK-NEXT: br i1 [[I31]], label [[BB34]], label [[BB32]]809; CHECK: bb32:810; CHECK-NEXT: [[I33:%.*]] = phi i64 [ [[I4]], [[BB3]] ], [ [[I11]], [[BB30]] ]811; CHECK-NEXT: br label [[BB35:%.*]]812; CHECK: bb34:813; CHECK-NEXT: ret void814; CHECK: bb35:815; CHECK-NEXT: [[I36:%.*]] = phi i64 [ [[I43:%.*]], [[BB35]] ], [ [[I33]], [[BB32]] ]816; CHECK-NEXT: [[I37:%.*]] = mul nsw i64 [[I36]], 5817; CHECK-NEXT: [[I38:%.*]] = getelementptr inbounds i8, ptr [[ARG1]], i64 [[I37]]818; CHECK-NEXT: [[I39:%.*]] = load i8, ptr [[I38]], align 1819; CHECK-NEXT: [[I40:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 [[I36]]820; CHECK-NEXT: [[I41:%.*]] = load i8, ptr [[I40]], align 1821; CHECK-NEXT: [[I42:%.*]] = add i8 [[I41]], [[I39]]822; CHECK-NEXT: store i8 [[I42]], ptr [[I40]], align 1823; CHECK-NEXT: [[I43]] = add nsw i64 [[I36]], 1824; CHECK-NEXT: [[I44:%.*]] = trunc i64 [[I43]] to i32825; CHECK-NEXT: [[I45:%.*]] = icmp eq i32 [[I44]], 1024826; CHECK-NEXT: br i1 [[I45]], label [[BB34]], label [[BB35]]827;828bb:829 %i = icmp eq i32 %arg2, 1024830 br i1 %i, label %bb34, label %bb3831 832bb3: ; preds = %bb833 %i4 = sext i32 %arg2 to i64834 %i5 = sub i32 1023, %arg2835 %i6 = zext i32 %i5 to i64836 %i7 = add nuw nsw i64 %i6, 1837 %i8 = icmp ult i32 %i5, 31838 br i1 %i8, label %bb32, label %bb9839 840bb9: ; preds = %bb3841 %i10 = and i64 %i7, 8589934560842 %i11 = add nsw i64 %i10, %i4843 %i12 = insertelement <32 x i64> poison, i64 %i4, i64 0844 %i13 = shufflevector <32 x i64> %i12, <32 x i64> poison, <32 x i32> zeroinitializer845 %i14 = add <32 x i64> %i13, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>846 br label %bb15847 848bb15: ; preds = %bb15, %bb9849 %i16 = phi i64 [ 0, %bb9 ], [ %i27, %bb15 ]850 %i17 = phi <32 x i64> [ %i14, %bb9 ], [ %i28, %bb15 ]851 %i18 = add i64 %i16, %i4852 %i19 = mul nsw <32 x i64> %i17, splat (i64 5)853 %i20 = getelementptr inbounds i8, ptr %arg1, <32 x i64> %i19854 %i21 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i20, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)855 %i22 = getelementptr inbounds i8, ptr %arg, i64 %i18856 %i24 = load <32 x i8>, ptr %i22, align 1857 %i25 = add <32 x i8> %i24, %i21858 store <32 x i8> %i25, ptr %i22, align 1859 %i27 = add nuw i64 %i16, 32860 %i28 = add <32 x i64> %i17, splat (i64 32)861 %i29 = icmp eq i64 %i27, %i10862 br i1 %i29, label %bb30, label %bb15863 864bb30: ; preds = %bb15865 %i31 = icmp eq i64 %i7, %i10866 br i1 %i31, label %bb34, label %bb32867 868bb32: ; preds = %bb30, %bb3869 %i33 = phi i64 [ %i4, %bb3 ], [ %i11, %bb30 ]870 br label %bb35871 872bb34: ; preds = %bb35, %bb30, %bb873 ret void874 875bb35: ; preds = %bb35, %bb32876 %i36 = phi i64 [ %i43, %bb35 ], [ %i33, %bb32 ]877 %i37 = mul nsw i64 %i36, 5878 %i38 = getelementptr inbounds i8, ptr %arg1, i64 %i37879 %i39 = load i8, ptr %i38, align 1880 %i40 = getelementptr inbounds i8, ptr %arg, i64 %i36881 %i41 = load i8, ptr %i40, align 1882 %i42 = add i8 %i41, %i39883 store i8 %i42, ptr %i40, align 1884 %i43 = add nsw i64 %i36, 1885 %i44 = trunc i64 %i43 to i32886 %i45 = icmp eq i32 %i44, 1024887 br i1 %i45, label %bb34, label %bb35888}889 890define void @gather_no_scalar_remainder(ptr noalias nocapture noundef %arg, ptr noalias nocapture noundef readonly %arg1, i64 noundef %arg2) {891; CHECK-LABEL: @gather_no_scalar_remainder(892; CHECK-NEXT: bb:893; CHECK-NEXT: [[I:%.*]] = shl i64 [[ARG2:%.*]], 4894; CHECK-NEXT: [[I3:%.*]] = icmp eq i64 [[I]], 0895; CHECK-NEXT: br i1 [[I3]], label [[BB16:%.*]], label [[BB2:%.*]]896; CHECK: bb2:897; CHECK-NEXT: br label [[BB4:%.*]]898; CHECK: bb4:899; CHECK-NEXT: [[I5:%.*]] = phi i64 [ [[I13:%.*]], [[BB4]] ], [ 0, [[BB2]] ]900; CHECK-NEXT: [[I6_SCALAR:%.*]] = phi i64 [ 0, [[BB2]] ], [ [[I14_SCALAR:%.*]], [[BB4]] ]901; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[ARG1:%.*]], i64 [[I6_SCALAR]]902; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr [[TMP0]], i64 5, <16 x i1> splat (i1 true), i32 16)903; CHECK-NEXT: [[I10:%.*]] = getelementptr inbounds i8, ptr [[ARG:%.*]], i64 [[I5]]904; CHECK-NEXT: [[I11:%.*]] = load <16 x i8>, ptr [[I10]], align 1905; CHECK-NEXT: [[I12:%.*]] = add <16 x i8> [[I11]], [[TMP1]]906; CHECK-NEXT: store <16 x i8> [[I12]], ptr [[I10]], align 1907; CHECK-NEXT: [[I13]] = add nuw i64 [[I5]], 16908; CHECK-NEXT: [[I14_SCALAR]] = add i64 [[I6_SCALAR]], 80909; CHECK-NEXT: [[I15:%.*]] = icmp eq i64 [[I13]], [[I]]910; CHECK-NEXT: br i1 [[I15]], label [[BB16]], label [[BB4]]911; CHECK: bb16:912; CHECK-NEXT: ret void913;914bb:915 %i = shl i64 %arg2, 4916 %i3 = icmp eq i64 %i, 0917 br i1 %i3, label %bb16, label %bb2918 919bb2: ; preds = %bb920 br label %bb4921 922bb4: ; preds = %bb4, %bb2923 %i5 = phi i64 [ %i13, %bb4 ], [ 0, %bb2 ]924 %i6 = phi <16 x i64> [ %i14, %bb4 ], [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %bb2 ]925 %i7 = mul <16 x i64> %i6, splat (i64 5)926 %i8 = getelementptr inbounds i8, ptr %arg1, <16 x i64> %i7927 %i9 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> %i8, i32 1, <16 x i1> splat (i1 true), <16 x i8> poison)928 %i10 = getelementptr inbounds i8, ptr %arg, i64 %i5929 %i11 = load <16 x i8>, ptr %i10, align 1930 %i12 = add <16 x i8> %i11, %i9931 store <16 x i8> %i12, ptr %i10, align 1932 %i13 = add nuw i64 %i5, 16933 %i14 = add <16 x i64> %i6, splat (i64 16)934 %i15 = icmp eq i64 %i13, %i935 br i1 %i15, label %bb16, label %bb4936 937bb16: ; preds = %bb4, %bb938 ret void939}940 941define <8 x i8> @broadcast_ptr_base(ptr %a) {942; CHECK-LABEL: @broadcast_ptr_base(943; CHECK-NEXT: entry:944; CHECK-NEXT: [[TMP0:%.*]] = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr [[A:%.*]], i64 64, <8 x i1> splat (i1 true), i32 8)945; CHECK-NEXT: ret <8 x i8> [[TMP0]]946;947entry:948 %0 = insertelement <8 x ptr> poison, ptr %a, i64 0949 %1 = shufflevector <8 x ptr> %0, <8 x ptr> poison, <8 x i32> zeroinitializer950 %2 = getelementptr i8, <8 x ptr> %1, <8 x i64> <i64 0, i64 64, i64 128, i64 192, i64 256, i64 320, i64 384, i64 448>951 %3 = tail call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> %2, i32 1, <8 x i1> splat (i1 true), <8 x i8> poison)952 ret <8 x i8> %3953}954 955define void @gather_narrow_idx(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {956; CHECK-LABEL: @gather_narrow_idx(957; CHECK-NEXT: entry:958; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]959; CHECK: vector.body:960; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]961; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 20, i16 21, i16 22, i16 23, i16 24, i16 25, i16 26, i16 27, i16 28, i16 29, i16 30, i16 31>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]962; CHECK-NEXT: [[I:%.*]] = mul nuw nsw <32 x i16> [[VEC_IND]], splat (i16 5)963; CHECK-NEXT: [[I1:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], <32 x i16> [[I]]964; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 [[I1]], <32 x i1> splat (i1 true), <32 x i8> poison)965; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]966; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I2]], align 1967; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]968; CHECK-NEXT: store <32 x i8> [[I4]], ptr [[I2]], align 1969; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32970; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i16> [[VEC_IND]], splat (i16 32)971; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024972; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]973; CHECK: for.cond.cleanup:974; CHECK-NEXT: ret void975;976entry:977 br label %vector.body978 979vector.body: ; preds = %vector.body, %entry980 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]981 %vec.ind = phi <32 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 20, i16 21, i16 22, i16 23, i16 24, i16 25, i16 26, i16 27, i16 28, i16 29, i16 30, i16 31>, %entry ], [ %vec.ind.next, %vector.body ]982 %i = mul nuw nsw <32 x i16> %vec.ind, splat (i16 5)983 %i1 = getelementptr inbounds i8, ptr %B, <32 x i16> %i984 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)985 %i2 = getelementptr inbounds i8, ptr %A, i64 %index986 %wide.load = load <32 x i8>, ptr %i2, align 1987 %i4 = add <32 x i8> %wide.load, %wide.masked.gather988 store <32 x i8> %i4, ptr %i2, align 1989 %index.next = add nuw i64 %index, 32990 %vec.ind.next = add <32 x i16> %vec.ind, splat (i16 32)991 %i6 = icmp eq i64 %index.next, 1024992 br i1 %i6, label %for.cond.cleanup, label %vector.body993 994for.cond.cleanup: ; preds = %vector.body995 ret void996}997 998define void @vp_gather(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {999; CHECK-LABEL: @vp_gather(1000; CHECK-NEXT: entry:1001; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1002; CHECK: vector.body:1003; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]1004; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]1005; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1006; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR1]]1007; CHECK-NEXT: [[ELEMS:%.*]] = sub i64 1024, [[VEC_IND_SCALAR]]1008; CHECK-NEXT: [[EVL:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[ELEMS]], i32 32, i1 false)1009; CHECK-NEXT: [[ODD:%.*]] = and <32 x i64> [[VEC_IND]], splat (i64 1)1010; CHECK-NEXT: [[MASK:%.*]] = icmp ne <32 x i64> [[ODD]], zeroinitializer1011; CHECK-NEXT: [[WIDE_VP_GATHER:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 5, <32 x i1> [[MASK]], i32 [[EVL]])1012; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]]1013; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I2]], align 11014; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_VP_GATHER]]1015; CHECK-NEXT: store <32 x i8> [[I4]], ptr [[I2]], align 11016; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], 321017; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], 1601018; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i64> [[VEC_IND]], splat (i64 32)1019; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[VEC_IND_NEXT_SCALAR]], 10241020; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]1021; CHECK: for.cond.cleanup:1022; CHECK-NEXT: ret void1023;1024entry:1025 br label %vector.body1026 1027vector.body: ; preds = %vector.body, %entry1028 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]1029 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]1030 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)1031 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i1032 1033 %elems = sub i64 1024, %index1034 %evl = call i32 @llvm.experimental.get.vector.length.i64(i64 %elems, i32 32, i1 false)1035 1036 %odd = and <32 x i64> %vec.ind, splat (i64 1)1037 %mask = icmp ne <32 x i64> %odd, splat (i64 0)1038 1039 %wide.vp.gather = call <32 x i8> @llvm.vp.gather(<32 x ptr> %i1, <32 x i1> %mask, i32 %evl)1040 %i2 = getelementptr inbounds i8, ptr %A, i64 %index1041 %wide.load = load <32 x i8>, ptr %i2, align 11042 %i4 = add <32 x i8> %wide.load, %wide.vp.gather1043 store <32 x i8> %i4, ptr %i2, align 11044 %index.next = add nuw i64 %index, 321045 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)1046 %i6 = icmp eq i64 %index.next, 10241047 br i1 %i6, label %for.cond.cleanup, label %vector.body1048 1049for.cond.cleanup: ; preds = %vector.body1050 ret void1051}1052 1053define void @vp_scatter(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {1054; CHECK-LABEL: @vp_scatter(1055; CHECK-NEXT: entry:1056; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1057; CHECK: vector.body:1058; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]1059; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]1060; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1061; CHECK-NEXT: [[I:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 [[VEC_IND_SCALAR]]1062; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I]], align 11063; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]]1064; CHECK-NEXT: [[ELEMS:%.*]] = sub i64 1024, [[VEC_IND_SCALAR]]1065; CHECK-NEXT: [[EVL:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[ELEMS]], i32 32, i1 false)1066; CHECK-NEXT: [[ODD:%.*]] = and <32 x i64> [[VEC_IND]], splat (i64 1)1067; CHECK-NEXT: [[MASK:%.*]] = icmp ne <32 x i64> [[ODD]], zeroinitializer1068; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.experimental.vp.strided.load.v32i8.p0.i64(ptr [[TMP0]], i64 5, <32 x i1> [[MASK]], i32 [[EVL]])1069; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]]1070; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v32i8.p0.i64(<32 x i8> [[I4]], ptr [[TMP0]], i64 5, <32 x i1> [[MASK]], i32 [[EVL]])1071; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], 321072; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], 1601073; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i64> [[VEC_IND]], splat (i64 32)1074; CHECK-NEXT: [[I5:%.*]] = icmp eq i64 [[VEC_IND_NEXT_SCALAR]], 10241075; CHECK-NEXT: br i1 [[I5]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]1076; CHECK: for.cond.cleanup:1077; CHECK-NEXT: ret void1078;1079entry:1080 br label %vector.body1081 1082vector.body: ; preds = %vector.body, %entry1083 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]1084 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]1085 %i = getelementptr inbounds i8, ptr %B, i64 %index1086 %wide.load = load <32 x i8>, ptr %i, align 11087 %i2 = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)1088 %i3 = getelementptr inbounds i8, ptr %A, <32 x i64> %i21089 1090 %elems = sub i64 1024, %index1091 %evl = call i32 @llvm.experimental.get.vector.length.i64(i64 %elems, i32 32, i1 false)1092 1093 %odd = and <32 x i64> %vec.ind, splat (i64 1)1094 %mask = icmp ne <32 x i64> %odd, splat (i64 0)1095 1096 %wide.masked.gather = call <32 x i8> @llvm.vp.gather(<32 x ptr> %i3, <32 x i1> %mask, i32 %evl)1097 %i4 = add <32 x i8> %wide.masked.gather, %wide.load1098 call void @llvm.vp.scatter(<32 x i8> %i4, <32 x ptr> %i3, <32 x i1> %mask, i32 %evl)1099 %index.next = add nuw i64 %index, 321100 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)1101 %i5 = icmp eq i64 %index.next, 10241102 br i1 %i5, label %for.cond.cleanup, label %vector.body1103 1104for.cond.cleanup: ; preds = %vector.body1105 ret void1106}1107