850 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt %s -S -riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+v | FileCheck %s --check-prefixes=CHECK3 4%struct.foo = type { i32, i32, i32, i32 }5 6define <vscale x 1 x i64> @gather(ptr %a, i32 %len) {7; CHECK-LABEL: @gather(8; CHECK-NEXT: vector.ph:9; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i6410; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()11; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]12; CHECK: vector.body:13; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]14; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]15; CHECK-NEXT: [[ACCUM:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[VECTOR_BODY]] ]16; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]], i32 317; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()18; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP1]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 [[TMP2]])19; CHECK-NEXT: [[ACCUM_NEXT]] = add <vscale x 1 x i64> [[ACCUM]], [[TMP3]]20; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]21; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], [[TMP0]]22; CHECK-NEXT: [[TMP4:%.*]] = icmp ne i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]23; CHECK-NEXT: br i1 [[TMP4]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]24; CHECK: for.cond.cleanup:25; CHECK-NEXT: ret <vscale x 1 x i64> [[ACCUM_NEXT]]26;27vector.ph:28 %wide.trip.count = zext i32 %len to i6429 %0 = tail call i64 @llvm.vscale.i64()30 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()31 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %0, i64 032 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer33 br label %vector.body34 35vector.body: ; preds = %vector.body, %vector.ph36 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]37 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]38 %accum = phi <vscale x 1 x i64> [ zeroinitializer, %vector.ph ], [ %accum.next, %vector.body ]39 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind, i32 340 %gather = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> %2, i32 8, <vscale x 1 x i1> splat (i1 true), <vscale x 1 x i64> poison)41 %accum.next = add <vscale x 1 x i64> %accum, %gather42 %index.next = add nuw i64 %index, %043 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat44 %3 = icmp ne i64 %index.next, %wide.trip.count45 br i1 %3, label %for.cond.cleanup, label %vector.body46 47for.cond.cleanup: ; preds = %vector.body48 ret <vscale x 1 x i64> %accum.next49}50 51define <vscale x 1 x i64> @gather_disjoint_or(ptr %a, i64 %len) {52; CHECK-LABEL: @gather_disjoint_or(53; CHECK-NEXT: vector.ph:54; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64()55; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]56; CHECK: vector.body:57; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]58; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 1, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]59; CHECK-NEXT: [[ACCUM:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[VECTOR_BODY]] ]60; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]]61; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()62; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP0]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 [[TMP1]])63; CHECK-NEXT: [[ACCUM_NEXT]] = add <vscale x 1 x i64> [[ACCUM]], [[TMP2]]64; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[VSCALE]]65; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], 266; CHECK-NEXT: [[EXIT:%.*]] = icmp ne i64 [[INDEX_NEXT]], [[LEN:%.*]]67; CHECK-NEXT: br i1 [[EXIT]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]68; CHECK: for.cond.cleanup:69; CHECK-NEXT: ret <vscale x 1 x i64> [[ACCUM_NEXT]]70;71vector.ph:72 %vscale = call i64 @llvm.vscale.i64()73 %step = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()74 %step.mul2 = shl <vscale x 1 x i64> %step, splat (i64 1)75 br label %vector.body76 77vector.body: ; preds = %vector.body, %vector.ph78 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]79 %vec.ind = phi <vscale x 1 x i64> [ %step.mul2, %vector.ph ], [ %vec.ind.next, %vector.body ]80 81 %accum = phi <vscale x 1 x i64> [ zeroinitializer, %vector.ph ], [ %accum.next, %vector.body ]82 83 %vec.ind.or = or disjoint <vscale x 1 x i64> %vec.ind, splat (i64 1)84 85 %gep = getelementptr i64, ptr %a, <vscale x 1 x i64> %vec.ind.or86 %gather = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(87 <vscale x 1 x ptr> %gep,88 i32 8,89 <vscale x 1 x i1> splat (i1 true),90 <vscale x 1 x i64> poison91 )92 93 %accum.next = add <vscale x 1 x i64> %accum, %gather94 %index.next = add nuw i64 %index, %vscale95 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, splat (i64 2)96 97 %exit = icmp ne i64 %index.next, %len98 br i1 %exit, label %for.cond.cleanup, label %vector.body99 100for.cond.cleanup: ; preds = %vector.body101 ret <vscale x 1 x i64> %accum.next102}103 104define <vscale x 1 x i64> @gather_non_invariant_step(ptr %a, ptr %b, i32 %len) {105; CHECK-LABEL: @gather_non_invariant_step(106; CHECK-NEXT: vector.ph:107; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64108; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]109; CHECK: vector.body:110; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]111; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]112; CHECK-NEXT: [[ACCUM:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[VECTOR_BODY]] ]113; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]], i32 3114; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()115; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP0]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 [[TMP1]])116; CHECK-NEXT: [[ACCUM_NEXT]] = add <vscale x 1 x i64> [[ACCUM]], [[TMP3]]117; CHECK-NEXT: [[B:%.*]] = getelementptr i64, ptr [[B1:%.*]], i64 [[VEC_IND_SCALAR]]118; CHECK-NEXT: [[STEP:%.*]] = load i64, ptr [[B]], align 8119; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], [[STEP]]120; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], [[STEP]]121; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i64 [[VEC_IND_NEXT_SCALAR]], [[WIDE_TRIP_COUNT]]122; CHECK-NEXT: br i1 [[TMP2]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]123; CHECK: for.cond.cleanup:124; CHECK-NEXT: ret <vscale x 1 x i64> [[ACCUM_NEXT]]125;126vector.ph:127 %wide.trip.count = zext i32 %len to i64128 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()129 br label %vector.body130 131vector.body: ; preds = %vector.body, %vector.ph132 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]133 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]134 %accum = phi <vscale x 1 x i64> [ zeroinitializer, %vector.ph ], [ %accum.next, %vector.body ]135 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind, i32 3136 %gather = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> %2, i32 8, <vscale x 1 x i1> splat (i1 true), <vscale x 1 x i64> poison)137 %accum.next = add <vscale x 1 x i64> %accum, %gather138 139 %b.gep = getelementptr i64, ptr %b, i64 %index140 %step = load i64, ptr %b.gep141 %index.next = add nuw i64 %index, %step142 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %step, i64 0143 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer144 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat145 %3 = icmp ne i64 %index.next, %wide.trip.count146 br i1 %3, label %for.cond.cleanup, label %vector.body147 148for.cond.cleanup: ; preds = %vector.body149 ret <vscale x 1 x i64> %accum.next150}151 152define <vscale x 1 x i64> @gather_non_invariant_step_shl(ptr %a, ptr %b, i32 %len) {153; CHECK-LABEL: @gather_non_invariant_step_shl(154; CHECK-NEXT: vector.ph:155; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64156; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]157; CHECK: vector.body:158; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]159; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 168, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]160; CHECK-NEXT: [[ACCUM:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[VECTOR_BODY]] ]161; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]], i32 3162; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()163; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP0]], i64 64, <vscale x 1 x i1> splat (i1 true), i32 [[TMP1]])164; CHECK-NEXT: [[ACCUM_NEXT]] = add <vscale x 1 x i64> [[ACCUM]], [[TMP3]]165; CHECK-NEXT: [[B:%.*]] = getelementptr i64, ptr [[B1:%.*]], i64 [[VEC_IND_SCALAR]]166; CHECK-NEXT: [[STEP:%.*]] = load i64, ptr [[B]], align 8167; CHECK-NEXT: [[STEP1:%.*]] = shl i64 [[STEP]], 2168; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], [[STEP]]169; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], [[STEP1]]170; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i64 [[VEC_IND_NEXT_SCALAR]], [[WIDE_TRIP_COUNT]]171; CHECK-NEXT: br i1 [[TMP2]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]172; CHECK: for.cond.cleanup:173; CHECK-NEXT: ret <vscale x 1 x i64> [[ACCUM_NEXT]]174;175vector.ph:176 %wide.trip.count = zext i32 %len to i64177 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()178 br label %vector.body179 180vector.body: ; preds = %vector.body, %vector.ph181 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]182 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]183 %accum = phi <vscale x 1 x i64> [ zeroinitializer, %vector.ph ], [ %accum.next, %vector.body ]184 185 %vec.ind.add = add <vscale x 1 x i64> %vec.ind, splat (i64 42)186 %vec.ind.shl = shl <vscale x 1 x i64> %vec.ind.add, splat (i64 2)187 188 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind.shl, i32 3189 %gather = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> %2, i32 8, <vscale x 1 x i1> splat (i1 true), <vscale x 1 x i64> poison)190 %accum.next = add <vscale x 1 x i64> %accum, %gather191 192 %b.gep = getelementptr i64, ptr %b, i64 %index193 %step = load i64, ptr %b.gep194 %index.next = add nuw i64 %index, %step195 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %step, i64 0196 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer197 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat198 %3 = icmp ne i64 %index.next, %wide.trip.count199 br i1 %3, label %for.cond.cleanup, label %vector.body200 201for.cond.cleanup: ; preds = %vector.body202 ret <vscale x 1 x i64> %accum.next203}204 205; Check that the operand of the binary op (%scale.splat in shl) always dominates206; the existing step value when we're adjusting it.207define <vscale x 1 x i64> @gather_splat_op_after_step(ptr %a, ptr %b, i32 %len) {208; CHECK-LABEL: @gather_splat_op_after_step(209; CHECK-NEXT: vector.ph:210; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64211; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()212; CHECK-NEXT: [[SCALE:%.*]] = load i64, ptr [[B:%.*]], align 8213; CHECK-NEXT: [[STRIDE:%.*]] = shl i64 1, [[SCALE]]214; CHECK-NEXT: [[STEP:%.*]] = shl i64 [[TMP0]], [[SCALE]]215; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[STRIDE]], 16216; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]217; CHECK: vector.body:218; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]219; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]220; CHECK-NEXT: [[ACCUM:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[VECTOR_BODY]] ]221; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]], i32 3222; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()223; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP2]], i64 [[TMP1]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP3]])224; CHECK-NEXT: [[ACCUM_NEXT]] = add <vscale x 1 x i64> [[ACCUM]], [[TMP4]]225; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]226; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], [[STEP]]227; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]228; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]229; CHECK: for.cond.cleanup:230; CHECK-NEXT: ret <vscale x 1 x i64> [[ACCUM_NEXT]]231;232vector.ph:233 %wide.trip.count = zext i32 %len to i64234 %0 = tail call i64 @llvm.vscale.i64()235 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()236 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %0, i64 0237 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer238 239 %scale = load i64, ptr %b240 %scale.head = insertelement <vscale x 1 x i64> poison, i64 %scale, i64 0241 %scale.splat = shufflevector <vscale x 1 x i64> %scale.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer242 br label %vector.body243 244vector.body: ; preds = %vector.body, %vector.ph245 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]246 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]247 %accum = phi <vscale x 1 x i64> [ zeroinitializer, %vector.ph ], [ %accum.next, %vector.body ]248 %vec.ind.shl = shl <vscale x 1 x i64> %vec.ind, %scale.splat249 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind.shl, i32 3250 %gather = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> %2, i32 8, <vscale x 1 x i1> splat (i1 true), <vscale x 1 x i64> poison)251 %accum.next = add <vscale x 1 x i64> %accum, %gather252 %index.next = add nuw i64 %index, %0253 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat254 %3 = icmp ne i64 %index.next, %wide.trip.count255 br i1 %3, label %for.cond.cleanup, label %vector.body256 257for.cond.cleanup: ; preds = %vector.body258 ret <vscale x 1 x i64> %accum.next259}260 261define void @scatter(ptr %a, i32 %len) {262; CHECK-LABEL: @scatter(263; CHECK-NEXT: vector.ph:264; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64265; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()266; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]267; CHECK: vector.body:268; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]269; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]270; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]], i32 3271; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()272; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv1i64.p0.i64(<vscale x 1 x i64> zeroinitializer, ptr [[TMP1]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 [[TMP2]])273; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]274; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], [[TMP0]]275; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[INDEX_NEXT]], [[WIDE_TRIP_COUNT]]276; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]277; CHECK: for.cond.cleanup:278; CHECK-NEXT: ret void279;280vector.ph:281 %wide.trip.count = zext i32 %len to i64282 %0 = tail call i64 @llvm.vscale.i64()283 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()284 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %0, i64 0285 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer286 br label %vector.body287 288vector.body: ; preds = %vector.body, %vector.ph289 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]290 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]291 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind, i32 3292 tail call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x ptr> %2, i32 8, <vscale x 1 x i1> splat (i1 true))293 %index.next = add nuw i64 %index, %0294 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat295 %3 = icmp ne i64 %index.next, %wide.trip.count296 br i1 %3, label %for.cond.cleanup, label %vector.body297 298for.cond.cleanup: ; preds = %vector.body299 ret void300}301 302define void @scatter_non_invariant_step(ptr %a, ptr %b, i32 %len) {303; CHECK-LABEL: @scatter_non_invariant_step(304; CHECK-NEXT: vector.ph:305; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64306; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]307; CHECK: vector.body:308; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]309; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]310; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]], i32 3311; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()312; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv1i64.p0.i64(<vscale x 1 x i64> zeroinitializer, ptr [[TMP0]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 [[TMP1]])313; CHECK-NEXT: [[B:%.*]] = getelementptr i64, ptr [[B1:%.*]], i64 [[VEC_IND_SCALAR]]314; CHECK-NEXT: [[STEP:%.*]] = load i64, ptr [[B]], align 8315; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], [[STEP]]316; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], [[STEP]]317; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i64 [[VEC_IND_NEXT_SCALAR]], [[WIDE_TRIP_COUNT]]318; CHECK-NEXT: br i1 [[TMP2]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]319; CHECK: for.cond.cleanup:320; CHECK-NEXT: ret void321;322vector.ph:323 %wide.trip.count = zext i32 %len to i64324 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()325 br label %vector.body326 327vector.body: ; preds = %vector.body, %vector.ph328 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]329 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]330 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind, i32 3331 tail call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x ptr> %2, i32 8, <vscale x 1 x i1> splat (i1 true))332 333 %b.gep = getelementptr i64, ptr %b, i64 %index334 %step = load i64, ptr %b.gep335 %index.next = add nuw i64 %index, %step336 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %step, i64 0337 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer338 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat339 %3 = icmp ne i64 %index.next, %wide.trip.count340 br i1 %3, label %for.cond.cleanup, label %vector.body341 342for.cond.cleanup: ; preds = %vector.body343 ret void344}345 346define void @scatter_non_invariant_step_add_shl(ptr %a, ptr %b, i32 %len) {347; CHECK-LABEL: @scatter_non_invariant_step_add_shl(348; CHECK-NEXT: vector.ph:349; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64350; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]351; CHECK: vector.body:352; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]353; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 168, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]354; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]], i32 3355; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()356; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv1i64.p0.i64(<vscale x 1 x i64> zeroinitializer, ptr [[TMP0]], i64 64, <vscale x 1 x i1> splat (i1 true), i32 [[TMP1]])357; CHECK-NEXT: [[B:%.*]] = getelementptr i64, ptr [[B1:%.*]], i64 [[VEC_IND_SCALAR]]358; CHECK-NEXT: [[STEP:%.*]] = load i64, ptr [[B]], align 8359; CHECK-NEXT: [[STEP1:%.*]] = shl i64 [[STEP]], 2360; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], [[STEP]]361; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], [[STEP1]]362; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i64 [[VEC_IND_NEXT_SCALAR]], [[WIDE_TRIP_COUNT]]363; CHECK-NEXT: br i1 [[TMP2]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]364; CHECK: for.cond.cleanup:365; CHECK-NEXT: ret void366;367vector.ph:368 %wide.trip.count = zext i32 %len to i64369 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()370 br label %vector.body371 372vector.body: ; preds = %vector.body, %vector.ph373 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]374 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]375 376 %vec.ind.add = add <vscale x 1 x i64> %vec.ind, splat (i64 42)377 %vec.ind.shl = shl <vscale x 1 x i64> %vec.ind.add, splat (i64 2)378 379 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind.shl, i32 3380 tail call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x ptr> %2, i32 8, <vscale x 1 x i1> splat (i1 true))381 382 %b.gep = getelementptr i64, ptr %b, i64 %index383 %step = load i64, ptr %b.gep384 %index.next = add nuw i64 %index, %step385 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %step, i64 0386 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer387 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat388 %3 = icmp ne i64 %index.next, %wide.trip.count389 br i1 %3, label %for.cond.cleanup, label %vector.body390 391for.cond.cleanup: ; preds = %vector.body392 ret void393}394 395define <vscale x 1 x i64> @gather_loopless(ptr %p, i64 %stride) {396; CHECK-LABEL: @gather_loopless(397; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[STRIDE:%.*]], 4398; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()399; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[P:%.*]], i64 [[TMP1]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP2]])400; CHECK-NEXT: ret <vscale x 1 x i64> [[TMP3]]401;402 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()403 %splat.insert = insertelement <vscale x 1 x i64> poison, i64 %stride, i64 0404 %splat = shufflevector <vscale x 1 x i64> %splat.insert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer405 %offsets = mul <vscale x 1 x i64> %step, %splat406 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> %offsets407 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(408 <vscale x 1 x ptr> %ptrs,409 i32 8,410 <vscale x 1 x i1> splat (i1 1),411 <vscale x 1 x i64> poison412 )413 ret <vscale x 1 x i64> %x414}415 416define <vscale x 1 x i64> @straightline_offset_add(ptr %p, i64 %offset) {417; CHECK-LABEL: @straightline_offset_add(418; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET:%.*]]419; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()420; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP1]], i64 4, <vscale x 1 x i1> splat (i1 true), i32 [[TMP2]])421; CHECK-NEXT: ret <vscale x 1 x i64> [[TMP3]]422;423 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()424 %splat.insert = insertelement <vscale x 1 x i64> poison, i64 %offset, i64 0425 %splat = shufflevector <vscale x 1 x i64> %splat.insert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer426 %offsetv = add <vscale x 1 x i64> %step, %splat427 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> %offsetv428 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(429 <vscale x 1 x ptr> %ptrs,430 i32 8,431 <vscale x 1 x i1> splat (i1 1),432 <vscale x 1 x i64> poison433 )434 ret <vscale x 1 x i64> %x435}436 437define <vscale x 1 x i64> @straightline_offset_disjoint_or_1(ptr %p) {438; CHECK-LABEL: @straightline_offset_disjoint_or_1(439; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 1440; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()441; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP1]], i64 8, <vscale x 1 x i1> splat (i1 true), i32 [[TMP2]])442; CHECK-NEXT: ret <vscale x 1 x i64> [[TMP3]]443;444 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()445 %step.shl = shl <vscale x 1 x i64> %step, splat (i64 1)446 %offsetv = or disjoint <vscale x 1 x i64> %step.shl, splat (i64 1)447 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> %offsetv448 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(449 <vscale x 1 x ptr> %ptrs,450 i32 8,451 <vscale x 1 x i1> splat (i1 true),452 <vscale x 1 x i64> poison453 )454 ret <vscale x 1 x i64> %x455}456 457define <vscale x 1 x i64> @straightline_offset_disjoint_or(ptr %p, i1 %offset) {458; CHECK-LABEL: @straightline_offset_disjoint_or(459; CHECK-NEXT: [[AND:%.*]] = zext i1 [[OFFSET:%.*]] to i64460; CHECK-NEXT: [[TMP4:%.*]] = or disjoint i64 4, [[AND]]461; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP4]]462; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()463; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP1]], i64 8, <vscale x 1 x i1> splat (i1 true), i32 [[TMP2]])464; CHECK-NEXT: ret <vscale x 1 x i64> [[TMP3]]465;466 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()467 %step.shl = shl <vscale x 1 x i64> %step, splat (i64 1)468 %add = add <vscale x 1 x i64> %step.shl, splat (i64 4)469 %zext = zext i1 %offset to i64470 %splat.insert = insertelement <vscale x 1 x i64> poison, i64 %zext, i64 0471 %splat = shufflevector <vscale x 1 x i64> %splat.insert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer472 %offsetv = or disjoint <vscale x 1 x i64> %add, %splat473 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> %offsetv474 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(475 <vscale x 1 x ptr> %ptrs,476 i32 8,477 <vscale x 1 x i1> splat (i1 true),478 <vscale x 1 x i64> poison479 )480 ret <vscale x 1 x i64> %x481}482 483define <vscale x 1 x i64> @straightline_offset_shl(ptr %p) {484; CHECK-LABEL: @straightline_offset_shl(485; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()486; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[P:%.*]], i64 32, <vscale x 1 x i1> splat (i1 true), i32 [[TMP1]])487; CHECK-NEXT: ret <vscale x 1 x i64> [[TMP2]]488;489 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()490 %offset = shl <vscale x 1 x i64> %step, splat (i64 3)491 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> %offset492 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(493 <vscale x 1 x ptr> %ptrs,494 i32 8,495 <vscale x 1 x i1> splat (i1 1),496 <vscale x 1 x i64> poison497 )498 ret <vscale x 1 x i64> %x499}500 501define <vscale x 1 x i64> @neg_shl_is_not_commutative(ptr %p) {502; CHECK-LABEL: @neg_shl_is_not_commutative(503; CHECK-NEXT: [[STEP:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()504; CHECK-NEXT: [[SPLAT_INSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 3, i64 0505; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[SPLAT_INSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer506; CHECK-NEXT: [[OFFSET:%.*]] = shl <vscale x 1 x i64> [[SPLAT]], [[STEP]]507; CHECK-NEXT: [[PTRS:%.*]] = getelementptr i32, ptr [[P:%.*]], <vscale x 1 x i64> [[OFFSET]]508; CHECK-NEXT: [[X:%.*]] = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[PTRS]], <vscale x 1 x i1> splat (i1 true), <vscale x 1 x i64> poison)509; CHECK-NEXT: ret <vscale x 1 x i64> [[X]]510;511 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()512 %splat.insert = insertelement <vscale x 1 x i64> poison, i64 3, i64 0513 %splat = shufflevector <vscale x 1 x i64> %splat.insert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer514 %offset = shl <vscale x 1 x i64> %splat, %step515 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> %offset516 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(517 <vscale x 1 x ptr> %ptrs,518 i32 8,519 <vscale x 1 x i1> splat (i1 1),520 <vscale x 1 x i64> poison521 )522 ret <vscale x 1 x i64> %x523}524 525define <vscale x 1 x i64> @straightline_offset_shl_nonc(ptr %p, i64 %shift) {526; CHECK-LABEL: @straightline_offset_shl_nonc(527; CHECK-NEXT: [[TMP1:%.*]] = shl i64 1, [[SHIFT:%.*]]528; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4529; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()530; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[P:%.*]], i64 [[TMP2]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP3]])531; CHECK-NEXT: ret <vscale x 1 x i64> [[TMP4]]532;533 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()534 %splat.insert = insertelement <vscale x 1 x i64> poison, i64 %shift, i64 0535 %splat = shufflevector <vscale x 1 x i64> %splat.insert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer536 %offset = shl <vscale x 1 x i64> %step, %splat537 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> %offset538 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(539 <vscale x 1 x ptr> %ptrs,540 i32 8,541 <vscale x 1 x i1> splat (i1 1),542 <vscale x 1 x i64> poison543 )544 ret <vscale x 1 x i64> %x545}546 547define void @scatter_loopless(<vscale x 1 x i64> %x, ptr %p, i64 %stride) {548; CHECK-LABEL: @scatter_loopless(549; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[STRIDE:%.*]], 4550; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()551; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv1i64.p0.i64(<vscale x 1 x i64> [[X:%.*]], ptr [[P:%.*]], i64 [[TMP1]], <vscale x 1 x i1> splat (i1 true), i32 [[TMP2]])552; CHECK-NEXT: ret void553;554 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()555 %splat.insert = insertelement <vscale x 1 x i64> poison, i64 %stride, i64 0556 %splat = shufflevector <vscale x 1 x i64> %splat.insert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer557 %offsets = mul <vscale x 1 x i64> %step, %splat558 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> %offsets559 call void @llvm.masked.scatter.nxv1i64.nxv1p0(560 <vscale x 1 x i64> %x,561 <vscale x 1 x ptr> %ptrs,562 i32 8,563 <vscale x 1 x i1> splat (i1 1)564 )565 ret void566}567 568; We previously crashed expecting a constant to be fixed length.569define void @constant_stride(<vscale x 1 x i64> %x, ptr %p, i64 %stride) {570; CHECK-LABEL: @constant_stride(571; CHECK-NEXT: [[PTRS:%.*]] = getelementptr i32, ptr [[P:%.*]], <vscale x 1 x i64> zeroinitializer572; CHECK-NEXT: call void @llvm.masked.scatter.nxv1i64.nxv1p0(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x ptr> align 8 [[PTRS]], <vscale x 1 x i1> splat (i1 true))573; CHECK-NEXT: ret void574;575 %ptrs = getelementptr i32, ptr %p, <vscale x 1 x i64> zeroinitializer576 call void @llvm.masked.scatter.nxv1i64.nxv1p0(577 <vscale x 1 x i64> %x,578 <vscale x 1 x ptr> %ptrs,579 i32 8,580 <vscale x 1 x i1> splat (i1 1)581 )582 ret void583}584 585define <vscale x 1 x i64> @vector_base_scalar_offset(ptr %p, i64 %offset) {586; CHECK-LABEL: @vector_base_scalar_offset(587; CHECK-NEXT: [[PTRS2OFFSET:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET:%.*]]588; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()589; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[PTRS2OFFSET]], i64 8, <vscale x 1 x i1> splat (i1 true), i32 [[TMP1]])590; CHECK-NEXT: ret <vscale x 1 x i64> [[TMP2]]591;592 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()593 %ptrs1 = getelementptr i64, ptr %p, <vscale x 1 x i64> %step594 %ptrs2 = getelementptr i64, <vscale x 1 x ptr> %ptrs1, i64 %offset595 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(596 <vscale x 1 x ptr> %ptrs2,597 i32 8,598 <vscale x 1 x i1> splat (i1 1),599 <vscale x 1 x i64> poison600 )601 ret <vscale x 1 x i64> %x602}603 604define <vscale x 1 x i64> @splat_base_scalar_offset(ptr %p, i64 %offset) {605; CHECK-LABEL: @splat_base_scalar_offset(606; CHECK-NEXT: [[PTRSOFFSET:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET:%.*]]607; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32()608; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[PTRSOFFSET]], i64 0, <vscale x 1 x i1> splat (i1 true), i32 [[TMP1]])609; CHECK-NEXT: ret <vscale x 1 x i64> [[TMP2]]610;611 %head = insertelement <vscale x 1 x ptr> poison, ptr %p, i32 0612 %splat = shufflevector <vscale x 1 x ptr> %head, <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer613 %ptrs = getelementptr i64, <vscale x 1 x ptr> %splat, i64 %offset614 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(615 <vscale x 1 x ptr> %ptrs,616 i32 8,617 <vscale x 1 x i1> splat (i1 1),618 <vscale x 1 x i64> poison619 )620 ret <vscale x 1 x i64> %x621}622 623; We shouldn't be able to determine a stride here.624define <vscale x 1 x i64> @nonstrided_base_scalar_offset(ptr %p, <vscale x 1 x i64> %v, i64 %offset) {625; CHECK-LABEL: @nonstrided_base_scalar_offset(626; CHECK-NEXT: [[PTRS1:%.*]] = getelementptr i64, ptr [[P:%.*]], <vscale x 1 x i64> [[V:%.*]]627; CHECK-NEXT: [[PTRS2:%.*]] = getelementptr i64, <vscale x 1 x ptr> [[PTRS1]], i64 [[OFFSET:%.*]]628; CHECK-NEXT: [[X:%.*]] = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[PTRS2]], <vscale x 1 x i1> splat (i1 true), <vscale x 1 x i64> poison)629; CHECK-NEXT: ret <vscale x 1 x i64> [[X]]630;631 %ptrs1 = getelementptr i64, ptr %p, <vscale x 1 x i64> %v632 %ptrs2 = getelementptr i64, <vscale x 1 x ptr> %ptrs1, i64 %offset633 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(634 <vscale x 1 x ptr> %ptrs2,635 i32 8,636 <vscale x 1 x i1> splat (i1 1),637 <vscale x 1 x i64> poison638 )639 ret <vscale x 1 x i64> %x640}641 642; We shouldn't be able to determine a scalar base here.643define <vscale x 1 x i64> @vector_base_vector_offset(ptr %p, <vscale x 1 x i64> %offset) {644; CHECK-LABEL: @vector_base_vector_offset(645; CHECK-NEXT: [[STEP:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()646; CHECK-NEXT: [[PTRS1:%.*]] = getelementptr i64, ptr [[P:%.*]], <vscale x 1 x i64> [[STEP]]647; CHECK-NEXT: [[PTRS2:%.*]] = getelementptr i64, <vscale x 1 x ptr> [[PTRS1]], <vscale x 1 x i64> [[OFFSET:%.*]]648; CHECK-NEXT: [[X:%.*]] = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> align 8 [[PTRS2]], <vscale x 1 x i1> splat (i1 true), <vscale x 1 x i64> poison)649; CHECK-NEXT: ret <vscale x 1 x i64> [[X]]650;651 %step = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()652 %ptrs1 = getelementptr i64, ptr %p, <vscale x 1 x i64> %step653 %ptrs2 = getelementptr i64, <vscale x 1 x ptr> %ptrs1, <vscale x 1 x i64> %offset654 %x = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(655 <vscale x 1 x ptr> %ptrs2,656 i32 8,657 <vscale x 1 x i1> splat (i1 1),658 <vscale x 1 x i64> poison659 )660 ret <vscale x 1 x i64> %x661}662 663define <vscale x 1 x i64> @vp_gather(ptr %a, i32 %len) {664; CHECK-LABEL: @vp_gather(665; CHECK-NEXT: vector.ph:666; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64667; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()668; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]669; CHECK: vector.body:670; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]671; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]672; CHECK-NEXT: [[ACCUM:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[VECTOR_BODY]] ]673; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]], i32 3674; CHECK-NEXT: [[GATHER:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP2]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 42)675; CHECK-NEXT: [[ACCUM_NEXT]] = add <vscale x 1 x i64> [[ACCUM]], [[GATHER]]676; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], [[TMP0]]677; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], [[TMP0]]678; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[VEC_IND_NEXT_SCALAR]], [[WIDE_TRIP_COUNT]]679; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]680; CHECK: for.cond.cleanup:681; CHECK-NEXT: ret <vscale x 1 x i64> [[ACCUM_NEXT]]682;683vector.ph:684 %wide.trip.count = zext i32 %len to i64685 %0 = tail call i64 @llvm.vscale.i64()686 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()687 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %0, i64 0688 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer689 br label %vector.body690 691vector.body: ; preds = %vector.body, %vector.ph692 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]693 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]694 %accum = phi <vscale x 1 x i64> [ zeroinitializer, %vector.ph ], [ %accum.next, %vector.body ]695 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind, i32 3696 %gather = call <vscale x 1 x i64> @llvm.vp.gather(<vscale x 1 x ptr> %2, <vscale x 1 x i1> splat (i1 true), i32 42)697 %accum.next = add <vscale x 1 x i64> %accum, %gather698 %index.next = add nuw i64 %index, %0699 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat700 %3 = icmp ne i64 %index.next, %wide.trip.count701 br i1 %3, label %for.cond.cleanup, label %vector.body702 703for.cond.cleanup: ; preds = %vector.body704 ret <vscale x 1 x i64> %accum.next705}706 707define void @vp_scatter(ptr %a, i32 %len) {708; CHECK-LABEL: @vp_scatter(709; CHECK-NEXT: vector.ph:710; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64711; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()712; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]713; CHECK: vector.body:714; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]715; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]716; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]], i32 3717; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv1i64.p0.i64(<vscale x 1 x i64> zeroinitializer, ptr [[TMP2]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 42)718; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], [[TMP0]]719; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], [[TMP0]]720; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[VEC_IND_NEXT_SCALAR]], [[WIDE_TRIP_COUNT]]721; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]722; CHECK: for.cond.cleanup:723; CHECK-NEXT: ret void724;725vector.ph:726 %wide.trip.count = zext i32 %len to i64727 %0 = tail call i64 @llvm.vscale.i64()728 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()729 %.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %0, i64 0730 %.splat = shufflevector <vscale x 1 x i64> %.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer731 br label %vector.body732 733vector.body: ; preds = %vector.body, %vector.ph734 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]735 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]736 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind, i32 3737 tail call void @llvm.vp.scatter(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x ptr> %2, <vscale x 1 x i1> splat (i1 true), i32 42)738 %index.next = add nuw i64 %index, %0739 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %.splat740 %3 = icmp ne i64 %index.next, %wide.trip.count741 br i1 %3, label %for.cond.cleanup, label %vector.body742 743for.cond.cleanup: ; preds = %vector.body744 ret void745}746 747; Test that reflects what the loop vectorizer will generate for an EVL tail748; folded loop749 750define <vscale x 1 x i64> @evl_gather(ptr %a, i32 %len) {751; CHECK-LABEL: @evl_gather(752; CHECK-NEXT: vector.ph:753; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64754; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]755; CHECK: vector.body:756; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]757; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]758; CHECK-NEXT: [[ACCUM:%.*]] = phi <vscale x 1 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[VECTOR_BODY]] ]759; CHECK-NEXT: [[ELEMS:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[VEC_IND_SCALAR]]760; CHECK-NEXT: [[EVL:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[ELEMS]], i32 1, i1 true)761; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR1]], i32 3762; CHECK-NEXT: [[GATHER:%.*]] = call <vscale x 1 x i64> @llvm.experimental.vp.strided.load.nxv1i64.p0.i64(ptr [[TMP0]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 [[EVL]])763; CHECK-NEXT: [[ACCUM_NEXT]] = add <vscale x 1 x i64> [[ACCUM]], [[GATHER]]764; CHECK-NEXT: [[EVL_ZEXT:%.*]] = zext i32 [[EVL]] to i64765; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add nuw i64 [[VEC_IND_SCALAR]], [[EVL_ZEXT]]766; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add i64 [[VEC_IND_SCALAR1]], [[EVL_ZEXT]]767; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[VEC_IND_NEXT_SCALAR]], [[WIDE_TRIP_COUNT]]768; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]769; CHECK: for.cond.cleanup:770; CHECK-NEXT: ret <vscale x 1 x i64> [[ACCUM_NEXT]]771;772vector.ph:773 %wide.trip.count = zext i32 %len to i64774 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()775 br label %vector.body776 777vector.body: ; preds = %vector.body, %vector.ph778 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]779 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]780 %accum = phi <vscale x 1 x i64> [ zeroinitializer, %vector.ph ], [ %accum.next, %vector.body ]781 782 %elems = sub i64 %wide.trip.count, %index783 %evl = call i32 @llvm.experimental.get.vector.length.i64(i64 %elems, i32 1, i1 true)784 785 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind, i32 3786 %gather = call <vscale x 1 x i64> @llvm.vp.gather(<vscale x 1 x ptr> %2, <vscale x 1 x i1> splat (i1 true), i32 %evl)787 %accum.next = add <vscale x 1 x i64> %accum, %gather788 789 %evl.zext = zext i32 %evl to i64790 %index.next = add nuw i64 %index, %evl.zext791 %evl.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %evl.zext, i64 0792 %evl.splat = shufflevector <vscale x 1 x i64> %evl.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer793 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %evl.splat794 %3 = icmp ne i64 %index.next, %wide.trip.count795 br i1 %3, label %for.cond.cleanup, label %vector.body796 797for.cond.cleanup: ; preds = %vector.body798 ret <vscale x 1 x i64> %accum.next799}800 801; Test that reflects what the loop vectorizer will generate for an EVL tail802; folded loop803 804define void @evl_scatter(ptr %a, i32 %len) {805; CHECK-LABEL: @evl_scatter(806; CHECK-NEXT: vector.ph:807; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN:%.*]] to i64808; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]809; CHECK: vector.body:810; CHECK-NEXT: [[VEC_IND_SCALAR1:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[VEC_IND_NEXT_SCALAR1:%.*]], [[VECTOR_BODY]] ]811; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]812; CHECK-NEXT: [[ELEMS:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[VEC_IND_SCALAR1]]813; CHECK-NEXT: [[EVL:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[ELEMS]], i32 1, i1 true)814; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr [[A:%.*]], i64 [[VEC_IND_SCALAR]], i32 3815; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv1i64.p0.i64(<vscale x 1 x i64> zeroinitializer, ptr [[TMP0]], i64 16, <vscale x 1 x i1> splat (i1 true), i32 [[EVL]])816; CHECK-NEXT: [[EVL_ZEXT:%.*]] = zext i32 [[EVL]] to i64817; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR1]] = add nuw i64 [[VEC_IND_SCALAR1]], [[EVL_ZEXT]]818; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i64 [[VEC_IND_SCALAR]], [[EVL_ZEXT]]819; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[VEC_IND_NEXT_SCALAR1]], [[WIDE_TRIP_COUNT]]820; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]821; CHECK: for.cond.cleanup:822; CHECK-NEXT: ret void823;824vector.ph:825 %wide.trip.count = zext i32 %len to i64826 %1 = tail call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()827 br label %vector.body828 829vector.body: ; preds = %vector.body, %vector.ph830 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]831 %vec.ind = phi <vscale x 1 x i64> [ %1, %vector.ph ], [ %vec.ind.next, %vector.body ]832 833 %elems = sub i64 %wide.trip.count, %index834 %evl = call i32 @llvm.experimental.get.vector.length.i64(i64 %elems, i32 1, i1 true)835 836 %2 = getelementptr inbounds %struct.foo, ptr %a, <vscale x 1 x i64> %vec.ind, i32 3837 tail call void @llvm.vp.scatter(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x ptr> %2, <vscale x 1 x i1> splat (i1 true), i32 %evl)838 839 %evl.zext = zext i32 %evl to i64840 %index.next = add nuw i64 %index, %evl.zext841 %evl.splatinsert = insertelement <vscale x 1 x i64> poison, i64 %evl.zext, i64 0842 %evl.splat = shufflevector <vscale x 1 x i64> %evl.splatinsert, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer843 %vec.ind.next = add <vscale x 1 x i64> %vec.ind, %evl.splat844 %3 = icmp ne i64 %index.next, %wide.trip.count845 br i1 %3, label %for.cond.cleanup, label %vector.body846 847for.cond.cleanup: ; preds = %vector.body848 ret void849}850