592 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"2; RUN: opt -S -hints-allow-reordering=false -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue < %s | FileCheck %s3 4target triple = "aarch64-unknown-linux-gnu"5 6 7define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {8; CHECK-LABEL: @simple_memset(9; CHECK-NEXT: entry:10; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)11; CHECK-NEXT: br label [[VECTOR_PH:%.*]]12; CHECK: vector.ph:13; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()14; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 415; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()16; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 217; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]18; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]19; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 020; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])21; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 022; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer23; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]24; CHECK: vector.body:25; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]26; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]27; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]28; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])29; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]30; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])31; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 032; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[TMP13]], true33; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]34; CHECK: middle.block:35; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT:%.*]]36; CHECK: while.end.loopexit:37; CHECK-NEXT: ret void38;39entry:40 br label %while.body41 42while.body: ; preds = %while.body, %entry43 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]44 %gep = getelementptr i32, ptr %ptr, i64 %index45 store i32 %val, ptr %gep46 %index.next = add nsw i64 %index, 147 %cmp10 = icmp ult i64 %index.next, %n48 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !049 50while.end.loopexit: ; preds = %while.body51 ret void52}53 54 55define void @simple_memset_v4i32(i32 %val, ptr %ptr, i64 %n) #0 {56; CHECK-LABEL: @simple_memset_v4i32(57; CHECK-NEXT: entry:58; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)59; CHECK-NEXT: br label [[VECTOR_PH:%.*]]60; CHECK: vector.ph:61; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[UMAX]], 462; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[UMAX]], 463; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[TMP0]], i64 064; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[UMAX]])65; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[VAL:%.*]], i64 066; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer67; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]68; CHECK: vector.body:69; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]70; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]71; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]72; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP4]], <4 x i1> [[ACTIVE_LANE_MASK]])73; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], 474; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX1]], i64 [[TMP2]])75; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 076; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP6]], true77; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]78; CHECK: middle.block:79; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT:%.*]]80; CHECK: while.end.loopexit:81; CHECK-NEXT: ret void82;83entry:84 br label %while.body85 86while.body: ; preds = %while.body, %entry87 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]88 %gep = getelementptr i32, ptr %ptr, i64 %index89 store i32 %val, ptr %gep90 %index.next = add nsw i64 %index, 191 %cmp10 = icmp ult i64 %index.next, %n92 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !393 94while.end.loopexit: ; preds = %while.body95 ret void96}97 98 99define void @simple_memcpy(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {100; CHECK-LABEL: @simple_memcpy(101; CHECK-NEXT: entry:102; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)103; CHECK-NEXT: br label [[VECTOR_PH:%.*]]104; CHECK: vector.ph:105; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()106; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4107; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()108; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2109; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]110; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]111; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0112; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])113; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]114; CHECK: vector.body:115; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]116; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]117; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[SRC:%.*]], i64 [[INDEX1]]118; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)119; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[DST:%.*]], i64 [[INDEX1]]120; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP13]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])121; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]122; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])123; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0124; CHECK-NEXT: [[TMP14:%.*]] = xor i1 [[TMP12]], true125; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]126; CHECK: middle.block:127; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT:%.*]]128; CHECK: while.end.loopexit:129; CHECK-NEXT: ret void130;131entry:132 br label %while.body133 134while.body: ; preds = %while.body, %entry135 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]136 %gep1 = getelementptr i32, ptr %src, i64 %index137 %val = load i32, ptr %gep1138 %gep2 = getelementptr i32, ptr %dst, i64 %index139 store i32 %val, ptr %gep2140 %index.next = add nsw i64 %index, 1141 %cmp10 = icmp ult i64 %index.next, %n142 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0143 144while.end.loopexit: ; preds = %while.body145 ret void146}147 148 149define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {150; CHECK-LABEL: @copy_stride4(151; CHECK-NEXT: entry:152; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 4)153; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1154; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 2155; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1156; CHECK-NEXT: br label [[VECTOR_PH:%.*]]157; CHECK: vector.ph:158; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()159; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4160; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()161; CHECK-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2162; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]]163; CHECK-NEXT: [[TMP11:%.*]] = icmp ugt i64 [[TMP2]], [[TMP9]]164; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0165; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP2]])166; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()167; CHECK-NEXT: [[TMP14:%.*]] = mul nsw <vscale x 4 x i64> [[TMP13]], splat (i64 4)168; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <vscale x 4 x i64> zeroinitializer, [[TMP14]]169; CHECK-NEXT: [[TMP18:%.*]] = mul nsw i64 4, [[TMP4]]170; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP18]], i64 0171; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer172; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]173; CHECK: vector.body:174; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]175; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]176; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]177; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[SRC:%.*]], <vscale x 4 x i64> [[VEC_IND]]178; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP19]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)179; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]180; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x ptr> align 4 [[TMP20]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])181; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP4]]182; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP12]])183; CHECK-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0184; CHECK-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true185; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]186; CHECK-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]187; CHECK: middle.block:188; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT:%.*]]189; CHECK: while.end.loopexit:190; CHECK-NEXT: ret void191;192entry:193 br label %while.body194 195while.body: ; preds = %while.body, %entry196 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]197 %gep1 = getelementptr i32, ptr %src, i64 %index198 %val = load i32, ptr %gep1199 %gep2 = getelementptr i32, ptr %dst, i64 %index200 store i32 %val, ptr %gep2201 %index.next = add nsw i64 %index, 4202 %cmp10 = icmp ult i64 %index.next, %n203 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0204 205while.end.loopexit: ; preds = %while.body206 ret void207}208 209 210define void @simple_gather_scatter(ptr noalias %dst, ptr noalias %src, ptr noalias %ind, i64 %n) #0 {211; CHECK-LABEL: @simple_gather_scatter(212; CHECK-NEXT: entry:213; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)214; CHECK-NEXT: br label [[VECTOR_PH:%.*]]215; CHECK: vector.ph:216; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()217; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4218; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()219; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2220; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]221; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]222; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0223; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])224; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]225; CHECK: vector.body:226; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]227; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]228; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[IND:%.*]], i64 [[INDEX1]]229; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)230; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[SRC:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]]231; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)232; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]]233; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x ptr> align 4 [[TMP14]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])234; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]235; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])236; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0237; CHECK-NEXT: [[TMP16:%.*]] = xor i1 [[TMP15]], true238; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]239; CHECK: middle.block:240; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT:%.*]]241; CHECK: while.end.loopexit:242; CHECK-NEXT: ret void243;244entry:245 br label %while.body246 247while.body: ; preds = %while.body, %entry248 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]249 %gep1 = getelementptr i32, ptr %ind, i64 %index250 %ind_val = load i32, ptr %gep1251 %gep2 = getelementptr i32, ptr %src, i32 %ind_val252 %val = load i32, ptr %gep2253 %gep3 = getelementptr i32, ptr %dst, i32 %ind_val254 store i32 %val, ptr %gep3255 %index.next = add nsw i64 %index, 1256 %cmp10 = icmp ult i64 %index.next, %n257 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0258 259while.end.loopexit: ; preds = %while.body260 ret void261}262 263 264; The original loop had an unconditional uniform load. Let's make sure265; we don't artificially create new predicated blocks for the load.266define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) #0 {267; CHECK-LABEL: @uniform_load(268; CHECK-NEXT: entry:269; CHECK-NEXT: br label [[VECTOR_PH:%.*]]270; CHECK: vector.ph:271; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()272; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4273; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()274; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2275; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N:%.*]], [[TMP6]]276; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]277; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0278; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])279; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]280; CHECK: vector.body:281; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]282; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]283; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[SRC:%.*]], align 4284; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0285; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer286; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]]287; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP12]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])288; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]289; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])290; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0291; CHECK-NEXT: [[TMP13:%.*]] = xor i1 [[TMP14]], true292; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]293; CHECK: middle.block:294; CHECK-NEXT: br label [[FOR_END:%.*]]295; CHECK: for.end:296; CHECK-NEXT: ret void297;298 299entry:300 br label %for.body301 302for.body: ; preds = %entry, %for.body303 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]304 %val = load i32, ptr %src, align 4305 %arrayidx = getelementptr inbounds i32, ptr %dst, i64 %indvars.iv306 store i32 %val, ptr %arrayidx, align 4307 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1308 %exitcond.not = icmp eq i64 %indvars.iv.next, %n309 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0310 311for.end: ; preds = %for.body, %entry312 ret void313}314 315 316; The original loop had a conditional uniform load. In this case we actually317; do need to perform conditional loads and so we end up using a gather instead.318; However, we at least ensure the mask is the overlap of the loop predicate319; and the original condition.320define void @cond_uniform_load(ptr noalias %dst, ptr noalias readonly %src, ptr noalias readonly %cond, i64 %n) #0 {321; CHECK-LABEL: @cond_uniform_load(322; CHECK-NEXT: entry:323; CHECK-NEXT: br label [[VECTOR_PH:%.*]]324; CHECK: vector.ph:325; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()326; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4327; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()328; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2329; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N:%.*]], [[TMP6]]330; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]331; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0332; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])333; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[SRC:%.*]], i64 0334; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer335; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]336; CHECK: vector.body:337; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]338; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]339; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX1]]340; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)341; CHECK-NEXT: [[TMP14:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer342; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> zeroinitializer343; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> poison)344; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32> zeroinitializer345; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX1]]346; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI]], ptr align 4 [[TMP16]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])347; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]348; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])349; CHECK-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0350; CHECK-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true351; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]352; CHECK: middle.block:353; CHECK-NEXT: br label [[FOR_END:%.*]]354; CHECK: for.end:355; CHECK-NEXT: ret void356;357 358entry:359 br label %for.body360 361for.body: ; preds = %entry, %if.end362 %index = phi i64 [ %index.next, %if.end ], [ 0, %entry ]363 %arrayidx = getelementptr inbounds i32, ptr %cond, i64 %index364 %0 = load i32, ptr %arrayidx, align 4365 %tobool.not = icmp eq i32 %0, 0366 br i1 %tobool.not, label %if.end, label %if.then367 368if.then: ; preds = %for.body369 %1 = load i32, ptr %src, align 4370 br label %if.end371 372if.end: ; preds = %if.then, %for.body373 %val.0 = phi i32 [ %1, %if.then ], [ 0, %for.body ]374 %arrayidx1 = getelementptr inbounds i32, ptr %dst, i64 %index375 store i32 %val.0, ptr %arrayidx1, align 4376 %index.next = add nuw i64 %index, 1377 %exitcond.not = icmp eq i64 %index.next, %n378 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0379 380for.end: ; preds = %for.inc, %entry381 ret void382}383 384 385; The original loop had an unconditional uniform store. Let's make sure386; we don't artificially create new predicated blocks for the load.387define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n) #0 {388; CHECK-LABEL: @uniform_store(389; CHECK-NEXT: entry:390; CHECK-NEXT: br label [[VECTOR_PH:%.*]]391; CHECK: vector.ph:392; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()393; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4394; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()395; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2396; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N:%.*]], [[TMP6]]397; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]398; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0399; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])400; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST:%.*]], i64 0401; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer402; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]403; CHECK: vector.body:404; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]405; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]406; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[INDEX]]407; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)408; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])409; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]410; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])411; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0412; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[TMP13]], true413; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]414; CHECK: middle.block:415; CHECK-NEXT: br label [[FOR_END:%.*]]416; CHECK: for.end:417; CHECK-NEXT: ret void418;419 420entry:421 br label %for.body422 423for.body: ; preds = %entry, %for.body424 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]425 %arrayidx = getelementptr inbounds i32, ptr %src, i64 %indvars.iv426 %val = load i32, ptr %arrayidx, align 4427 store i32 %val, ptr %dst, align 4428 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1429 %exitcond.not = icmp eq i64 %indvars.iv.next, %n430 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0431 432for.end: ; preds = %for.body, %entry433 ret void434}435 436 437define void @simple_fdiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {438; CHECK-LABEL: @simple_fdiv(439; CHECK-NEXT: entry:440; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)441; CHECK-NEXT: br label [[VECTOR_PH:%.*]]442; CHECK: vector.ph:443; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()444; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4445; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()446; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2447; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]448; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]449; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0450; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])451; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]452; CHECK: vector.body:453; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]454; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]455; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[SRC:%.*]], i64 [[INDEX1]]456; CHECK-NEXT: [[TMP12:%.*]] = getelementptr float, ptr [[DST:%.*]], i64 [[INDEX1]]457; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)458; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)459; CHECK-NEXT: [[TMP15:%.*]] = fdiv <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD2]]460; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[TMP15]], ptr align 4 [[TMP12]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])461; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP1]]462; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])463; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0464; CHECK-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true465; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]466; CHECK: middle.block:467; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT:%.*]]468; CHECK: while.end.loopexit:469; CHECK-NEXT: ret void470;471entry:472 br label %while.body473 474while.body: ; preds = %while.body, %entry475 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]476 %gep1 = getelementptr float, ptr %src, i64 %index477 %gep2 = getelementptr float, ptr %dst, i64 %index478 %val1 = load float, ptr %gep1479 %val2 = load float, ptr %gep2480 %res = fdiv float %val1, %val2481 store float %res, ptr %gep2482 %index.next = add nsw i64 %index, 1483 %cmp10 = icmp ult i64 %index.next, %n484 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0485 486while.end.loopexit: ; preds = %while.body487 ret void488}489 490; Integer divides can throw exceptions; if we vectorize, we must ensure491; that speculated lanes don't fault.492define void @simple_idiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {493; CHECK-LABEL: @simple_idiv(494; CHECK-NEXT: entry:495; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)496; CHECK-NEXT: br label [[VECTOR_PH:%.*]]497; CHECK: vector.ph:498; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()499; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4500; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()501; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2502; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]503; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]504; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0505; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])506; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]507; CHECK: vector.body:508; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]509; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]510; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[SRC:%.*]], i64 [[INDEX1]]511; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[DST:%.*]], i64 [[INDEX1]]512; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)513; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)514; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD2]], <vscale x 4 x i32> splat (i32 1)515; CHECK-NEXT: [[TMP16:%.*]] = udiv <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP15]]516; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP16]], ptr align 4 [[TMP12]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])517; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP1]]518; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])519; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0520; CHECK-NEXT: [[TMP17:%.*]] = xor i1 [[TMP14]], true521; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]522; CHECK: middle.block:523; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT:%.*]]524; CHECK: while.end.loopexit:525; CHECK-NEXT: ret void526;527entry:528 br label %while.body529 530while.body: ; preds = %while.body, %entry531 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]532 %gep1 = getelementptr i32, ptr %src, i64 %index533 %gep2 = getelementptr i32, ptr %dst, i64 %index534 %val1 = load i32, ptr %gep1535 %val2 = load i32, ptr %gep2536 %res = udiv i32 %val1, %val2537 store i32 %res, ptr %gep2538 %index.next = add nsw i64 %index, 1539 %cmp10 = icmp ult i64 %index.next, %n540 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0541 542while.end.loopexit: ; preds = %while.body543 ret void544}545 546define void @simple_memset_trip1024(i32 %val, ptr %ptr, i64 %n) #0 {547; CHECK-LABEL: @simple_memset_trip1024(548; CHECK-NEXT: entry:549; CHECK-NEXT: br label [[VECTOR_PH:%.*]]550; CHECK: vector.ph:551; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()552; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4553; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]554; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]555; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0556; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer557; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]558; CHECK: vector.body:559; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]560; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]561; CHECK-NEXT: store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP7]], align 4562; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP3]]563; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]564; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]565; CHECK: middle.block:566; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]567; CHECK-NEXT: br i1 [[CMP_N]], label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH:%.*]]568; CHECK: scalar.ph:569;570entry:571 br label %while.body572 573while.body: ; preds = %while.body, %entry574 %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]575 %gep = getelementptr i32, ptr %ptr, i64 %index576 store i32 %val, ptr %gep577 %index.next = add nsw i64 %index, 1578 %cmp10 = icmp ult i64 %index.next, 1024579 br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0580 581while.end.loopexit: ; preds = %while.body582 ret void583}584 585!0 = distinct !{!0, !1, !2}586!1 = !{!"llvm.loop.vectorize.width", i32 4}587!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}588!3 = distinct !{!3, !4}589!4 = !{!"llvm.loop.vectorize.width", i32 4}590 591attributes #0 = { "target-features"="+sve" vscale_range(1,16) }592