792 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=loop-vectorize \3; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \4; RUN: -mtriple=riscv64 -mattr=+v -S < %s | FileCheck --check-prefix=IF-EVL %s5 6; RUN: opt -passes=loop-vectorize \7; RUN: -prefer-predicate-over-epilogue=scalar-epilogue \8; RUN: -mtriple=riscv64 -mattr=+v -S < %s | FileCheck --check-prefix=NO-VP %s9 10define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {11; IF-EVL-LABEL: @interleave(12; IF-EVL-NEXT: entry:13; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]14; IF-EVL: vector.ph:15; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]16; IF-EVL: vector.body:17; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]18; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]19; IF-EVL-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)20; IF-EVL-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[B:%.*]], i64 [[EVL_BASED_IV]], i32 021; IF-EVL-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP16]], 222; IF-EVL-NEXT: [[WIDE_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])23; IF-EVL-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])24; IF-EVL-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 025; IF-EVL-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 126; IF-EVL-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[TMP15]], [[TMP14]]27; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]28; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP9]], ptr align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])29; IF-EVL-NEXT: [[TMP8:%.*]] = zext i32 [[TMP16]] to i6430; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP8]], [[EVL_BASED_IV]]31; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]32; IF-EVL-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 033; IF-EVL-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]34; IF-EVL: middle.block:35; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]36; IF-EVL: for.cond.cleanup:37; IF-EVL-NEXT: ret void38;39; NO-VP-LABEL: @interleave(40; NO-VP-NEXT: entry:41; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()42; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 243; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]44; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]45; NO-VP: vector.ph:46; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()47; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 448; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]49; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]50; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]51; NO-VP: vector.body:52; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]53; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i32], ptr [[B:%.*]], i64 [[INDEX]], i32 054; NO-VP-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x i32>, ptr [[TMP13]], align 455; NO-VP-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC1]])56; NO-VP-NEXT: [[TMP18:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 057; NO-VP-NEXT: [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 158; NO-VP-NEXT: [[TMP21:%.*]] = add nsw <vscale x 4 x i32> [[TMP19]], [[TMP18]]59; NO-VP-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]60; NO-VP-NEXT: store <vscale x 4 x i32> [[TMP21]], ptr [[TMP22]], align 461; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]62; NO-VP-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]63; NO-VP-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]64; NO-VP: middle.block:65; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]66; NO-VP-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]67; NO-VP: scalar.ph:68; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]69; NO-VP-NEXT: br label [[FOR_BODY:%.*]]70; NO-VP: for.body:71; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]72; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i64 [[IV]], i32 073; NO-VP-NEXT: [[TMP29:%.*]] = load i32, ptr [[ARRAYIDX]], align 474; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i64 [[IV]], i32 175; NO-VP-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX2]], align 476; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP30]], [[TMP29]]77; NO-VP-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]78; NO-VP-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX4]], align 479; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 180; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]81; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]82; NO-VP: for.cond.cleanup:83; NO-VP-NEXT: ret void84;85entry:86 br label %for.body87 88for.body:89 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]90 %arrayidx = getelementptr inbounds [2 x i32], ptr %b, i64 %iv, i32 091 %0 = load i32, ptr %arrayidx, align 492 %arrayidx2 = getelementptr inbounds [2 x i32], ptr %b, i64 %iv, i32 193 %1 = load i32, ptr %arrayidx2, align 494 %add = add nsw i32 %1, %095 %arrayidx4 = getelementptr inbounds i32, ptr %a, i64 %iv96 store i32 %add, ptr %arrayidx4, align 497 %iv.next = add nuw nsw i64 %iv, 198 %exitcond.not = icmp eq i64 %iv.next, %N99 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !0100 101for.cond.cleanup:102 ret void103}104 105; Interleaved group with gap but without tail gap106; E.g.107; int (*a)[4];108; int rdx = 0;109; for (int i = 0; i < n; i++) {110; rdx += a[i][0];111; rdx += a[i][1];112; // No access a[i][2]113; rdx += a[i][3];114; }115;116define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {117; IF-EVL-LABEL: @load_factor_4_with_gap(118; IF-EVL-NEXT: entry:119; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]120; IF-EVL: vector.ph:121; IF-EVL-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()122; IF-EVL-NEXT: [[TMP1:%.*]] = mul <vscale x 4 x i64> [[TMP2]], splat (i64 1)123; IF-EVL-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP1]]124; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]125; IF-EVL: vector.body:126; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]127; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]128; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]129; IF-EVL-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)130; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64131; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0132; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer133; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0134; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])135; IF-EVL-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]136; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])137; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[WIDE_MASKED_GATHER1]]138; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3139; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])140; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]141; IF-EVL-NEXT: [[TMP12]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP4]])142; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]143; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]144; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0145; IF-EVL-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]146; IF-EVL: middle.block:147; IF-EVL-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP12]])148; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]149; IF-EVL: exit:150; IF-EVL-NEXT: ret i32 [[TMP15]]151;152; NO-VP-LABEL: @load_factor_4_with_gap(153; NO-VP-NEXT: entry:154; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()155; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2156; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N:%.*]], [[TMP1]]157; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]158; NO-VP: vector.ph:159; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()160; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4161; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]162; NO-VP-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0163; NO-VP-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]164; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP5]]165; NO-VP-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()166; NO-VP-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP6]], splat (i64 1)167; NO-VP-NEXT: [[INDUCTION:%.*]] = add nuw nsw <vscale x 4 x i64> zeroinitializer, [[TMP7]]168; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0169; NO-VP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer170; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]171; NO-VP: vector.body:172; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]173; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]174; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]175; NO-VP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0176; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)177; NO-VP-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]178; NO-VP-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)179; NO-VP-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER1]]180; NO-VP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3181; NO-VP-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)182; NO-VP-NEXT: [[TMP13]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER2]]183; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]184; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]185; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]186; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]187; NO-VP: middle.block:188; NO-VP-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])189; NO-VP-NEXT: br label [[SCALAR_PH]]190; NO-VP: scalar.ph:191; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]192; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]193; NO-VP-NEXT: br label [[FOR_BODY:%.*]]194; NO-VP: for.body:195; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]196; NO-VP-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]197; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0198; NO-VP-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4199; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP16]]200; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1201; NO-VP-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4202; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP17]]203; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3204; NO-VP-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4205; NO-VP-NEXT: [[ADD2]] = add nsw i32 [[ADD1]], [[TMP18]]206; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1207; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]208; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]209; NO-VP: exit:210; NO-VP-NEXT: [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[FOR_BODY]] ]211; NO-VP-NEXT: ret i32 [[ADD2_LCSSA]]212;213entry:214 br label %for.body215 216for.body:217 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]218 %rdx = phi i32 [ 0, %entry ], [ %add2, %for.body ]219 %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 0220 %0 = load i32, ptr %arrayidx, align 4221 %add = add nsw i32 %rdx, %0222 %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 1223 %1 = load i32, ptr %arrayidx, align 4224 %add1 = add nsw i32 %add, %1225 %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 3226 %2 = load i32, ptr %arrayidx2, align 4227 %add2 = add nsw i32 %add1, %2228 %iv.next = add nuw nsw i64 %iv, 1229 %exitcond.not = icmp eq i64 %iv.next, %n230 br i1 %exitcond.not, label %exit, label %for.body231 232exit:233 ret i32 %add2234}235 236; Interleaved group with gap but without tail gap237; E.g.238; int (*a)[4];239; for (int i = 0; i < n; i++) {240; a[i][0] = i;241; a[i][1] = i;242; // No access a[i][2]243; a[i][3] = i;244; }245;246define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) {247; IF-EVL-LABEL: @store_factor_4_with_gap(248; IF-EVL-NEXT: entry:249; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]250; IF-EVL: vector.ph:251; IF-EVL-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()252; IF-EVL-NEXT: [[TMP1:%.*]] = mul <vscale x 4 x i32> [[TMP4]], splat (i32 1)253; IF-EVL-NEXT: [[INDUCTION1:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP1]]254; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]255; IF-EVL: vector.body:256; IF-EVL-NEXT: [[VEC_IND2:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]257; IF-EVL-NEXT: [[AVL:%.*]] = phi i32 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]258; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)259; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP6]], i64 0260; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer261; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND2]], i32 0262; IF-EVL-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])263; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND2]], i32 1264; IF-EVL-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])265; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND2]], i32 3266; IF-EVL-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])267; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]268; IF-EVL-NEXT: [[VEC_IND_NEXT5]] = add <vscale x 4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT]]269; IF-EVL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0270; IF-EVL-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]271; IF-EVL: middle.block:272; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]273; IF-EVL: exit:274; IF-EVL-NEXT: ret void275;276; NO-VP-LABEL: @store_factor_4_with_gap(277; NO-VP-NEXT: entry:278; NO-VP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()279; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2280; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP1]]281; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]282; NO-VP: vector.ph:283; NO-VP-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()284; NO-VP-NEXT: [[TMP9:%.*]] = mul nuw i32 [[TMP2]], 4285; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP9]]286; NO-VP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]287; NO-VP-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()288; NO-VP-NEXT: [[TMP5:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP7]], splat (i32 1)289; NO-VP-NEXT: [[INDUCTION1:%.*]] = add nuw nsw <vscale x 4 x i32> zeroinitializer, [[TMP5]]290; NO-VP-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0291; NO-VP-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer292; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]293; NO-VP: vector.body:294; NO-VP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]295; NO-VP-NEXT: [[VEC_IND4:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]296; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND4]], i32 0297; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true))298; NO-VP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 1299; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true))300; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 3301; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true))302; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP9]]303; NO-VP-NEXT: [[VEC_IND_NEXT5]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND4]], [[BROADCAST_SPLAT3]]304; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]305; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]306; NO-VP: middle.block:307; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]308; NO-VP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]309; NO-VP: scalar.ph:310; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]311; NO-VP-NEXT: br label [[FOR_BODY:%.*]]312; NO-VP: for.body:313; NO-VP-NEXT: [[TMP15:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]314; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 0315; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4316; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 1317; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4318; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 3319; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX2]], align 4320; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1321; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]322; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]323; NO-VP: exit:324; NO-VP-NEXT: ret void325;326entry:327 br label %for.body328 329for.body:330 %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]331 %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 0332 store i32 %iv, ptr %arrayidx, align 4333 %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 1334 store i32 %iv, ptr %arrayidx1, align 4335 %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 3336 store i32 %iv, ptr %arrayidx2, align 4337 %iv.next = add nuw nsw i32 %iv, 1338 %exitcond.not = icmp eq i32 %iv.next, %n339 br i1 %exitcond.not, label %exit, label %for.body340 341exit:342 ret void343}344 345; Interleaved group with tail gap346; E.g.347; int (*a)[4];348; int rdx = 0;349; for (int i = 0; i < n; i++) {350; rdx += a[i][0];351; rdx += a[i][1];352; rdx += a[i][2];353; // No access a[i][3]354; }355;356define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {357; IF-EVL-LABEL: @load_factor_4_with_tail_gap(358; IF-EVL-NEXT: entry:359; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]360; IF-EVL: vector.ph:361; IF-EVL-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()362; IF-EVL-NEXT: [[TMP1:%.*]] = mul <vscale x 4 x i64> [[TMP2]], splat (i64 1)363; IF-EVL-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP1]]364; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]365; IF-EVL: vector.body:366; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]367; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]368; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]369; IF-EVL-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)370; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64371; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0372; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer373; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0374; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])375; IF-EVL-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]376; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])377; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[WIDE_MASKED_GATHER1]]378; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2379; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])380; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]381; IF-EVL-NEXT: [[TMP12]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP4]])382; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]383; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]384; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0385; IF-EVL-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]386; IF-EVL: middle.block:387; IF-EVL-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP12]])388; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]389; IF-EVL: exit:390; IF-EVL-NEXT: ret i32 [[TMP15]]391;392; NO-VP-LABEL: @load_factor_4_with_tail_gap(393; NO-VP-NEXT: entry:394; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()395; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2396; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N:%.*]], [[TMP1]]397; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]398; NO-VP: vector.ph:399; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()400; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4401; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]402; NO-VP-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0403; NO-VP-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]404; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP5]]405; NO-VP-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()406; NO-VP-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP6]], splat (i64 1)407; NO-VP-NEXT: [[INDUCTION:%.*]] = add nuw nsw <vscale x 4 x i64> zeroinitializer, [[TMP7]]408; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0409; NO-VP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer410; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]411; NO-VP: vector.body:412; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]413; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]414; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]415; NO-VP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0416; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)417; NO-VP-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]418; NO-VP-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)419; NO-VP-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER1]]420; NO-VP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2421; NO-VP-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)422; NO-VP-NEXT: [[TMP13]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER2]]423; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]424; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]425; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]426; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]427; NO-VP: middle.block:428; NO-VP-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])429; NO-VP-NEXT: br label [[SCALAR_PH]]430; NO-VP: scalar.ph:431; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]432; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]433; NO-VP-NEXT: br label [[FOR_BODY:%.*]]434; NO-VP: for.body:435; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]436; NO-VP-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]437; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0438; NO-VP-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4439; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP16]]440; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1441; NO-VP-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4442; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP17]]443; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2444; NO-VP-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4445; NO-VP-NEXT: [[ADD2]] = add nsw i32 [[ADD1]], [[TMP18]]446; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1447; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]448; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]449; NO-VP: exit:450; NO-VP-NEXT: [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[FOR_BODY]] ]451; NO-VP-NEXT: ret i32 [[ADD2_LCSSA]]452;453entry:454 br label %for.body455 456for.body:457 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]458 %rdx = phi i32 [ 0, %entry ], [ %add2, %for.body ]459 %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 0460 %0 = load i32, ptr %arrayidx, align 4461 %add = add nsw i32 %rdx, %0462 %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 1463 %1 = load i32, ptr %arrayidx, align 4464 %add1 = add nsw i32 %add, %1465 %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 2466 %2 = load i32, ptr %arrayidx2, align 4467 %add2 = add nsw i32 %add1, %2468 %iv.next = add nuw nsw i64 %iv, 1469 %exitcond.not = icmp eq i64 %iv.next, %n470 br i1 %exitcond.not, label %exit, label %for.body471 472exit:473 ret i32 %add2474}475 476; Interleaved group with tail gap477; E.g.478; int (*a)[4];479; int rdx = 0;480; for (int i = 0; i < n; i++) {481; a[i][0] = i;482; a[i][1] = i;483; a[i][2] = i;484; // No access a[i][3]485; }486;487define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) {488; IF-EVL-LABEL: @store_factor_4_with_tail_gap(489; IF-EVL-NEXT: entry:490; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]491; IF-EVL: vector.ph:492; IF-EVL-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()493; IF-EVL-NEXT: [[TMP1:%.*]] = mul <vscale x 4 x i32> [[TMP4]], splat (i32 1)494; IF-EVL-NEXT: [[INDUCTION1:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP1]]495; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]496; IF-EVL: vector.body:497; IF-EVL-NEXT: [[VEC_IND2:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]498; IF-EVL-NEXT: [[AVL:%.*]] = phi i32 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]499; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)500; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP6]], i64 0501; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer502; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND2]], i32 0503; IF-EVL-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])504; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND2]], i32 1505; IF-EVL-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])506; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND2]], i32 2507; IF-EVL-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])508; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]509; IF-EVL-NEXT: [[VEC_IND_NEXT5]] = add <vscale x 4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT]]510; IF-EVL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0511; IF-EVL-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]512; IF-EVL: middle.block:513; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]514; IF-EVL: exit:515; IF-EVL-NEXT: ret void516;517; NO-VP-LABEL: @store_factor_4_with_tail_gap(518; NO-VP-NEXT: entry:519; NO-VP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()520; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2521; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP1]]522; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]523; NO-VP: vector.ph:524; NO-VP-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()525; NO-VP-NEXT: [[TMP9:%.*]] = mul nuw i32 [[TMP2]], 4526; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP9]]527; NO-VP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]528; NO-VP-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()529; NO-VP-NEXT: [[TMP5:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP7]], splat (i32 1)530; NO-VP-NEXT: [[INDUCTION1:%.*]] = add nuw nsw <vscale x 4 x i32> zeroinitializer, [[TMP5]]531; NO-VP-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0532; NO-VP-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer533; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]534; NO-VP: vector.body:535; NO-VP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]536; NO-VP-NEXT: [[VEC_IND4:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]537; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND4]], i32 0538; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true))539; NO-VP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 1540; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true))541; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 2542; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true))543; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP9]]544; NO-VP-NEXT: [[VEC_IND_NEXT5]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND4]], [[BROADCAST_SPLAT3]]545; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]546; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]547; NO-VP: middle.block:548; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]549; NO-VP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]550; NO-VP: scalar.ph:551; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]552; NO-VP-NEXT: br label [[FOR_BODY:%.*]]553; NO-VP: for.body:554; NO-VP-NEXT: [[TMP15:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]555; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 0556; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4557; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 1558; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4559; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 2560; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX2]], align 4561; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1562; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]563; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]564; NO-VP: exit:565; NO-VP-NEXT: ret void566;567entry:568 br label %for.body569 570for.body:571 %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]572 %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 0573 store i32 %iv, ptr %arrayidx, align 4574 %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 1575 store i32 %iv, ptr %arrayidx1, align 4576 %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 2577 store i32 %iv, ptr %arrayidx2, align 4578 %iv.next = add nuw nsw i32 %iv, 1579 %exitcond.not = icmp eq i32 %iv.next, %n580 br i1 %exitcond.not, label %exit, label %for.body581 582exit:583 ret void584}585 586define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {587; IF-EVL-LABEL: @load_factor_4_reverse(588; IF-EVL-NEXT: entry:589; IF-EVL-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1590; IF-EVL-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)591; IF-EVL-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]592; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]593; IF-EVL: vector.ph:594; IF-EVL-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()595; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0596; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer597; IF-EVL-NEXT: [[TMP3:%.*]] = mul nsw <vscale x 4 x i64> [[TMP4]], splat (i64 -1)598; IF-EVL-NEXT: [[INDUCTION:%.*]] = add nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP3]]599; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]600; IF-EVL: vector.body:601; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]602; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]603; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP1]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]604; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)605; IF-EVL-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64606; IF-EVL-NEXT: [[TMP8:%.*]] = mul nsw i64 -1, [[TMP7]]607; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0608; IF-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer609; IF-EVL-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0610; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])611; IF-EVL-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]612; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])613; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER3]]614; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2615; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])616; IF-EVL-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER4]]617; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3618; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])619; IF-EVL-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP13]], [[WIDE_MASKED_GATHER5]]620; IF-EVL-NEXT: [[TMP16]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP6]])621; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]622; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]623; IF-EVL-NEXT: [[TMP18:%.*]] = icmp eq i64 [[AVL_NEXT]], 0624; IF-EVL-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]625; IF-EVL: middle.block:626; IF-EVL-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])627; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]628; IF-EVL: exit:629; IF-EVL-NEXT: ret i32 [[TMP19]]630;631; NO-VP-LABEL: @load_factor_4_reverse(632; NO-VP-NEXT: entry:633; NO-VP-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1634; NO-VP-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)635; NO-VP-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]636; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()637; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2638; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]639; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]640; NO-VP: vector.ph:641; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()642; NO-VP-NEXT: [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4643; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]644; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]645; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[N_VEC]]646; NO-VP-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()647; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0648; NO-VP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer649; NO-VP-NEXT: [[TMP8:%.*]] = mul nsw <vscale x 4 x i64> [[TMP7]], splat (i64 -1)650; NO-VP-NEXT: [[INDUCTION:%.*]] = add nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP8]]651; NO-VP-NEXT: [[TMP9:%.*]] = mul nsw i64 -1, [[TMP5]]652; NO-VP-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0653; NO-VP-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer654; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]655; NO-VP: vector.body:656; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]657; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]658; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]659; NO-VP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0660; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)661; NO-VP-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]662; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)663; NO-VP-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER3]]664; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2665; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)666; NO-VP-NEXT: [[TMP14:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[WIDE_MASKED_GATHER4]]667; NO-VP-NEXT: [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3668; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)669; NO-VP-NEXT: [[TMP16]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_GATHER5]]670; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]671; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]672; NO-VP-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]673; NO-VP-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]674; NO-VP: middle.block:675; NO-VP-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])676; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]677; NO-VP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]678; NO-VP: scalar.ph:679; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ [[N]], [[ENTRY:%.*]] ]680; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]681; NO-VP-NEXT: br label [[FOR_BODY:%.*]]682; NO-VP: for.body:683; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]684; NO-VP-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD3:%.*]], [[FOR_BODY]] ]685; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0686; NO-VP-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4687; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP19]]688; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1689; NO-VP-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4690; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP20]]691; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2692; NO-VP-NEXT: [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4693; NO-VP-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD1]], [[TMP21]]694; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3695; NO-VP-NEXT: [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4696; NO-VP-NEXT: [[ADD3]] = add nsw i32 [[ADD2]], [[TMP22]]697; NO-VP-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1698; NO-VP-NEXT: [[EXITCOND:%.*]] = icmp sgt i64 [[IV_NEXT]], 0699; NO-VP-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]700; NO-VP: exit:701; NO-VP-NEXT: [[ADD3_LCSSA:%.*]] = phi i32 [ [[ADD3]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]702; NO-VP-NEXT: ret i32 [[ADD3_LCSSA]]703;704entry:705 br label %for.body706 707for.body:708 %iv = phi i64 [ %n, %entry ], [ %iv.next, %for.body ]709 %rdx = phi i32 [ 0, %entry ], [ %add3, %for.body ]710 %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 0711 %0 = load i32, ptr %arrayidx, align 4712 %add = add nsw i32 %rdx, %0713 %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 1714 %1 = load i32, ptr %arrayidx, align 4715 %add1 = add nsw i32 %add, %1716 %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 2717 %2 = load i32, ptr %arrayidx2, align 4718 %add2 = add nsw i32 %add1, %2719 %arrayidx3 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 3720 %3 = load i32, ptr %arrayidx3, align 4721 %add3 = add nsw i32 %add2, %3722 %iv.next = add nsw i64 %iv, -1723 %exitcond = icmp sgt i64 %iv.next, 0724 br i1 %exitcond, label %for.body, label %exit725 726exit:727 ret i32 %add3728}729 730define void @store_factor_4_reverse(i32 %n, ptr noalias %a) {731; IF-EVL-LABEL: @store_factor_4_reverse(732; IF-EVL-NEXT: entry:733; IF-EVL-NEXT: br label [[FOR_BODY:%.*]]734; IF-EVL: for.body:735; IF-EVL-NEXT: [[TMP19:%.*]] = phi i32 [ [[TMP6:%.*]], [[SCALAR_PH:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]736; IF-EVL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i32 [[TMP19]], i32 0737; IF-EVL-NEXT: store i32 [[TMP19]], ptr [[ARRAYIDX]], align 4738; IF-EVL-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP19]], i32 1739; IF-EVL-NEXT: store i32 [[TMP19]], ptr [[ARRAYIDX1]], align 4740; IF-EVL-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP19]], i32 2741; IF-EVL-NEXT: store i32 [[TMP19]], ptr [[ARRAYIDX2]], align 4742; IF-EVL-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP19]], i32 3743; IF-EVL-NEXT: store i32 [[TMP19]], ptr [[ARRAYIDX3]], align 4744; IF-EVL-NEXT: [[IV_NEXT]] = add nsw i32 [[TMP19]], -1745; IF-EVL-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV_NEXT]], 0746; IF-EVL-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT:%.*]]747; IF-EVL: exit:748; IF-EVL-NEXT: ret void749;750; NO-VP-LABEL: @store_factor_4_reverse(751; NO-VP-NEXT: entry:752; NO-VP-NEXT: br label [[FOR_BODY:%.*]]753; NO-VP: for.body:754; NO-VP-NEXT: [[TMP20:%.*]] = phi i32 [ [[BC_RESUME_VAL:%.*]], [[SCALAR_PH:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]755; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i32 [[TMP20]], i32 0756; NO-VP-NEXT: store i32 [[TMP20]], ptr [[ARRAYIDX]], align 4757; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 1758; NO-VP-NEXT: store i32 [[TMP20]], ptr [[ARRAYIDX1]], align 4759; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 2760; NO-VP-NEXT: store i32 [[TMP20]], ptr [[ARRAYIDX2]], align 4761; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 3762; NO-VP-NEXT: store i32 [[TMP20]], ptr [[ARRAYIDX3]], align 4763; NO-VP-NEXT: [[IV_NEXT]] = add nsw i32 [[TMP20]], -1764; NO-VP-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV_NEXT]], 0765; NO-VP-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT:%.*]]766; NO-VP: exit:767; NO-VP-NEXT: ret void768;769entry:770 br label %for.body771 772for.body:773 %iv = phi i32 [ %n, %entry ], [ %iv.next, %for.body ]774 %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 0775 store i32 %iv, ptr %arrayidx, align 4776 %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 1777 store i32 %iv, ptr %arrayidx1, align 4778 %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 2779 store i32 %iv, ptr %arrayidx2, align 4780 %arrayidx3 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 3781 store i32 %iv, ptr %arrayidx3, align 4782 %iv.next = add nsw i32 %iv, -1783 %exitcond = icmp sgt i32 %iv.next, 0784 br i1 %exitcond, label %for.body, label %exit785 786exit:787 ret void788}789 790!0 = distinct !{!0, !1}791!1 = !{!"llvm.loop.vectorize.enable", i1 true}792