409 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -passes=loop-vectorize \3; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \4; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=IF-EVL5 6; RUN: opt -passes=loop-vectorize \7; RUN: -prefer-predicate-over-epilogue=scalar-epilogue \8; RUN: -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s --check-prefix=NO-VP9 10define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {11; IF-EVL-LABEL: define void @test_sdiv(12; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {13; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]14; IF-EVL-NEXT: br label %[[VECTOR_PH:.*]]15; IF-EVL: [[VECTOR_PH]]:16; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]17; IF-EVL: [[VECTOR_BODY]]:18; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]19; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]20; IF-EVL-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)21; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]22; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])23; IF-EVL-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[B]], i64 [[EVL_BASED_IV]]24; IF-EVL-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])25; IF-EVL-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> [[VP_OP_LOAD1]], <vscale x 2 x i64> splat (i64 1), i32 [[TMP5]])26; IF-EVL-NEXT: [[VP_OP:%.*]] = sdiv <vscale x 2 x i64> [[VP_OP_LOAD]], [[TMP11]]27; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]28; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])29; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP5]] to i6430; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]31; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]32; IF-EVL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 033; IF-EVL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]34; IF-EVL: [[MIDDLE_BLOCK]]:35; IF-EVL-NEXT: br label %[[EXIT:.*]]36; IF-EVL: [[EXIT]]:37; IF-EVL-NEXT: ret void38;39; NO-VP-LABEL: define void @test_sdiv(40; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {41; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:42; NO-VP-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()43; NO-VP-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 144; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]45; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]46; NO-VP: [[VECTOR_PH]]:47; NO-VP-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()48; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP13]], 249; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]50; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]51; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]52; NO-VP: [[VECTOR_BODY]]:53; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]54; NO-VP-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]55; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 856; NO-VP-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]57; NO-VP-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 858; NO-VP-NEXT: [[TMP8:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]59; NO-VP-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]60; NO-VP-NEXT: store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 861; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]62; NO-VP-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]63; NO-VP-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]64; NO-VP: [[MIDDLE_BLOCK]]:65; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]66; NO-VP-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]67; NO-VP: [[SCALAR_PH]]:68; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]69; NO-VP-NEXT: br label %[[LOOP:.*]]70; NO-VP: [[LOOP]]:71; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]72; NO-VP-NEXT: [[A_GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]73; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[A_GEP]], align 874; NO-VP-NEXT: [[B_GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]75; NO-VP-NEXT: [[TMP1:%.*]] = load i64, ptr [[B_GEP]], align 876; NO-VP-NEXT: [[TMP2:%.*]] = sdiv i64 [[TMP0]], [[TMP1]]77; NO-VP-NEXT: [[C_GEP:%.*]] = getelementptr i64, ptr [[C]], i64 [[IV]]78; NO-VP-NEXT: store i64 [[TMP2]], ptr [[C_GEP]], align 879; NO-VP-NEXT: [[IV_NEXT]] = add i64 [[IV]], 180; NO-VP-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], 102481; NO-VP-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]82; NO-VP: [[EXIT]]:83; NO-VP-NEXT: ret void84;85loop.preheader:86 br label %loop87 88loop:89 %iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]90 %a.gep = getelementptr i64, ptr %a, i64 %iv91 %0 = load i64, ptr %a.gep92 %b.gep = getelementptr i64, ptr %b, i64 %iv93 %1 = load i64, ptr %b.gep94 %2 = sdiv i64 %0, %195 %c.gep = getelementptr i64, ptr %c, i64 %iv96 store i64 %2, ptr %c.gep97 %iv.next = add i64 %iv, 198 %done = icmp eq i64 %iv.next, 102499 br i1 %done, label %exit, label %loop100 101exit:102 ret void103}104 105 106define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {107; IF-EVL-LABEL: define void @test_udiv(108; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {109; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]110; IF-EVL-NEXT: br label %[[VECTOR_PH:.*]]111; IF-EVL: [[VECTOR_PH]]:112; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]113; IF-EVL: [[VECTOR_BODY]]:114; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]115; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]116; IF-EVL-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)117; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]118; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])119; IF-EVL-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[B]], i64 [[EVL_BASED_IV]]120; IF-EVL-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])121; IF-EVL-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> [[VP_OP_LOAD1]], <vscale x 2 x i64> splat (i64 1), i32 [[TMP5]])122; IF-EVL-NEXT: [[VP_OP:%.*]] = udiv <vscale x 2 x i64> [[VP_OP_LOAD]], [[TMP11]]123; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]124; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])125; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP5]] to i64126; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]127; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]128; IF-EVL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0129; IF-EVL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]130; IF-EVL: [[MIDDLE_BLOCK]]:131; IF-EVL-NEXT: br label %[[EXIT:.*]]132; IF-EVL: [[EXIT]]:133; IF-EVL-NEXT: ret void134;135; NO-VP-LABEL: define void @test_udiv(136; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {137; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:138; NO-VP-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()139; NO-VP-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1140; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]141; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]142; NO-VP: [[VECTOR_PH]]:143; NO-VP-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()144; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP13]], 2145; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]146; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]147; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]148; NO-VP: [[VECTOR_BODY]]:149; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]150; NO-VP-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]151; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8152; NO-VP-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]153; NO-VP-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8154; NO-VP-NEXT: [[TMP8:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]155; NO-VP-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]156; NO-VP-NEXT: store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 8157; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]158; NO-VP-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]159; NO-VP-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]160; NO-VP: [[MIDDLE_BLOCK]]:161; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]162; NO-VP-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]163; NO-VP: [[SCALAR_PH]]:164; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]165; NO-VP-NEXT: br label %[[LOOP:.*]]166; NO-VP: [[LOOP]]:167; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]168; NO-VP-NEXT: [[A_GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]169; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[A_GEP]], align 8170; NO-VP-NEXT: [[B_GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]171; NO-VP-NEXT: [[TMP1:%.*]] = load i64, ptr [[B_GEP]], align 8172; NO-VP-NEXT: [[TMP2:%.*]] = udiv i64 [[TMP0]], [[TMP1]]173; NO-VP-NEXT: [[C_GEP:%.*]] = getelementptr i64, ptr [[C]], i64 [[IV]]174; NO-VP-NEXT: store i64 [[TMP2]], ptr [[C_GEP]], align 8175; NO-VP-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1176; NO-VP-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], 1024177; NO-VP-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]178; NO-VP: [[EXIT]]:179; NO-VP-NEXT: ret void180;181loop.preheader:182 br label %loop183 184loop:185 %iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]186 %a.gep = getelementptr i64, ptr %a, i64 %iv187 %0 = load i64, ptr %a.gep188 %b.gep = getelementptr i64, ptr %b, i64 %iv189 %1 = load i64, ptr %b.gep190 %2 = udiv i64 %0, %1191 %c.gep = getelementptr i64, ptr %c, i64 %iv192 store i64 %2, ptr %c.gep193 %iv.next = add i64 %iv, 1194 %done = icmp eq i64 %iv.next, 1024195 br i1 %done, label %exit, label %loop196 197exit:198 ret void199}200 201define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {202; IF-EVL-LABEL: define void @test_srem(203; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {204; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]205; IF-EVL-NEXT: br label %[[VECTOR_PH:.*]]206; IF-EVL: [[VECTOR_PH]]:207; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]208; IF-EVL: [[VECTOR_BODY]]:209; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]210; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]211; IF-EVL-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)212; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]213; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])214; IF-EVL-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[B]], i64 [[EVL_BASED_IV]]215; IF-EVL-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])216; IF-EVL-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> [[VP_OP_LOAD1]], <vscale x 2 x i64> splat (i64 1), i32 [[TMP5]])217; IF-EVL-NEXT: [[VP_OP:%.*]] = srem <vscale x 2 x i64> [[VP_OP_LOAD]], [[TMP11]]218; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]219; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])220; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP5]] to i64221; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]222; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]223; IF-EVL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0224; IF-EVL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]225; IF-EVL: [[MIDDLE_BLOCK]]:226; IF-EVL-NEXT: br label %[[EXIT:.*]]227; IF-EVL: [[EXIT]]:228; IF-EVL-NEXT: ret void229;230; NO-VP-LABEL: define void @test_srem(231; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {232; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:233; NO-VP-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()234; NO-VP-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1235; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]236; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]237; NO-VP: [[VECTOR_PH]]:238; NO-VP-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()239; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP13]], 2240; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]241; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]242; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]243; NO-VP: [[VECTOR_BODY]]:244; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]245; NO-VP-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]246; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8247; NO-VP-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]248; NO-VP-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8249; NO-VP-NEXT: [[TMP8:%.*]] = srem <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]250; NO-VP-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]251; NO-VP-NEXT: store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 8252; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]253; NO-VP-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]254; NO-VP-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]255; NO-VP: [[MIDDLE_BLOCK]]:256; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]257; NO-VP-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]258; NO-VP: [[SCALAR_PH]]:259; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]260; NO-VP-NEXT: br label %[[LOOP:.*]]261; NO-VP: [[LOOP]]:262; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]263; NO-VP-NEXT: [[A_GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]264; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[A_GEP]], align 8265; NO-VP-NEXT: [[B_GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]266; NO-VP-NEXT: [[TMP1:%.*]] = load i64, ptr [[B_GEP]], align 8267; NO-VP-NEXT: [[TMP2:%.*]] = srem i64 [[TMP0]], [[TMP1]]268; NO-VP-NEXT: [[C_GEP:%.*]] = getelementptr i64, ptr [[C]], i64 [[IV]]269; NO-VP-NEXT: store i64 [[TMP2]], ptr [[C_GEP]], align 8270; NO-VP-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1271; NO-VP-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], 1024272; NO-VP-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]273; NO-VP: [[EXIT]]:274; NO-VP-NEXT: ret void275;276loop.preheader:277 br label %loop278 279loop:280 %iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]281 %a.gep = getelementptr i64, ptr %a, i64 %iv282 %0 = load i64, ptr %a.gep283 %b.gep = getelementptr i64, ptr %b, i64 %iv284 %1 = load i64, ptr %b.gep285 %2 = srem i64 %0, %1286 %c.gep = getelementptr i64, ptr %c, i64 %iv287 store i64 %2, ptr %c.gep288 %iv.next = add i64 %iv, 1289 %done = icmp eq i64 %iv.next, 1024290 br i1 %done, label %exit, label %loop291 292exit:293 ret void294}295 296define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {297; IF-EVL-LABEL: define void @test_urem(298; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {299; IF-EVL-NEXT: [[LOOP_PREHEADER:.*:]]300; IF-EVL-NEXT: br label %[[VECTOR_PH:.*]]301; IF-EVL: [[VECTOR_PH]]:302; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]303; IF-EVL: [[VECTOR_BODY]]:304; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]305; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]306; IF-EVL-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)307; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[EVL_BASED_IV]]308; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])309; IF-EVL-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[B]], i64 [[EVL_BASED_IV]]310; IF-EVL-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])311; IF-EVL-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> [[VP_OP_LOAD1]], <vscale x 2 x i64> splat (i64 1), i32 [[TMP5]])312; IF-EVL-NEXT: [[VP_OP:%.*]] = urem <vscale x 2 x i64> [[VP_OP_LOAD]], [[TMP11]]313; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[EVL_BASED_IV]]314; IF-EVL-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])315; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP5]] to i64316; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[EVL_BASED_IV]]317; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]318; IF-EVL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0319; IF-EVL-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]320; IF-EVL: [[MIDDLE_BLOCK]]:321; IF-EVL-NEXT: br label %[[EXIT:.*]]322; IF-EVL: [[EXIT]]:323; IF-EVL-NEXT: ret void324;325; NO-VP-LABEL: define void @test_urem(326; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {327; NO-VP-NEXT: [[LOOP_PREHEADER:.*]]:328; NO-VP-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()329; NO-VP-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1330; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]331; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]332; NO-VP: [[VECTOR_PH]]:333; NO-VP-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64()334; NO-VP-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP13]], 2335; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]336; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]337; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]338; NO-VP: [[VECTOR_BODY]]:339; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]340; NO-VP-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]341; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8342; NO-VP-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]343; NO-VP-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8344; NO-VP-NEXT: [[TMP8:%.*]] = urem <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]345; NO-VP-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]346; NO-VP-NEXT: store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 8347; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]348; NO-VP-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]349; NO-VP-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]350; NO-VP: [[MIDDLE_BLOCK]]:351; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]352; NO-VP-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]353; NO-VP: [[SCALAR_PH]]:354; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ]355; NO-VP-NEXT: br label %[[LOOP:.*]]356; NO-VP: [[LOOP]]:357; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]358; NO-VP-NEXT: [[A_GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]359; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[A_GEP]], align 8360; NO-VP-NEXT: [[B_GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]361; NO-VP-NEXT: [[TMP1:%.*]] = load i64, ptr [[B_GEP]], align 8362; NO-VP-NEXT: [[TMP2:%.*]] = urem i64 [[TMP0]], [[TMP1]]363; NO-VP-NEXT: [[C_GEP:%.*]] = getelementptr i64, ptr [[C]], i64 [[IV]]364; NO-VP-NEXT: store i64 [[TMP2]], ptr [[C_GEP]], align 8365; NO-VP-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1366; NO-VP-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], 1024367; NO-VP-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]368; NO-VP: [[EXIT]]:369; NO-VP-NEXT: ret void370;371loop.preheader:372 br label %loop373 374loop:375 %iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]376 %a.gep = getelementptr i64, ptr %a, i64 %iv377 %0 = load i64, ptr %a.gep378 %b.gep = getelementptr i64, ptr %b, i64 %iv379 %1 = load i64, ptr %b.gep380 %2 = urem i64 %0, %1381 %c.gep = getelementptr i64, ptr %c, i64 %iv382 store i64 %2, ptr %c.gep383 %iv.next = add i64 %iv, 1384 %done = icmp eq i64 %iv.next, 1024385 br i1 %done, label %exit, label %loop386 387exit:388 ret void389}390;.391; IF-EVL: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}392; IF-EVL: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}393; IF-EVL: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}394; IF-EVL: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}395; IF-EVL: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}396; IF-EVL: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}397;.398; NO-VP: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}399; NO-VP: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}400; NO-VP: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}401; NO-VP: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}402; NO-VP: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}403; NO-VP: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}404; NO-VP: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}405; NO-VP: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}406; NO-VP: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}407; NO-VP: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}408;.409