738 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=on -mtriple riscv64-linux-gnu -mattr=+v,+f -S 2>%t | FileCheck %s3; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=off -mtriple riscv64-linux-gnu -mattr=+v,+f -S 2>%t | FileCheck --check-prefix=FIXED %s4 5; Tests specific to div/rem handling - both predicated and not6 7target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"8target triple = "riscv64"9 10define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {11; CHECK-LABEL: @vector_udiv(12; CHECK-NEXT: entry:13; CHECK-NEXT: br label [[VECTOR_PH:%.*]]14; CHECK: vector.ph:15; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 016; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer17; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]18; CHECK: vector.body:19; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]20; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]21; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)22; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]23; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])24; CHECK-NEXT: [[TMP9:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]25; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])26; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i6427; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]28; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]29; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 030; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]31; CHECK: middle.block:32; CHECK-NEXT: br label [[FOR_BODY:%.*]]33; CHECK: for.end:34; CHECK-NEXT: ret void35;36; FIXED-LABEL: @vector_udiv(37; FIXED-NEXT: entry:38; FIXED-NEXT: br label [[VECTOR_PH:%.*]]39; FIXED: vector.ph:40; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 041; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer42; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]43; FIXED: vector.body:44; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]45; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]46; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 847; FIXED-NEXT: [[TMP5:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]48; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 849; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 450; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 102451; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]52; FIXED: middle.block:53; FIXED-NEXT: br label [[FOR_BODY:%.*]]54; FIXED: for.end:55; FIXED-NEXT: ret void56;57entry:58 br label %for.body59 60for.body:61 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]62 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv63 %elem = load i64, ptr %arrayidx64 %divrem = udiv i64 %elem, %v65 store i64 %divrem, ptr %arrayidx66 %iv.next = add nuw nsw i64 %iv, 167 %exitcond.not = icmp eq i64 %iv.next, 102468 br i1 %exitcond.not, label %for.end, label %for.body69 70for.end:71 ret void72}73 74define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {75; CHECK-LABEL: @vector_sdiv(76; CHECK-NEXT: entry:77; CHECK-NEXT: br label [[VECTOR_PH:%.*]]78; CHECK: vector.ph:79; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 080; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer81; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]82; CHECK: vector.body:83; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]84; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]85; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)86; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]87; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])88; CHECK-NEXT: [[TMP9:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]89; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])90; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i6491; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]92; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]93; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 094; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]95; CHECK: middle.block:96; CHECK-NEXT: br label [[FOR_BODY:%.*]]97; CHECK: for.end:98; CHECK-NEXT: ret void99;100; FIXED-LABEL: @vector_sdiv(101; FIXED-NEXT: entry:102; FIXED-NEXT: br label [[VECTOR_PH:%.*]]103; FIXED: vector.ph:104; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0105; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer106; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]107; FIXED: vector.body:108; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]109; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]110; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8111; FIXED-NEXT: [[TMP5:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]112; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8113; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4114; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024115; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]116; FIXED: middle.block:117; FIXED-NEXT: br label [[FOR_BODY:%.*]]118; FIXED: for.end:119; FIXED-NEXT: ret void120;121entry:122 br label %for.body123 124for.body:125 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]126 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv127 %elem = load i64, ptr %arrayidx128 %divrem = sdiv i64 %elem, %v129 store i64 %divrem, ptr %arrayidx130 %iv.next = add nuw nsw i64 %iv, 1131 %exitcond.not = icmp eq i64 %iv.next, 1024132 br i1 %exitcond.not, label %for.end, label %for.body133 134for.end:135 ret void136}137 138define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {139; CHECK-LABEL: @vector_urem(140; CHECK-NEXT: entry:141; CHECK-NEXT: br label [[VECTOR_PH:%.*]]142; CHECK: vector.ph:143; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0144; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer145; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]146; CHECK: vector.body:147; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]148; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]149; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)150; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]151; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])152; CHECK-NEXT: [[TMP9:%.*]] = urem <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]153; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])154; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i64155; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]156; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]157; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0158; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]159; CHECK: middle.block:160; CHECK-NEXT: br label [[FOR_BODY:%.*]]161; CHECK: for.end:162; CHECK-NEXT: ret void163;164; FIXED-LABEL: @vector_urem(165; FIXED-NEXT: entry:166; FIXED-NEXT: br label [[VECTOR_PH:%.*]]167; FIXED: vector.ph:168; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0169; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer170; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]171; FIXED: vector.body:172; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]173; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]174; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8175; FIXED-NEXT: [[TMP5:%.*]] = urem <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]176; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8177; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4178; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024179; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]180; FIXED: middle.block:181; FIXED-NEXT: br label [[FOR_BODY:%.*]]182; FIXED: for.end:183; FIXED-NEXT: ret void184;185entry:186 br label %for.body187 188for.body:189 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]190 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv191 %elem = load i64, ptr %arrayidx192 %divrem = urem i64 %elem, %v193 store i64 %divrem, ptr %arrayidx194 %iv.next = add nuw nsw i64 %iv, 1195 %exitcond.not = icmp eq i64 %iv.next, 1024196 br i1 %exitcond.not, label %for.end, label %for.body197 198for.end:199 ret void200}201 202define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {203; CHECK-LABEL: @vector_srem(204; CHECK-NEXT: entry:205; CHECK-NEXT: br label [[VECTOR_PH:%.*]]206; CHECK: vector.ph:207; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0208; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer209; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]210; CHECK: vector.body:211; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]212; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]213; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)214; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]215; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])216; CHECK-NEXT: [[TMP9:%.*]] = srem <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]217; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])218; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP10]] to i64219; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]220; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]221; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0222; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]223; CHECK: middle.block:224; CHECK-NEXT: br label [[FOR_BODY:%.*]]225; CHECK: for.end:226; CHECK-NEXT: ret void227;228; FIXED-LABEL: @vector_srem(229; FIXED-NEXT: entry:230; FIXED-NEXT: br label [[VECTOR_PH:%.*]]231; FIXED: vector.ph:232; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0233; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer234; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]235; FIXED: vector.body:236; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]237; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]238; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8239; FIXED-NEXT: [[TMP5:%.*]] = srem <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]240; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8241; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4242; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024243; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]244; FIXED: middle.block:245; FIXED-NEXT: br label [[FOR_BODY:%.*]]246; FIXED: for.end:247; FIXED-NEXT: ret void248;249entry:250 br label %for.body251 252for.body:253 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]254 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv255 %elem = load i64, ptr %arrayidx256 %divrem = srem i64 %elem, %v257 store i64 %divrem, ptr %arrayidx258 %iv.next = add nuw nsw i64 %iv, 1259 %exitcond.not = icmp eq i64 %iv.next, 1024260 br i1 %exitcond.not, label %for.end, label %for.body261 262for.end:263 ret void264}265 266define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {267; CHECK-LABEL: @predicated_udiv(268; CHECK-NEXT: entry:269; CHECK-NEXT: br label [[VECTOR_PH:%.*]]270; CHECK: vector.ph:271; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0272; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer273; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer274; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]275; CHECK: vector.body:276; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]277; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]278; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)279; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]280; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])281; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i64> splat (i64 1), i32 [[TMP12]])282; CHECK-NEXT: [[TMP11:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[TMP10]]283; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP6]], i32 0284; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]285; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])286; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP12]] to i64287; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]288; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]289; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0290; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]291; CHECK: middle.block:292; CHECK-NEXT: br label [[LATCH:%.*]]293; CHECK: for.end:294; CHECK-NEXT: ret void295;296; FIXED-LABEL: @predicated_udiv(297; FIXED-NEXT: entry:298; FIXED-NEXT: br label [[VECTOR_PH:%.*]]299; FIXED: vector.ph:300; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0301; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer302; FIXED-NEXT: [[TMP0:%.*]] = icmp ne <4 x i64> [[BROADCAST_SPLAT]], zeroinitializer303; FIXED-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0304; FIXED-NEXT: [[TMP5:%.*]] = select i1 [[TMP1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> splat (i64 1)305; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]306; FIXED: vector.body:307; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]308; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]309; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8310; FIXED-NEXT: [[TMP8:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], [[TMP5]]311; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0312; FIXED-NEXT: [[PREDPHI2:%.*]] = select i1 [[TMP6]], <4 x i64> [[TMP8]], <4 x i64> [[WIDE_LOAD1]]313; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8314; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4315; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024316; FIXED-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]317; FIXED: middle.block:318; FIXED-NEXT: br label [[LATCH:%.*]]319; FIXED: for.end:320; FIXED-NEXT: ret void321;322entry:323 br label %for.body324 325for.body:326 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]327 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv328 %elem = load i64, ptr %arrayidx329 %c = icmp ne i64 %v, 0330 br i1 %c, label %do_op, label %latch331do_op:332 %divrem = udiv i64 %elem, %v333 br label %latch334latch:335 %phi = phi i64 [%elem, %for.body], [%divrem, %do_op]336 store i64 %phi, ptr %arrayidx337 %iv.next = add nuw nsw i64 %iv, 1338 %exitcond.not = icmp eq i64 %iv.next, 1024339 br i1 %exitcond.not, label %for.end, label %for.body340 341for.end:342 ret void343}344 345define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {346; CHECK-LABEL: @predicated_sdiv(347; CHECK-NEXT: entry:348; CHECK-NEXT: br label [[VECTOR_PH:%.*]]349; CHECK: vector.ph:350; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0351; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer352; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer353; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]354; CHECK: vector.body:355; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]356; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]357; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)358; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]359; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])360; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i64> splat (i64 1), i32 [[TMP12]])361; CHECK-NEXT: [[TMP11:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], [[TMP10]]362; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP6]], i32 0363; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]364; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])365; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP12]] to i64366; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]367; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]368; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0369; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]370; CHECK: middle.block:371; CHECK-NEXT: br label [[LATCH:%.*]]372; CHECK: for.end:373; CHECK-NEXT: ret void374;375; FIXED-LABEL: @predicated_sdiv(376; FIXED-NEXT: entry:377; FIXED-NEXT: br label [[VECTOR_PH:%.*]]378; FIXED: vector.ph:379; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0380; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer381; FIXED-NEXT: [[TMP0:%.*]] = icmp ne <4 x i64> [[BROADCAST_SPLAT]], zeroinitializer382; FIXED-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0383; FIXED-NEXT: [[TMP5:%.*]] = select i1 [[TMP1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> splat (i64 1)384; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]385; FIXED: vector.body:386; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]387; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]388; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8389; FIXED-NEXT: [[TMP8:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], [[TMP5]]390; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0391; FIXED-NEXT: [[PREDPHI2:%.*]] = select i1 [[TMP6]], <4 x i64> [[TMP8]], <4 x i64> [[WIDE_LOAD1]]392; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8393; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4394; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024395; FIXED-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]396; FIXED: middle.block:397; FIXED-NEXT: br label [[LATCH:%.*]]398; FIXED: for.end:399; FIXED-NEXT: ret void400;401entry:402 br label %for.body403 404for.body:405 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]406 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv407 %elem = load i64, ptr %arrayidx408 %c = icmp ne i64 %v, 0409 br i1 %c, label %do_op, label %latch410do_op:411 %divrem = sdiv i64 %elem, %v412 br label %latch413latch:414 %phi = phi i64 [%elem, %for.body], [%divrem, %do_op]415 store i64 %phi, ptr %arrayidx416 %iv.next = add nuw nsw i64 %iv, 1417 %exitcond.not = icmp eq i64 %iv.next, 1024418 br i1 %exitcond.not, label %for.end, label %for.body419 420for.end:421 ret void422}423 424define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {425; CHECK-LABEL: @predicated_udiv_by_constant(426; CHECK-NEXT: entry:427; CHECK-NEXT: br label [[VECTOR_PH:%.*]]428; CHECK: vector.ph:429; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]430; CHECK: vector.body:431; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]432; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]433; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)434; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]435; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])436; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)437; CHECK-NEXT: [[TMP10:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 27)438; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP2]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[WIDE_LOAD]]439; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])440; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP14]] to i64441; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]442; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]443; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0444; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]445; CHECK: middle.block:446; CHECK-NEXT: br label [[LATCH:%.*]]447; CHECK: for.end:448; CHECK-NEXT: ret void449;450; FIXED-LABEL: @predicated_udiv_by_constant(451; FIXED-NEXT: entry:452; FIXED-NEXT: br label [[VECTOR_PH:%.*]]453; FIXED: vector.ph:454; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]455; FIXED: vector.body:456; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]457; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]458; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8459; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD1]], splat (i64 42)460; FIXED-NEXT: [[TMP7:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], splat (i64 27)461; FIXED-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> [[TMP7]], <4 x i64> [[WIDE_LOAD1]]462; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP1]], align 8463; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4464; FIXED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024465; FIXED-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]466; FIXED: middle.block:467; FIXED-NEXT: br label [[LATCH:%.*]]468; FIXED: for.end:469; FIXED-NEXT: ret void470;471entry:472 br label %for.body473 474for.body:475 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]476 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv477 %elem = load i64, ptr %arrayidx478 %c = icmp ne i64 %elem, 42479 br i1 %c, label %do_op, label %latch480do_op:481 %divrem = udiv i64 %elem, 27482 br label %latch483latch:484 %phi = phi i64 [%elem, %for.body], [%divrem, %do_op]485 store i64 %phi, ptr %arrayidx486 %iv.next = add nuw nsw i64 %iv, 1487 %exitcond.not = icmp eq i64 %iv.next, 1024488 br i1 %exitcond.not, label %for.end, label %for.body489 490for.end:491 ret void492}493 494define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {495; CHECK-LABEL: @predicated_sdiv_by_constant(496; CHECK-NEXT: entry:497; CHECK-NEXT: br label [[VECTOR_PH:%.*]]498; CHECK: vector.ph:499; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]500; CHECK: vector.body:501; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]502; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]503; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)504; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]505; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])506; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)507; CHECK-NEXT: [[TMP10:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 27)508; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP2]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[WIDE_LOAD]]509; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])510; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP14]] to i64511; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]512; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]513; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0514; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]515; CHECK: middle.block:516; CHECK-NEXT: br label [[LATCH:%.*]]517; CHECK: for.end:518; CHECK-NEXT: ret void519;520; FIXED-LABEL: @predicated_sdiv_by_constant(521; FIXED-NEXT: entry:522; FIXED-NEXT: br label [[VECTOR_PH:%.*]]523; FIXED: vector.ph:524; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]525; FIXED: vector.body:526; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]527; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]528; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8529; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD1]], splat (i64 42)530; FIXED-NEXT: [[TMP7:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], splat (i64 27)531; FIXED-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> [[TMP7]], <4 x i64> [[WIDE_LOAD1]]532; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP1]], align 8533; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4534; FIXED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024535; FIXED-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]536; FIXED: middle.block:537; FIXED-NEXT: br label [[LATCH:%.*]]538; FIXED: for.end:539; FIXED-NEXT: ret void540;541entry:542 br label %for.body543 544for.body:545 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]546 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv547 %elem = load i64, ptr %arrayidx548 %c = icmp ne i64 %elem, 42549 br i1 %c, label %do_op, label %latch550do_op:551 %divrem = sdiv i64 %elem, 27552 br label %latch553latch:554 %phi = phi i64 [%elem, %for.body], [%divrem, %do_op]555 store i64 %phi, ptr %arrayidx556 %iv.next = add nuw nsw i64 %iv, 1557 %exitcond.not = icmp eq i64 %iv.next, 1024558 br i1 %exitcond.not, label %for.end, label %for.body559 560for.end:561 ret void562}563 564define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {565; CHECK-LABEL: @predicated_sdiv_by_minus_one(566; CHECK-NEXT: entry:567; CHECK-NEXT: br label [[VECTOR_PH:%.*]]568; CHECK: vector.ph:569; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]570; CHECK: vector.body:571; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]572; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]573; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)574; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]575; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP12]])576; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 -128)577; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.vp.merge.nxv16i8(<vscale x 16 x i1> [[TMP9]], <vscale x 16 x i8> splat (i8 -1), <vscale x 16 x i8> splat (i8 1), i32 [[TMP12]])578; CHECK-NEXT: [[TMP11:%.*]] = sdiv <vscale x 16 x i8> [[WIDE_LOAD]], [[TMP10]]579; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[WIDE_LOAD]]580; CHECK-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[PREDPHI]], ptr align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP12]])581; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP12]] to i64582; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]583; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]584; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0585; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]586; CHECK: middle.block:587; CHECK-NEXT: br label [[LATCH:%.*]]588; CHECK: for.end:589; CHECK-NEXT: ret void590;591; FIXED-LABEL: @predicated_sdiv_by_minus_one(592; FIXED-NEXT: entry:593; FIXED-NEXT: br label [[VECTOR_PH:%.*]]594; FIXED: vector.ph:595; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]596; FIXED: vector.body:597; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]598; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]599; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1600; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD1]], splat (i8 -128)601; FIXED-NEXT: [[TMP7:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> splat (i8 -1), <32 x i8> splat (i8 1)602; FIXED-NEXT: [[TMP9:%.*]] = sdiv <32 x i8> [[WIDE_LOAD1]], [[TMP7]]603; FIXED-NEXT: [[PREDPHI2:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[TMP9]], <32 x i8> [[WIDE_LOAD1]]604; FIXED-NEXT: store <32 x i8> [[PREDPHI2]], ptr [[TMP1]], align 1605; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32606; FIXED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024607; FIXED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]608; FIXED: middle.block:609; FIXED-NEXT: br label [[LATCH:%.*]]610; FIXED: for.end:611; FIXED-NEXT: ret void612;613entry:614 br label %for.body615 616for.body:617 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]618 %arrayidx = getelementptr inbounds i8, ptr %a, i64 %iv619 %elem = load i8, ptr %arrayidx620 %c = icmp ne i8 %elem, 128621 br i1 %c, label %do_op, label %latch622do_op:623 %divrem = sdiv i8 %elem, -1 ;; UB if %elem = INT_MIN624 br label %latch625latch:626 %phi = phi i8 [%elem, %for.body], [%divrem, %do_op]627 store i8 %phi, ptr %arrayidx628 %iv.next = add nuw nsw i64 %iv, 1629 %exitcond.not = icmp eq i64 %iv.next, 1024630 br i1 %exitcond.not, label %for.end, label %for.body631 632for.end:633 ret void634}635 636; Test for https://github.com/llvm/llvm-project/issues/159402. For invariant divisors,637; selects can be introduced outside the vector loop and their cost should not be638; considered for each loop iteration.639define i32 @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c) {640; CHECK-LABEL: @udiv_sdiv_with_invariant_divisors(641; CHECK-NEXT: entry:642; CHECK-NEXT: br label [[VECTOR_PH:%.*]]643; CHECK: vector.ph:644; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i1> poison, i1 [[C:%.*]], i64 0645; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer646; CHECK-NEXT: [[TMP0:%.*]] = xor <vscale x 8 x i1> [[BROADCAST_SPLAT]], splat (i1 true)647; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[X:%.*]], i64 0648; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer649; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[Y:%.*]], i64 0650; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer651; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()652; CHECK-NEXT: [[TMP2:%.*]] = mul <vscale x 8 x i8> [[TMP1]], splat (i8 1)653; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 8 x i8> splat (i8 -12), [[TMP2]]654; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]655; CHECK: vector.body:656; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i8> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]657; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]658; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true)659; CHECK-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8660; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[TMP4]], i64 0661; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT5]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer662; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i8> @llvm.vp.merge.nxv8i8(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> [[BROADCAST_SPLAT2]], <vscale x 8 x i8> splat (i8 1), i32 [[TMP3]])663; CHECK-NEXT: [[TMP9:%.*]] = udiv <vscale x 8 x i8> [[VEC_IND]], [[TMP8]]664; CHECK-NEXT: [[TMP10:%.*]] = zext <vscale x 8 x i8> [[TMP9]] to <vscale x 8 x i16>665; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 8 x i16> @llvm.vp.merge.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[BROADCAST_SPLAT4]], <vscale x 8 x i16> splat (i16 1), i32 [[TMP3]])666; CHECK-NEXT: [[TMP12:%.*]] = sdiv <vscale x 8 x i16> [[TMP10]], [[TMP11]]667; CHECK-NEXT: [[TMP13:%.*]] = sext <vscale x 8 x i16> [[TMP12]] to <vscale x 8 x i32>668; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> [[TMP13]]669; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]]670; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i8> [[VEC_IND]], [[BROADCAST_SPLAT6]]671; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[AVL_NEXT]], 0672; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]673; CHECK: middle.block:674; CHECK-NEXT: [[TMP16:%.*]] = zext i32 [[TMP3]] to i64675; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[TMP16]], 1676; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64()677; CHECK-NEXT: [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 8678; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 0679; CHECK-NEXT: [[MERGE_LCSSA:%.*]] = extractelement <vscale x 8 x i32> [[PREDPHI]], i64 [[TMP17]]680; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]681; CHECK: exit:682; CHECK-NEXT: ret i32 [[MERGE_LCSSA]]683;684; FIXED-LABEL: @udiv_sdiv_with_invariant_divisors(685; FIXED-NEXT: entry:686; FIXED-NEXT: br label [[VECTOR_PH:%.*]]687; FIXED: vector.ph:688; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[Y:%.*]], i64 0689; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer690; FIXED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i8> poison, i8 [[X:%.*]], i64 0691; FIXED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT1]], <4 x i8> poison, <4 x i32> zeroinitializer692; FIXED-NEXT: [[TMP0:%.*]] = select i1 [[C:%.*]], <4 x i8> splat (i8 1), <4 x i8> [[BROADCAST_SPLAT2]]693; FIXED-NEXT: [[TMP1:%.*]] = select i1 [[C]], <4 x i16> splat (i16 1), <4 x i16> [[BROADCAST_SPLAT]]694; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]695; FIXED: vector.body:696; FIXED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]697; FIXED-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 -12, i8 -11, i8 -10, i8 -9>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]698; FIXED-NEXT: [[TMP2:%.*]] = udiv <4 x i8> [[VEC_IND]], [[TMP0]]699; FIXED-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16>700; FIXED-NEXT: [[TMP4:%.*]] = sdiv <4 x i16> [[TMP3]], [[TMP1]]701; FIXED-NEXT: [[TMP5:%.*]] = sext <4 x i16> [[TMP4]] to <4 x i32>702; FIXED-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> zeroinitializer, <4 x i32> [[TMP5]]703; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4704; FIXED-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)705; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12706; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]707; FIXED: middle.block:708; FIXED-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3709; FIXED-NEXT: br label [[LOOP_LATCH:%.*]]710; FIXED: exit:711; FIXED-NEXT: ret i32 [[TMP7]]712;713entry:714 br label %loop.header715 716loop.header:717 %iv = phi i16 [ -12, %entry ], [ %iv.next, %loop.latch ]718 %narrow.iv = phi i8 [ -12, %entry ], [ %iv.next.trunc, %loop.latch ]719 br i1 %c, label %loop.latch, label %then720 721then:722 %ud = udiv i8 %narrow.iv, %x723 %ud.ext = zext i8 %ud to i16724 %sd = sdiv i16 %ud.ext, %y725 %sd.ext = sext i16 %sd to i32726 br label %loop.latch727 728loop.latch:729 %merge = phi i32 [ 0, %loop.header ], [ %sd.ext, %then ]730 %iv.next = add nsw i16 %iv, 1731 %ec = icmp eq i16 %iv.next, 0732 %iv.next.trunc = trunc i16 %iv.next to i8733 br i1 %ec, label %exit, label %loop.header734 735exit:736 ret i32 %merge737}738