1854 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg -simplifycfg-require-and-preserve-domtree=1 -tail-predication=enabled < %s -S -o - | FileCheck %s3 4target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"5target triple = "thumbv8.1m.main-arm-none-eabi"6 7; Should not be vectorized8define i64 @add_i64_i64(ptr nocapture readonly %x, i32 %n) #0 {9; CHECK-LABEL: @add_i64_i64(10; CHECK-NEXT: entry:11; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 012; CHECK-NEXT: br i1 [[CMP6]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]13; CHECK: for.body:14; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]15; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]16; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[X:%.*]], i32 [[I_08]]17; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 818; CHECK-NEXT: [[ADD]] = add nsw i64 [[TMP0]], [[R_07]]19; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 120; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]21; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]22; CHECK: for.cond.cleanup:23; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ]24; CHECK-NEXT: ret i64 [[R_0_LCSSA]]25;26entry:27 %cmp6 = icmp sgt i32 %n, 028 br i1 %cmp6, label %for.body, label %for.cond.cleanup29 30for.body: ; preds = %entry, %for.body31 %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]32 %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]33 %arrayidx = getelementptr inbounds i64, ptr %x, i32 %i.0834 %0 = load i64, ptr %arrayidx, align 835 %add = add nsw i64 %0, %r.0736 %inc = add nuw nsw i32 %i.08, 137 %exitcond = icmp eq i32 %inc, %n38 br i1 %exitcond, label %for.cond.cleanup, label %for.body39 40for.cond.cleanup: ; preds = %for.body, %entry41 %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]42 ret i64 %r.0.lcssa43}44 45; 4x to use VADDLV46; FIXME: TailPredicate47define i64 @add_i32_i64(ptr nocapture readonly %x, i32 %n) #0 {48; CHECK-LABEL: @add_i32_i64(49; CHECK-NEXT: entry:50; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 051; CHECK-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]52; CHECK: for.body.preheader:53; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 454; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]55; CHECK: vector.ph:56; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 214748364457; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]58; CHECK: vector.body:59; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]60; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]61; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]62; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 463; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>64; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])65; CHECK-NEXT: [[TMP3]] = add i64 [[VEC_PHI]], [[TMP2]]66; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 467; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]68; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]69; CHECK: middle.block:70; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]71; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]72; CHECK: scalar.ph:73; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]74; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]75; CHECK-NEXT: br label [[FOR_BODY:%.*]]76; CHECK: for.body:77; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]78; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]79; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i32 [[I_08]]80; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 481; CHECK-NEXT: [[CONV:%.*]] = sext i32 [[TMP5]] to i6482; CHECK-NEXT: [[ADD]] = add nsw i64 [[R_07]], [[CONV]]83; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 184; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]85; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]86; CHECK: for.cond.cleanup:87; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]88; CHECK-NEXT: ret i64 [[R_0_LCSSA]]89;90entry:91 %cmp6 = icmp sgt i32 %n, 092 br i1 %cmp6, label %for.body, label %for.cond.cleanup93 94for.body: ; preds = %entry, %for.body95 %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]96 %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]97 %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.0898 %0 = load i32, ptr %arrayidx, align 499 %conv = sext i32 %0 to i64100 %add = add nsw i64 %r.07, %conv101 %inc = add nuw nsw i32 %i.08, 1102 %exitcond = icmp eq i32 %inc, %n103 br i1 %exitcond, label %for.cond.cleanup, label %for.body104 105for.cond.cleanup: ; preds = %for.body, %entry106 %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]107 ret i64 %r.0.lcssa108}109 110; 4x to use VADDLV111; FIXME: TailPredicate112define i64 @add_i16_i64(ptr nocapture readonly %x, i32 %n) #0 {113; CHECK-LABEL: @add_i16_i64(114; CHECK-NEXT: entry:115; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0116; CHECK-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]117; CHECK: for.body.preheader:118; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4119; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]120; CHECK: vector.ph:121; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483644122; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]123; CHECK: vector.body:124; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]125; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]126; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]127; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2128; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>129; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])130; CHECK-NEXT: [[TMP3]] = add i64 [[VEC_PHI]], [[TMP2]]131; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4132; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]133; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]134; CHECK: middle.block:135; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]136; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]137; CHECK: scalar.ph:138; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]139; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]140; CHECK-NEXT: br label [[FOR_BODY:%.*]]141; CHECK: for.body:142; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]143; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]144; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[I_08]]145; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX]], align 2146; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP5]] to i64147; CHECK-NEXT: [[ADD]] = add nsw i64 [[R_07]], [[CONV]]148; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1149; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]150; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]151; CHECK: for.cond.cleanup:152; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]153; CHECK-NEXT: ret i64 [[R_0_LCSSA]]154;155entry:156 %cmp6 = icmp sgt i32 %n, 0157 br i1 %cmp6, label %for.body, label %for.cond.cleanup158 159for.body: ; preds = %entry, %for.body160 %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]161 %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]162 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.08163 %0 = load i16, ptr %arrayidx, align 2164 %conv = sext i16 %0 to i64165 %add = add nsw i64 %r.07, %conv166 %inc = add nuw nsw i32 %i.08, 1167 %exitcond = icmp eq i32 %inc, %n168 br i1 %exitcond, label %for.cond.cleanup, label %for.body169 170for.cond.cleanup: ; preds = %for.body, %entry171 %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]172 ret i64 %r.0.lcssa173}174 175; 4x to use VADDLV176; FIXME: TailPredicate177define i64 @add_i8_i64(ptr nocapture readonly %x, i32 %n) #0 {178; CHECK-LABEL: @add_i8_i64(179; CHECK-NEXT: entry:180; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0181; CHECK-NEXT: br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]182; CHECK: for.body.preheader:183; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4184; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]185; CHECK: vector.ph:186; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483644187; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]188; CHECK: vector.body:189; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]190; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]191; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]192; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1193; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i64>194; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])195; CHECK-NEXT: [[TMP3]] = add i64 [[VEC_PHI]], [[TMP2]]196; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4197; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]198; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]199; CHECK: middle.block:200; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]201; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]202; CHECK: scalar.ph:203; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]204; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]205; CHECK-NEXT: br label [[FOR_BODY:%.*]]206; CHECK: for.body:207; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]208; CHECK-NEXT: [[R_07:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]209; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i32 [[I_08]]210; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX]], align 1211; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP5]] to i64212; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[R_07]], [[CONV]]213; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_08]], 1214; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]215; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]216; CHECK: for.cond.cleanup:217; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]218; CHECK-NEXT: ret i64 [[R_0_LCSSA]]219;220entry:221 %cmp6 = icmp sgt i32 %n, 0222 br i1 %cmp6, label %for.body, label %for.cond.cleanup223 224for.body: ; preds = %entry, %for.body225 %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]226 %r.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]227 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.08228 %0 = load i8, ptr %arrayidx, align 1229 %conv = zext i8 %0 to i64230 %add = add nuw nsw i64 %r.07, %conv231 %inc = add nuw nsw i32 %i.08, 1232 %exitcond = icmp eq i32 %inc, %n233 br i1 %exitcond, label %for.cond.cleanup, label %for.body234 235for.cond.cleanup: ; preds = %for.body, %entry236 %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]237 ret i64 %r.0.lcssa238}239 240; 4x to use VADDV.u32241define i32 @add_i32_i32(ptr nocapture readonly %x, i32 %n) #0 {242; CHECK-LABEL: @add_i32_i32(243; CHECK-NEXT: entry:244; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0245; CHECK-NEXT: br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]246; CHECK: vector.ph:247; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 3248; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4249; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]250; CHECK: vector.body:251; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]252; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]253; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])254; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]255; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> zeroinitializer)256; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_MASKED_LOAD]])257; CHECK-NEXT: [[TMP2]] = add i32 [[VEC_PHI]], [[TMP1]]258; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4259; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]260; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]261; CHECK: for.cond.cleanup:262; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[VECTOR_BODY]] ]263; CHECK-NEXT: ret i32 [[R_0_LCSSA]]264;265entry:266 %cmp6 = icmp sgt i32 %n, 0267 br i1 %cmp6, label %for.body, label %for.cond.cleanup268 269for.body: ; preds = %entry, %for.body270 %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]271 %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]272 %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.08273 %0 = load i32, ptr %arrayidx, align 4274 %add = add nsw i32 %0, %r.07275 %inc = add nuw nsw i32 %i.08, 1276 %exitcond = icmp eq i32 %inc, %n277 br i1 %exitcond, label %for.cond.cleanup, label %for.body278 279for.cond.cleanup: ; preds = %for.body, %entry280 %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]281 ret i32 %r.0.lcssa282}283 284; 8x to use VADDV.u16285define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) #0 {286; CHECK-LABEL: @add_i16_i32(287; CHECK-NEXT: entry:288; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0289; CHECK-NEXT: br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]290; CHECK: vector.ph:291; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7292; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8293; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]294; CHECK: vector.body:295; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]296; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]297; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])298; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]299; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)300; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD]] to <8 x i32>301; CHECK-NEXT: [[TMP2:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP1]], <8 x i32> zeroinitializer302; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP2]])303; CHECK-NEXT: [[TMP4]] = add i32 [[VEC_PHI]], [[TMP3]]304; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8305; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]306; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]307; CHECK: for.cond.cleanup:308; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]309; CHECK-NEXT: ret i32 [[R_0_LCSSA]]310;311entry:312 %cmp6 = icmp sgt i32 %n, 0313 br i1 %cmp6, label %for.body, label %for.cond.cleanup314 315for.body: ; preds = %entry, %for.body316 %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]317 %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]318 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.08319 %0 = load i16, ptr %arrayidx, align 2320 %conv = sext i16 %0 to i32321 %add = add nsw i32 %r.07, %conv322 %inc = add nuw nsw i32 %i.08, 1323 %exitcond = icmp eq i32 %inc, %n324 br i1 %exitcond, label %for.cond.cleanup, label %for.body325 326for.cond.cleanup: ; preds = %for.body, %entry327 %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]328 ret i32 %r.0.lcssa329}330 331; 16x to use VADDV.u16332define i32 @add_i8_i32(ptr nocapture readonly %x, i32 %n) #0 {333; CHECK-LABEL: @add_i8_i32(334; CHECK-NEXT: entry:335; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0336; CHECK-NEXT: br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]337; CHECK: vector.ph:338; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15339; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16340; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]341; CHECK: vector.body:342; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]343; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]344; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])345; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]346; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)347; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>348; CHECK-NEXT: [[TMP2:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP1]], <16 x i32> zeroinitializer349; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP2]])350; CHECK-NEXT: [[TMP4]] = add i32 [[VEC_PHI]], [[TMP3]]351; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16352; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]353; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]354; CHECK: for.cond.cleanup:355; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]356; CHECK-NEXT: ret i32 [[R_0_LCSSA]]357;358entry:359 %cmp6 = icmp sgt i32 %n, 0360 br i1 %cmp6, label %for.body, label %for.cond.cleanup361 362for.body: ; preds = %entry, %for.body363 %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]364 %r.07 = phi i32 [ %add, %for.body ], [ 0, %entry ]365 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.08366 %0 = load i8, ptr %arrayidx, align 1367 %conv = zext i8 %0 to i32368 %add = add nuw nsw i32 %r.07, %conv369 %inc = add nuw nsw i32 %i.08, 1370 %exitcond = icmp eq i32 %inc, %n371 br i1 %exitcond, label %for.cond.cleanup, label %for.body372 373for.cond.cleanup: ; preds = %for.body, %entry374 %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]375 ret i32 %r.0.lcssa376}377 378; 8x to use VADDV.u16379define signext i16 @add_i16_i16(ptr nocapture readonly %x, i32 %n) #0 {380; CHECK-LABEL: @add_i16_i16(381; CHECK-NEXT: entry:382; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0383; CHECK-NEXT: br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]384; CHECK: vector.ph:385; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7386; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8387; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]388; CHECK: vector.body:389; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]390; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]391; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])392; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]393; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> zeroinitializer)394; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[WIDE_MASKED_LOAD]])395; CHECK-NEXT: [[TMP2]] = add i16 [[VEC_PHI]], [[TMP1]]396; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8397; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]398; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]399; CHECK: for.cond.cleanup:400; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[VECTOR_BODY]] ]401; CHECK-NEXT: ret i16 [[R_0_LCSSA]]402;403entry:404 %cmp8 = icmp sgt i32 %n, 0405 br i1 %cmp8, label %for.body, label %for.cond.cleanup406 407for.body: ; preds = %entry, %for.body408 %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]409 %r.09 = phi i16 [ %add, %for.body ], [ 0, %entry ]410 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.010411 %0 = load i16, ptr %arrayidx, align 2412 %add = add i16 %0, %r.09413 %inc = add nuw nsw i32 %i.010, 1414 %exitcond = icmp eq i32 %inc, %n415 br i1 %exitcond, label %for.cond.cleanup, label %for.body416 417for.cond.cleanup: ; preds = %for.body, %entry418 %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]419 ret i16 %r.0.lcssa420}421 422; 16x to use VADDV.u8423define signext i16 @add_i8_i16(ptr nocapture readonly %x, i32 %n) #0 {424; CHECK-LABEL: @add_i8_i16(425; CHECK-NEXT: entry:426; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0427; CHECK-NEXT: br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]428; CHECK: vector.ph:429; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15430; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16431; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]432; CHECK: vector.body:433; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]434; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]435; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])436; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]437; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)438; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i16>439; CHECK-NEXT: [[TMP2:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i16> [[TMP1]], <16 x i16> zeroinitializer440; CHECK-NEXT: [[TMP3:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[TMP2]])441; CHECK-NEXT: [[TMP4]] = add i16 [[VEC_PHI]], [[TMP3]]442; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16443; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]444; CHECK-NEXT: br i1 [[TMP5]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]445; CHECK: for.cond.cleanup:446; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[VECTOR_BODY]] ]447; CHECK-NEXT: ret i16 [[R_0_LCSSA]]448;449entry:450 %cmp8 = icmp sgt i32 %n, 0451 br i1 %cmp8, label %for.body, label %for.cond.cleanup452 453for.body: ; preds = %entry, %for.body454 %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]455 %r.09 = phi i16 [ %add, %for.body ], [ 0, %entry ]456 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.010457 %0 = load i8, ptr %arrayidx, align 1458 %conv = zext i8 %0 to i16459 %add = add i16 %r.09, %conv460 %inc = add nuw nsw i32 %i.010, 1461 %exitcond = icmp eq i32 %inc, %n462 br i1 %exitcond, label %for.cond.cleanup, label %for.body463 464for.cond.cleanup: ; preds = %for.body, %entry465 %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]466 ret i16 %r.0.lcssa467}468 469; 16x to use VADDV.u8470define zeroext i8 @add_i8_i8(ptr nocapture readonly %x, i32 %n) #0 {471; CHECK-LABEL: @add_i8_i8(472; CHECK-NEXT: entry:473; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[N:%.*]], 0474; CHECK-NEXT: br i1 [[CMP7]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]475; CHECK: vector.ph:476; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15477; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16478; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]479; CHECK: vector.body:480; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]481; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i8 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]482; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])483; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]484; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> zeroinitializer)485; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[WIDE_MASKED_LOAD]])486; CHECK-NEXT: [[TMP2]] = add i8 [[VEC_PHI]], [[TMP1]]487; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16488; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]489; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]490; CHECK: for.cond.cleanup:491; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[VECTOR_BODY]] ]492; CHECK-NEXT: ret i8 [[R_0_LCSSA]]493;494entry:495 %cmp7 = icmp sgt i32 %n, 0496 br i1 %cmp7, label %for.body, label %for.cond.cleanup497 498for.body: ; preds = %entry, %for.body499 %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]500 %r.08 = phi i8 [ %add, %for.body ], [ 0, %entry ]501 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.09502 %0 = load i8, ptr %arrayidx, align 1503 %add = add i8 %0, %r.08504 %inc = add nuw nsw i32 %i.09, 1505 %exitcond = icmp eq i32 %inc, %n506 br i1 %exitcond, label %for.cond.cleanup, label %for.body507 508for.cond.cleanup: ; preds = %for.body, %entry509 %r.0.lcssa = phi i8 [ 0, %entry ], [ %add, %for.body ]510 ret i8 %r.0.lcssa511}512 513; Not vectorized514define i64 @mla_i64_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {515; CHECK-LABEL: @mla_i64_i64(516; CHECK-NEXT: entry:517; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0518; CHECK-NEXT: br i1 [[CMP8]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]519; CHECK: for.body:520; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]521; CHECK-NEXT: [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]522; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[X:%.*]], i32 [[I_010]]523; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8524; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i64, ptr [[Y:%.*]], i32 [[I_010]]525; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8526; CHECK-NEXT: [[MUL:%.*]] = mul nsw i64 [[TMP1]], [[TMP0]]527; CHECK-NEXT: [[ADD]] = add nsw i64 [[MUL]], [[R_09]]528; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_010]], 1529; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]530; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]531; CHECK: for.cond.cleanup:532; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[ADD]], [[FOR_BODY]] ]533; CHECK-NEXT: ret i64 [[R_0_LCSSA]]534;535entry:536 %cmp8 = icmp sgt i32 %n, 0537 br i1 %cmp8, label %for.body, label %for.cond.cleanup538 539for.body: ; preds = %entry, %for.body540 %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]541 %r.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]542 %arrayidx = getelementptr inbounds i64, ptr %x, i32 %i.010543 %0 = load i64, ptr %arrayidx, align 8544 %arrayidx1 = getelementptr inbounds i64, ptr %y, i32 %i.010545 %1 = load i64, ptr %arrayidx1, align 8546 %mul = mul nsw i64 %1, %0547 %add = add nsw i64 %mul, %r.09548 %inc = add nuw nsw i32 %i.010, 1549 %exitcond = icmp eq i32 %inc, %n550 br i1 %exitcond, label %for.cond.cleanup, label %for.body551 552for.cond.cleanup: ; preds = %for.body, %entry553 %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]554 ret i64 %r.0.lcssa555}556 557; 4x to use VMLAL.u32558; FIXME: TailPredicate559define i64 @mla_i32_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {560; CHECK-LABEL: @mla_i32_i64(561; CHECK-NEXT: entry:562; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0563; CHECK-NEXT: br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]564; CHECK: for.body.preheader:565; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4566; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]567; CHECK: vector.ph:568; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483644569; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]570; CHECK: vector.body:571; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]572; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]573; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]574; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4575; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[Y:%.*]], i32 [[INDEX]]576; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4577; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]578; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>579; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])580; CHECK-NEXT: [[TMP5]] = add i64 [[VEC_PHI]], [[TMP4]]581; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4582; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]583; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]584; CHECK: middle.block:585; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]586; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]587; CHECK: scalar.ph:588; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]589; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]590; CHECK-NEXT: br label [[FOR_BODY:%.*]]591; CHECK: for.body:592; CHECK-NEXT: [[I_010:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]593; CHECK-NEXT: [[R_09:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]594; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i32 [[I_010]]595; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4596; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i32, ptr [[Y]], i32 [[I_010]]597; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4598; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]599; CHECK-NEXT: [[CONV:%.*]] = sext i32 [[MUL]] to i64600; CHECK-NEXT: [[ADD]] = add nsw i64 [[R_09]], [[CONV]]601; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_010]], 1602; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]603; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]604; CHECK: for.cond.cleanup:605; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]606; CHECK-NEXT: ret i64 [[R_0_LCSSA]]607;608entry:609 %cmp8 = icmp sgt i32 %n, 0610 br i1 %cmp8, label %for.body, label %for.cond.cleanup611 612for.body: ; preds = %entry, %for.body613 %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]614 %r.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]615 %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.010616 %0 = load i32, ptr %arrayidx, align 4617 %arrayidx1 = getelementptr inbounds i32, ptr %y, i32 %i.010618 %1 = load i32, ptr %arrayidx1, align 4619 %mul = mul nsw i32 %1, %0620 %conv = sext i32 %mul to i64621 %add = add nsw i64 %r.09, %conv622 %inc = add nuw nsw i32 %i.010, 1623 %exitcond = icmp eq i32 %inc, %n624 br i1 %exitcond, label %for.cond.cleanup, label %for.body625 626for.cond.cleanup: ; preds = %for.body, %entry627 %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]628 ret i64 %r.0.lcssa629}630 631; 8x to use VMLAL.u16632; FIXME: TailPredicate633define i64 @mla_i16_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {634; CHECK-LABEL: @mla_i16_i64(635; CHECK-NEXT: entry:636; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0637; CHECK-NEXT: br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]638; CHECK: for.body.preheader:639; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8640; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]641; CHECK: vector.ph:642; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483640643; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]644; CHECK: vector.body:645; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]646; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]647; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]648; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2649; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i32 [[INDEX]]650; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2651; CHECK-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD1]] to <8 x i64>652; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>653; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <8 x i64> [[TMP4]], [[TMP3]]654; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])655; CHECK-NEXT: [[TMP7]] = add i64 [[VEC_PHI]], [[TMP6]]656; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8657; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]658; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]659; CHECK: middle.block:660; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]661; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]662; CHECK: scalar.ph:663; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]664; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]665; CHECK-NEXT: br label [[FOR_BODY:%.*]]666; CHECK: for.body:667; CHECK-NEXT: [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]668; CHECK-NEXT: [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]669; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[I_012]]670; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX]], align 2671; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP9]] to i32672; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i16, ptr [[Y]], i32 [[I_012]]673; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2674; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP10]] to i32675; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]676; CHECK-NEXT: [[CONV3:%.*]] = sext i32 [[MUL]] to i64677; CHECK-NEXT: [[ADD]] = add nsw i64 [[R_011]], [[CONV3]]678; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_012]], 1679; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]680; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]681; CHECK: for.cond.cleanup:682; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]683; CHECK-NEXT: ret i64 [[R_0_LCSSA]]684;685entry:686 %cmp10 = icmp sgt i32 %n, 0687 br i1 %cmp10, label %for.body, label %for.cond.cleanup688 689for.body: ; preds = %entry, %for.body690 %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]691 %r.011 = phi i64 [ %add, %for.body ], [ 0, %entry ]692 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.012693 %0 = load i16, ptr %arrayidx, align 2694 %conv = sext i16 %0 to i32695 %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %i.012696 %1 = load i16, ptr %arrayidx1, align 2697 %conv2 = sext i16 %1 to i32698 %mul = mul nsw i32 %conv2, %conv699 %conv3 = sext i32 %mul to i64700 %add = add nsw i64 %r.011, %conv3701 %inc = add nuw nsw i32 %i.012, 1702 %exitcond = icmp eq i32 %inc, %n703 br i1 %exitcond, label %for.cond.cleanup, label %for.body704 705for.cond.cleanup: ; preds = %for.body, %entry706 %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]707 ret i64 %r.0.lcssa708}709 710; 8x to use VMLAL.u16711; FIXME: TailPredicate712define i64 @mla_i8_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {713; CHECK-LABEL: @mla_i8_i64(714; CHECK-NEXT: entry:715; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 0716; CHECK-NEXT: br i1 [[CMP10]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]717; CHECK: for.body.preheader:718; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8719; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]720; CHECK: vector.ph:721; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 2147483640722; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]723; CHECK: vector.body:724; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]725; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]726; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]727; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1728; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i32 [[INDEX]]729; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1730; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i8> [[WIDE_LOAD1]] to <8 x i64>731; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i64>732; CHECK-NEXT: [[TMP5:%.*]] = mul nuw nsw <8 x i64> [[TMP4]], [[TMP3]]733; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])734; CHECK-NEXT: [[TMP7]] = add i64 [[VEC_PHI]], [[TMP6]]735; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8736; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]737; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]738; CHECK: middle.block:739; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]740; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]741; CHECK: scalar.ph:742; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]743; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]744; CHECK-NEXT: br label [[FOR_BODY:%.*]]745; CHECK: for.body:746; CHECK-NEXT: [[I_012:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]747; CHECK-NEXT: [[R_011:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]748; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i32 [[I_012]]749; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX]], align 1750; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP9]] to i32751; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i32 [[I_012]]752; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1753; CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP10]] to i32754; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]]755; CHECK-NEXT: [[CONV3:%.*]] = zext nneg i32 [[MUL]] to i64756; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[R_011]], [[CONV3]]757; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_012]], 1758; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]759; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]760; CHECK: for.cond.cleanup:761; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]762; CHECK-NEXT: ret i64 [[R_0_LCSSA]]763;764entry:765 %cmp10 = icmp sgt i32 %n, 0766 br i1 %cmp10, label %for.body, label %for.cond.cleanup767 768for.body: ; preds = %entry, %for.body769 %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]770 %r.011 = phi i64 [ %add, %for.body ], [ 0, %entry ]771 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.012772 %0 = load i8, ptr %arrayidx, align 1773 %conv = zext i8 %0 to i32774 %arrayidx1 = getelementptr inbounds i8, ptr %y, i32 %i.012775 %1 = load i8, ptr %arrayidx1, align 1776 %conv2 = zext i8 %1 to i32777 %mul = mul nuw nsw i32 %conv2, %conv778 %conv3 = zext i32 %mul to i64779 %add = add nuw nsw i64 %r.011, %conv3780 %inc = add nuw nsw i32 %i.012, 1781 %exitcond = icmp eq i32 %inc, %n782 br i1 %exitcond, label %for.cond.cleanup, label %for.body783 784for.cond.cleanup: ; preds = %for.body, %entry785 %r.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]786 ret i64 %r.0.lcssa787}788 789; 4x to use VMLA.i32790define i32 @mla_i32_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {791; CHECK-LABEL: @mla_i32_i32(792; CHECK-NEXT: entry:793; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N:%.*]], 0794; CHECK-NEXT: br i1 [[CMP8]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]795; CHECK: vector.ph:796; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 3797; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -4798; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]799; CHECK: vector.body:800; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]801; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]802; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])803; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[Y:%.*]], i32 [[INDEX]]804; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)805; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[Y1:%.*]], i32 [[INDEX]]806; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP7]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)807; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD1]]808; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> zeroinitializer809; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])810; CHECK-NEXT: [[TMP5]] = add i32 [[VEC_PHI]], [[TMP4]]811; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4812; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]813; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]814; CHECK: for.cond.cleanup:815; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]816; CHECK-NEXT: ret i32 [[R_0_LCSSA]]817;818entry:819 %cmp8 = icmp sgt i32 %n, 0820 br i1 %cmp8, label %for.body, label %for.cond.cleanup821 822for.body: ; preds = %entry, %for.body823 %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]824 %r.09 = phi i32 [ %add, %for.body ], [ 0, %entry ]825 %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.010826 %0 = load i32, ptr %arrayidx, align 4827 %arrayidx1 = getelementptr inbounds i32, ptr %y, i32 %i.010828 %1 = load i32, ptr %arrayidx1, align 4829 %mul = mul nsw i32 %1, %0830 %add = add nsw i32 %mul, %r.09831 %inc = add nuw nsw i32 %i.010, 1832 %exitcond = icmp eq i32 %inc, %n833 br i1 %exitcond, label %for.cond.cleanup, label %for.body834 835for.cond.cleanup: ; preds = %for.body, %entry836 %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]837 ret i32 %r.0.lcssa838}839 840; 8x to use VMLA.i16841define i32 @mla_i16_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {842; CHECK-LABEL: @mla_i16_i32(843; CHECK-NEXT: entry:844; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0845; CHECK-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]846; CHECK: vector.ph:847; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7848; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8849; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]850; CHECK: vector.body:851; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]852; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]853; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])854; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]855; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)856; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i32 [[INDEX]]857; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP2]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)858; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD1]] to <8 x i32>859; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD]] to <8 x i32>860; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <8 x i32> [[TMP3]], [[TMP1]]861; CHECK-NEXT: [[TMP5:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP4]], <8 x i32> zeroinitializer862; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP5]])863; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]864; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8865; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]866; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]867; CHECK: for.cond.cleanup:868; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]869; CHECK-NEXT: ret i32 [[R_0_LCSSA]]870;871entry:872 %cmp9 = icmp sgt i32 %n, 0873 br i1 %cmp9, label %for.body, label %for.cond.cleanup874 875for.body: ; preds = %entry, %for.body876 %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]877 %r.010 = phi i32 [ %add, %for.body ], [ 0, %entry ]878 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.011879 %0 = load i16, ptr %arrayidx, align 2880 %conv = sext i16 %0 to i32881 %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %i.011882 %1 = load i16, ptr %arrayidx1, align 2883 %conv2 = sext i16 %1 to i32884 %mul = mul nsw i32 %conv2, %conv885 %add = add nsw i32 %mul, %r.010886 %inc = add nuw nsw i32 %i.011, 1887 %exitcond = icmp eq i32 %inc, %n888 br i1 %exitcond, label %for.cond.cleanup, label %for.body889 890for.cond.cleanup: ; preds = %for.body, %entry891 %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]892 ret i32 %r.0.lcssa893}894 895; 16x to use VMLA.i8896define i32 @mla_i8_i32(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {897; CHECK-LABEL: @mla_i8_i32(898; CHECK-NEXT: entry:899; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0900; CHECK-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]901; CHECK: vector.ph:902; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 15903; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16904; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]905; CHECK: vector.body:906; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]907; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]908; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])909; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]910; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)911; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i32 [[INDEX]]912; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)913; CHECK-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD1]] to <16 x i32>914; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>915; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <16 x i32> [[TMP2]], [[TMP3]]916; CHECK-NEXT: [[TMP5:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP4]], <16 x i32> zeroinitializer917; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP5]])918; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]919; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16920; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]921; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]922; CHECK: for.cond.cleanup:923; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]924; CHECK-NEXT: ret i32 [[R_0_LCSSA]]925;926entry:927 %cmp9 = icmp sgt i32 %n, 0928 br i1 %cmp9, label %for.body, label %for.cond.cleanup929 930for.body: ; preds = %entry, %for.body931 %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]932 %r.010 = phi i32 [ %add, %for.body ], [ 0, %entry ]933 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.011934 %0 = load i8, ptr %arrayidx, align 1935 %conv = zext i8 %0 to i32936 %arrayidx1 = getelementptr inbounds i8, ptr %y, i32 %i.011937 %1 = load i8, ptr %arrayidx1, align 1938 %conv2 = zext i8 %1 to i32939 %mul = mul nuw nsw i32 %conv2, %conv940 %add = add nuw nsw i32 %mul, %r.010941 %inc = add nuw nsw i32 %i.011, 1942 %exitcond = icmp eq i32 %inc, %n943 br i1 %exitcond, label %for.cond.cleanup, label %for.body944 945for.cond.cleanup: ; preds = %for.body, %entry946 %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]947 ret i32 %r.0.lcssa948}949 950; 8x to use VMLA.i16951define signext i16 @mla_i16_i16(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {952; CHECK-LABEL: @mla_i16_i16(953; CHECK-NEXT: entry:954; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 0955; CHECK-NEXT: br i1 [[CMP11]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]956; CHECK: vector.ph:957; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 7958; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8959; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]960; CHECK: vector.body:961; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]962; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]963; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])964; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i32 [[INDEX]]965; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP1]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)966; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[Y1:%.*]], i32 [[INDEX]]967; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP7]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)968; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i16> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD1]]969; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> [[TMP2]], <8 x i16> zeroinitializer970; CHECK-NEXT: [[TMP4:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP3]])971; CHECK-NEXT: [[TMP5]] = add i16 [[VEC_PHI]], [[TMP4]]972; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8973; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]974; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]975; CHECK: for.cond.cleanup:976; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]977; CHECK-NEXT: ret i16 [[R_0_LCSSA]]978;979entry:980 %cmp11 = icmp sgt i32 %n, 0981 br i1 %cmp11, label %for.body, label %for.cond.cleanup982 983for.body: ; preds = %entry, %for.body984 %i.013 = phi i32 [ %inc, %for.body ], [ 0, %entry ]985 %r.012 = phi i16 [ %add, %for.body ], [ 0, %entry ]986 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.013987 %0 = load i16, ptr %arrayidx, align 2988 %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %i.013989 %1 = load i16, ptr %arrayidx1, align 2990 %mul = mul i16 %1, %0991 %add = add i16 %mul, %r.012992 %inc = add nuw nsw i32 %i.013, 1993 %exitcond = icmp eq i32 %inc, %n994 br i1 %exitcond, label %for.cond.cleanup, label %for.body995 996for.cond.cleanup: ; preds = %for.body, %entry997 %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]998 ret i16 %r.0.lcssa999}1000 1001; 16x to use VMLA.i81002define signext i16 @mla_i8_i16(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {1003; CHECK-LABEL: @mla_i8_i16(1004; CHECK-NEXT: entry:1005; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 01006; CHECK-NEXT: br i1 [[CMP11]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]1007; CHECK: vector.ph:1008; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 151009; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -161010; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1011; CHECK: vector.body:1012; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1013; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i16 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]1014; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])1015; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]1016; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)1017; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i32 [[INDEX]]1018; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)1019; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD1]] to <16 x i16>1020; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i16>1021; CHECK-NEXT: [[TMP4:%.*]] = mul nuw <16 x i16> [[TMP3]], [[TMP1]]1022; CHECK-NEXT: [[TMP5:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i16> [[TMP4]], <16 x i16> zeroinitializer1023; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[TMP5]])1024; CHECK-NEXT: [[TMP7]] = add i16 [[VEC_PHI]], [[TMP6]]1025; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 161026; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1027; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]1028; CHECK: for.cond.cleanup:1029; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]1030; CHECK-NEXT: ret i16 [[R_0_LCSSA]]1031;1032entry:1033 %cmp11 = icmp sgt i32 %n, 01034 br i1 %cmp11, label %for.body, label %for.cond.cleanup1035 1036for.body: ; preds = %entry, %for.body1037 %i.013 = phi i32 [ %inc, %for.body ], [ 0, %entry ]1038 %r.012 = phi i16 [ %add, %for.body ], [ 0, %entry ]1039 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.0131040 %0 = load i8, ptr %arrayidx, align 11041 %conv = zext i8 %0 to i161042 %arrayidx1 = getelementptr inbounds i8, ptr %y, i32 %i.0131043 %1 = load i8, ptr %arrayidx1, align 11044 %conv2 = zext i8 %1 to i161045 %mul = mul nuw i16 %conv2, %conv1046 %add = add i16 %mul, %r.0121047 %inc = add nuw nsw i32 %i.013, 11048 %exitcond = icmp eq i32 %inc, %n1049 br i1 %exitcond, label %for.cond.cleanup, label %for.body1050 1051for.cond.cleanup: ; preds = %for.body, %entry1052 %r.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]1053 ret i16 %r.0.lcssa1054}1055 1056; 16x to use VMLA.i81057define zeroext i8 @mla_i8_i8(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {1058; CHECK-LABEL: @mla_i8_i8(1059; CHECK-NEXT: entry:1060; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N:%.*]], 01061; CHECK-NEXT: br i1 [[CMP10]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]1062; CHECK: vector.ph:1063; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 151064; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -161065; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1066; CHECK: vector.body:1067; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1068; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i8 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]1069; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])1070; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i32 [[INDEX]]1071; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)1072; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[Y1:%.*]], i32 [[INDEX]]1073; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP7]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)1074; CHECK-NEXT: [[TMP2:%.*]] = mul <16 x i8> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD1]]1075; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> [[TMP2]], <16 x i8> zeroinitializer1076; CHECK-NEXT: [[TMP4:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[TMP3]])1077; CHECK-NEXT: [[TMP5]] = add i8 [[VEC_PHI]], [[TMP4]]1078; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 161079; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1080; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]1081; CHECK: for.cond.cleanup:1082; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]1083; CHECK-NEXT: ret i8 [[R_0_LCSSA]]1084;1085entry:1086 %cmp10 = icmp sgt i32 %n, 01087 br i1 %cmp10, label %for.body, label %for.cond.cleanup1088 1089for.body: ; preds = %entry, %for.body1090 %i.012 = phi i32 [ %inc, %for.body ], [ 0, %entry ]1091 %r.011 = phi i8 [ %add, %for.body ], [ 0, %entry ]1092 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.0121093 %0 = load i8, ptr %arrayidx, align 11094 %arrayidx1 = getelementptr inbounds i8, ptr %y, i32 %i.0121095 %1 = load i8, ptr %arrayidx1, align 11096 %mul = mul i8 %1, %01097 %add = add i8 %mul, %r.0111098 %inc = add nuw nsw i32 %i.012, 11099 %exitcond = icmp eq i32 %inc, %n1100 br i1 %exitcond, label %for.cond.cleanup, label %for.body1101 1102for.cond.cleanup: ; preds = %for.body, %entry1103 %r.0.lcssa = phi i8 [ 0, %entry ], [ %add, %for.body ]1104 ret i8 %r.0.lcssa1105}1106 1107; 8x as different types1108define i32 @red_mla_ext_s8_s16_s32(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, i32 %n) #0 {1109; CHECK-LABEL: @red_mla_ext_s8_s16_s32(1110; CHECK-NEXT: entry:1111; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N:%.*]], 01112; CHECK-NEXT: br i1 [[CMP9_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_PH:%.*]]1113; CHECK: vector.ph:1114; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 71115; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -81116; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1117; CHECK: vector.body:1118; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1119; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]1120; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[N]])1121; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 [[INDEX]]1122; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP0]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i8> poison)1123; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i32 [[INDEX]]1124; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[TMP2]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)1125; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[WIDE_MASKED_LOAD1]] to <8 x i32>1126; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>1127; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <8 x i32> [[TMP3]], [[TMP1]]1128; CHECK-NEXT: [[TMP5:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i32> [[TMP4]], <8 x i32> zeroinitializer1129; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP5]])1130; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]1131; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 81132; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1133; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]1134; CHECK: for.cond.cleanup:1135; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]1136; CHECK-NEXT: ret i32 [[S_0_LCSSA]]1137;1138entry:1139 %cmp9.not = icmp eq i32 %n, 01140 br i1 %cmp9.not, label %for.cond.cleanup, label %for.body.preheader1141 1142for.body.preheader: ; preds = %entry1143 br label %for.body1144 1145for.body: ; preds = %for.body.preheader, %for.body1146 %i.011 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]1147 %s.010 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]1148 %arrayidx = getelementptr inbounds i8, ptr %A, i32 %i.0111149 %0 = load i8, ptr %arrayidx, align 11150 %conv = sext i8 %0 to i321151 %arrayidx1 = getelementptr inbounds i16, ptr %B, i32 %i.0111152 %1 = load i16, ptr %arrayidx1, align 21153 %conv2 = sext i16 %1 to i321154 %mul = mul nsw i32 %conv2, %conv1155 %add = add nsw i32 %mul, %s.0101156 %inc = add nuw i32 %i.011, 11157 %exitcond.not = icmp eq i32 %inc, %n1158 br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body1159 1160for.cond.cleanup.loopexit: ; preds = %for.body1161 %add.lcssa = phi i32 [ %add, %for.body ]1162 br label %for.cond.cleanup1163 1164for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry1165 %s.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]1166 ret i32 %s.0.lcssa1167}1168 1169; 4x as different sext vs zext1170define i64 @red_mla_ext_s16_u16_s64(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, i32 %n) #0 {1171; CHECK-LABEL: @red_mla_ext_s16_u16_s64(1172; CHECK-NEXT: entry:1173; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N:%.*]], 01174; CHECK-NEXT: br i1 [[CMP9_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]1175; CHECK: for.body.preheader:1176; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 41177; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1178; CHECK: vector.ph:1179; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -41180; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1181; CHECK: vector.body:1182; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1183; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]1184; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i32 [[INDEX]]1185; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 11186; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i32>1187; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i16, ptr [[B1:%.*]], i32 [[INDEX]]1188; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP11]], align 21189; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[WIDE_LOAD1]] to <4 x i32>1190; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP1]]1191; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i32> [[TMP4]] to <4 x i64>1192; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])1193; CHECK-NEXT: [[TMP7]] = add i64 [[VEC_PHI]], [[TMP6]]1194; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 41195; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1196; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]1197; CHECK: middle.block:1198; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]1199; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]1200; CHECK: scalar.ph:1201; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]1202; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]1203; CHECK-NEXT: br label [[FOR_BODY:%.*]]1204; CHECK: for.body:1205; CHECK-NEXT: [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1206; CHECK-NEXT: [[S_010:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]1207; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[I_011]]1208; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX]], align 11209; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP9]] to i321210; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[B1]], i32 [[I_011]]1211; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX1]], align 21212; CHECK-NEXT: [[CONV2:%.*]] = zext i16 [[TMP10]] to i321213; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]1214; CHECK-NEXT: [[MUL2:%.*]] = zext i32 [[MUL]] to i641215; CHECK-NEXT: [[ADD]] = add nsw i64 [[S_010]], [[MUL2]]1216; CHECK-NEXT: [[INC]] = add nuw i32 [[I_011]], 11217; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]1218; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]1219; CHECK: for.cond.cleanup:1220; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]1221; CHECK-NEXT: ret i64 [[S_0_LCSSA]]1222;1223entry:1224 %cmp9.not = icmp eq i32 %n, 01225 br i1 %cmp9.not, label %for.cond.cleanup, label %for.body.preheader1226 1227for.body.preheader: ; preds = %entry1228 br label %for.body1229 1230for.body: ; preds = %for.body.preheader, %for.body1231 %i.011 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]1232 %s.010 = phi i64 [ %add, %for.body ], [ 0, %for.body.preheader ]1233 %arrayidx = getelementptr inbounds i16, ptr %A, i32 %i.0111234 %0 = load i16, ptr %arrayidx, align 11235 %conv = sext i16 %0 to i321236 %arrayidx1 = getelementptr inbounds i16, ptr %B, i32 %i.0111237 %1 = load i16, ptr %arrayidx1, align 21238 %conv2 = zext i16 %1 to i321239 %mul = mul nsw i32 %conv2, %conv1240 %mul2 = zext i32 %mul to i641241 %add = add nsw i64 %mul2, %s.0101242 %inc = add nuw i32 %i.011, 11243 %exitcond.not = icmp eq i32 %inc, %n1244 br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body1245 1246for.cond.cleanup.loopexit: ; preds = %for.body1247 %add.lcssa = phi i64 [ %add, %for.body ]1248 br label %for.cond.cleanup1249 1250for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry1251 %s.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]1252 ret i64 %s.0.lcssa1253}1254 1255; 4x as different sext vs zext1256define i32 @red_mla_u8_s8_u32(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, i32 %n) #0 {1257; CHECK-LABEL: @red_mla_u8_s8_u32(1258; CHECK-NEXT: entry:1259; CHECK-NEXT: [[CMP9_NOT:%.*]] = icmp eq i32 [[N:%.*]], 01260; CHECK-NEXT: br i1 [[CMP9_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_PH:%.*]]1261; CHECK: vector.ph:1262; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 31263; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -41264; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1265; CHECK: vector.body:1266; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1267; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]1268; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])1269; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i32 [[INDEX]]1270; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)1271; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_MASKED_LOAD1]] to <4 x i32>1272; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[B1:%.*]], i32 [[INDEX]]1273; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP9]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)1274; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[WIDE_MASKED_LOAD2]] to <4 x i32>1275; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP1]]1276; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer1277; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])1278; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]1279; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 41280; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1281; CHECK-NEXT: br i1 [[TMP8]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]1282; CHECK: for.cond.cleanup:1283; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[VECTOR_BODY]] ]1284; CHECK-NEXT: ret i32 [[S_0_LCSSA]]1285;1286entry:1287 %cmp9.not = icmp eq i32 %n, 01288 br i1 %cmp9.not, label %for.cond.cleanup, label %for.body.preheader1289 1290for.body.preheader: ; preds = %entry1291 br label %for.body1292 1293for.body: ; preds = %for.body.preheader, %for.body1294 %i.011 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]1295 %s.010 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]1296 %arrayidx = getelementptr inbounds i8, ptr %A, i32 %i.0111297 %0 = load i8, ptr %arrayidx, align 11298 %conv = zext i8 %0 to i321299 %arrayidx1 = getelementptr inbounds i8, ptr %B, i32 %i.0111300 %1 = load i8, ptr %arrayidx1, align 11301 %conv2 = sext i8 %1 to i321302 %mul = mul nsw i32 %conv2, %conv1303 %add = add i32 %mul, %s.0101304 %inc = add nuw i32 %i.011, 11305 %exitcond.not = icmp eq i32 %inc, %n1306 br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body1307 1308for.cond.cleanup.loopexit: ; preds = %for.body1309 %add.lcssa = phi i32 [ %add, %for.body ]1310 br label %for.cond.cleanup1311 1312for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry1313 %s.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]1314 ret i32 %s.0.lcssa1315}1316 1317; Make sure interleave group members feeding in-loop reductions can be handled.1318define i32 @reduction_interleave_group(i32 %n, ptr %arr) #0 {1319; CHECK-LABEL: @reduction_interleave_group(1320; CHECK-NEXT: entry:1321; CHECK-NEXT: [[GUARD:%.*]] = icmp sgt i32 [[N:%.*]], 01322; CHECK-NEXT: br i1 [[GUARD]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]1323; CHECK: for.body.preheader:1324; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[N]], -11325; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 11326; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[TMP1]], 11327; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 71328; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1329; CHECK: vector.ph:1330; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP2]], -41331; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC]], 11332; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1333; CHECK: vector.body:1334; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1335; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]1336; CHECK-NEXT: [[DOTIDX:%.*]] = shl i32 [[INDEX]], 31337; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[ARR:%.*]], i32 [[DOTIDX]]1338; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 41339; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>1340; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>1341; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC1]])1342; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[VEC_PHI]], [[TMP6]]1343; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC]])1344; CHECK-NEXT: [[TMP9]] = add i32 [[TMP7]], [[TMP8]]1345; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 41346; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1347; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]1348; CHECK: middle.block:1349; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]1350; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT]], label [[SCALAR_PH]]1351; CHECK: scalar.ph:1352; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]1353; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]1354; CHECK-NEXT: br label [[FOR_BODY:%.*]]1355; CHECK: for.body:1356; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1357; CHECK-NEXT: [[RED_PHI:%.*]] = phi i32 [ [[RED_2:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]1358; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IV]]1359; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr i8, ptr [[TMP11]], i32 41360; CHECK-NEXT: [[L_0:%.*]] = load i32, ptr [[GEP_0]], align 41361; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i32 [[IV]]1362; CHECK-NEXT: [[L_1:%.*]] = load i32, ptr [[GEP_1]], align 41363; CHECK-NEXT: [[RED_1:%.*]] = add i32 [[L_0]], [[RED_PHI]]1364; CHECK-NEXT: [[RED_2]] = add i32 [[RED_1]], [[L_1]]1365; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 21366; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]1367; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]1368; CHECK: exit:1369; CHECK-NEXT: [[RET_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RED_2]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]1370; CHECK-NEXT: ret i32 [[RET_LCSSA]]1371;1372entry:1373 %guard = icmp sgt i32 %n, 01374 br i1 %guard , label %for.body, label %exit1375 1376for.body: ; preds = %for.body.preheader, %for.body1377 %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]1378 %red.phi = phi i32 [ %red.2, %for.body ], [ 0, %entry ]1379 %add = or disjoint i32 %iv, 11380 %gep.0 = getelementptr inbounds i32, ptr %arr, i32 %add1381 %l.0 = load i32, ptr %gep.0, align 41382 %gep.1 = getelementptr inbounds i32, ptr %arr, i32 %iv1383 %l.1 = load i32, ptr %gep.1, align 41384 %red.1 = add i32 %l.0, %red.phi1385 %red.2 = add i32 %red.1, %l.11386 %iv.next = add nuw nsw i32 %iv, 21387 %cmp = icmp slt i32 %iv.next, %n1388 br i1 %cmp, label %for.body, label %exit1389 1390exit:1391 %ret.lcssa = phi i32 [ 0, %entry ], [ %red.2, %for.body ]1392 ret i32 %ret.lcssa1393}1394 1395; 16x to use VMLA.i8, same as mla_i8_i32 with multiple uses of the ext `add(mul(x, x))`1396define i32 @mla_i8_i32_multiuse(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {1397; CHECK-LABEL: @mla_i8_i32_multiuse(1398; CHECK-NEXT: entry:1399; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 01400; CHECK-NEXT: br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]1401; CHECK: vector.ph:1402; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw i32 [[N]], 151403; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -161404; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1405; CHECK: vector.body:1406; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1407; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]1408; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])1409; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i32 [[INDEX]]1410; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP0]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)1411; CHECK-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>1412; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw <16 x i32> [[TMP7]], [[TMP7]]1413; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i32> [[TMP2]], <16 x i32> zeroinitializer1414; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP3]])1415; CHECK-NEXT: [[TMP5]] = add i32 [[VEC_PHI]], [[TMP4]]1416; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 161417; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1418; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]1419; CHECK: for.cond.cleanup:1420; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP5]], [[VECTOR_BODY]] ]1421; CHECK-NEXT: ret i32 [[R_0_LCSSA]]1422;1423entry:1424 %cmp9 = icmp sgt i32 %n, 01425 br i1 %cmp9, label %for.body, label %for.cond.cleanup1426 1427for.body: ; preds = %entry, %for.body1428 %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]1429 %r.010 = phi i32 [ %add, %for.body ], [ 0, %entry ]1430 %arrayidx = getelementptr inbounds i8, ptr %x, i32 %i.0111431 %0 = load i8, ptr %arrayidx, align 11432 %conv = zext i8 %0 to i321433 %mul = mul nuw nsw i32 %conv, %conv1434 %add = add nuw nsw i32 %mul, %r.0101435 %inc = add nuw nsw i32 %i.011, 11436 %exitcond = icmp eq i32 %inc, %n1437 br i1 %exitcond, label %for.cond.cleanup, label %for.body1438 1439for.cond.cleanup: ; preds = %for.body, %entry1440 %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]1441 ret i32 %r.0.lcssa1442}1443 1444define i64 @mla_xx_sext_zext(ptr nocapture noundef readonly %x, i32 %n) #0 {1445; CHECK-LABEL: @mla_xx_sext_zext(1446; CHECK-NEXT: entry:1447; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 01448; CHECK-NEXT: br i1 [[CMP9]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]1449; CHECK: for.body.preheader:1450; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 81451; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1452; CHECK: vector.ph:1453; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 21474836401454; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1455; CHECK: vector.body:1456; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1457; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]1458; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]1459; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 21460; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>1461; CHECK-NEXT: [[TMP3:%.*]] = mul nsw <8 x i64> [[TMP1]], [[TMP1]]1462; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP3]])1463; CHECK-NEXT: [[TMP5]] = add i64 [[VEC_PHI]], [[TMP4]]1464; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 81465; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1466; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]1467; CHECK: middle.block:1468; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]1469; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]1470; CHECK: scalar.ph:1471; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]1472; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]1473; CHECK-NEXT: br label [[FOR_BODY:%.*]]1474; CHECK: for.cond.cleanup:1475; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]1476; CHECK-NEXT: ret i64 [[S_0_LCSSA]]1477; CHECK: for.body:1478; CHECK-NEXT: [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1479; CHECK-NEXT: [[S_010:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]1480; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[I_011]]1481; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 21482; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP7]] to i321483; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV]], [[CONV]]1484; CHECK-NEXT: [[CONV3:%.*]] = zext nneg i32 [[MUL]] to i641485; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[S_010]], [[CONV3]]1486; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_011]], 11487; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]1488; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]1489;1490entry:1491 %cmp9 = icmp sgt i32 %n, 01492 br i1 %cmp9, label %for.body, label %for.cond.cleanup1493 1494for.cond.cleanup: ; preds = %for.body, %entry1495 %s.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]1496 ret i64 %s.0.lcssa1497 1498for.body: ; preds = %entry, %for.body1499 %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]1500 %s.010 = phi i64 [ %add, %for.body ], [ 0, %entry ]1501 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.0111502 %0 = load i16, ptr %arrayidx, align 21503 %conv = sext i16 %0 to i321504 %mul = mul nsw i32 %conv, %conv1505 %conv3 = zext nneg i32 %mul to i641506 %add = add nuw nsw i64 %s.010, %conv31507 %inc = add nuw nsw i32 %i.011, 11508 %exitcond.not = icmp eq i32 %inc, %n1509 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body1510}1511 1512define i64 @mla_and_add_together_16_64(ptr nocapture noundef readonly %x, i32 noundef %n) #0 {1513; CHECK-LABEL: @mla_and_add_together_16_64(1514; CHECK-NEXT: entry:1515; CHECK-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[N:%.*]], 01516; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP16]])1517; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i32 [[N]], 81518; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1519; CHECK: vector.ph:1520; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 21474836401521; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1522; CHECK: vector.body:1523; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1524; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]1525; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]1526; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]1527; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 21528; CHECK-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>1529; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <8 x i64> [[TMP4]], [[TMP4]]1530; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP2]])1531; CHECK-NEXT: [[TMP5]] = add i64 [[VEC_PHI]], [[TMP3]]1532; CHECK-NEXT: [[TMP10:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>1533; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP10]])1534; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI1]], [[TMP6]]1535; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 81536; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1537; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]1538; CHECK: middle.block:1539; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]1540; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]1541; CHECK: scalar.ph:1542; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]1543; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]1544; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]1545; CHECK-NEXT: br label [[FOR_BODY:%.*]]1546; CHECK: for.cond.cleanup:1547; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]1548; CHECK-NEXT: [[ADD6_LCSSA:%.*]] = phi i32 [ [[ADD6:%.*]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]1549; CHECK-NEXT: [[CONV7:%.*]] = sext i32 [[ADD6_LCSSA]] to i641550; CHECK-NEXT: [[DIV:%.*]] = sdiv i64 [[CONV7]], [[ADD_LCSSA]]1551; CHECK-NEXT: ret i64 [[DIV]]1552; CHECK: for.body:1553; CHECK-NEXT: [[I_019:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1554; CHECK-NEXT: [[T_018:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]1555; CHECK-NEXT: [[S_017:%.*]] = phi i32 [ [[ADD6]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ]1556; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[I_019]]1557; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX]], align 21558; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP9]] to i321559; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV]], [[CONV]]1560; CHECK-NEXT: [[CONV3:%.*]] = zext nneg i32 [[MUL]] to i641561; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[T_018]], [[CONV3]]1562; CHECK-NEXT: [[ADD6]] = add nsw i32 [[S_017]], [[CONV]]1563; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_019]], 11564; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]1565; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]1566;1567entry:1568 %cmp16 = icmp sgt i32 %n, 01569 tail call void @llvm.assume(i1 %cmp16)1570 br label %for.body1571 1572for.cond.cleanup:1573 %conv7 = sext i32 %add6 to i641574 %div = sdiv i64 %conv7, %add1575 ret i64 %div1576 1577for.body:1578 %i.019 = phi i32 [ %inc, %for.body ], [ 0, %entry ]1579 %t.018 = phi i64 [ %add, %for.body ], [ 0, %entry ]1580 %s.017 = phi i32 [ %add6, %for.body ], [ 0, %entry ]1581 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.0191582 %0 = load i16, ptr %arrayidx, align 21583 %conv = sext i16 %0 to i321584 %mul = mul nsw i32 %conv, %conv1585 %conv3 = zext nneg i32 %mul to i641586 %add = add nuw nsw i64 %t.018, %conv31587 %add6 = add nsw i32 %s.017, %conv1588 %inc = add nuw nsw i32 %i.019, 11589 %exitcond.not = icmp eq i32 %inc, %n1590 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body1591}1592 1593define i64 @interleave_doublereduct_i16_i64(ptr %x, ptr %y, i32 %n) {1594; CHECK-LABEL: @interleave_doublereduct_i16_i64(1595; CHECK-NEXT: entry:1596; CHECK-NEXT: [[CMP23:%.*]] = icmp sgt i32 [[N:%.*]], 01597; CHECK-NEXT: br i1 [[CMP23]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]1598; CHECK: for.cond.cleanup:1599; CHECK-NEXT: [[T_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD12:%.*]], [[FOR_BODY]] ]1600; CHECK-NEXT: ret i64 [[T_0_LCSSA]]1601; CHECK: for.body:1602; CHECK-NEXT: [[I_025:%.*]] = phi i32 [ [[ADD13:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]1603; CHECK-NEXT: [[T_024:%.*]] = phi i64 [ [[ADD12]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]1604; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i16, ptr [[X:%.*]], i32 [[I_025]]1605; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX]], align 21606; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP0]] to i321607; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i16, ptr [[Y:%.*]], i32 [[I_025]]1608; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 21609; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP1]] to i321610; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]1611; CHECK-NEXT: [[CONV3:%.*]] = sext i32 [[MUL]] to i641612; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[T_024]], [[CONV3]]1613; CHECK-NEXT: [[ADD4:%.*]] = or disjoint i32 [[I_025]], 11614; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i32 [[ADD4]]1615; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX5]], align 21616; CHECK-NEXT: [[CONV6:%.*]] = sext i16 [[TMP2]] to i321617; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i16, ptr [[Y]], i32 [[ADD4]]1618; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[ARRAYIDX8]], align 21619; CHECK-NEXT: [[CONV9:%.*]] = sext i16 [[TMP3]] to i321620; CHECK-NEXT: [[MUL10:%.*]] = mul nsw i32 [[CONV9]], [[CONV6]]1621; CHECK-NEXT: [[CONV11:%.*]] = sext i32 [[MUL10]] to i641622; CHECK-NEXT: [[ADD12]] = add nsw i64 [[ADD]], [[CONV11]]1623; CHECK-NEXT: [[ADD13]] = add nuw nsw i32 [[I_025]], 21624; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[ADD13]], [[N]]1625; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]1626;1627entry:1628 %cmp23 = icmp sgt i32 %n, 01629 br i1 %cmp23, label %for.body, label %for.cond.cleanup1630 1631for.cond.cleanup:1632 %t.0.lcssa = phi i64 [ 0, %entry ], [ %add12, %for.body ]1633 ret i64 %t.0.lcssa1634 1635for.body:1636 %i.025 = phi i32 [ %add13, %for.body ], [ 0, %entry ]1637 %t.024 = phi i64 [ %add12, %for.body ], [ 0, %entry ]1638 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.0251639 %0 = load i16, ptr %arrayidx, align 21640 %conv = sext i16 %0 to i321641 %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %i.0251642 %1 = load i16, ptr %arrayidx1, align 21643 %conv2 = sext i16 %1 to i321644 %mul = mul nsw i32 %conv2, %conv1645 %conv3 = sext i32 %mul to i641646 %add = add nsw i64 %t.024, %conv31647 %add4 = or disjoint i32 %i.025, 11648 %arrayidx5 = getelementptr inbounds i16, ptr %x, i32 %add41649 %2 = load i16, ptr %arrayidx5, align 21650 %conv6 = sext i16 %2 to i321651 %arrayidx8 = getelementptr inbounds i16, ptr %y, i32 %add41652 %3 = load i16, ptr %arrayidx8, align 21653 %conv9 = sext i16 %3 to i321654 %mul10 = mul nsw i32 %conv9, %conv61655 %conv11 = sext i32 %mul10 to i641656 %add12 = add nsw i64 %add, %conv111657 %add13 = add nuw nsw i32 %i.025, 21658 %cmp = icmp slt i32 %add13, %n1659 br i1 %cmp, label %for.body, label %for.cond.cleanup1660}1661 1662define i64 @test_std_q31(ptr %x, i32 %n) #0 {1663; CHECK-LABEL: @test_std_q31(1664; CHECK-NEXT: entry:1665; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], 01666; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP11]])1667; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i32 [[N]], 41668; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1669; CHECK: vector.ph:1670; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], 21474836441671; CHECK-NEXT: br label [[FOR_BODY1:%.*]]1672; CHECK: vector.body:1673; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY1]] ]1674; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[FOR_BODY1]] ]1675; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[FOR_BODY1]] ]1676; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]1677; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 41678; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[WIDE_LOAD]], splat (i32 8)1679; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[TMP1]] to <4 x i64>1680; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP2]])1681; CHECK-NEXT: [[TMP4]] = add i64 [[VEC_PHI]], [[TMP3]]1682; CHECK-NEXT: [[TMP6:%.*]] = mul nsw <4 x i64> [[TMP2]], [[TMP2]]1683; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP6]])1684; CHECK-NEXT: [[TMP8]] = add i64 [[VEC_PHI1]], [[TMP7]]1685; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 41686; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1687; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY1]], !llvm.loop [[LOOP37:![0-9]+]]1688; CHECK: middle.block:1689; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]1690; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]1691; CHECK: scalar.ph:1692; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]1693; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]1694; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]1695; CHECK-NEXT: br label [[FOR_BODY:%.*]]1696; CHECK: for.cond.cleanup:1697; CHECK-NEXT: [[ADD:%.*]] = phi i64 [ [[ADD1:%.*]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]1698; CHECK-NEXT: [[ADD3:%.*]] = phi i64 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]1699; CHECK-NEXT: [[DIV:%.*]] = sdiv i64 [[ADD3]], [[ADD]]1700; CHECK-NEXT: ret i64 [[DIV]]1701; CHECK: for.body:1702; CHECK-NEXT: [[S_014:%.*]] = phi i64 [ [[ADD1]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]1703; CHECK-NEXT: [[I_013:%.*]] = phi i32 [ [[ADD4:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1704; CHECK-NEXT: [[T_012:%.*]] = phi i64 [ [[ADD5]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ]1705; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i32 [[I_013]]1706; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 41707; CHECK-NEXT: [[SHR:%.*]] = ashr i32 [[TMP0]], 81708; CHECK-NEXT: [[CONV:%.*]] = sext i32 [[SHR]] to i641709; CHECK-NEXT: [[ADD1]] = add nsw i64 [[S_014]], [[CONV]]1710; CHECK-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV]]1711; CHECK-NEXT: [[ADD5]] = add nuw nsw i64 [[MUL]], [[T_012]]1712; CHECK-NEXT: [[ADD4]] = add nuw nsw i32 [[I_013]], 11713; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[ADD4]], [[N]]1714; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]1715;1716entry:1717 %cmp11 = icmp sgt i32 %n, 01718 tail call void @llvm.assume(i1 %cmp11)1719 br label %for.body1720 1721for.cond.cleanup: ; preds = %for.body1722 %div = sdiv i64 %add3, %add1723 ret i64 %div1724 1725for.body: ; preds = %entry, %for.body1726 %s.014 = phi i64 [ %add, %for.body ], [ 0, %entry ]1727 %i.013 = phi i32 [ %add4, %for.body ], [ 0, %entry ]1728 %t.012 = phi i64 [ %add3, %for.body ], [ 0, %entry ]1729 %arrayidx = getelementptr inbounds i32, ptr %x, i32 %i.0131730 %0 = load i32, ptr %arrayidx, align 41731 %shr = ashr i32 %0, 81732 %conv = sext i32 %shr to i641733 %add = add nsw i64 %s.014, %conv1734 %mul = mul nsw i64 %conv, %conv1735 %add3 = add nuw nsw i64 %mul, %t.0121736 %add4 = add nuw nsw i32 %i.013, 11737 %exitcond.not = icmp eq i32 %add4, %n1738 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body1739}1740 1741define i64 @test_fir_q15(ptr %x, ptr %y, i32 %n) #0 {1742; CHECK-LABEL: @test_fir_q15(1743; CHECK-NEXT: entry:1744; CHECK-NEXT: [[CMP23:%.*]] = icmp sgt i32 [[N:%.*]], 01745; CHECK-NEXT: br i1 [[CMP23]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]1746; CHECK: for.body.preheader:1747; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[N]], -11748; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 11749; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[TMP1]], 11750; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 151751; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1752; CHECK: vector.ph:1753; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP2]], -81754; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC]], 11755; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1756; CHECK: vector.body:1757; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1758; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]1759; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 11760; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[OFFSET_IDX]]1761; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[TMP3]], align 21762; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>1763; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>1764; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i32 [[OFFSET_IDX]]1765; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <16 x i16>, ptr [[TMP4]], align 21766; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>1767; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i16> [[WIDE_VEC2]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>1768; CHECK-NEXT: [[TMP6:%.*]] = sext <8 x i16> [[STRIDED_VEC3]] to <8 x i64>1769; CHECK-NEXT: [[TMP7:%.*]] = sext <8 x i16> [[STRIDED_VEC]] to <8 x i64>1770; CHECK-NEXT: [[TMP8:%.*]] = mul nsw <8 x i64> [[TMP6]], [[TMP7]]1771; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP8]])1772; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[VEC_PHI]], [[TMP9]]1773; CHECK-NEXT: [[TMP11:%.*]] = sext <8 x i16> [[STRIDED_VEC4]] to <8 x i64>1774; CHECK-NEXT: [[TMP12:%.*]] = sext <8 x i16> [[STRIDED_VEC1]] to <8 x i64>1775; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <8 x i64> [[TMP11]], [[TMP12]]1776; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP13]])1777; CHECK-NEXT: [[TMP16]] = add i64 [[TMP10]], [[TMP15]]1778; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 81779; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1780; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]1781; CHECK: middle.block:1782; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]1783; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]1784; CHECK: scalar.ph:1785; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]1786; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]1787; CHECK-NEXT: br label [[FOR_BODY:%.*]]1788; CHECK: for.cond.cleanup:1789; CHECK-NEXT: [[S_0_LCSSA:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD12:%.*]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]1790; CHECK-NEXT: ret i64 [[S_0_LCSSA]]1791; CHECK: for.body:1792; CHECK-NEXT: [[I_025:%.*]] = phi i32 [ [[ADD13:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1793; CHECK-NEXT: [[S_024:%.*]] = phi i64 [ [[ADD12]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]1794; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[I_025]]1795; CHECK-NEXT: [[TMP18:%.*]] = load i16, ptr [[ARRAYIDX]], align 21796; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP18]] to i321797; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[Y]], i32 [[I_025]]1798; CHECK-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX1]], align 21799; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP19]] to i321800; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]1801; CHECK-NEXT: [[CONV3:%.*]] = sext i32 [[MUL]] to i641802; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[S_024]], [[CONV3]]1803; CHECK-NEXT: [[ADD4:%.*]] = or disjoint i32 [[I_025]], 11804; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[ADD4]]1805; CHECK-NEXT: [[TMP20:%.*]] = load i16, ptr [[ARRAYIDX5]], align 21806; CHECK-NEXT: [[CONV6:%.*]] = sext i16 [[TMP20]] to i321807; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i16, ptr [[Y]], i32 [[ADD4]]1808; CHECK-NEXT: [[TMP21:%.*]] = load i16, ptr [[ARRAYIDX8]], align 21809; CHECK-NEXT: [[CONV9:%.*]] = sext i16 [[TMP21]] to i321810; CHECK-NEXT: [[MUL10:%.*]] = mul nsw i32 [[CONV9]], [[CONV6]]1811; CHECK-NEXT: [[CONV11:%.*]] = sext i32 [[MUL10]] to i641812; CHECK-NEXT: [[ADD12]] = add nsw i64 [[ADD]], [[CONV11]]1813; CHECK-NEXT: [[ADD13]] = add nuw nsw i32 [[I_025]], 21814; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[ADD13]], [[N]]1815; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]], !llvm.loop [[LOOP40:![0-9]+]]1816;1817entry:1818 %cmp23 = icmp sgt i32 %n, 01819 br i1 %cmp23, label %for.body, label %for.cond.cleanup1820 1821for.cond.cleanup: ; preds = %for.body, %entry1822 %s.0.lcssa = phi i64 [ 0, %entry ], [ %add12, %for.body ]1823 ret i64 %s.0.lcssa1824 1825for.body: ; preds = %entry, %for.body1826 %i.025 = phi i32 [ %add13, %for.body ], [ 0, %entry ]1827 %s.024 = phi i64 [ %add12, %for.body ], [ 0, %entry ]1828 %arrayidx = getelementptr inbounds i16, ptr %x, i32 %i.0251829 %0 = load i16, ptr %arrayidx, align 21830 %conv = sext i16 %0 to i321831 %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %i.0251832 %1 = load i16, ptr %arrayidx1, align 21833 %conv2 = sext i16 %1 to i321834 %mul = mul nsw i32 %conv2, %conv1835 %conv3 = sext i32 %mul to i641836 %add = add nsw i64 %s.024, %conv31837 %add4 = or disjoint i32 %i.025, 11838 %arrayidx5 = getelementptr inbounds i16, ptr %x, i32 %add41839 %2 = load i16, ptr %arrayidx5, align 21840 %conv6 = sext i16 %2 to i321841 %arrayidx8 = getelementptr inbounds i16, ptr %y, i32 %add41842 %3 = load i16, ptr %arrayidx8, align 21843 %conv9 = sext i16 %3 to i321844 %mul10 = mul nsw i32 %conv9, %conv61845 %conv11 = sext i32 %mul10 to i641846 %add12 = add nsw i64 %add, %conv111847 %add13 = add nuw nsw i32 %i.025, 21848 %cmp = icmp slt i32 %add13, %n1849 br i1 %cmp, label %for.body, label %for.cond.cleanup1850}1851 1852 1853attributes #0 = { "target-features"="+mve" }1854