428 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --prefix-filecheck-ir-name INST --version 62; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve %s -S -o - | FileCheck %s3 4define i16 @reduction_i32(ptr nocapture readonly %A, ptr nocapture readonly %B, i32 %N) {5; CHECK-LABEL: define i16 @reduction_i32(6; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {7; CHECK-NEXT: [[ENTRY:.*]]:8; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 09; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]10; CHECK: [[VECTOR_PH]]:11; CHECK-NEXT: [[TMP:%.*]] = add i32 [[N]], -112; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP]], 813; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -814; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N_VEC]], -815; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 316; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 117; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP2]])18; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]19; CHECK: [[VECTOR_BODY]]:20; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]21; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]22; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]23; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]24; CHECK-NEXT: [[INSTTMP2:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[INDEX]]25; CHECK-NEXT: [[TMP5:%.*]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[TMP4]])26; CHECK-NEXT: [[TMP6]] = sub i32 [[TMP4]], 827; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 4 [[INSTTMP2]], <8 x i1> [[TMP5]], <8 x i16> undef)28; CHECK-NEXT: [[INSTTMP5:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[INDEX]]29; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 4 [[INSTTMP5]], <8 x i1> [[TMP5]], <8 x i16> undef)30; CHECK-NEXT: [[TMP7:%.*]] = add <8 x i16> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]31; CHECK-NEXT: [[TMP8]] = add <8 x i16> [[TMP7]], [[WIDE_MASKED_LOAD3]]32; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 833; CHECK-NEXT: [[TMP9]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP3]], i32 1)34; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i32 [[TMP9]], 035; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_BODY]], label %[[MIDDLE_BLOCK:.*]]36; CHECK: [[MIDDLE_BLOCK]]:37; CHECK-NEXT: [[VEC_PHI_LCSSA:%.*]] = phi <8 x i16> [ [[VEC_PHI]], %[[VECTOR_BODY]] ]38; CHECK-NEXT: [[DOTLCSSA3:%.*]] = phi <8 x i1> [ [[TMP5]], %[[VECTOR_BODY]] ]39; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi <8 x i16> [ [[TMP8]], %[[VECTOR_BODY]] ]40; CHECK-NEXT: [[TMP10:%.*]] = select <8 x i1> [[DOTLCSSA3]], <8 x i16> [[DOTLCSSA]], <8 x i16> [[VEC_PHI_LCSSA]]41; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i16> [[TMP10]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>42; CHECK-NEXT: [[BIN_RDX:%.*]] = add <8 x i16> [[RDX_SHUF]], [[TMP10]]43; CHECK-NEXT: [[RDX_SHUF4:%.*]] = shufflevector <8 x i16> [[BIN_RDX]], <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>44; CHECK-NEXT: [[BIN_RDX5:%.*]] = add <8 x i16> [[RDX_SHUF4]], [[BIN_RDX]]45; CHECK-NEXT: [[RDX_SHUF6:%.*]] = shufflevector <8 x i16> [[BIN_RDX5]], <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>46; CHECK-NEXT: [[BIN_RDX7:%.*]] = add <8 x i16> [[RDX_SHUF6]], [[BIN_RDX5]]47; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i16> [[BIN_RDX7]], i32 048; CHECK-NEXT: ret i16 [[TMP11]]49; CHECK: [[FOR_COND_CLEANUP]]:50; CHECK-NEXT: [[RES_0:%.*]] = phi i16 [ 0, %[[ENTRY]] ]51; CHECK-NEXT: ret i16 [[RES_0]]52;53entry:54 %cmp8 = icmp eq i32 %N, 055 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph56 57vector.ph:58 %tmp = add i32 %N, -159 %n.rnd.up = add i32 %tmp, 860 %n.vec = and i32 %n.rnd.up, -861 %0 = add i32 %n.vec, -862 %1 = lshr i32 %0, 363 %2 = add i32 %1, 164 %start = call i32 @llvm.start.loop.iterations.i32(i32 %2)65 br label %vector.body66 67vector.body: ; preds = %vector.body, %vector.ph68 %index = phi i32 [ 0, %vector.ph], [ %index.next, %vector.body ]69 %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph], [ %tmp8, %vector.body ]70 %3 = phi i32 [ %start, %vector.ph], [ %4, %vector.body ]71 %tmp2 = getelementptr inbounds i16, ptr %A, i32 %index72 %tmp3 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %N)73 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp2, i32 4, <8 x i1> %tmp3, <8 x i16> undef)74 %tmp5 = getelementptr inbounds i16, ptr %B, i32 %index75 %wide.masked.load3 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp5, i32 4, <8 x i1> %tmp3, <8 x i16> undef)76 %tmp7 = add <8 x i16> %wide.masked.load, %vec.phi77 %tmp8 = add <8 x i16> %tmp7, %wide.masked.load378 %index.next = add i32 %index, 879 %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1)80 %5 = icmp ne i32 %4, 081 br i1 %5, label %vector.body, label %middle.block82 83middle.block: ; preds = %vector.body84 %vec.phi.lcssa = phi <8 x i16> [ %vec.phi, %vector.body ]85 %.lcssa3 = phi <8 x i1> [ %tmp3, %vector.body ]86 %.lcssa = phi <8 x i16> [ %tmp8, %vector.body ]87 %tmp10 = select <8 x i1> %.lcssa3, <8 x i16> %.lcssa, <8 x i16> %vec.phi.lcssa88 %rdx.shuf = shufflevector <8 x i16> %tmp10, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>89 %bin.rdx = add <8 x i16> %rdx.shuf, %tmp1090 %rdx.shuf4 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>91 %bin.rdx5 = add <8 x i16> %rdx.shuf4, %bin.rdx92 %rdx.shuf6 = shufflevector <8 x i16> %bin.rdx5, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>93 %bin.rdx7 = add <8 x i16> %rdx.shuf6, %bin.rdx594 %tmp11 = extractelement <8 x i16> %bin.rdx7, i32 095 ret i16 %tmp1196 97for.cond.cleanup:98 %res.0 = phi i16 [ 0, %entry ]99 ret i16 %res.0100}101 102define i16 @reduction_i32_with_scalar(ptr nocapture readonly %A, i16 %B, i32 %N) local_unnamed_addr {103; CHECK-LABEL: define i16 @reduction_i32_with_scalar(104; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], i16 [[B:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {105; CHECK-NEXT: [[ENTRY:.*]]:106; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 0107; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]108; CHECK: [[VECTOR_PH]]:109; CHECK-NEXT: [[TMP:%.*]] = add i32 [[N]], -1110; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i32 [[TMP]], 8111; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8112; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i16> undef, i16 [[B]], i32 0113; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT3]], <8 x i16> undef, <8 x i32> zeroinitializer114; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N_VEC]], -8115; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 3116; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1117; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP2]])118; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]119; CHECK: [[VECTOR_BODY]]:120; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]121; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[INSTTMP6:%.*]], %[[VECTOR_BODY]] ]122; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]123; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]124; CHECK-NEXT: [[INSTTMP2:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[INDEX]]125; CHECK-NEXT: [[TMP5:%.*]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[TMP4]])126; CHECK-NEXT: [[TMP6]] = sub i32 [[TMP4]], 8127; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 4 [[INSTTMP2]], <8 x i1> [[TMP5]], <8 x i16> undef)128; CHECK-NEXT: [[INSTTMP5:%.*]] = add <8 x i16> [[VEC_PHI]], [[BROADCAST_SPLAT4]]129; CHECK-NEXT: [[INSTTMP6]] = add <8 x i16> [[INSTTMP5]], [[WIDE_MASKED_LOAD]]130; CHECK-NEXT: [[INDEX_NEXT]] = add nuw nsw i32 [[INDEX]], 8131; CHECK-NEXT: [[TMP7]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP3]], i32 1)132; CHECK-NEXT: [[TMP8:%.*]] = icmp ne i32 [[TMP7]], 0133; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_BODY]], label %[[MIDDLE_BLOCK:.*]]134; CHECK: [[MIDDLE_BLOCK]]:135; CHECK-NEXT: [[INSTTMP8:%.*]] = select <8 x i1> [[TMP5]], <8 x i16> [[INSTTMP6]], <8 x i16> [[VEC_PHI]]136; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i16> [[INSTTMP8]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>137; CHECK-NEXT: [[BIN_RDX:%.*]] = add <8 x i16> [[RDX_SHUF]], [[INSTTMP8]]138; CHECK-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <8 x i16> [[BIN_RDX]], <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>139; CHECK-NEXT: [[BIN_RDX6:%.*]] = add <8 x i16> [[RDX_SHUF5]], [[BIN_RDX]]140; CHECK-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <8 x i16> [[BIN_RDX6]], <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>141; CHECK-NEXT: [[BIN_RDX8:%.*]] = add <8 x i16> [[RDX_SHUF7]], [[BIN_RDX6]]142; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[BIN_RDX8]], i32 0143; CHECK-NEXT: ret i16 [[TMP9]]144; CHECK: [[FOR_COND_CLEANUP]]:145; CHECK-NEXT: [[RES_0:%.*]] = phi i16 [ 0, %[[ENTRY]] ]146; CHECK-NEXT: ret i16 [[RES_0]]147;148entry:149 %cmp8 = icmp eq i32 %N, 0150 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph151 152vector.ph:153 %tmp = add i32 %N, -1154 %n.rnd.up = add nuw nsw i32 %tmp, 8155 %n.vec = and i32 %n.rnd.up, -8156 %broadcast.splatinsert3 = insertelement <8 x i16> undef, i16 %B, i32 0157 %broadcast.splat4 = shufflevector <8 x i16> %broadcast.splatinsert3, <8 x i16> undef, <8 x i32> zeroinitializer158 %0 = add i32 %n.vec, -8159 %1 = lshr i32 %0, 3160 %2 = add nuw nsw i32 %1, 1161 %start = call i32 @llvm.start.loop.iterations.i32(i32 %2)162 br label %vector.body163 164vector.body: ; preds = %vector.body, %vector.ph165 %index = phi i32 [ 0, %vector.ph], [ %index.next, %vector.body ]166 %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph], [ %tmp6, %vector.body ]167 %3 = phi i32 [ %start, %vector.ph], [ %4, %vector.body ]168 %tmp2 = getelementptr inbounds i16, ptr %A, i32 %index169 %tmp3 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %N)170 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp2, i32 4, <8 x i1> %tmp3, <8 x i16> undef)171 %tmp5 = add <8 x i16> %vec.phi, %broadcast.splat4172 %tmp6 = add <8 x i16> %tmp5, %wide.masked.load173 %index.next = add nuw nsw i32 %index, 8174 %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1)175 %5 = icmp ne i32 %4, 0176 br i1 %5, label %vector.body, label %middle.block177 178middle.block: ; preds = %vector.body179 %tmp8 = select <8 x i1> %tmp3, <8 x i16> %tmp6, <8 x i16> %vec.phi180 %rdx.shuf = shufflevector <8 x i16> %tmp8, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>181 %bin.rdx = add <8 x i16> %rdx.shuf, %tmp8182 %rdx.shuf5 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>183 %bin.rdx6 = add <8 x i16> %rdx.shuf5, %bin.rdx184 %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx6, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>185 %bin.rdx8 = add <8 x i16> %rdx.shuf7, %bin.rdx6186 %tmp9 = extractelement <8 x i16> %bin.rdx8, i32 0187 ret i16 %tmp9188 189for.cond.cleanup:190 %res.0 = phi i16 [ 0, %entry ]191 ret i16 %res.0192}193 194; The vector loop is not guarded with an entry check (N == 0). Check that195; despite this we can still calculate a precise enough range so that the196; the overflow checks for get.active.active.lane.mask don't reject197; tail-predication.198define i16 @reduction_not_guarded(ptr nocapture readonly %A, i16 %B, i32 %N) local_unnamed_addr {199; CHECK-LABEL: define i16 @reduction_not_guarded(200; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], i16 [[B:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {201; CHECK-NEXT: [[ENTRY:.*]]:202; CHECK-NEXT: [[TMP:%.*]] = add i32 [[N]], -1203; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i32 [[TMP]], 8204; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8205; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i16> undef, i16 [[B]], i32 0206; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT3]], <8 x i16> undef, <8 x i32> zeroinitializer207; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N_VEC]], -8208; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 3209; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1210; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP2]])211; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]212; CHECK: [[VECTOR_BODY]]:213; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]214; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, %[[ENTRY]] ], [ [[INSTTMP6:%.*]], %[[VECTOR_BODY]] ]215; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[START]], %[[ENTRY]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]216; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ [[N]], %[[ENTRY]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]217; CHECK-NEXT: [[INSTTMP2:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[INDEX]]218; CHECK-NEXT: [[TMP5:%.*]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[TMP4]])219; CHECK-NEXT: [[TMP6]] = sub i32 [[TMP4]], 8220; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 4 [[INSTTMP2]], <8 x i1> [[TMP5]], <8 x i16> undef)221; CHECK-NEXT: [[INSTTMP5:%.*]] = add <8 x i16> [[VEC_PHI]], [[BROADCAST_SPLAT4]]222; CHECK-NEXT: [[INSTTMP6]] = add <8 x i16> [[INSTTMP5]], [[WIDE_MASKED_LOAD]]223; CHECK-NEXT: [[INDEX_NEXT]] = add nuw nsw i32 [[INDEX]], 8224; CHECK-NEXT: [[TMP7]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP3]], i32 1)225; CHECK-NEXT: [[TMP8:%.*]] = icmp ne i32 [[TMP7]], 0226; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_BODY]], label %[[MIDDLE_BLOCK:.*]]227; CHECK: [[MIDDLE_BLOCK]]:228; CHECK-NEXT: [[INSTTMP8:%.*]] = select <8 x i1> [[TMP5]], <8 x i16> [[INSTTMP6]], <8 x i16> [[VEC_PHI]]229; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i16> [[INSTTMP8]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>230; CHECK-NEXT: [[BIN_RDX:%.*]] = add <8 x i16> [[RDX_SHUF]], [[INSTTMP8]]231; CHECK-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <8 x i16> [[BIN_RDX]], <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>232; CHECK-NEXT: [[BIN_RDX6:%.*]] = add <8 x i16> [[RDX_SHUF5]], [[BIN_RDX]]233; CHECK-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <8 x i16> [[BIN_RDX6]], <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>234; CHECK-NEXT: [[BIN_RDX8:%.*]] = add <8 x i16> [[RDX_SHUF7]], [[BIN_RDX6]]235; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[BIN_RDX8]], i32 0236; CHECK-NEXT: ret i16 [[TMP9]]237;238entry:239 %tmp = add i32 %N, -1240 %n.rnd.up = add nuw nsw i32 %tmp, 8241 %n.vec = and i32 %n.rnd.up, -8242 %broadcast.splatinsert3 = insertelement <8 x i16> undef, i16 %B, i32 0243 %broadcast.splat4 = shufflevector <8 x i16> %broadcast.splatinsert3, <8 x i16> undef, <8 x i32> zeroinitializer244 %0 = add i32 %n.vec, -8245 %1 = lshr i32 %0, 3246 %2 = add nuw nsw i32 %1, 1247 %start = call i32 @llvm.start.loop.iterations.i32(i32 %2)248 br label %vector.body249 250vector.body: ; preds = %vector.body, %vector.ph251 %index = phi i32 [ 0, %entry], [ %index.next, %vector.body ]252 %vec.phi = phi <8 x i16> [ zeroinitializer, %entry], [ %tmp6, %vector.body ]253 %3 = phi i32 [ %start, %entry ], [ %4, %vector.body ]254 %tmp2 = getelementptr inbounds i16, ptr %A, i32 %index255 %tmp3 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %N)256 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp2, i32 4, <8 x i1> %tmp3, <8 x i16> undef)257 %tmp5 = add <8 x i16> %vec.phi, %broadcast.splat4258 %tmp6 = add <8 x i16> %tmp5, %wide.masked.load259 %index.next = add nuw nsw i32 %index, 8260 %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1)261 %5 = icmp ne i32 %4, 0262 br i1 %5, label %vector.body, label %middle.block263 264middle.block: ; preds = %vector.body265 %tmp8 = select <8 x i1> %tmp3, <8 x i16> %tmp6, <8 x i16> %vec.phi266 %rdx.shuf = shufflevector <8 x i16> %tmp8, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>267 %bin.rdx = add <8 x i16> %rdx.shuf, %tmp8268 %rdx.shuf5 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>269 %bin.rdx6 = add <8 x i16> %rdx.shuf5, %bin.rdx270 %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx6, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>271 %bin.rdx8 = add <8 x i16> %rdx.shuf7, %bin.rdx6272 %tmp9 = extractelement <8 x i16> %bin.rdx8, i32 0273 ret i16 %tmp9274}275 276define dso_local void @Correlation(ptr nocapture readonly %Input, ptr nocapture %Output, i16 signext %Size, i16 signext %N, i16 signext %Scale) local_unnamed_addr #0 {277; CHECK-LABEL: define dso_local void @Correlation(278; CHECK-SAME: ptr readonly captures(none) [[INPUT:%.*]], ptr captures(none) [[OUTPUT:%.*]], i16 signext [[SIZE:%.*]], i16 signext [[N:%.*]], i16 signext [[SCALE:%.*]]) local_unnamed_addr #[[ATTR0]] {279; CHECK-NEXT: [[ENTRY:.*:]]280; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[N]] to i32281; CHECK-NEXT: [[CMP36:%.*]] = icmp sgt i16 [[N]], 0282; CHECK-NEXT: br i1 [[CMP36]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END17:.*]]283; CHECK: [[FOR_BODY_LR_PH]]:284; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[SIZE]] to i32285; CHECK-NEXT: [[CONV1032:%.*]] = zext i16 [[SCALE]] to i32286; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[CONV2]], 3287; CHECK-NEXT: br label %[[FOR_BODY:.*]]288; CHECK: [[FOR_BODY]]:289; CHECK-NEXT: [[LSR_IV51:%.*]] = phi i32 [ [[LSR_IV_NEXT:%.*]], %[[FOR_END:.*]] ], [ [[TMP0]], %[[FOR_BODY_LR_PH]] ]290; CHECK-NEXT: [[LSR_IV46:%.*]] = phi ptr [ [[SCEVGEP47:%.*]], %[[FOR_END]] ], [ [[INPUT]], %[[FOR_BODY_LR_PH]] ]291; CHECK-NEXT: [[I_037:%.*]] = phi i32 [ 0, %[[FOR_BODY_LR_PH]] ], [ [[INC16:%.*]], %[[FOR_END]] ]292; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i32 [[I_037]], -1293; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[CONV2]], [[TMP1]]294; CHECK-NEXT: [[TMP3:%.*]] = mul nsw i32 [[I_037]], -1295; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP0]], [[TMP3]]296; CHECK-NEXT: [[TMP5:%.*]] = lshr i32 [[TMP4]], 2297; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i32 [[TMP5]], 2298; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[TMP6]], -4299; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP7]], 2300; CHECK-NEXT: [[TMP9:%.*]] = add nuw nsw i32 [[TMP8]], 1301; CHECK-NEXT: [[CMP433:%.*]] = icmp slt i32 [[I_037]], [[CONV2]]302; CHECK-NEXT: br i1 [[CMP433]], label %[[VECTOR_PH:.*]], label %[[FOR_END]]303; CHECK: [[VECTOR_PH]]:304; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP9]])305; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]306; CHECK: [[VECTOR_BODY]]:307; CHECK-NEXT: [[LSR_IV48:%.*]] = phi ptr [ [[SCEVGEP49:%.*]], %[[VECTOR_BODY]] ], [ [[LSR_IV46]], %[[VECTOR_PH]] ]308; CHECK-NEXT: [[LSR_IV:%.*]] = phi ptr [ [[SCEVGEP:%.*]], %[[VECTOR_BODY]] ], [ [[INPUT]], %[[VECTOR_PH]] ]309; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]310; CHECK-NEXT: [[TMP10:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]311; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]312; CHECK-NEXT: [[TMP12:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP11]])313; CHECK-NEXT: [[TMP13]] = sub i32 [[TMP11]], 4314; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[LSR_IV]], <4 x i1> [[TMP12]], <4 x i16> undef)315; CHECK-NEXT: [[TMP14:%.*]] = sext <4 x i16> [[WIDE_MASKED_LOAD]] to <4 x i32>316; CHECK-NEXT: [[WIDE_MASKED_LOAD42:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[LSR_IV48]], <4 x i1> [[TMP12]], <4 x i16> undef)317; CHECK-NEXT: [[TMP15:%.*]] = sext <4 x i16> [[WIDE_MASKED_LOAD42]] to <4 x i32>318; CHECK-NEXT: [[TMP16:%.*]] = mul nsw <4 x i32> [[TMP15]], [[TMP14]]319; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> undef, i32 [[CONV1032]], i32 0320; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <4 x i32> [[TMP17]], <4 x i32> undef, <4 x i32> zeroinitializer321; CHECK-NEXT: [[TMP19:%.*]] = ashr <4 x i32> [[TMP16]], [[TMP18]]322; CHECK-NEXT: [[TMP20]] = add <4 x i32> [[TMP19]], [[VEC_PHI]]323; CHECK-NEXT: [[SCEVGEP]] = getelementptr i16, ptr [[LSR_IV]], i32 4324; CHECK-NEXT: [[SCEVGEP49]] = getelementptr i16, ptr [[LSR_IV48]], i32 4325; CHECK-NEXT: [[TMP21]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP10]], i32 1)326; CHECK-NEXT: [[TMP22:%.*]] = icmp ne i32 [[TMP21]], 0327; CHECK-NEXT: br i1 [[TMP22]], label %[[VECTOR_BODY]], label %[[MIDDLE_BLOCK:.*]]328; CHECK: [[MIDDLE_BLOCK]]:329; CHECK-NEXT: [[TMP23:%.*]] = select <4 x i1> [[TMP12]], <4 x i32> [[TMP20]], <4 x i32> [[VEC_PHI]]330; CHECK-NEXT: [[TMP24:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP23]])331; CHECK-NEXT: br label %[[FOR_END]]332; CHECK: [[FOR_END]]:333; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[FOR_BODY]] ], [ [[TMP24]], %[[MIDDLE_BLOCK]] ]334; CHECK-NEXT: [[TMP25:%.*]] = lshr i32 [[SUM_0_LCSSA]], 16335; CHECK-NEXT: [[CONV13:%.*]] = trunc i32 [[TMP25]] to i16336; CHECK-NEXT: [[ARRAYIDX14:%.*]] = getelementptr inbounds i16, ptr [[OUTPUT]], i32 [[I_037]]337; CHECK-NEXT: store i16 [[CONV13]], ptr [[ARRAYIDX14]], align 2338; CHECK-NEXT: [[INC16]] = add nuw nsw i32 [[I_037]], 1339; CHECK-NEXT: [[SCEVGEP47]] = getelementptr i16, ptr [[LSR_IV46]], i32 1340; CHECK-NEXT: [[LSR_IV_NEXT]] = add i32 [[LSR_IV51]], -1341; CHECK-NEXT: [[EXITCOND39:%.*]] = icmp eq i32 [[INC16]], [[CONV]]342; CHECK-NEXT: br i1 [[EXITCOND39]], label %[[FOR_END17]], label %[[FOR_BODY]]343; CHECK: [[FOR_END17]]:344; CHECK-NEXT: ret void345;346entry:347 %conv = sext i16 %N to i32348 %cmp36 = icmp sgt i16 %N, 0349 br i1 %cmp36, label %for.body.lr.ph, label %for.end17350 351for.body.lr.ph:352 %conv2 = sext i16 %Size to i32353 %conv1032 = zext i16 %Scale to i32354 %0 = add i32 %conv2, 3355 br label %for.body356 357for.body:358 %lsr.iv51 = phi i32 [ %lsr.iv.next, %for.end ], [ %0, %for.body.lr.ph ]359 %lsr.iv46 = phi ptr [ %scevgep47, %for.end ], [ %Input, %for.body.lr.ph ]360 %i.037 = phi i32 [ 0, %for.body.lr.ph ], [ %inc16, %for.end ]361 %1 = mul nsw i32 %i.037, -1362 %2 = add i32 %0, %1363 %3 = lshr i32 %2, 2364 %4 = shl nuw i32 %3, 2365 %5 = add i32 %4, -4366 %6 = lshr i32 %5, 2367 %7 = add nuw nsw i32 %6, 1368 %8 = sub i32 %conv2, %i.037369 %cmp433 = icmp slt i32 %i.037, %conv2370 br i1 %cmp433, label %vector.ph, label %for.end371 372vector.ph: ; preds = %for.body373 %start = call i32 @llvm.start.loop.iterations.i32(i32 %7)374 br label %vector.body375 376vector.body: ; preds = %vector.body, %vector.ph377 %lsr.iv48 = phi ptr [ %scevgep49, %vector.body ], [ %lsr.iv46, %vector.ph ]378 %lsr.iv = phi ptr [ %scevgep, %vector.body ], [ %Input, %vector.ph ]379 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]380 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %16, %vector.body ]381 %9 = phi i32 [ %start, %vector.ph ], [ %17, %vector.body ]382 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %8)383 %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %lsr.iv, i32 2, <4 x i1> %active.lane.mask, <4 x i16> undef)384 %10 = sext <4 x i16> %wide.masked.load to <4 x i32>385 %wide.masked.load42 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %lsr.iv48, i32 2, <4 x i1> %active.lane.mask, <4 x i16> undef)386 %11 = sext <4 x i16> %wide.masked.load42 to <4 x i32>387 %12 = mul nsw <4 x i32> %11, %10388 %13 = insertelement <4 x i32> undef, i32 %conv1032, i32 0389 %14 = shufflevector <4 x i32> %13, <4 x i32> undef, <4 x i32> zeroinitializer390 %15 = ashr <4 x i32> %12, %14391 %16 = add <4 x i32> %15, %vec.phi392 %index.next = add i32 %index, 4393 %scevgep = getelementptr i16, ptr %lsr.iv, i32 4394 %scevgep49 = getelementptr i16, ptr %lsr.iv48, i32 4395 %17 = call i32 @llvm.loop.decrement.reg.i32(i32 %9, i32 1)396 %18 = icmp ne i32 %17, 0397 br i1 %18, label %vector.body, label %middle.block398 399middle.block: ; preds = %vector.body400 %19 = select <4 x i1> %active.lane.mask, <4 x i32> %16, <4 x i32> %vec.phi401 %20 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %19)402 br label %for.end403 404for.end: ; preds = %middle.block, %for.body405 %Sum.0.lcssa = phi i32 [ 0, %for.body ], [ %20, %middle.block ]406 %21 = lshr i32 %Sum.0.lcssa, 16407 %conv13 = trunc i32 %21 to i16408 %arrayidx14 = getelementptr inbounds i16, ptr %Output, i32 %i.037409 store i16 %conv13, ptr %arrayidx14, align 2410 %inc16 = add nuw nsw i32 %i.037, 1411 %scevgep47 = getelementptr i16, ptr %lsr.iv46, i32 1412 %lsr.iv.next = add i32 %lsr.iv51, -1413 %exitcond39 = icmp eq i32 %inc16, %conv414 br i1 %exitcond39, label %for.end17, label %for.body415 416for.end17: ; preds = %for.end, %entry417 ret void418}419 420declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32 immarg, <8 x i1>, <8 x i16>)421declare i32 @llvm.start.loop.iterations.i32(i32)422declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)423declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)424declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)425declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)426declare i32 @llvm.loop.decrement.reg.i32(i32, i32)427declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32 immarg, <4 x i1>, <4 x i16>)428