470 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 62; RUN: opt --mattr=+neon,+dotprod -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S %s | FileCheck %s3 4target triple = "arm64-apple-macosx"5 6define i32 @red_zext_mul_by_63(ptr %start, ptr %end) {7; CHECK-LABEL: define i32 @red_zext_mul_by_63(8; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0:[0-9]+]] {9; CHECK-NEXT: [[ENTRY:.*]]:10; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i6411; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i6412; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], 113; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]14; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 1615; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]16; CHECK: [[VECTOR_PH]]:17; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 1618; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]19; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]20; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]21; CHECK: [[VECTOR_BODY]]:22; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]23; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]24; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]25; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 126; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>27; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i32> [[TMP3]], splat (i32 63)28; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP4]])29; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1630; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]31; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]32; CHECK: [[MIDDLE_BLOCK]]:33; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])34; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]35; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]36; CHECK: [[SCALAR_PH]]:37; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]38; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]39; CHECK-NEXT: br label %[[LOOP:.*]]40; CHECK: [[LOOP]]:41; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[GEP_IV_NEXT:%.*]], %[[LOOP]] ]42; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]43; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[PTR_IV]], align 144; CHECK-NEXT: [[L_EXT:%.*]] = zext i8 [[L]] to i3245; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[L_EXT]], 6346; CHECK-NEXT: [[RED_NEXT]] = add i32 [[RED]], [[MUL]]47; CHECK-NEXT: [[GEP_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 148; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]49; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]50; CHECK: [[EXIT]]:51; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]52; CHECK-NEXT: ret i32 [[RED_NEXT_LCSSA]]53;54entry:55 br label %loop56 57loop:58 %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]59 %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]60 %l = load i8, ptr %ptr.iv, align 161 %l.ext = zext i8 %l to i3262 %mul = mul i32 %l.ext, 6363 %red.next = add i32 %red, %mul64 %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 165 %ec = icmp eq ptr %ptr.iv, %end66 br i1 %ec, label %exit, label %loop67 68exit:69 ret i32 %red.next70}71 72define i32 @red_zext_mul_by_255(ptr %start, ptr %end) {73; CHECK-LABEL: define i32 @red_zext_mul_by_255(74; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {75; CHECK-NEXT: [[ENTRY:.*]]:76; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i6477; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i6478; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], 179; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]80; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 1681; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]82; CHECK: [[VECTOR_PH]]:83; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 1684; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]85; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]86; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]87; CHECK: [[VECTOR_BODY]]:88; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]89; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]90; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]91; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 192; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>93; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i32> [[TMP3]], splat (i32 255)94; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP4]])95; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1696; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]97; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]98; CHECK: [[MIDDLE_BLOCK]]:99; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])100; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]101; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]102; CHECK: [[SCALAR_PH]]:103; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]104; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]105; CHECK-NEXT: br label %[[LOOP:.*]]106; CHECK: [[LOOP]]:107; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[GEP_IV_NEXT:%.*]], %[[LOOP]] ]108; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]109; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[PTR_IV]], align 1110; CHECK-NEXT: [[L_EXT:%.*]] = zext i8 [[L]] to i32111; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[L_EXT]], 255112; CHECK-NEXT: [[RED_NEXT]] = add i32 [[RED]], [[MUL]]113; CHECK-NEXT: [[GEP_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 1114; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]115; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]116; CHECK: [[EXIT]]:117; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]118; CHECK-NEXT: ret i32 [[RED_NEXT_LCSSA]]119;120entry:121 br label %loop122 123loop:124 %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]125 %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]126 %l = load i8, ptr %ptr.iv, align 1127 %l.ext = zext i8 %l to i32128 %mul = mul i32 %l.ext, 255129 %red.next = add i32 %red, %mul130 %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1131 %ec = icmp eq ptr %ptr.iv, %end132 br i1 %ec, label %exit, label %loop133 134exit:135 ret i32 %red.next136}137 138define i32 @red_zext_mul_by_256(ptr %start, ptr %end) {139; CHECK-LABEL: define i32 @red_zext_mul_by_256(140; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {141; CHECK-NEXT: [[ENTRY:.*]]:142; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64143; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64144; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], 1145; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]146; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 16147; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]148; CHECK: [[VECTOR_PH]]:149; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 16150; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]151; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]152; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]153; CHECK: [[VECTOR_BODY]]:154; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]155; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]156; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]157; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 1158; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>159; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i32> [[TMP3]], splat (i32 256)160; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[VEC_PHI]], [[TMP4]]161; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16162; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]163; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]164; CHECK: [[MIDDLE_BLOCK]]:165; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP5]])166; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]167; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]168; CHECK: [[SCALAR_PH]]:169; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]170; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]171; CHECK-NEXT: br label %[[LOOP:.*]]172; CHECK: [[LOOP]]:173; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[GEP_IV_NEXT:%.*]], %[[LOOP]] ]174; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]175; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[PTR_IV]], align 1176; CHECK-NEXT: [[L_EXT:%.*]] = zext i8 [[L]] to i32177; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[L_EXT]], 256178; CHECK-NEXT: [[RED_NEXT]] = add i32 [[RED]], [[MUL]]179; CHECK-NEXT: [[GEP_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 1180; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]181; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]182; CHECK: [[EXIT]]:183; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]184; CHECK-NEXT: ret i32 [[RED_NEXT_LCSSA]]185;186entry:187 br label %loop188 189loop:190 %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]191 %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]192 %l = load i8, ptr %ptr.iv, align 1193 %l.ext = zext i8 %l to i32194 %mul = mul i32 %l.ext, 256195 %red.next = add i32 %red, %mul196 %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1197 %ec = icmp eq ptr %ptr.iv, %end198 br i1 %ec, label %exit, label %loop199 200exit:201 ret i32 %red.next202}203 204define i32 @red_sext_mul_by_63(ptr %start, ptr %end) {205; CHECK-LABEL: define i32 @red_sext_mul_by_63(206; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {207; CHECK-NEXT: [[ENTRY:.*]]:208; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64209; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64210; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], 1211; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]212; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 16213; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]214; CHECK: [[VECTOR_PH]]:215; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 16216; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]217; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]218; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]219; CHECK: [[VECTOR_BODY]]:220; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]221; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]222; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]223; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 1224; CHECK-NEXT: [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>225; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i32> [[TMP3]], splat (i32 63)226; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP4]])227; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16228; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]229; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]230; CHECK: [[MIDDLE_BLOCK]]:231; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])232; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]233; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]234; CHECK: [[SCALAR_PH]]:235; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]236; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]237; CHECK-NEXT: br label %[[LOOP:.*]]238; CHECK: [[LOOP]]:239; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[GEP_IV_NEXT:%.*]], %[[LOOP]] ]240; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]241; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[PTR_IV]], align 1242; CHECK-NEXT: [[L_EXT:%.*]] = sext i8 [[L]] to i32243; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[L_EXT]], 63244; CHECK-NEXT: [[RED_NEXT]] = add i32 [[RED]], [[MUL]]245; CHECK-NEXT: [[GEP_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 1246; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]247; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]248; CHECK: [[EXIT]]:249; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]250; CHECK-NEXT: ret i32 [[RED_NEXT_LCSSA]]251;252entry:253 br label %loop254 255loop:256 %ptr_iv = phi ptr [ %start, %entry ], [ %gep_iv_next, %loop ]257 %red = phi i32 [ 0, %entry ], [ %red_next, %loop ]258 %l = load i8, ptr %ptr_iv, align 1259 %l_ext = sext i8 %l to i32260 %mul = mul i32 %l_ext, 63261 %red_next = add i32 %red, %mul262 %gep_iv_next = getelementptr i8, ptr %ptr_iv, i64 1263 %ec = icmp eq ptr %ptr_iv, %end264 br i1 %ec, label %exit, label %loop265 266exit:267 %red_next_lcssa = phi i32 [ %red_next, %loop ]268 ret i32 %red_next_lcssa269}270 271; Constants >= 128 cannot be treated as sign-extended.272define i32 @red_sext_mul_by_128(ptr %start, ptr %end) {273; CHECK-LABEL: define i32 @red_sext_mul_by_128(274; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {275; CHECK-NEXT: [[ENTRY:.*]]:276; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64277; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64278; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], 1279; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]280; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 16281; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]282; CHECK: [[VECTOR_PH]]:283; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 16284; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]285; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]286; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]287; CHECK: [[VECTOR_BODY]]:288; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]289; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]290; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]291; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 1292; CHECK-NEXT: [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>293; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i32> [[TMP3]], splat (i32 128)294; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[VEC_PHI]], [[TMP4]]295; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16296; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]297; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]298; CHECK: [[MIDDLE_BLOCK]]:299; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP5]])300; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]301; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]302; CHECK: [[SCALAR_PH]]:303; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]304; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]305; CHECK-NEXT: br label %[[LOOP:.*]]306; CHECK: [[LOOP]]:307; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[GEP_IV_NEXT:%.*]], %[[LOOP]] ]308; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]309; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[PTR_IV]], align 1310; CHECK-NEXT: [[L_EXT:%.*]] = sext i8 [[L]] to i32311; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[L_EXT]], 128312; CHECK-NEXT: [[RED_NEXT]] = add i32 [[RED]], [[MUL]]313; CHECK-NEXT: [[GEP_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 1314; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]315; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]316; CHECK: [[EXIT]]:317; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]318; CHECK-NEXT: ret i32 [[RED_NEXT_LCSSA]]319;320entry:321 br label %loop322 323loop:324 %ptr_iv = phi ptr [ %start, %entry ], [ %gep_iv_next, %loop ]325 %red = phi i32 [ 0, %entry ], [ %red_next, %loop ]326 %l = load i8, ptr %ptr_iv, align 1327 %l_ext = sext i8 %l to i32328 %mul = mul i32 %l_ext, 128329 %red_next = add i32 %red, %mul330 %gep_iv_next = getelementptr i8, ptr %ptr_iv, i64 1331 %ec = icmp eq ptr %ptr_iv, %end332 br i1 %ec, label %exit, label %loop333 334exit:335 %red_next_lcssa = phi i32 [ %red_next, %loop ]336 ret i32 %red_next_lcssa337}338 339define i32 @red_sext_mul_by_255(ptr %start, ptr %end) {340; CHECK-LABEL: define i32 @red_sext_mul_by_255(341; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {342; CHECK-NEXT: [[ENTRY:.*]]:343; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64344; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64345; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], 1346; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]347; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 16348; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]349; CHECK: [[VECTOR_PH]]:350; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 16351; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]352; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]353; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]354; CHECK: [[VECTOR_BODY]]:355; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]356; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]357; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]358; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 1359; CHECK-NEXT: [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>360; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i32> [[TMP3]], splat (i32 255)361; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[VEC_PHI]], [[TMP4]]362; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16363; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]364; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]365; CHECK: [[MIDDLE_BLOCK]]:366; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP5]])367; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]368; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]369; CHECK: [[SCALAR_PH]]:370; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]371; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]372; CHECK-NEXT: br label %[[LOOP:.*]]373; CHECK: [[LOOP]]:374; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[GEP_IV_NEXT:%.*]], %[[LOOP]] ]375; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]376; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[PTR_IV]], align 1377; CHECK-NEXT: [[L_EXT:%.*]] = sext i8 [[L]] to i32378; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[L_EXT]], 255379; CHECK-NEXT: [[RED_NEXT]] = add i32 [[RED]], [[MUL]]380; CHECK-NEXT: [[GEP_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 1381; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]382; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]383; CHECK: [[EXIT]]:384; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]385; CHECK-NEXT: ret i32 [[RED_NEXT_LCSSA]]386;387entry:388 br label %loop389 390loop:391 %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]392 %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]393 %l = load i8, ptr %ptr.iv, align 1394 %l.ext = sext i8 %l to i32395 %mul = mul i32 %l.ext, 255396 %red.next = add i32 %red, %mul397 %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1398 %ec = icmp eq ptr %ptr.iv, %end399 br i1 %ec, label %exit, label %loop400 401exit:402 ret i32 %red.next403}404 405define i32 @red_sext_mul_by_256(ptr %start, ptr %end) {406; CHECK-LABEL: define i32 @red_sext_mul_by_256(407; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR0]] {408; CHECK-NEXT: [[ENTRY:.*]]:409; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64410; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64411; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], 1412; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]413; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 16414; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]415; CHECK: [[VECTOR_PH]]:416; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 16417; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]418; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]419; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]420; CHECK: [[VECTOR_BODY]]:421; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]422; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]423; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]424; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP]], align 1425; CHECK-NEXT: [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>426; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i32> [[TMP3]], splat (i32 256)427; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[VEC_PHI]], [[TMP4]]428; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16429; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]430; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]431; CHECK: [[MIDDLE_BLOCK]]:432; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP5]])433; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]434; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]435; CHECK: [[SCALAR_PH]]:436; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]437; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]438; CHECK-NEXT: br label %[[LOOP:.*]]439; CHECK: [[LOOP]]:440; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[GEP_IV_NEXT:%.*]], %[[LOOP]] ]441; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]442; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[PTR_IV]], align 1443; CHECK-NEXT: [[L_EXT:%.*]] = sext i8 [[L]] to i32444; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[L_EXT]], 256445; CHECK-NEXT: [[RED_NEXT]] = add i32 [[RED]], [[MUL]]446; CHECK-NEXT: [[GEP_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 1447; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[END]]448; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]449; CHECK: [[EXIT]]:450; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]451; CHECK-NEXT: ret i32 [[RED_NEXT_LCSSA]]452;453entry:454 br label %loop455 456loop:457 %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]458 %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]459 %l = load i8, ptr %ptr.iv, align 1460 %l.ext = sext i8 %l to i32461 %mul = mul i32 %l.ext, 256462 %red.next = add i32 %red, %mul463 %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1464 %ec = icmp eq ptr %ptr.iv, %end465 br i1 %ec, label %exit, label %loop466 467exit:468 ret i32 %red.next469}470