504 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -p loop-unroll -unroll-add-parallel-reductions -S %s | FileCheck %s3 4define i32 @test_add_reduction(ptr %a, i64 %n) {5; CHECK-LABEL: define i32 @test_add_reduction(6; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {7; CHECK-NEXT: [[ENTRY:.*]]:8; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -19; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 110; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 111; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]12; CHECK: [[ENTRY_NEW]]:13; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]14; CHECK-NEXT: br label %[[LOOP:.*]]15; CHECK: [[LOOP]]:16; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]17; CHECK-NEXT: [[RDX_1:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]18; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]19; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]20; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV]]21; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 222; CHECK-NEXT: [[RDX_NEXT]] = add i32 [[RDX]], [[TMP2]]23; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 124; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT]]25; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 226; CHECK-NEXT: [[RDX_NEXT_1]] = add i32 [[RDX_1]], [[TMP3]]27; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 228; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 229; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]30; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]31; CHECK: [[EXIT_UNR_LCSSA]]:32; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]33; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]34; CHECK-NEXT: [[RDX_UNR1:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]35; CHECK-NEXT: [[BIN_RDX:%.*]] = add i32 [[RDX_NEXT_1]], [[RDX_NEXT]]36; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 037; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]38; CHECK: [[LOOP_EPIL_PREHEADER]]:39; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]40; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ]41; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 042; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])43; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]44; CHECK: [[LOOP_EPIL]]:45; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_EPIL_INIT]]46; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 247; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = add nuw nsw i32 [[RDX_UNR]], [[TMP4]]48; CHECK-NEXT: br label %[[EXIT]]49; CHECK: [[EXIT]]:50; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]51; CHECK-NEXT: ret i32 [[RES]]52;53entry:54 br label %loop55 56loop:57 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]58 %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]59 %gep.a = getelementptr inbounds nuw i32, ptr %a, i64 %iv60 %1 = load i32, ptr %gep.a, align 261 %rdx.next = add nuw nsw i32 %rdx, %162 %iv.next = add nuw nsw i64 %iv, 163 %ec = icmp eq i64 %iv.next, %n64 br i1 %ec, label %exit, label %loop, !llvm.loop !065 66exit:67 %res = phi i32 [ %rdx.next, %loop ]68 ret i32 %res69}70 71define i32 @test_add_reduction_constant_op(ptr %a, i64 %n) {72; CHECK-LABEL: define i32 @test_add_reduction_constant_op(73; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {74; CHECK-NEXT: [[ENTRY:.*]]:75; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -176; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 177; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 178; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]79; CHECK: [[ENTRY_NEW]]:80; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]81; CHECK-NEXT: br label %[[LOOP:.*]]82; CHECK: [[LOOP]]:83; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]84; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]85; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]86; CHECK-NEXT: [[RDX_NEXT_1]] = add nuw nsw i32 [[RDX]], 287; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 288; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 289; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]90; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP2:![0-9]+]]91; CHECK: [[EXIT_UNR_LCSSA]]:92; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]93; CHECK-NEXT: [[RDX_UNR1:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]94; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 095; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]96; CHECK: [[LOOP_EPIL_PREHEADER]]:97; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_UNR1]], %[[EXIT_UNR_LCSSA]] ]98; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 099; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])100; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]101; CHECK: [[LOOP_EPIL]]:102; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = add nuw nsw i32 [[RDX_UNR]], 1103; CHECK-NEXT: br label %[[EXIT]]104; CHECK: [[EXIT]]:105; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]106; CHECK-NEXT: ret i32 [[RES]]107;108entry:109 br label %loop110 111loop:112 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]113 %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]114 %rdx.next = add nuw nsw i32 %rdx, 1115 %iv.next = add nuw nsw i64 %iv, 1116 %ec = icmp eq i64 %iv.next, %n117 br i1 %ec, label %exit, label %loop, !llvm.loop !0118 119exit:120 %res = phi i32 [ %rdx.next, %loop ]121 ret i32 %res122}123 124define i32 @test_add_reduction_8x_unroll(ptr %a, i64 %n) {125; CHECK-LABEL: define i32 @test_add_reduction_8x_unroll(126; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {127; CHECK-NEXT: [[ENTRY:.*]]:128; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1129; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 7130; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7131; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]132; CHECK: [[ENTRY_NEW]]:133; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]134; CHECK-NEXT: br label %[[LOOP:.*]]135; CHECK: [[LOOP]]:136; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_7:%.*]], %[[LOOP]] ]137; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_7:%.*]], %[[LOOP]] ]138; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_7:%.*]], %[[LOOP]] ]139; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV]]140; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 2141; CHECK-NEXT: [[RDX_NEXT:%.*]] = add nuw nsw i32 [[RDX]], [[TMP2]]142; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1143; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT]]144; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 2145; CHECK-NEXT: [[RDX_2:%.*]] = add nuw nsw i32 [[RDX_NEXT]], [[TMP3]]146; CHECK-NEXT: [[IV_NEXT_1:%.*]] = add nuw nsw i64 [[IV]], 2147; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT_1]]148; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_2]], align 2149; CHECK-NEXT: [[RDX_NEXT_2:%.*]] = add nuw nsw i32 [[RDX_2]], [[TMP4]]150; CHECK-NEXT: [[IV_NEXT_2:%.*]] = add nuw nsw i64 [[IV]], 3151; CHECK-NEXT: [[GEP_A_3:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT_2]]152; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[GEP_A_3]], align 2153; CHECK-NEXT: [[RDX_4:%.*]] = add nuw nsw i32 [[RDX_NEXT_2]], [[TMP5]]154; CHECK-NEXT: [[IV_NEXT_3:%.*]] = add nuw nsw i64 [[IV]], 4155; CHECK-NEXT: [[GEP_A_4:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT_3]]156; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[GEP_A_4]], align 2157; CHECK-NEXT: [[RDX_NEXT_4:%.*]] = add nuw nsw i32 [[RDX_4]], [[TMP6]]158; CHECK-NEXT: [[IV_NEXT_4:%.*]] = add nuw nsw i64 [[IV]], 5159; CHECK-NEXT: [[GEP_A_5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT_4]]160; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[GEP_A_5]], align 2161; CHECK-NEXT: [[RDX_6:%.*]] = add nuw nsw i32 [[RDX_NEXT_4]], [[TMP7]]162; CHECK-NEXT: [[IV_NEXT_5:%.*]] = add nuw nsw i64 [[IV]], 6163; CHECK-NEXT: [[GEP_A_6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT_5]]164; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[GEP_A_6]], align 2165; CHECK-NEXT: [[RDX_NEXT_6:%.*]] = add nuw nsw i32 [[RDX_6]], [[TMP8]]166; CHECK-NEXT: [[IV_NEXT_6:%.*]] = add nuw nsw i64 [[IV]], 7167; CHECK-NEXT: [[GEP_A_7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_NEXT_6]]168; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[GEP_A_7]], align 2169; CHECK-NEXT: [[RDX_NEXT_7]] = add nuw nsw i32 [[RDX_NEXT_6]], [[TMP9]]170; CHECK-NEXT: [[IV_NEXT_7]] = add nuw nsw i64 [[IV]], 8171; CHECK-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8172; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]]173; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]174; CHECK: [[EXIT_UNR_LCSSA]]:175; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_7]], %[[LOOP]] ]176; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[LOOP]] ]177; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_7]], %[[LOOP]] ]178; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0179; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]180; CHECK: [[LOOP_EPIL_PREHEADER]]:181; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]182; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_UNR]], %[[EXIT_UNR_LCSSA]] ]183; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0184; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])185; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]186; CHECK: [[LOOP_EPIL]]:187; CHECK-NEXT: [[IV_EPIL:%.*]] = phi i64 [ [[IV_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ]188; CHECK-NEXT: [[RDX_EPIL:%.*]] = phi i32 [ [[RDX_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[RDX_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ]189; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, %[[LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_EPIL]] ]190; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_EPIL]]191; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 2192; CHECK-NEXT: [[RDX_NEXT_EPIL]] = add nuw nsw i32 [[RDX_EPIL]], [[TMP10]]193; CHECK-NEXT: [[IV_NEXT_EPIL]] = add nuw nsw i64 [[IV_EPIL]], 1194; CHECK-NEXT: [[EC_EPIL:%.*]] = icmp eq i64 [[IV_NEXT_EPIL]], [[N]]195; CHECK-NEXT: [[EPIL_ITER_NEXT]] = add i64 [[EPIL_ITER]], 1196; CHECK-NEXT: [[EPIL_ITER_CMP:%.*]] = icmp ne i64 [[EPIL_ITER_NEXT]], [[XTRAITER]]197; CHECK-NEXT: br i1 [[EPIL_ITER_CMP]], label %[[LOOP_EPIL]], label %[[EXIT_EPILOG_LCSSA:.*]], !llvm.loop [[LOOP4:![0-9]+]]198; CHECK: [[EXIT_EPILOG_LCSSA]]:199; CHECK-NEXT: [[RES_PH1:%.*]] = phi i32 [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]200; CHECK-NEXT: br label %[[EXIT]]201; CHECK: [[EXIT]]:202; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RES_PH1]], %[[EXIT_EPILOG_LCSSA]] ]203; CHECK-NEXT: ret i32 [[RES]]204;205entry:206 br label %loop207 208loop:209 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]210 %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]211 %gep.a = getelementptr inbounds nuw i32, ptr %a, i64 %iv212 %1 = load i32, ptr %gep.a, align 2213 %rdx.next = add nuw nsw i32 %rdx, %1214 %iv.next = add nuw nsw i64 %iv, 1215 %ec = icmp eq i64 %iv.next, %n216 br i1 %ec, label %exit, label %loop, !llvm.loop !2217 218exit:219 %res = phi i32 [ %rdx.next, %loop ]220 ret i32 %res221}222 223define <4 x i32> @test_vector_add_reduction(ptr %a, i64 %n) {224; CHECK-LABEL: define <4 x i32> @test_vector_add_reduction(225; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {226; CHECK-NEXT: [[ENTRY:.*]]:227; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1228; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1229; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1230; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]231; CHECK: [[ENTRY_NEW]]:232; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]233; CHECK-NEXT: br label %[[LOOP:.*]]234; CHECK: [[LOOP]]:235; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]236; CHECK-NEXT: [[RDX_1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]237; CHECK-NEXT: [[RDX:%.*]] = phi <4 x i32> [ zeroinitializer, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]238; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]239; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw <4 x i32>, ptr [[A]], i64 [[IV]]240; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr [[GEP_A]], align 16241; CHECK-NEXT: [[RDX_NEXT]] = add <4 x i32> [[RDX]], [[TMP2]]242; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1243; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw <4 x i32>, ptr [[A]], i64 [[IV_NEXT]]244; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr [[GEP_A_1]], align 16245; CHECK-NEXT: [[RDX_NEXT_1]] = add <4 x i32> [[RDX_1]], [[TMP3]]246; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2247; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2248; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]249; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]250; CHECK: [[EXIT_UNR_LCSSA]]:251; CHECK-NEXT: [[RES_PH:%.*]] = phi <4 x i32> [ [[RDX_NEXT_1]], %[[LOOP]] ]252; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]253; CHECK-NEXT: [[RDX_UNR:%.*]] = phi <4 x i32> [ [[RDX_NEXT_1]], %[[LOOP]] ]254; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[RDX_NEXT_1]], [[RDX_NEXT]]255; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0256; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]257; CHECK: [[LOOP_EPIL_PREHEADER]]:258; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]259; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi <4 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ]260; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0261; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])262; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]263; CHECK: [[LOOP_EPIL]]:264; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw <4 x i32>, ptr [[A]], i64 [[IV_EPIL_INIT]]265; CHECK-NEXT: [[TMP4:%.*]] = load <4 x i32>, ptr [[GEP_A_EPIL]], align 16266; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = add <4 x i32> [[RDX_EPIL_INIT]], [[TMP4]]267; CHECK-NEXT: br label %[[EXIT]]268; CHECK: [[EXIT]]:269; CHECK-NEXT: [[RES:%.*]] = phi <4 x i32> [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]270; CHECK-NEXT: ret <4 x i32> [[RES]]271;272entry:273 br label %loop274 275loop:276 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]277 %rdx = phi <4 x i32> [ zeroinitializer, %entry ], [ %rdx.next, %loop ]278 %gep.a = getelementptr inbounds nuw <4 x i32>, ptr %a, i64 %iv279 %1 = load <4 x i32>, ptr %gep.a, align 16280 %rdx.next = add <4 x i32> %rdx, %1281 %iv.next = add nuw nsw i64 %iv, 1282 %ec = icmp eq i64 %iv.next, %n283 br i1 %ec, label %exit, label %loop, !llvm.loop !0284 285exit:286 %res = phi <4 x i32> [ %rdx.next, %loop ]287 ret <4 x i32> %res288}289 290define float @test_fadd_reduction(ptr %a, i64 %n) {291; CHECK-LABEL: define float @test_fadd_reduction(292; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {293; CHECK-NEXT: [[ENTRY:.*]]:294; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1295; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1296; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1297; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]298; CHECK: [[ENTRY_NEW]]:299; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]300; CHECK-NEXT: br label %[[LOOP:.*]]301; CHECK: [[LOOP]]:302; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]303; CHECK-NEXT: [[RDX_1:%.*]] = phi float [ -0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]304; CHECK-NEXT: [[RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]305; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]306; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV]]307; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 16308; CHECK-NEXT: [[RDX_NEXT]] = fadd reassoc float [[RDX]], [[TMP2]]309; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1310; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_NEXT]]311; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 16312; CHECK-NEXT: [[RDX_NEXT_1]] = fadd reassoc float [[RDX_1]], [[TMP3]]313; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2314; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2315; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]316; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]317; CHECK: [[EXIT_UNR_LCSSA]]:318; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]319; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]320; CHECK-NEXT: [[RDX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]321; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc float [[RDX_NEXT_1]], [[RDX_NEXT]]322; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0323; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]324; CHECK: [[LOOP_EPIL_PREHEADER]]:325; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]326; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ]327; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0328; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])329; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]330; CHECK: [[LOOP_EPIL]]:331; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_EPIL_INIT]]332; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 16333; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = fadd reassoc float [[RDX_EPIL_INIT]], [[TMP4]]334; CHECK-NEXT: br label %[[EXIT]]335; CHECK: [[EXIT]]:336; CHECK-NEXT: [[RES:%.*]] = phi float [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]337; CHECK-NEXT: ret float [[RES]]338;339entry:340 br label %loop341 342loop:343 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]344 %rdx = phi float [ 0.0, %entry ], [ %rdx.next, %loop ]345 %gep.a = getelementptr inbounds nuw float, ptr %a, i64 %iv346 %1 = load float, ptr %gep.a, align 16347 %rdx.next = fadd reassoc float %rdx, %1348 %iv.next = add nuw nsw i64 %iv, 1349 %ec = icmp eq i64 %iv.next, %n350 br i1 %ec, label %exit, label %loop, !llvm.loop !0351 352exit:353 %res = phi float [ %rdx.next, %loop ]354 ret float %res355}356 357define float @test_fadd_no_reassoc(ptr %a, i64 %n) {358; CHECK-LABEL: define float @test_fadd_no_reassoc(359; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {360; CHECK-NEXT: [[ENTRY:.*]]:361; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1362; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1363; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1364; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]365; CHECK: [[ENTRY_NEW]]:366; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]367; CHECK-NEXT: br label %[[LOOP:.*]]368; CHECK: [[LOOP]]:369; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]370; CHECK-NEXT: [[RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]371; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]372; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV]]373; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 16374; CHECK-NEXT: [[RDX_NEXT:%.*]] = fadd float [[RDX]], [[TMP2]]375; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1376; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_NEXT]]377; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 16378; CHECK-NEXT: [[RDX_NEXT_1]] = fadd float [[RDX_NEXT]], [[TMP3]]379; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2380; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2381; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]382; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]383; CHECK: [[EXIT_UNR_LCSSA]]:384; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]385; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]386; CHECK-NEXT: [[RDX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]387; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0388; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]389; CHECK: [[LOOP_EPIL_PREHEADER]]:390; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]391; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RDX_UNR]], %[[EXIT_UNR_LCSSA]] ]392; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0393; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])394; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]395; CHECK: [[LOOP_EPIL]]:396; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_EPIL_INIT]]397; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 16398; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = fadd float [[RDX_EPIL_INIT]], [[TMP4]]399; CHECK-NEXT: br label %[[EXIT]]400; CHECK: [[EXIT]]:401; CHECK-NEXT: [[RES:%.*]] = phi float [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]402; CHECK-NEXT: ret float [[RES]]403;404entry:405 br label %loop406 407loop:408 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]409 %rdx = phi float [ 0.0, %entry ], [ %rdx.next, %loop ]410 %gep.a = getelementptr inbounds nuw float, ptr %a, i64 %iv411 %1 = load float, ptr %gep.a, align 16412 %rdx.next = fadd float %rdx, %1413 %iv.next = add nuw nsw i64 %iv, 1414 %ec = icmp eq i64 %iv.next, %n415 br i1 %ec, label %exit, label %loop, !llvm.loop !0416 417exit:418 %res = phi float [ %rdx.next, %loop ]419 ret float %res420}421 422define float @test_fadd_other_fastmath(ptr %a, i64 %n) {423; CHECK-LABEL: define float @test_fadd_other_fastmath(424; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {425; CHECK-NEXT: [[ENTRY:.*]]:426; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1427; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1428; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1429; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]430; CHECK: [[ENTRY_NEW]]:431; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]432; CHECK-NEXT: br label %[[LOOP:.*]]433; CHECK: [[LOOP]]:434; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]435; CHECK-NEXT: [[RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]436; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]437; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV]]438; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 16439; CHECK-NEXT: [[RDX_NEXT:%.*]] = fadd contract float [[RDX]], [[TMP2]]440; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1441; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_NEXT]]442; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 16443; CHECK-NEXT: [[RDX_NEXT_1]] = fadd contract float [[RDX_NEXT]], [[TMP3]]444; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2445; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2446; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]447; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]448; CHECK: [[EXIT_UNR_LCSSA]]:449; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]450; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]451; CHECK-NEXT: [[RDX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]452; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0453; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]454; CHECK: [[LOOP_EPIL_PREHEADER]]:455; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]456; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RDX_UNR]], %[[EXIT_UNR_LCSSA]] ]457; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0458; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])459; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]460; CHECK: [[LOOP_EPIL]]:461; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_EPIL_INIT]]462; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 16463; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = fadd contract float [[RDX_EPIL_INIT]], [[TMP4]]464; CHECK-NEXT: br label %[[EXIT]]465; CHECK: [[EXIT]]:466; CHECK-NEXT: [[RES:%.*]] = phi float [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]467; CHECK-NEXT: ret float [[RES]]468;469entry:470 br label %loop471 472loop:473 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]474 %rdx = phi float [ 0.0, %entry ], [ %rdx.next, %loop ]475 %gep.a = getelementptr inbounds nuw float, ptr %a, i64 %iv476 %1 = load float, ptr %gep.a, align 16477 %rdx.next = fadd contract float %rdx, %1478 %iv.next = add nuw nsw i64 %iv, 1479 %ec = icmp eq i64 %iv.next, %n480 br i1 %ec, label %exit, label %loop, !llvm.loop !0481 482exit:483 %res = phi float [ %rdx.next, %loop ]484 ret float %res485}486 487!0 = distinct !{!0, !1}488!1 = !{!"llvm.loop.unroll.count", i32 2}489 490!2 = distinct !{!2, !3}491!3 = !{!"llvm.loop.unroll.count", i32 8}492 493;.494; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}495; CHECK: [[META1]] = !{!"llvm.loop.unroll.disable"}496; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}497; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}498; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}499; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}500; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}501; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}502; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]]}503;.504