270 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 52; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s3 4target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"5target triple = "x86_64-unknown-linux-gnu"6 7; Test case for https://github.com/llvm/llvm-project/issues/111042.8define void @replicate_udiv_with_only_first_lane_used(i32 %x, ptr %dst, i64 %d) {9; CHECK-LABEL: define void @replicate_udiv_with_only_first_lane_used(10; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[D:%.*]]) {11; CHECK-NEXT: [[ENTRY:.*:]]12; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[X]], 1013; CHECK-NEXT: br label %[[VECTOR_PH:.*]]14; CHECK: [[VECTOR_PH]]:15; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]16; CHECK: [[VECTOR_BODY]]:17; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]18; CHECK-NEXT: store i16 0, ptr [[DST]], align 219; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 820; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9621; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]22; CHECK: [[MIDDLE_BLOCK]]:23; CHECK-NEXT: br label %[[SCALAR_PH:.*]]24; CHECK: [[SCALAR_PH]]:25; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]26; CHECK: [[LOOP_HEADER]]:27; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]28; CHECK-NEXT: br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]29; CHECK: [[ELSE]]:30; CHECK-NEXT: [[DIV_I:%.*]] = udiv i64 99, [[D]]31; CHECK-NEXT: br label %[[LOOP_LATCH]]32; CHECK: [[LOOP_LATCH]]:33; CHECK-NEXT: [[RETVAL_0_I:%.*]] = phi i64 [ [[DIV_I]], %[[ELSE]] ], [ 0, %[[LOOP_HEADER]] ]34; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i64 [[RETVAL_0_I]]35; CHECK-NEXT: store i16 0, ptr [[GEP]], align 236; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 137; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 10138; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[EXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]39; CHECK: [[EXIT]]:40; CHECK-NEXT: ret void41;42entry:43 %c = icmp eq i32 %x, 1044 br label %loop.header45 46loop.header: ; preds = %loop.latch, %entry47 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]48 br i1 true, label %loop.latch, label %else49 50else:51 %div.i = udiv i64 99, %d52 br label %loop.latch53 54loop.latch:55 %retval.0.i = phi i64 [ %div.i, %else ], [ 0, %loop.header ]56 %gep = getelementptr i16, ptr %dst, i64 %retval.0.i57 store i16 0, ptr %gep, align 258 %iv.next = add i64 %iv, 159 %cmp = icmp ult i64 %iv.next, 10160 br i1 %cmp, label %loop.header, label %exit61 62exit:63 ret void64}65 66; Variant where ptradd cannot be const-folded.67define void @replicate_udiv_with_only_first_lane_used2(i32 %x, ptr %dst, i64 %d) {68; CHECK-LABEL: define void @replicate_udiv_with_only_first_lane_used2(69; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[D:%.*]]) {70; CHECK-NEXT: [[ENTRY:.*:]]71; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[X]], 1072; CHECK-NEXT: br label %[[VECTOR_PH:.*]]73; CHECK: [[VECTOR_PH]]:74; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 075; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer76; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)77; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]78; CHECK: [[VECTOR_BODY]]:79; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE14:.*]] ]80; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 081; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]82; CHECK: [[PRED_UDIV_IF]]:83; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 99, [[D]]84; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]85; CHECK: [[PRED_UDIV_CONTINUE]]:86; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[PRED_UDIV_IF]] ]87; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i32 188; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]]89; CHECK: [[PRED_UDIV_IF1]]:90; CHECK-NEXT: [[TMP6:%.*]] = udiv i64 99, [[D]]91; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]92; CHECK: [[PRED_UDIV_CONTINUE2]]:93; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP0]], i32 294; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]]95; CHECK: [[PRED_UDIV_IF3]]:96; CHECK-NEXT: [[TMP10:%.*]] = udiv i64 99, [[D]]97; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE4]]98; CHECK: [[PRED_UDIV_CONTINUE4]]:99; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3100; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6:.*]]101; CHECK: [[PRED_UDIV_IF5]]:102; CHECK-NEXT: [[TMP14:%.*]] = udiv i64 99, [[D]]103; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE6]]104; CHECK: [[PRED_UDIV_CONTINUE6]]:105; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0106; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_UDIV_IF7:.*]], label %[[PRED_UDIV_CONTINUE8:.*]]107; CHECK: [[PRED_UDIV_IF7]]:108; CHECK-NEXT: [[TMP18:%.*]] = udiv i64 99, [[D]]109; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE8]]110; CHECK: [[PRED_UDIV_CONTINUE8]]:111; CHECK-NEXT: [[TMP15:%.*]] = phi i64 [ poison, %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP18]], %[[PRED_UDIV_IF7]] ]112; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1113; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_UDIV_IF9:.*]], label %[[PRED_UDIV_CONTINUE10:.*]]114; CHECK: [[PRED_UDIV_IF9]]:115; CHECK-NEXT: [[TMP22:%.*]] = udiv i64 99, [[D]]116; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE10]]117; CHECK: [[PRED_UDIV_CONTINUE10]]:118; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2119; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_UDIV_IF11:.*]], label %[[PRED_UDIV_CONTINUE12:.*]]120; CHECK: [[PRED_UDIV_IF11]]:121; CHECK-NEXT: [[TMP26:%.*]] = udiv i64 99, [[D]]122; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE12]]123; CHECK: [[PRED_UDIV_CONTINUE12]]:124; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3125; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_UDIV_IF13:.*]], label %[[PRED_UDIV_CONTINUE14]]126; CHECK: [[PRED_UDIV_IF13]]:127; CHECK-NEXT: [[TMP30:%.*]] = udiv i64 99, [[D]]128; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE14]]129; CHECK: [[PRED_UDIV_CONTINUE14]]:130; CHECK-NEXT: [[TMP45:%.*]] = select i1 [[C]], i64 0, i64 [[TMP3]]131; CHECK-NEXT: [[TMP47:%.*]] = select i1 [[C]], i64 0, i64 [[TMP15]]132; CHECK-NEXT: [[TMP46:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP45]]133; CHECK-NEXT: [[TMP48:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP47]]134; CHECK-NEXT: store i16 0, ptr [[TMP46]], align 2135; CHECK-NEXT: store i16 0, ptr [[TMP48]], align 2136; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8137; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96138; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]139; CHECK: [[MIDDLE_BLOCK]]:140; CHECK-NEXT: br label %[[SCALAR_PH:.*]]141; CHECK: [[SCALAR_PH]]:142; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]143; CHECK: [[LOOP_HEADER]]:144; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]145; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]146; CHECK: [[ELSE]]:147; CHECK-NEXT: [[DIV_I:%.*]] = udiv i64 99, [[D]]148; CHECK-NEXT: br label %[[LOOP_LATCH]]149; CHECK: [[LOOP_LATCH]]:150; CHECK-NEXT: [[RETVAL_0_I:%.*]] = phi i64 [ [[DIV_I]], %[[ELSE]] ], [ 0, %[[LOOP_HEADER]] ]151; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i64 [[RETVAL_0_I]]152; CHECK-NEXT: store i16 0, ptr [[GEP]], align 2153; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1154; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 101155; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[EXIT:.*]], !llvm.loop [[LOOP5:![0-9]+]]156; CHECK: [[EXIT]]:157; CHECK-NEXT: ret void158;159entry:160 %c = icmp eq i32 %x, 10161 br label %loop.header162 163loop.header: ; preds = %loop.latch, %entry164 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]165 br i1 %c, label %loop.latch, label %else166 167else:168 %div.i = udiv i64 99, %d169 br label %loop.latch170 171loop.latch:172 %retval.0.i = phi i64 [ %div.i, %else ], [ 0, %loop.header ]173 %gep = getelementptr i16, ptr %dst, i64 %retval.0.i174 store i16 0, ptr %gep, align 2175 %iv.next = add i64 %iv, 1176 %cmp = icmp ult i64 %iv.next, 101177 br i1 %cmp, label %loop.header, label %exit178 179exit:180 ret void181}182 183define float @uniform_load_replicating_select(ptr %A, ptr %B, i64 %1) {184; CHECK-LABEL: define float @uniform_load_replicating_select(185; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[TMP0:%.*]]) {186; CHECK-NEXT: [[ENTRY:.*]]:187; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 1188; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 8189; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]190; CHECK: [[VECTOR_PH]]:191; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 8192; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]193; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]194; CHECK: [[VECTOR_BODY]]:195; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]196; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 4197; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 5198; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 6199; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 7200; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr [[A]], align 4201; CHECK-NEXT: [[TMP10:%.*]] = fcmp ogt float [[TMP6]], 0.000000e+00202; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP2]]203; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP3]]204; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]]205; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]]206; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP15]]207; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP16]]208; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP17]]209; CHECK-NEXT: [[TMP22:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP18]]210; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP10]], <4 x float> splat (float 1.000000e+01), <4 x float> splat (float 1.000000e+00)211; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[TMP19]], align 4212; CHECK-NEXT: [[TMP25:%.*]] = load float, ptr [[TMP20]], align 4213; CHECK-NEXT: [[TMP26:%.*]] = load float, ptr [[TMP21]], align 4214; CHECK-NEXT: [[TMP27:%.*]] = load float, ptr [[TMP22]], align 4215; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x float> poison, float [[TMP24]], i32 0216; CHECK-NEXT: [[TMP29:%.*]] = insertelement <4 x float> [[TMP28]], float [[TMP25]], i32 1217; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x float> [[TMP29]], float [[TMP26]], i32 2218; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x float> [[TMP30]], float [[TMP27]], i32 3219; CHECK-NEXT: [[TMP32:%.*]] = fdiv <4 x float> splat (float 4.000000e+00), [[TMP31]]220; CHECK-NEXT: [[TMP33:%.*]] = call <4 x float> @llvm.pow.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP32]])221; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8222; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]223; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]224; CHECK: [[MIDDLE_BLOCK]]:225; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x float> [[TMP33]], i32 3226; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]227; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]228; CHECK: [[SCALAR_PH]]:229; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]230; CHECK-NEXT: br label %[[LOOP:.*]]231; CHECK: [[LOOP]]:232; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]233; CHECK-NEXT: [[L:%.*]] = load float, ptr [[A]], align 4234; CHECK-NEXT: [[C:%.*]] = fcmp ogt float [[L]], 0.000000e+00235; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]236; CHECK-NEXT: [[SEL_PTR:%.*]] = select i1 [[C]], ptr [[A]], ptr [[GEP_B]]237; CHECK-NEXT: [[BASE:%.*]] = select i1 [[C]], float 1.000000e+01, float 1.000000e+00238; CHECK-NEXT: [[L_2:%.*]] = load float, ptr [[SEL_PTR]], align 4239; CHECK-NEXT: [[DIV:%.*]] = fdiv float 4.000000e+00, [[L_2]]240; CHECK-NEXT: [[POW:%.*]] = tail call float @llvm.pow.f32(float [[BASE]], float [[DIV]])241; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1242; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[TMP0]]243; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]244; CHECK: [[EXIT]]:245; CHECK-NEXT: [[POW_LCSSA:%.*]] = phi float [ [[POW]], %[[LOOP]] ], [ [[TMP35]], %[[MIDDLE_BLOCK]] ]246; CHECK-NEXT: ret float [[POW_LCSSA]]247;248entry:249 br label %loop250 251loop:252 %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]253 %l = load float, ptr %A, align 4254 %c = fcmp ogt float %l, 0.000000e+00255 %gep.B = getelementptr inbounds float, ptr %B, i64 %iv256 %sel.ptr = select i1 %c, ptr %A, ptr %gep.B257 %base = select i1 %c, float 10.000000e+00, float 1.000000e+00258 %l.2 = load float, ptr %sel.ptr, align 4259 %div = fdiv float 4.000000e+00, %l.2260 %pow = tail call float @llvm.pow.f32(float %base, float %div)261 %iv.next = add i64 %iv, 1262 %ec = icmp eq i64 %iv, %1263 br i1 %ec, label %exit, label %loop264 265exit:266 ret float %pow267}268 269declare float @llvm.pow.f32(float, float)270