238 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 52; REQUIRES: asserts3; RUN: opt < %s -aa-pipeline=basic-aa -passes=loop-vectorize,instcombine -S -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s4; RUN: opt < %s -passes=loop-vectorize -force-vector-width=2 -S | FileCheck %s -check-prefix=FORCE5 6target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"7target triple = "x86_64-unknown-linux-gnu"8 9; CHECK-LABEL: PR3167110;11; Check a pointer in which one of its uses is consecutive-like and another of12; its uses is non-consecutive-like. In the test case below, %tmp3 is the13; pointer operand of an interleaved load, making it consecutive-like. However,14; it is also the pointer operand of a non-interleaved store that will become a15; scatter operation. %tmp3 (and the induction variable) should not be marked16; uniform-after-vectorization.17;18; CHECK: LV: Found uniform instruction: %tmp0 = getelementptr inbounds %data, ptr %d, i64 0, i32 3, i64 %i19; CHECK-NOT: LV: Found uniform instruction: %tmp3 = getelementptr inbounds %data, ptr %d, i64 0, i32 0, i64 %i20; CHECK-NOT: LV: Found uniform instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]21; CHECK-NOT: LV: Found uniform instruction: %i.next = add nuw nsw i64 %i, 522 23 24%data = type { [32000 x float], [3 x i32], [4 x i8], [32000 x float] }25 26define void @PR31671(float %x, ptr %d) #0 {27; CHECK-LABEL: define void @PR31671(28; CHECK-SAME: float [[X:%.*]], ptr [[D:%.*]]) #[[ATTR0:[0-9]+]] {29; CHECK-NEXT: [[ENTRY:.*:]]30; CHECK-NEXT: br label %[[VECTOR_PH:.*]]31; CHECK: [[VECTOR_PH]]:32; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x float> poison, float [[X]], i64 033; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x float> [[BROADCAST_SPLATINSERT]], <16 x float> poison, <16 x i32> zeroinitializer34; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]35; CHECK: [[VECTOR_BODY]]:36; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]37; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 5, i64 10, i64 15, i64 20, i64 25, i64 30, i64 35, i64 40, i64 45, i64 50, i64 55, i64 60, i64 65, i64 70, i64 75>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]38; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds nuw i8, ptr [[D]], i64 12801639; CHECK-NEXT: [[DOTIDX:%.*]] = mul i64 [[INDEX]], 2040; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[DOTSPLIT]], i64 [[DOTIDX]]41; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <80 x float>, ptr [[TMP0]], align 442; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <80 x float> [[WIDE_VEC]], <80 x float> poison, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>43; CHECK-NEXT: [[TMP1:%.*]] = fmul <16 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]44; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[DATA:%.*]], ptr [[D]], i64 0, i32 0, <16 x i64> [[VEC_IND]]45; CHECK-NEXT: [[TMP3:%.*]] = extractelement <16 x ptr> [[TMP2]], i64 046; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <80 x float>, ptr [[TMP3]], align 447; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <80 x float> [[WIDE_VEC1]], <80 x float> poison, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>48; CHECK-NEXT: [[TMP4:%.*]] = fadd <16 x float> [[STRIDED_VEC2]], [[TMP1]]49; CHECK-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[TMP4]], <16 x ptr> align 4 [[TMP2]], <16 x i1> splat (i1 true))50; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1651; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 80)52; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 638453; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]54; CHECK: [[MIDDLE_BLOCK]]:55; CHECK-NEXT: br label %[[SCALAR_PH:.*]]56; CHECK: [[SCALAR_PH]]:57;58; FORCE-LABEL: define void @PR31671(59; FORCE-SAME: float [[X:%.*]], ptr [[D:%.*]]) #[[ATTR0:[0-9]+]] {60; FORCE-NEXT: [[ENTRY:.*:]]61; FORCE-NEXT: br label %[[VECTOR_PH:.*]]62; FORCE: [[VECTOR_PH]]:63; FORCE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[X]], i64 064; FORCE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer65; FORCE-NEXT: br label %[[VECTOR_BODY:.*]]66; FORCE: [[VECTOR_BODY]]:67; FORCE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]68; FORCE-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 569; FORCE-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 070; FORCE-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 571; FORCE-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 1072; FORCE-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 1573; FORCE-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2074; FORCE-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 2575; FORCE-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 3076; FORCE-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 3577; FORCE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[DATA:%.*]], ptr [[D]], i64 0, i32 3, i64 [[TMP0]]78; FORCE-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 3, i64 [[TMP2]]79; FORCE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 3, i64 [[TMP4]]80; FORCE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 3, i64 [[TMP6]]81; FORCE-NEXT: [[WIDE_VEC:%.*]] = load <10 x float>, ptr [[TMP8]], align 482; FORCE-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <10 x float> [[WIDE_VEC]], <10 x float> poison, <2 x i32> <i32 0, i32 5>83; FORCE-NEXT: [[WIDE_VEC1:%.*]] = load <10 x float>, ptr [[TMP9]], align 484; FORCE-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <10 x float> [[WIDE_VEC1]], <10 x float> poison, <2 x i32> <i32 0, i32 5>85; FORCE-NEXT: [[WIDE_VEC3:%.*]] = load <10 x float>, ptr [[TMP10]], align 486; FORCE-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <10 x float> [[WIDE_VEC3]], <10 x float> poison, <2 x i32> <i32 0, i32 5>87; FORCE-NEXT: [[WIDE_VEC5:%.*]] = load <10 x float>, ptr [[TMP11]], align 488; FORCE-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <10 x float> [[WIDE_VEC5]], <10 x float> poison, <2 x i32> <i32 0, i32 5>89; FORCE-NEXT: [[TMP12:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]90; FORCE-NEXT: [[TMP13:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC2]]91; FORCE-NEXT: [[TMP14:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC4]]92; FORCE-NEXT: [[TMP15:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC6]]93; FORCE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP0]]94; FORCE-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP1]]95; FORCE-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP2]]96; FORCE-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP3]]97; FORCE-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP4]]98; FORCE-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP5]]99; FORCE-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP6]]100; FORCE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP7]]101; FORCE-NEXT: [[WIDE_VEC7:%.*]] = load <10 x float>, ptr [[TMP16]], align 4102; FORCE-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <10 x float> [[WIDE_VEC7]], <10 x float> poison, <2 x i32> <i32 0, i32 5>103; FORCE-NEXT: [[WIDE_VEC9:%.*]] = load <10 x float>, ptr [[TMP18]], align 4104; FORCE-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <10 x float> [[WIDE_VEC9]], <10 x float> poison, <2 x i32> <i32 0, i32 5>105; FORCE-NEXT: [[WIDE_VEC11:%.*]] = load <10 x float>, ptr [[TMP20]], align 4106; FORCE-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <10 x float> [[WIDE_VEC11]], <10 x float> poison, <2 x i32> <i32 0, i32 5>107; FORCE-NEXT: [[WIDE_VEC13:%.*]] = load <10 x float>, ptr [[TMP22]], align 4108; FORCE-NEXT: [[STRIDED_VEC14:%.*]] = shufflevector <10 x float> [[WIDE_VEC13]], <10 x float> poison, <2 x i32> <i32 0, i32 5>109; FORCE-NEXT: [[TMP24:%.*]] = fadd <2 x float> [[STRIDED_VEC8]], [[TMP12]]110; FORCE-NEXT: [[TMP28:%.*]] = extractelement <2 x float> [[TMP24]], i32 0111; FORCE-NEXT: [[TMP29:%.*]] = extractelement <2 x float> [[TMP24]], i32 1112; FORCE-NEXT: [[TMP25:%.*]] = fadd <2 x float> [[STRIDED_VEC10]], [[TMP13]]113; FORCE-NEXT: [[TMP30:%.*]] = extractelement <2 x float> [[TMP25]], i32 0114; FORCE-NEXT: [[TMP31:%.*]] = extractelement <2 x float> [[TMP25]], i32 1115; FORCE-NEXT: [[TMP26:%.*]] = fadd <2 x float> [[STRIDED_VEC12]], [[TMP14]]116; FORCE-NEXT: [[TMP32:%.*]] = extractelement <2 x float> [[TMP26]], i32 0117; FORCE-NEXT: [[TMP33:%.*]] = extractelement <2 x float> [[TMP26]], i32 1118; FORCE-NEXT: [[TMP27:%.*]] = fadd <2 x float> [[STRIDED_VEC14]], [[TMP15]]119; FORCE-NEXT: [[TMP34:%.*]] = extractelement <2 x float> [[TMP27]], i32 0120; FORCE-NEXT: [[TMP35:%.*]] = extractelement <2 x float> [[TMP27]], i32 1121; FORCE-NEXT: store float [[TMP28]], ptr [[TMP16]], align 4122; FORCE-NEXT: store float [[TMP29]], ptr [[TMP17]], align 4123; FORCE-NEXT: store float [[TMP30]], ptr [[TMP18]], align 4124; FORCE-NEXT: store float [[TMP31]], ptr [[TMP19]], align 4125; FORCE-NEXT: store float [[TMP32]], ptr [[TMP20]], align 4126; FORCE-NEXT: store float [[TMP33]], ptr [[TMP21]], align 4127; FORCE-NEXT: store float [[TMP34]], ptr [[TMP22]], align 4128; FORCE-NEXT: store float [[TMP35]], ptr [[TMP23]], align 4129; FORCE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8130; FORCE-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 6392131; FORCE-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]132; FORCE: [[MIDDLE_BLOCK]]:133; FORCE-NEXT: br label %[[SCALAR_PH:.*]]134; FORCE: [[SCALAR_PH]]:135;136entry:137 br label %for.body138 139for.body:140 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]141 %tmp0 = getelementptr inbounds %data, ptr %d, i64 0, i32 3, i64 %i142 %tmp1 = load float, ptr %tmp0, align 4143 %tmp2 = fmul float %x, %tmp1144 %tmp3 = getelementptr inbounds %data, ptr %d, i64 0, i32 0, i64 %i145 %tmp4 = load float, ptr %tmp3, align 4146 %tmp5 = fadd float %tmp4, %tmp2147 store float %tmp5, ptr %tmp3, align 4148 %i.next = add nuw nsw i64 %i, 5149 %cond = icmp slt i64 %i.next, 32000150 br i1 %cond, label %for.body, label %for.end151 152for.end:153 ret void154}155 156attributes #0 = { "target-cpu"="knl" }157 158; CHECK-LABEL: PR40816159;160; Check that scalar with predication instructions are not considered uniform161; after vectorization, because that results in replicating a region instead of162; having a single instance (out of VF). The predication stems from a tiny count163; of 3 leading to folding the tail by masking using icmp ule <i, i+1> <= <2, 2>.164;165; CHECK: LV: Found trip count: 3166; CHECK: LV: Found uniform instruction: {{%.*}} = icmp eq i32 {{%.*}}, 0167; CHECK-NOT: LV: Found uniform instruction: {{%.*}} = load i32, ptr {{%.*}}, align 1168; CHECK: LV: Found not uniform due to requiring predication: {{%.*}} = load i32, ptr {{%.*}}, align 1169;170;171@a = internal constant [3 x i32] [i32 7, i32 7, i32 0], align 1172@b = external global i32, align 1173 174define void @PR40816() #1 {175; CHECK-LABEL: define void @PR40816(176; CHECK-SAME: ) #[[ATTR1:[0-9]+]] {177; CHECK-NEXT: [[ENTRY:.*]]:178; CHECK-NEXT: br label %[[FOR_BODY:.*]]179; CHECK: [[FOR_BODY]]:180; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]181; CHECK-NEXT: store i32 [[TMP0]], ptr @b, align 1182; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP0]], 2183; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[TMP0]], 1184; CHECK-NEXT: br i1 [[CMP2]], label %[[RETURN:.*]], label %[[FOR_BODY]]185; CHECK: [[RETURN]]:186; CHECK-NEXT: ret void187;188; FORCE-LABEL: define void @PR40816(189; FORCE-SAME: ) #[[ATTR1:[0-9]+]] {190; FORCE-NEXT: [[ENTRY:.*:]]191; FORCE-NEXT: br label %[[VECTOR_PH:.*]]192; FORCE: [[VECTOR_PH]]:193; FORCE-NEXT: br label %[[VECTOR_BODY:.*]]194; FORCE: [[VECTOR_BODY]]:195; FORCE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]196; FORCE-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]197; FORCE-NEXT: [[TMP2:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 2)198; FORCE-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0199; FORCE-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]200; FORCE: [[PRED_STORE_IF]]:201; FORCE-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0202; FORCE-NEXT: store i32 [[TMP0]], ptr @b, align 1203; FORCE-NEXT: br label %[[PRED_STORE_CONTINUE]]204; FORCE: [[PRED_STORE_CONTINUE]]:205; FORCE-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1206; FORCE-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]207; FORCE: [[PRED_STORE_IF1]]:208; FORCE-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1209; FORCE-NEXT: store i32 [[TMP1]], ptr @b, align 1210; FORCE-NEXT: br label %[[PRED_STORE_CONTINUE2]]211; FORCE: [[PRED_STORE_CONTINUE2]]:212; FORCE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2213; FORCE-NEXT: [[VEC_IND_NEXT]] = add <2 x i8> [[VEC_IND]], splat (i8 2)214; FORCE-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4215; FORCE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]216; FORCE: [[MIDDLE_BLOCK]]:217; FORCE-NEXT: br label %[[RETURN:.*]]218; FORCE: [[RETURN]]:219; FORCE-NEXT: ret void220;221entry:222 br label %for.body223 224for.body: ; preds = %for.body, %entry225 %0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]226 store i32 %0, ptr @b, align 1227 %arrayidx1 = getelementptr inbounds [3 x i32], ptr @a, i32 0, i32 %0228 %1 = load i32, ptr %arrayidx1, align 1229 %cmp2 = icmp eq i32 %1, 0230 %inc = add nuw nsw i32 %0, 1231 br i1 %cmp2, label %return, label %for.body232 233return: ; preds = %for.body234 ret void235}236 237attributes #1 = { "target-cpu"="core2" }238