1155 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 52; RUN: opt < %s -passes="loop-vectorize" -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s3 4target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"5 6; This test checks that we can vectorize loop with reduction variable7; stored in an invariant address.8;9; int sum = 0;10; for(i=0..N) {11; sum += src[i];12; dst[42] = sum;13; }14define void @reduc_store(ptr %dst, ptr readonly %src) {15; CHECK-LABEL: define void @reduc_store(16; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {17; CHECK-NEXT: [[ENTRY:.*:]]18; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 4219; CHECK-NEXT: store i32 0, ptr [[GEP_DST]], align 420; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]21; CHECK: [[VECTOR_MEMCHECK]]:22; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 17223; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 400024; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_DST]], [[SCEVGEP1]]25; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]26; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]27; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]28; CHECK: [[VECTOR_PH]]:29; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]30; CHECK: [[VECTOR_BODY]]:31; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]32; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]33; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]34; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META0:![0-9]+]]35; CHECK-NEXT: [[TMP1]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]36; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 437; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100038; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]39; CHECK: [[MIDDLE_BLOCK]]:40; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])41; CHECK-NEXT: store i32 [[TMP3]], ptr [[GEP_DST]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]42; CHECK-NEXT: br [[EXIT:label %.*]]43; CHECK: [[SCALAR_PH]]:44;45entry:46 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 4247 store i32 0, ptr %gep.dst, align 448 br label %for.body49 50for.body:51 %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]52 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]53 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv54 %0 = load i32, ptr %gep.src, align 455 %add = add nsw i32 %sum, %056 store i32 %add, ptr %gep.dst, align 457 %iv.next = add nuw nsw i64 %iv, 158 %exitcond = icmp eq i64 %iv.next, 100059 br i1 %exitcond, label %exit, label %for.body60 61exit:62 ret void63}64 65; Same as above but with floating point numbers instead.66;67; float sum = 0;68; for(i=0..N) {69; sum += src[i];70; dst[42] = sum;71; }72define void @reduc_store_fadd_fast(ptr %dst, ptr readonly %src) {73; CHECK-LABEL: define void @reduc_store_fadd_fast(74; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {75; CHECK-NEXT: [[ENTRY:.*:]]76; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 4277; CHECK-NEXT: store float 0.000000e+00, ptr [[GEP_DST]], align 478; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]79; CHECK: [[VECTOR_MEMCHECK]]:80; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 17281; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 400082; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_DST]], [[SCEVGEP1]]83; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]84; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]85; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]86; CHECK: [[VECTOR_PH]]:87; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]88; CHECK: [[VECTOR_BODY]]:89; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]90; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]91; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]]92; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4, !alias.scope [[META9:![0-9]+]]93; CHECK-NEXT: [[TMP1]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]94; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 495; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100096; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]97; CHECK: [[MIDDLE_BLOCK]]:98; CHECK-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP1]])99; CHECK-NEXT: store float [[TMP3]], ptr [[GEP_DST]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META9]]100; CHECK-NEXT: br [[EXIT:label %.*]]101; CHECK: [[SCALAR_PH]]:102;103entry:104 %gep.dst = getelementptr inbounds float, ptr %dst, i64 42105 store float 0.000000e+00, ptr %gep.dst, align 4106 br label %for.body107 108for.body:109 %sum = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]110 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]111 %gep.src = getelementptr inbounds float, ptr %src, i64 %iv112 %0 = load float, ptr %gep.src, align 4113 %add = fadd fast float %sum, %0114 store float %add, ptr %gep.dst, align 4115 %iv.next = add nuw nsw i64 %iv, 1116 %exitcond = icmp eq i64 %iv.next, 1000117 br i1 %exitcond, label %exit, label %for.body118 119exit:120 ret void121}122 123; Check that if we have a read from an invariant address, we do not vectorize.124;125; int sum = 0;126; for(i=0..N) {127; sum += src[i];128; dst.2[i] = dst[42];129; dst[42] = sum;130; }131define void @reduc_store_load(ptr %dst, ptr readonly %src, ptr noalias %dst.2) {132; CHECK-LABEL: define void @reduc_store_load(133; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]], ptr noalias [[DST_2:%.*]]) {134; CHECK-NEXT: [[ENTRY:.*]]:135; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42136; CHECK-NEXT: store i32 0, ptr [[GEP_DST]], align 4137; CHECK-NEXT: br label %[[FOR_BODY:.*]]138; CHECK: [[FOR_BODY]]:139; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]140; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]141; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]142; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP_SRC]], align 4143; CHECK-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[TMP0]]144; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP_DST]], align 4145; CHECK-NEXT: [[GEP_DST_2:%.*]] = getelementptr inbounds i32, ptr [[DST_2]], i64 [[IV]]146; CHECK-NEXT: store i32 [[LV]], ptr [[GEP_DST_2]], align 4147; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_DST]], align 4148; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1149; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000150; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]151; CHECK: [[EXIT]]:152; CHECK-NEXT: ret void153;154entry:155 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42156 store i32 0, ptr %gep.dst, align 4157 br label %for.body158 159for.body:160 %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]161 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]162 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv163 %0 = load i32, ptr %gep.src, align 4164 %add = add nsw i32 %sum, %0165 %lv = load i32, ptr %gep.dst166 %gep.dst.2 = getelementptr inbounds i32, ptr %dst.2, i64 %iv167 store i32 %lv, ptr %gep.dst.2, align 4168 store i32 %add, ptr %gep.dst, align 4169 %iv.next = add nuw nsw i64 %iv, 1170 %exitcond = icmp eq i64 %iv.next, 1000171 br i1 %exitcond, label %exit, label %for.body172 173exit:174 ret void175}176 177; Check that if we have a read from an invariant address, we do not vectorize,178; even if we vectorize with runtime checks. The test below is a variant of179; @reduc_store_load with a non-constant dependence distance, resulting in180; vectorization with runtime checks.181;182define void @reduc_store_load_with_non_constant_distance_dependence(ptr %dst, ptr noalias %dst.2, i64 %off) {183; CHECK-LABEL: define void @reduc_store_load_with_non_constant_distance_dependence(184; CHECK-SAME: ptr [[DST:%.*]], ptr noalias [[DST_2:%.*]], i64 [[OFF:%.*]]) {185; CHECK-NEXT: [[ENTRY:.*]]:186; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42187; CHECK-NEXT: [[DST_2_OFF:%.*]] = getelementptr inbounds i32, ptr [[DST_2]], i64 [[OFF]]188; CHECK-NEXT: store i32 0, ptr [[GEP_DST]], align 4189; CHECK-NEXT: br label %[[FOR_BODY:.*]]190; CHECK: [[FOR_BODY]]:191; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]192; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]193; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[DST_2]], i64 [[IV]]194; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP_SRC]], align 4195; CHECK-NEXT: [[IV_OFF:%.*]] = mul i64 [[IV]], 2196; CHECK-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[TMP0]]197; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP_DST]], align 4198; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_DST]], align 4199; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[DST_2_OFF]], i64 [[IV]]200; CHECK-NEXT: store i32 [[LV]], ptr [[GEP_SRC_2]], align 4201; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1202; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000203; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]204; CHECK: [[EXIT]]:205; CHECK-NEXT: ret void206;207entry:208 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42209 %dst.2.off = getelementptr inbounds i32, ptr %dst.2, i64 %off210 store i32 0, ptr %gep.dst, align 4211 br label %for.body212 213for.body:214 %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]215 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]216 %gep.src = getelementptr inbounds i32, ptr %dst.2, i64 %iv217 %0 = load i32, ptr %gep.src, align 4218 %iv.off = mul i64 %iv, 2219 %add = add nsw i32 %sum, %0220 %lv = load i32, ptr %gep.dst221 store i32 %add, ptr %gep.dst, align 4222 %gep.src.2 = getelementptr inbounds i32, ptr %dst.2.off, i64 %iv223 store i32 %lv, ptr %gep.src.2, align 4224 %iv.next = add nuw nsw i64 %iv, 1225 %exitcond = icmp eq i64 %iv.next, 1000226 br i1 %exitcond, label %exit, label %for.body227 228exit:229 ret void230}231 232 233; Final value is not guaranteed to be stored in an invariant address.234; We don't vectorize in that case.235;236; int sum = 0;237; for(i=0..N) {238; int diff = y[i] - x[i];239; if (diff > 0) {240; sum = += diff;241; *t = sum;242; }243; }244define void @reduc_cond_store(ptr %t, ptr readonly %x, ptr readonly %y) {245; CHECK-LABEL: define void @reduc_cond_store(246; CHECK-SAME: ptr [[T:%.*]], ptr readonly [[X:%.*]], ptr readonly [[Y:%.*]]) {247; CHECK-NEXT: [[ENTRY:.*]]:248; CHECK-NEXT: store i32 0, ptr [[T]], align 4249; CHECK-NEXT: br label %[[FOR_BODY:.*]]250; CHECK: [[FOR_BODY]]:251; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_2:%.*]], %[[IF_END:.*]] ]252; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[IF_END]] ]253; CHECK-NEXT: [[GEP_Y:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IV]]254; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP_Y]], align 4255; CHECK-NEXT: [[GEP_X:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IV]]256; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[GEP_X]], align 4257; CHECK-NEXT: [[DIFF:%.*]] = sub nsw i32 [[TMP0]], [[TMP1]]258; CHECK-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[DIFF]], 0259; CHECK-NEXT: br i1 [[CMP2]], label %[[IF_THEN:.*]], label %[[IF_END]]260; CHECK: [[IF_THEN]]:261; CHECK-NEXT: [[SUM_1:%.*]] = add nsw i32 [[DIFF]], [[SUM]]262; CHECK-NEXT: store i32 [[SUM_1]], ptr [[T]], align 4263; CHECK-NEXT: br label %[[IF_END]]264; CHECK: [[IF_END]]:265; CHECK-NEXT: [[SUM_2]] = phi i32 [ [[SUM_1]], %[[IF_THEN]] ], [ [[TMP0]], %[[FOR_BODY]] ]266; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1267; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000268; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]269; CHECK: [[FOR_END]]:270; CHECK-NEXT: ret void271;272entry:273 store i32 0, ptr %t, align 4274 br label %for.body275 276for.body:277 %sum = phi i32 [ 0, %entry ], [ %sum.2, %if.end ]278 %iv = phi i64 [ 0, %entry ], [ %iv.next, %if.end ]279 %gep.y = getelementptr inbounds i32, ptr %y, i64 %iv280 %0 = load i32, ptr %gep.y, align 4281 %gep.x = getelementptr inbounds i32, ptr %x, i64 %iv282 %1 = load i32, ptr %gep.x, align 4283 %diff = sub nsw i32 %0, %1284 %cmp2 = icmp sgt i32 %diff, 0285 br i1 %cmp2, label %if.then, label %if.end286 287if.then:288 %sum.1 = add nsw i32 %diff, %sum289 store i32 %sum.1, ptr %t, align 4290 br label %if.end291 292if.end:293 %sum.2 = phi i32 [ %sum.1, %if.then ], [ %0, %for.body ]294 %iv.next = add nuw nsw i64 %iv, 1295 %exitcond = icmp eq i64 %iv.next, 1000296 br i1 %exitcond, label %for.end, label %for.body297 298for.end:299 ret void300}301 302; Check that we can vectorize code with several stores to an invariant address303; with condition that final reduction value is stored too.304;305; int sum = 0;306; for(int i=0; i < 1000; i+=2) {307; sum += src[i];308; dst[42] = sum;309; sum += src[i+1];310; dst[42] = sum;311; }312define void @reduc_store_inside_unrolled(ptr %dst, ptr readonly %src) {313; CHECK-LABEL: define void @reduc_store_inside_unrolled(314; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {315; CHECK-NEXT: [[ENTRY:.*:]]316; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42317; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]318; CHECK: [[VECTOR_MEMCHECK]]:319; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 172320; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4000321; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_DST]], [[SCEVGEP1]]322; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]323; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]324; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]325; CHECK: [[VECTOR_PH]]:326; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]327; CHECK: [[VECTOR_BODY]]:328; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]329; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 2, i64 4, i64 6>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]330; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[VECTOR_BODY]] ]331; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2332; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0333; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2334; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4335; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6336; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP0]]337; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]338; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP2]]339; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP3]]340; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META16:![0-9]+]]341; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META16]]342; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META16]]343; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META16]]344; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i32 0345; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 1346; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 2347; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i32 3348; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[TMP15]], [[VEC_PHI]]349; CHECK-NEXT: [[TMP17:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 1)350; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP17]], i32 0351; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP17]], i32 1352; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP17]], i32 2353; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP17]], i32 3354; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP18]]355; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP20]]356; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP22]]357; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP24]]358; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP37]], align 4, !alias.scope [[META16]]359; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP23]], align 4, !alias.scope [[META16]]360; CHECK-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP38]], align 4, !alias.scope [[META16]]361; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP25]], align 4, !alias.scope [[META16]]362; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP26]], i32 0363; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP27]], i32 1364; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 2365; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 3366; CHECK-NEXT: [[TMP34]] = add <4 x i32> [[TMP33]], [[TMP16]]367; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4368; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 8)369; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500370; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]371; CHECK: [[MIDDLE_BLOCK]]:372; CHECK-NEXT: [[TMP36:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP34]])373; CHECK-NEXT: store i32 [[TMP36]], ptr [[GEP_DST]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META16]]374; CHECK-NEXT: br [[EXIT:label %.*]]375; CHECK: [[SCALAR_PH]]:376;377entry:378 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42379 br label %for.body380 381for.body:382 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]383 %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.body ]384 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv385 %0 = load i32, ptr %gep.src, align 4386 %sum.1 = add nsw i32 %0, %sum387 store i32 %sum.1, ptr %gep.dst, align 4388 %1 = or disjoint i64 %iv, 1389 %gep.src.1 = getelementptr inbounds i32, ptr %src, i64 %1390 %2 = load i32, ptr %gep.src.1, align 4391 %sum.2 = add nsw i32 %2, %sum.1392 store i32 %sum.2, ptr %gep.dst, align 4393 %iv.next = add nuw nsw i64 %iv, 2394 %cmp = icmp slt i64 %iv.next, 1000395 br i1 %cmp, label %for.body, label %exit396 397exit:398 ret void399}400 401; Check that we cannot vectorize code if stored value is not the final reduction402; value403;404; int sum = 0;405; for(int i=0; i < 1000; i++) {406; sum += src[i];407; dst[42] = sum + 1;408; }409define void @reduc_store_not_final_value(ptr %dst, ptr readonly %src) {410; CHECK-LABEL: define void @reduc_store_not_final_value(411; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {412; CHECK-NEXT: [[ENTRY:.*]]:413; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42414; CHECK-NEXT: store i32 0, ptr [[GEP_DST]], align 4415; CHECK-NEXT: br label %[[FOR_BODY:.*]]416; CHECK: [[FOR_BODY]]:417; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]418; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]419; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]420; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP_SRC]], align 4421; CHECK-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[TMP0]]422; CHECK-NEXT: [[SUM_PLUS_ONE:%.*]] = add i32 [[ADD]], 1423; CHECK-NEXT: store i32 [[SUM_PLUS_ONE]], ptr [[GEP_DST]], align 4424; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1425; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000426; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]427; CHECK: [[EXIT]]:428; CHECK-NEXT: ret void429;430entry:431 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42432 store i32 0, ptr %gep.dst, align 4433 br label %for.body434 435for.body:436 %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]437 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]438 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv439 %0 = load i32, ptr %gep.src, align 4440 %add = add nsw i32 %sum, %0441 %sum_plus_one = add i32 %add, 1442 store i32 %sum_plus_one, ptr %gep.dst, align 4443 %iv.next = add nuw nsw i64 %iv, 1444 %exitcond = icmp eq i64 %iv.next, 1000445 br i1 %exitcond, label %exit, label %for.body446 447exit:448 ret void449}450 451; We cannot vectorize if two (or more) invariant stores exist in a loop.452;453; int sum = 0;454; for(int i=0; i < 1000; i+=2) {455; sum += src[i];456; dst[42] = sum;457; sum += src[i+1];458; other_dst[42] = sum;459; }460define void @reduc_double_invariant_store(ptr %dst, ptr %other_dst, ptr readonly %src) {461; CHECK-LABEL: define void @reduc_double_invariant_store(462; CHECK-SAME: ptr [[DST:%.*]], ptr [[OTHER_DST:%.*]], ptr readonly [[SRC:%.*]]) {463; CHECK-NEXT: [[ENTRY:.*]]:464; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42465; CHECK-NEXT: [[GEP_OTHER_DST:%.*]] = getelementptr inbounds i32, ptr [[OTHER_DST]], i64 42466; CHECK-NEXT: br label %[[FOR_BODY:.*]]467; CHECK: [[FOR_BODY]]:468; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]469; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]470; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]471; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4472; CHECK-NEXT: [[SUM_1:%.*]] = add nsw i32 [[TMP0]], [[SUM]]473; CHECK-NEXT: store i32 [[SUM_1]], ptr [[GEP_DST]], align 4474; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[IV]], 1475; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]476; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4477; CHECK-NEXT: [[SUM_2]] = add nsw i32 [[TMP2]], [[SUM_1]]478; CHECK-NEXT: store i32 [[SUM_2]], ptr [[GEP_OTHER_DST]], align 4479; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2480; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], 1000481; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[EXIT:.*]]482; CHECK: [[EXIT]]:483; CHECK-NEXT: ret void484;485entry:486 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42487 %gep.other_dst = getelementptr inbounds i32, ptr %other_dst, i64 42488 br label %for.body489 490for.body:491 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]492 %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.body ]493 %arrayidx = getelementptr inbounds i32, ptr %src, i64 %iv494 %0 = load i32, ptr %arrayidx, align 4495 %sum.1 = add nsw i32 %0, %sum496 store i32 %sum.1, ptr %gep.dst, align 4497 %1 = or disjoint i64 %iv, 1498 %arrayidx4 = getelementptr inbounds i32, ptr %src, i64 %1499 %2 = load i32, ptr %arrayidx4, align 4500 %sum.2 = add nsw i32 %2, %sum.1501 store i32 %sum.2, ptr %gep.other_dst, align 4502 %iv.next = add nuw nsw i64 %iv, 2503 %cmp = icmp slt i64 %iv.next, 1000504 br i1 %cmp, label %for.body, label %exit505 506exit:507 ret void508}509 510; int sum = 0;511; for(int i=0; i < 1000; i+=2) {512; sum += src[i];513; if (src[i+1] > 0)514; dst[42] = sum;515; sum += src[i+1];516; dst[42] = sum;517; }518define void @reduc_store_middle_store_predicated(ptr %dst, ptr readonly %src) {519; CHECK-LABEL: define void @reduc_store_middle_store_predicated(520; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {521; CHECK-NEXT: [[ENTRY:.*:]]522; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 42523; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]524; CHECK: [[VECTOR_MEMCHECK]]:525; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 172526; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4000527; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_DST]], [[SCEVGEP1]]528; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]529; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]530; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]531; CHECK: [[VECTOR_PH]]:532; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]533; CHECK: [[VECTOR_BODY]]:534; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]535; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 2, i64 4, i64 6>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]536; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[VECTOR_BODY]] ]537; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2538; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0539; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2540; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4541; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6542; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP0]]543; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]544; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP2]]545; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP3]]546; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META23:![0-9]+]]547; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META23]]548; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META23]]549; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META23]]550; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i32 0551; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 1552; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 2553; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i32 3554; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[TMP15]], [[VEC_PHI]]555; CHECK-NEXT: [[TMP17:%.*]] = or disjoint <4 x i64> [[VEC_IND]], splat (i64 1)556; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP17]], i32 0557; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP17]], i32 1558; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP17]], i32 2559; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP17]], i32 3560; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP18]]561; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP20]]562; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP22]]563; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP24]]564; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP37]], align 4, !alias.scope [[META23]]565; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP23]], align 4, !alias.scope [[META23]]566; CHECK-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP38]], align 4, !alias.scope [[META23]]567; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP25]], align 4, !alias.scope [[META23]]568; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP26]], i32 0569; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP27]], i32 1570; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 2571; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 3572; CHECK-NEXT: [[TMP34]] = add <4 x i32> [[TMP33]], [[TMP16]]573; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4574; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 8)575; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500576; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]577; CHECK: [[MIDDLE_BLOCK]]:578; CHECK-NEXT: [[TMP36:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP34]])579; CHECK-NEXT: store i32 [[TMP36]], ptr [[GEP_DST]], align 4, !alias.scope [[META27:![0-9]+]], !noalias [[META23]]580; CHECK-NEXT: br [[EXIT:label %.*]]581; CHECK: [[SCALAR_PH]]:582;583entry:584 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42585 br label %for.body586 587for.body: ; preds = %latch, %entry588 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]589 %sum = phi i32 [ 0, %entry ], [ %sum.2, %latch ]590 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv591 %0 = load i32, ptr %gep.src, align 4592 %sum.1 = add nsw i32 %0, %sum593 %cmp = icmp sgt i32 %0, 0594 br i1 %cmp, label %predicated, label %latch595 596predicated: ; preds = %for.body597 store i32 %sum.1, ptr %gep.dst, align 4598 br label %latch599 600latch: ; preds = %predicated, %for.body601 %1 = or disjoint i64 %iv, 1602 %gep.src.1 = getelementptr inbounds i32, ptr %src, i64 %1603 %2 = load i32, ptr %gep.src.1, align 4604 %sum.2 = add nsw i32 %2, %sum.1605 store i32 %sum.2, ptr %gep.dst, align 4606 %iv.next = add nuw nsw i64 %iv, 2607 %cmp.1 = icmp slt i64 %iv.next, 1000608 br i1 %cmp.1, label %for.body, label %exit609 610exit: ; preds = %latch611 ret void612}613 614; int sum = 0;615; for(int i=0; i < 1000; i+=2) {616; sum += src[i];617; dst[42] = sum;618; sum += src[i+1];619; if (src[i+1] > 0)620; dst[42] = sum;621; }622define void @reduc_store_final_store_predicated(ptr %dst, ptr readonly %src) {623; CHECK-LABEL: define void @reduc_store_final_store_predicated(624; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {625; CHECK-NEXT: [[ENTRY:.*]]:626; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42627; CHECK-NEXT: br label %[[FOR_BODY:.*]]628; CHECK: [[FOR_BODY]]:629; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]630; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_1:%.*]], %[[LATCH]] ]631; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]632; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4633; CHECK-NEXT: [[SUM_1]] = add nsw i32 [[TMP0]], [[SUM]]634; CHECK-NEXT: store i32 [[SUM_1]], ptr [[GEP_DST]], align 4635; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[IV]], 1636; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]637; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_SRC_1]], align 4638; CHECK-NEXT: [[SUM_2:%.*]] = add nsw i32 [[TMP2]], [[SUM_1]]639; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP2]], 0640; CHECK-NEXT: br i1 [[CMP1]], label %[[PREDICATED:.*]], label %[[LATCH]]641; CHECK: [[PREDICATED]]:642; CHECK-NEXT: store i32 [[SUM_2]], ptr [[GEP_DST]], align 4643; CHECK-NEXT: br label %[[LATCH]]644; CHECK: [[LATCH]]:645; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2646; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], 1000647; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[EXIT:.*]]648; CHECK: [[EXIT]]:649; CHECK-NEXT: ret void650;651entry:652 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42653 br label %for.body654 655for.body: ; preds = %latch, %entry656 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]657 %sum = phi i32 [ 0, %entry ], [ %sum.1, %latch ]658 %arrayidx = getelementptr inbounds i32, ptr %src, i64 %iv659 %0 = load i32, ptr %arrayidx, align 4660 %sum.1 = add nsw i32 %0, %sum661 store i32 %sum.1, ptr %gep.dst, align 4662 %1 = or disjoint i64 %iv, 1663 %gep.src.1 = getelementptr inbounds i32, ptr %src, i64 %1664 %2 = load i32, ptr %gep.src.1, align 4665 %sum.2 = add nsw i32 %2, %sum.1666 %cmp1 = icmp sgt i32 %2, 0667 br i1 %cmp1, label %predicated, label %latch668 669predicated: ; preds = %for.body670 store i32 %sum.2, ptr %gep.dst, align 4671 br label %latch672 673latch: ; preds = %predicated, %for.body674 %iv.next = add nuw nsw i64 %iv, 2675 %cmp = icmp slt i64 %iv.next, 1000676 br i1 %cmp, label %for.body, label %exit677 678exit: ; preds = %latch679 ret void680}681 682; Final reduction value is overwritten inside loop683;684; for(int i=0; i < 1000; i++) {685; sum += src[i];686; dst[42] = sum;687; dst[42] = 0;688; }689define void @reduc_store_final_store_overwritten(ptr %dst, ptr readonly %src) {690; CHECK-LABEL: define void @reduc_store_final_store_overwritten(691; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {692; CHECK-NEXT: [[ENTRY:.*]]:693; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42694; CHECK-NEXT: br label %[[FOR_BODY:.*]]695; CHECK: [[FOR_BODY]]:696; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]697; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]698; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]699; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP_SRC]], align 4700; CHECK-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[TMP0]]701; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_DST]], align 4702; CHECK-NEXT: store i32 0, ptr [[GEP_DST]], align 4703; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1704; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000705; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]706; CHECK: [[EXIT]]:707; CHECK-NEXT: ret void708;709entry:710 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42711 br label %for.body712 713for.body:714 %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]715 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]716 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv717 %0 = load i32, ptr %gep.src, align 4718 %add = add nsw i32 %sum, %0719 store i32 %add, ptr %gep.dst, align 4720 store i32 0, ptr %gep.dst, align 4721 %iv.next = add nuw nsw i64 %iv, 1722 %exitcond = icmp eq i64 %iv.next, 1000723 br i1 %exitcond, label %exit, label %for.body724 725exit:726 ret void727}728 729; Final value used outside of loop does not prevent vectorization730;731; int sum = 0;732; for(int i=0; i < 1000; i++) {733; sum += src[i];734; dst[42] = sum;735; }736; dst[43] = sum;737define void @reduc_store_inoutside(ptr %dst, ptr readonly %src) {738; CHECK-LABEL: define void @reduc_store_inoutside(739; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {740; CHECK-NEXT: [[ENTRY:.*:]]741; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42742; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]743; CHECK: [[VECTOR_MEMCHECK]]:744; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 172745; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4000746; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_DST]], [[SCEVGEP1]]747; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]748; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]749; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]750; CHECK: [[VECTOR_PH]]:751; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]752; CHECK: [[VECTOR_BODY]]:753; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]754; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]755; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]756; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META30:![0-9]+]]757; CHECK-NEXT: [[TMP1]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]758; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4759; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000760; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]761; CHECK: [[MIDDLE_BLOCK]]:762; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])763; CHECK-NEXT: store i32 [[TMP3]], ptr [[GEP_DST]], align 4, !alias.scope [[META34:![0-9]+]], !noalias [[META30]]764; CHECK-NEXT: br [[EXIT:label %.*]]765; CHECK: [[SCALAR_PH]]:766;767entry:768 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42769 br label %for.body770 771for.body:772 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]773 %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]774 %arrayidx = getelementptr inbounds i32, ptr %src, i64 %iv775 %0 = load i32, ptr %arrayidx, align 4776 %sum.1 = add nsw i32 %0, %sum777 store i32 %sum.1, ptr %gep.dst, align 4778 %iv.next = add nuw nsw i64 %iv, 1779 %exitcond = icmp eq i64 %iv.next, 1000780 br i1 %exitcond, label %exit, label %for.body781 782exit:783 %sum.lcssa = phi i32 [ %sum.1, %for.body ]784 %gep.dst.1 = getelementptr inbounds i32, ptr %dst, i64 43785 store i32 %sum.lcssa, ptr %gep.dst.1, align 4786 ret void787}788 789; Test for PR55540.790define void @test_drop_poison_generating_dead_recipe(ptr %dst) {791; CHECK-LABEL: define void @test_drop_poison_generating_dead_recipe(792; CHECK-SAME: ptr [[DST:%.*]]) {793; CHECK-NEXT: [[ENTRY:.*:]]794; CHECK-NEXT: br label %[[VECTOR_PH:.*]]795; CHECK: [[VECTOR_PH]]:796; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]797; CHECK: [[VECTOR_BODY]]:798; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]799; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ]800; CHECK-NEXT: [[TMP0]] = add <4 x i64> [[VEC_PHI]], splat (i64 -31364)801; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4802; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 360803; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]804; CHECK: [[MIDDLE_BLOCK]]:805; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP0]])806; CHECK-NEXT: store i64 [[TMP2]], ptr [[DST]], align 8807; CHECK-NEXT: br label %[[SCALAR_PH:.*]]808; CHECK: [[SCALAR_PH]]:809;810entry:811 br label %body812 813body:814 %red = phi i64 [ 0, %entry ], [ %red.next, %body ]815 %iv = phi i32 [ 2, %entry ], [ %iv.next, %body ]816 %add.1 = add nuw i64 %red, -23523817 store i64 %add.1, ptr %dst, align 8818 %red.next = add nuw i64 %red, -31364819 store i64 %red.next, ptr %dst, align 8820 %iv.next = add nuw nsw i32 %iv, 1821 %ec = icmp ugt i32 %iv, 363822 br i1 %ec, label %exit, label %body823 824exit:825 ret void826}827 828define void @reduc_store_invariant_addr_not_hoisted(ptr %dst, ptr readonly %src) {829; CHECK-LABEL: define void @reduc_store_invariant_addr_not_hoisted(830; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {831; CHECK-NEXT: [[ENTRY:.*]]:832; CHECK-NEXT: br label %[[FOR_BODY:.*]]833; CHECK: [[FOR_BODY]]:834; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]835; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]836; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]837; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP_SRC]], align 4838; CHECK-NEXT: [[ADD]] = add nsw i32 [[SUM]], [[TMP0]]839; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42840; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP_DST]], align 4841; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1842; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000843; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[FOR_BODY]]844; CHECK: [[EXIT]]:845; CHECK-NEXT: ret void846;847entry:848 br label %for.body849 850for.body:851 %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]852 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]853 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv854 %0 = load i32, ptr %gep.src, align 4855 %add = add nsw i32 %sum, %0856 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42857 store i32 %add, ptr %gep.dst, align 4858 %iv.next = add nuw nsw i64 %iv, 1859 %exitcond = icmp eq i64 %iv.next, 1000860 br i1 %exitcond, label %exit, label %for.body861 862exit:863 ret void864}865 866; Make sure we can vectorize loop with a non-reduction value stored to an867; invariant address that is calculated inside loop.868define i32 @non_reduc_store_invariant_addr_not_hoisted(ptr %dst, ptr readonly %src) {869; CHECK-LABEL: define i32 @non_reduc_store_invariant_addr_not_hoisted(870; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {871; CHECK-NEXT: [[ENTRY:.*:]]872; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]873; CHECK: [[VECTOR_MEMCHECK]]:874; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[DST]], i64 168875; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 172876; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 4000877; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]]878; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]879; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]880; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]881; CHECK: [[VECTOR_PH]]:882; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 42883; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]884; CHECK: [[VECTOR_BODY]]:885; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]886; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]887; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]888; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META39:![0-9]+]]889; CHECK-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]890; CHECK-NEXT: store i32 0, ptr [[TMP0]], align 4, !alias.scope [[META42:![0-9]+]], !noalias [[META39]]891; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4892; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000893; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]894; CHECK: [[MIDDLE_BLOCK]]:895; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])896; CHECK-NEXT: br [[EXIT:label %.*]]897; CHECK: [[SCALAR_PH]]:898;899entry:900 br label %for.body901 902for.body: ; preds = %for.body, %entry903 %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]904 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]905 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv906 %0 = load i32, ptr %gep.src, align 4907 %add = add nsw i32 %sum, %0908 %gep.dst = getelementptr inbounds i32, ptr %dst, i64 42909 store i32 0, ptr %gep.dst, align 4910 %iv.next = add nuw nsw i64 %iv, 1911 %exitcond = icmp eq i64 %iv.next, 1000912 br i1 %exitcond, label %exit, label %for.body913 914exit: ; preds = %for.body915 %add.lcssa = phi i32 [ %add, %for.body ]916 ret i32 %add.lcssa917}918 919; Make sure that if there are several reductions in the loop, the order of invariant stores sank outside of the loop is preserved920; See https://github.com/llvm/llvm-project/issues/64047921define void @reduc_add_mul_store_same_ptr(ptr %dst, ptr readonly %src) {922; CHECK-LABEL: define void @reduc_add_mul_store_same_ptr(923; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {924; CHECK-NEXT: [[ENTRY:.*:]]925; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]926; CHECK: [[VECTOR_MEMCHECK]]:927; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4928; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4000929; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]930; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]931; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]932; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]933; CHECK: [[VECTOR_PH]]:934; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]935; CHECK: [[VECTOR_BODY]]:936; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]937; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]938; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 1), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]939; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]940; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META46:![0-9]+]]941; CHECK-NEXT: [[TMP1]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]942; CHECK-NEXT: [[TMP2]] = mul <4 x i32> [[VEC_PHI2]], [[WIDE_LOAD]]943; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4944; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000945; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]946; CHECK: [[MIDDLE_BLOCK]]:947; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])948; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP2]])949; CHECK-NEXT: store i32 [[TMP4]], ptr [[DST]], align 4, !alias.scope [[META50:![0-9]+]], !noalias [[META46]]950; CHECK-NEXT: store i32 [[TMP5]], ptr [[DST]], align 4, !alias.scope [[META50]], !noalias [[META46]]951; CHECK-NEXT: br [[EXIT:label %.*]]952; CHECK: [[SCALAR_PH]]:953;954entry:955 br label %for.body956 957for.body:958 %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.body ]959 %mul = phi i32 [ 1, %entry ], [ %mul.next, %for.body ]960 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]961 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv962 %0 = load i32, ptr %gep.src, align 4963 %sum.next = add nsw i32 %sum, %0964 store i32 %sum.next, ptr %dst, align 4965 %mul.next = mul nsw i32 %mul, %0966 store i32 %mul.next, ptr %dst, align 4967 %iv.next = add nuw nsw i64 %iv, 1968 %exitcond = icmp eq i64 %iv.next, 1000969 br i1 %exitcond, label %exit, label %for.body970 971exit:972 ret void973}974 975define void @reduc_mul_add_store_same_ptr(ptr %dst, ptr readonly %src) {976; CHECK-LABEL: define void @reduc_mul_add_store_same_ptr(977; CHECK-SAME: ptr [[DST:%.*]], ptr readonly [[SRC:%.*]]) {978; CHECK-NEXT: [[ENTRY:.*:]]979; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]980; CHECK: [[VECTOR_MEMCHECK]]:981; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4982; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4000983; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]984; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]985; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]986; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]987; CHECK: [[VECTOR_PH]]:988; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]989; CHECK: [[VECTOR_BODY]]:990; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]991; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]992; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 1), %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]993; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]994; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META53:![0-9]+]]995; CHECK-NEXT: [[TMP1]] = mul <4 x i32> [[VEC_PHI2]], [[WIDE_LOAD]]996; CHECK-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]997; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4998; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000999; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP56:![0-9]+]]1000; CHECK: [[MIDDLE_BLOCK]]:1001; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])1002; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1]])1003; CHECK-NEXT: store i32 [[TMP5]], ptr [[DST]], align 4, !alias.scope [[META57:![0-9]+]], !noalias [[META53]]1004; CHECK-NEXT: store i32 [[TMP4]], ptr [[DST]], align 4, !alias.scope [[META57]], !noalias [[META53]]1005; CHECK-NEXT: br [[EXIT:label %.*]]1006; CHECK: [[SCALAR_PH]]:1007;1008entry:1009 br label %for.body1010 1011for.body:1012 %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.body ]1013 %mul = phi i32 [ 1, %entry ], [ %mul.next, %for.body ]1014 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1015 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv1016 %0 = load i32, ptr %gep.src, align 41017 %mul.next = mul nsw i32 %mul, %01018 store i32 %mul.next, ptr %dst, align 41019 %sum.next = add nsw i32 %sum, %01020 store i32 %sum.next, ptr %dst, align 41021 %iv.next = add nuw nsw i64 %iv, 11022 %exitcond = icmp eq i64 %iv.next, 10001023 br i1 %exitcond, label %exit, label %for.body1024 1025exit:1026 ret void1027}1028 1029; Same as above but storing is done to two different pointers and they can be aliased1030define void @reduc_add_mul_store_different_ptr(ptr %dst1, ptr %dst2, ptr readonly %src) {1031; CHECK-LABEL: define void @reduc_add_mul_store_different_ptr(1032; CHECK-SAME: ptr [[DST1:%.*]], ptr [[DST2:%.*]], ptr readonly [[SRC:%.*]]) {1033; CHECK-NEXT: [[ENTRY:.*:]]1034; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]1035; CHECK: [[VECTOR_MEMCHECK]]:1036; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST1]], i64 41037; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST2]], i64 41038; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 40001039; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST1]], [[SCEVGEP1]]1040; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST2]], [[SCEVGEP]]1041; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]1042; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST1]], [[SCEVGEP2]]1043; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]1044; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]1045; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]1046; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[DST2]], [[SCEVGEP2]]1047; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]1048; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]1049; CHECK-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]1050; CHECK-NEXT: br i1 [[CONFLICT_RDX9]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1051; CHECK: [[VECTOR_PH]]:1052; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1053; CHECK: [[VECTOR_BODY]]:1054; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1055; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]1056; CHECK-NEXT: [[VEC_PHI10:%.*]] = phi <4 x i32> [ splat (i32 1), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]1057; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]1058; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META60:![0-9]+]]1059; CHECK-NEXT: [[TMP1]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]1060; CHECK-NEXT: [[TMP2]] = mul <4 x i32> [[VEC_PHI10]], [[WIDE_LOAD]]1061; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41062; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10001063; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP63:![0-9]+]]1064; CHECK: [[MIDDLE_BLOCK]]:1065; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])1066; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP2]])1067; CHECK-NEXT: store i32 [[TMP4]], ptr [[DST1]], align 4, !alias.scope [[META64:![0-9]+]], !noalias [[META66:![0-9]+]]1068; CHECK-NEXT: store i32 [[TMP5]], ptr [[DST2]], align 4, !alias.scope [[META68:![0-9]+]], !noalias [[META60]]1069; CHECK-NEXT: br [[EXIT:label %.*]]1070; CHECK: [[SCALAR_PH]]:1071;1072entry:1073 br label %for.body1074 1075for.body:1076 %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.body ]1077 %mul = phi i32 [ 1, %entry ], [ %mul.next, %for.body ]1078 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1079 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv1080 %0 = load i32, ptr %gep.src, align 41081 %sum.next = add nsw i32 %sum, %01082 store i32 %sum.next, ptr %dst1, align 41083 %mul.next = mul nsw i32 %mul, %01084 store i32 %mul.next, ptr %dst2, align 41085 %iv.next = add nuw nsw i64 %iv, 11086 %exitcond = icmp eq i64 %iv.next, 10001087 br i1 %exitcond, label %exit, label %for.body1088 1089exit:1090 ret void1091}1092 1093define void @reduc_mul_add_store_different_ptr(ptr %dst1, ptr %dst2, ptr readonly %src) {1094; CHECK-LABEL: define void @reduc_mul_add_store_different_ptr(1095; CHECK-SAME: ptr [[DST1:%.*]], ptr [[DST2:%.*]], ptr readonly [[SRC:%.*]]) {1096; CHECK-NEXT: [[ENTRY:.*:]]1097; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]1098; CHECK: [[VECTOR_MEMCHECK]]:1099; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST1]], i64 41100; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST2]], i64 41101; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 40001102; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST1]], [[SCEVGEP1]]1103; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST2]], [[SCEVGEP]]1104; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]1105; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST1]], [[SCEVGEP2]]1106; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]1107; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]1108; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]1109; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[DST2]], [[SCEVGEP2]]1110; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]1111; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]1112; CHECK-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]1113; CHECK-NEXT: br i1 [[CONFLICT_RDX9]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1114; CHECK: [[VECTOR_PH]]:1115; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1116; CHECK: [[VECTOR_BODY]]:1117; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1118; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]1119; CHECK-NEXT: [[VEC_PHI10:%.*]] = phi <4 x i32> [ splat (i32 1), %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]1120; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]1121; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META70:![0-9]+]]1122; CHECK-NEXT: [[TMP1]] = mul <4 x i32> [[VEC_PHI10]], [[WIDE_LOAD]]1123; CHECK-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]1124; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41125; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10001126; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP73:![0-9]+]]1127; CHECK: [[MIDDLE_BLOCK]]:1128; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])1129; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1]])1130; CHECK-NEXT: store i32 [[TMP5]], ptr [[DST1]], align 4, !alias.scope [[META74:![0-9]+]], !noalias [[META76:![0-9]+]]1131; CHECK-NEXT: store i32 [[TMP4]], ptr [[DST2]], align 4, !alias.scope [[META78:![0-9]+]], !noalias [[META70]]1132; CHECK-NEXT: br [[EXIT:label %.*]]1133; CHECK: [[SCALAR_PH]]:1134;1135entry:1136 br label %for.body1137 1138for.body:1139 %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.body ]1140 %mul = phi i32 [ 1, %entry ], [ %mul.next, %for.body ]1141 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1142 %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv1143 %0 = load i32, ptr %gep.src, align 41144 %mul.next = mul nsw i32 %mul, %01145 store i32 %mul.next, ptr %dst1, align 41146 %sum.next = add nsw i32 %sum, %01147 store i32 %sum.next, ptr %dst2, align 41148 %iv.next = add nuw nsw i64 %iv, 11149 %exitcond = icmp eq i64 %iv.next, 10001150 br i1 %exitcond, label %exit, label %for.body1151 1152exit:1153 ret void1154}1155