780 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 62; RUN: opt -p loop-vectorize -S %s | FileCheck %s3 4target triple = "arm64-apple-macosx15.0.0"5 6define void @replicating_load_used_as_store_addr(ptr noalias %A) {7; CHECK-LABEL: define void @replicating_load_used_as_store_addr(8; CHECK-SAME: ptr noalias [[A:%.*]]) {9; CHECK-NEXT: [[ENTRY:.*:]]10; CHECK-NEXT: br label %[[VECTOR_PH:.*]]11; CHECK: [[VECTOR_PH]]:12; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]13; CHECK: [[VECTOR_BODY]]:14; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]15; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 116; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 117; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP0]], 118; CHECK-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[A]], i64 [[INDEX]]19; CHECK-NEXT: [[TMP4:%.*]] = getelementptr ptr, ptr [[A]], i64 [[TMP0]]20; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP3]], align 821; CHECK-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP4]], align 822; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP1]] to i3223; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP2]] to i3224; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP5]], align 425; CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP6]], align 426; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 227; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10028; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]29; CHECK: [[MIDDLE_BLOCK]]:30; CHECK-NEXT: br label %[[SCALAR_PH:.*]]31; CHECK: [[SCALAR_PH]]:32;33entry:34 br label %loop35 36loop:37 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]38 %iv.next = add i64 %iv, 139 %gep.A = getelementptr ptr, ptr %A, i64 %iv40 %l.p = load ptr, ptr %gep.A, align 841 %iv.trunc = trunc i64 %iv.next to i3242 store i32 %iv.trunc, ptr %l.p, align 443 %ec = icmp eq i64 %iv, 10044 br i1 %ec, label %exit, label %loop45 46exit:47 ret void48}49 50define void @replicating_load_used_as_store_addr_2(ptr noalias %invar.dst, ptr noalias %invar.src, ptr noalias %src) {51; CHECK-LABEL: define void @replicating_load_used_as_store_addr_2(52; CHECK-SAME: ptr noalias [[INVAR_DST:%.*]], ptr noalias [[INVAR_SRC:%.*]], ptr noalias [[SRC:%.*]]) {53; CHECK-NEXT: [[ENTRY:.*:]]54; CHECK-NEXT: br label %[[VECTOR_PH:.*]]55; CHECK: [[VECTOR_PH]]:56; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]57; CHECK: [[VECTOR_BODY]]:58; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]59; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[INVAR_SRC]], align 460; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i6461; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i128, ptr [[SRC]], i64 [[TMP1]]62; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 463; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 12364; CHECK-NEXT: store i32 [[TMP4]], ptr [[INVAR_DST]], align 865; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 266; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10067; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]68; CHECK: [[MIDDLE_BLOCK]]:69; CHECK-NEXT: br label %[[EXIT:.*]]70; CHECK: [[EXIT]]:71; CHECK-NEXT: ret void72;73entry:74 br label %loop75 76loop:77 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]78 %l.offset = load i32, ptr %invar.src, align 479 %offset.ext = sext i32 %l.offset to i6480 %gep.src = getelementptr i128, ptr %src, i64 %offset.ext81 %l.v = load i32, ptr %gep.src, align 482 %add = add i32 %l.v, 12383 store i32 %add, ptr %invar.dst, align 884 %iv.next = add i64 %iv, 185 %exitcond41.not = icmp eq i64 %iv.next, 10086 br i1 %exitcond41.not, label %exit, label %loop87 88exit:89 ret void90}91 92 93define void @replicating_load_used_as_store_addr_3(ptr noalias %src, ptr noalias %dst, ptr noalias %invar.dst, i8 %x) {94; CHECK-LABEL: define void @replicating_load_used_as_store_addr_3(95; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], ptr noalias [[INVAR_DST:%.*]], i8 [[X:%.*]]) {96; CHECK-NEXT: [[ENTRY:.*:]]97; CHECK-NEXT: br label %[[VECTOR_PH:.*]]98; CHECK: [[VECTOR_PH]]:99; CHECK-NEXT: [[TMP0:%.*]] = xor i8 [[X]], 10100; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i64101; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]102; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]103; CHECK: [[VECTOR_BODY]]:104; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]105; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 1106; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i32107; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP4]], 111108; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP4]] to i64109; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]110; CHECK-NEXT: store i8 0, ptr [[TMP7]], align 1111; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP5]] to i8112; CHECK-NEXT: store i8 [[TMP8]], ptr [[INVAR_DST]], align 1113; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2114; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100115; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]116; CHECK: [[MIDDLE_BLOCK]]:117; CHECK-NEXT: br label %[[EXIT:.*]]118; CHECK: [[EXIT]]:119; CHECK-NEXT: ret void120;121entry:122 br label %loop123 124loop:125 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]126 %xor = xor i8 %x, 10127 %ext = zext i8 %xor to i64128 %gep.src = getelementptr i8, ptr %src, i64 %ext129 %l = load i8, ptr %gep.src, align 1130 %l.ext = zext i8 %l to i32131 %xor.2 = xor i32 %l.ext, 111132 %idx2.ext = zext i32 %l.ext to i64133 %gep.dst = getelementptr i8, ptr %dst, i64 %idx2.ext134 store i8 0, ptr %gep.dst, align 1135 %xor.2.trunc = trunc i32 %xor.2 to i8136 store i8 %xor.2.trunc, ptr %invar.dst, align 1137 %iv.next = add i64 %iv, 1138 %ec = icmp eq i64 %iv.next, 100139 br i1 %ec, label %exit, label %loop140 141exit:142 ret void143}144 145define void @uniform_gep_for_replicating_gep(ptr %dst) {146; CHECK-LABEL: define void @uniform_gep_for_replicating_gep(147; CHECK-SAME: ptr [[DST:%.*]]) {148; CHECK-NEXT: [[ENTRY:.*:]]149; CHECK-NEXT: br label %[[VECTOR_PH:.*]]150; CHECK: [[VECTOR_PH]]:151; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]152; CHECK: [[VECTOR_BODY]]:153; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]154; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]155; CHECK-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)156; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2157; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <2 x i32> [[VEC_IND]], zeroinitializer158; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i32> [[STEP_ADD]], zeroinitializer159; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[INDEX]], 1160; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP2]], 1161; CHECK-NEXT: [[TMP11:%.*]] = zext <2 x i1> [[TMP5]] to <2 x i8>162; CHECK-NEXT: [[TMP6:%.*]] = zext <2 x i1> [[TMP3]] to <2 x i8>163; CHECK-NEXT: [[TMP14:%.*]] = zext i32 [[TMP8]] to i64164; CHECK-NEXT: [[TMP15:%.*]] = zext i32 [[TMP9]] to i64165; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP14]]166; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP15]]167; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i8> [[TMP11]], i32 1168; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1169; CHECK-NEXT: store i8 [[TMP22]], ptr [[TMP18]], align 1170; CHECK-NEXT: store i8 [[TMP12]], ptr [[TMP19]], align 1171; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4172; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2)173; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128174; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]175; CHECK: [[MIDDLE_BLOCK]]:176; CHECK-NEXT: br label %[[SCALAR_PH:.*]]177; CHECK: [[SCALAR_PH]]:178;179entry:180 br label %loop181 182loop:183 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]184 %c = icmp eq i32 %iv, 0185 %shift = lshr i32 %iv, 1186 %ext = zext i1 %c to i8187 %ext.shift = zext i32 %shift to i64188 %gep = getelementptr i64, ptr %dst, i64 %ext.shift189 store i8 %ext, ptr %gep, align 1190 %iv.next = add i32 %iv, 1191 %ec = icmp eq i32 %iv, 128192 br i1 %ec, label %exit, label %loop193 194exit:195 ret void196}197 198define void @test_load_gep_widen_induction(ptr noalias %dst, ptr noalias %dst2) #0 {199; CHECK-LABEL: define void @test_load_gep_widen_induction(200; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[DST2:%.*]]) #[[ATTR0:[0-9]+]] {201; CHECK-NEXT: [[ENTRY:.*:]]202; CHECK-NEXT: br label %[[VECTOR_PH:.*]]203; CHECK: [[VECTOR_PH]]:204; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]205; CHECK: [[VECTOR_BODY]]:206; CHECK-NEXT: [[OFFSET_IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]207; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]208; CHECK-NEXT: [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 2)209; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add <2 x i64> [[STEP_ADD]], splat (i64 2)210; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add <2 x i64> [[STEP_ADD_2]], splat (i64 2)211; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i128, ptr [[DST]], <2 x i64> [[VEC_IND]]212; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x ptr> [[TMP0]], i32 0213; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x ptr> [[TMP0]], i32 1214; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i128, ptr [[DST]], <2 x i64> [[STEP_ADD]]215; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x ptr> [[TMP1]], i32 0216; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x ptr> [[TMP1]], i32 1217; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i128, ptr [[DST]], <2 x i64> [[STEP_ADD_2]]218; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x ptr> [[TMP2]], i32 0219; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x ptr> [[TMP2]], i32 1220; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i128, ptr [[DST]], <2 x i64> [[STEP_ADD_3]]221; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x ptr> [[TMP3]], i32 0222; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x ptr> [[TMP3]], i32 1223; CHECK-NEXT: store ptr null, ptr [[TMP5]], align 8224; CHECK-NEXT: store ptr null, ptr [[TMP6]], align 8225; CHECK-NEXT: store ptr null, ptr [[TMP7]], align 8226; CHECK-NEXT: store ptr null, ptr [[TMP8]], align 8227; CHECK-NEXT: store ptr null, ptr [[TMP9]], align 8228; CHECK-NEXT: store ptr null, ptr [[TMP10]], align 8229; CHECK-NEXT: store ptr null, ptr [[TMP11]], align 8230; CHECK-NEXT: store ptr null, ptr [[TMP17]], align 8231; CHECK-NEXT: [[TMP12:%.*]] = getelementptr ptr, ptr [[DST2]], i64 [[OFFSET_IDX]]232; CHECK-NEXT: [[TMP13:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 2233; CHECK-NEXT: [[TMP14:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 4234; CHECK-NEXT: [[TMP15:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 6235; CHECK-NEXT: store <2 x ptr> [[TMP0]], ptr [[TMP12]], align 8236; CHECK-NEXT: store <2 x ptr> [[TMP1]], ptr [[TMP13]], align 8237; CHECK-NEXT: store <2 x ptr> [[TMP2]], ptr [[TMP14]], align 8238; CHECK-NEXT: store <2 x ptr> [[TMP3]], ptr [[TMP15]], align 8239; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[OFFSET_IDX]], 8240; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD_3]], splat (i64 2)241; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96242; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]243; CHECK: [[MIDDLE_BLOCK]]:244; CHECK-NEXT: br label %[[SCALAR_PH:.*]]245; CHECK: [[SCALAR_PH]]:246;247entry:248 br label %loop249 250loop: ; preds = %loop, %entry251 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]252 %gep.dst.iv = getelementptr i128, ptr %dst, i64 %iv253 %iv.next = add i64 %iv, 1254 store ptr null, ptr %gep.dst.iv, align 8255 %gep.dst2.iv = getelementptr ptr, ptr %dst2, i64 %iv256 store ptr %gep.dst.iv, ptr %gep.dst2.iv257 %exitcond.not = icmp eq i64 %iv.next, 100258 br i1 %exitcond.not, label %exit, label %loop259 260exit:261 ret void262}263 264 265define ptr @replicating_store_in_conditional_latch(ptr %p, i32 %n) #0 {266; CHECK-LABEL: define ptr @replicating_store_in_conditional_latch(267; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {268; CHECK-NEXT: [[ENTRY:.*:]]269; CHECK-NEXT: [[TMP0:%.*]] = sub i32 0, [[N]]270; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64271; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 1272; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1273; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], 4274; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]275; CHECK: [[VECTOR_PH]]:276; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4277; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0278; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 4, i64 [[N_MOD_VF]]279; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP5]]280; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32281; CHECK-NEXT: [[TMP6:%.*]] = mul i32 [[DOTCAST]], -2282; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[N_VEC]], 48283; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]]284; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]285; CHECK: [[VECTOR_BODY]]:286; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]287; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 48288; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 48289; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 96290; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 144291; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[OFFSET_IDX]]292; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]293; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP10]]294; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]295; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 24296; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 24297; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 24298; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 24299; CHECK-NEXT: store ptr null, ptr [[TMP12]], align 8300; CHECK-NEXT: store ptr null, ptr [[TMP13]], align 8301; CHECK-NEXT: store ptr null, ptr [[TMP14]], align 8302; CHECK-NEXT: store ptr null, ptr [[TMP15]], align 8303; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4304; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]305; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]306; CHECK: [[MIDDLE_BLOCK]]:307; CHECK-NEXT: br label %[[SCALAR_PH]]308; CHECK: [[SCALAR_PH]]:309;310entry:311 br label %loop.header312 313loop.header:314 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]315 %ptr.iv = phi ptr [ %p, %entry ], [ %ptr.iv.next, %loop.latch ]316 %gep.ptr.iv = getelementptr i8, ptr %ptr.iv, i64 24317 %c = icmp eq i32 %iv, %n318 br i1 %c, label %exit, label %loop.latch319 320loop.latch:321 %iv.next = add nsw i32 %iv, -2322 store ptr null, ptr %gep.ptr.iv, align 8323 %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 48324 br label %loop.header325 326exit:327 ret ptr %gep.ptr.iv328}329 330declare void @init(ptr)331 332define void @scalar_store_cost_after_discarding_interleave_group(ptr %dst, i32 %x, ptr %src) {333; CHECK-LABEL: define void @scalar_store_cost_after_discarding_interleave_group(334; CHECK-SAME: ptr [[DST:%.*]], i32 [[X:%.*]], ptr [[SRC:%.*]]) {335; CHECK-NEXT: [[ENTRY:.*]]:336; CHECK-NEXT: [[TEMP1:%.*]] = alloca [64 x i32], align 4337; CHECK-NEXT: call void @init(ptr [[TEMP1]])338; CHECK-NEXT: br label %[[LOOP:.*]]339; CHECK: [[LOOP]]:340; CHECK-NEXT: [[TMP21:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]341; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TEMP1]], align 4342; CHECK-NEXT: [[SHR_0:%.*]] = lshr i32 [[X]], 1343; CHECK-NEXT: [[MUL_0:%.*]] = mul i32 [[X]], -171254344; CHECK-NEXT: [[SHR_1:%.*]] = lshr i32 [[MUL_0]], 1345; CHECK-NEXT: [[ADD_0:%.*]] = add i32 [[SHR_0]], [[SHR_1]]346; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP21]]347; CHECK-NEXT: store i16 0, ptr [[TMP30]], align 2348; CHECK-NEXT: [[GEP_0_1:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP21]]349; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[GEP_0_1]], i64 14350; CHECK-NEXT: store i16 0, ptr [[TMP38]], align 2351; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], 1352; CHECK-NEXT: [[SHR_2:%.*]] = lshr i32 [[ADD_1]], 1353; CHECK-NEXT: [[TMP54:%.*]] = trunc i32 [[SHR_2]] to i16354; CHECK-NEXT: [[TMP46:%.*]] = getelementptr i8, ptr [[TMP30]], i64 2355; CHECK-NEXT: store i16 [[TMP54]], ptr [[TMP46]], align 2356; CHECK-NEXT: [[SUB_0:%.*]] = sub i32 0, [[MUL_0]]357; CHECK-NEXT: [[SHR_3:%.*]] = lshr i32 [[SUB_0]], 1358; CHECK-NEXT: [[TMP70:%.*]] = trunc i32 [[SHR_3]] to i16359; CHECK-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[TMP30]], i64 12360; CHECK-NEXT: store i16 [[TMP70]], ptr [[TMP62]], align 2361; CHECK-NEXT: [[OR_0:%.*]] = or i32 [[X]], 1362; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[OR_0]], 1363; CHECK-NEXT: [[SHR_4:%.*]] = lshr i32 [[ADD_2]], 1364; CHECK-NEXT: [[TMP86:%.*]] = trunc i32 [[SHR_4]] to i16365; CHECK-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[TMP30]], i64 4366; CHECK-NEXT: store i16 [[TMP86]], ptr [[TMP78]], align 2367; CHECK-NEXT: [[GEP_0_2:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP21]]368; CHECK-NEXT: [[TMP94:%.*]] = getelementptr i8, ptr [[GEP_0_2]], i64 10369; CHECK-NEXT: store i16 0, ptr [[TMP94]], align 2370; CHECK-NEXT: [[TRUNC_3:%.*]] = trunc i32 [[TMP22]] to i16371; CHECK-NEXT: [[OR_1:%.*]] = or i16 [[TRUNC_3]], 1372; CHECK-NEXT: [[TMP113:%.*]] = add i16 [[OR_1]], 1373; CHECK-NEXT: [[TMP105:%.*]] = getelementptr i8, ptr [[TMP30]], i64 8374; CHECK-NEXT: store i16 [[TMP113]], ptr [[TMP105]], align 2375; CHECK-NEXT: [[TMP121:%.*]] = getelementptr i8, ptr [[TMP30]], i64 6376; CHECK-NEXT: store i16 0, ptr [[TMP121]], align 2377; CHECK-NEXT: [[IV_NEXT]] = add i64 [[TMP21]], 8378; CHECK-NEXT: [[EC:%.*]] = icmp ult i64 [[TMP21]], 128379; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]380; CHECK: [[EXIT]]:381; CHECK-NEXT: ret void382;383entry:384 %temp1 = alloca [64 x i32], align 4385 call void @init(ptr %temp1)386 br label %loop387 388loop:389 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]390 %1 = load i32, ptr %temp1, align 4391 %shr.0 = lshr i32 %x, 1392 %mul.0 = mul i32 %x, -171254393 %shr.1 = lshr i32 %mul.0, 1394 %add.0 = add i32 %shr.0, %shr.1395 %gep.0 = getelementptr i16, ptr %dst, i64 %iv396 store i16 0, ptr %gep.0, align 2397 %gep.0.1 = getelementptr i16, ptr %dst, i64 %iv398 %gep.14 = getelementptr i8, ptr %gep.0.1, i64 14399 store i16 0, ptr %gep.14, align 2400 %add.1 = add i32 %add.0, 1401 %shr.2 = lshr i32 %add.1, 1402 %trunc.0 = trunc i32 %shr.2 to i16403 %gep.2 = getelementptr i8, ptr %gep.0, i64 2404 store i16 %trunc.0, ptr %gep.2, align 2405 %sub.0 = sub i32 0, %mul.0406 %shr.3 = lshr i32 %sub.0, 1407 %trunc.1 = trunc i32 %shr.3 to i16408 %gep.12 = getelementptr i8, ptr %gep.0, i64 12409 store i16 %trunc.1, ptr %gep.12, align 2410 %or.0 = or i32 %x, 1411 %add.2 = add i32 %or.0, 1412 %shr.4 = lshr i32 %add.2, 1413 %trunc.2 = trunc i32 %shr.4 to i16414 %gep.4 = getelementptr i8, ptr %gep.0, i64 4415 store i16 %trunc.2, ptr %gep.4, align 2416 %gep.0.2 = getelementptr i16, ptr %dst, i64 %iv417 %gep.10 = getelementptr i8, ptr %gep.0.2, i64 10418 store i16 0, ptr %gep.10, align 2419 %trunc.3 = trunc i32 %1 to i16420 %or.1 = or i16 %trunc.3, 1421 %add.3 = add i16 %or.1, 1422 %gep.8 = getelementptr i8, ptr %gep.0, i64 8423 store i16 %add.3, ptr %gep.8, align 2424 %gep.6 = getelementptr i8, ptr %gep.0, i64 6425 store i16 0, ptr %gep.6, align 2426 %iv.next = add i64 %iv, 8427 %ec = icmp ult i64 %iv, 128428 br i1 %ec, label %loop, label %exit429 430exit:431 ret void432}433 434define void @test_prefer_vector_addressing(ptr %start, ptr %ms, ptr noalias %src) #0 {435; CHECK-LABEL: define void @test_prefer_vector_addressing(436; CHECK-SAME: ptr [[START:%.*]], ptr [[MS:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {437; CHECK-NEXT: [[ENTRY:.*:]]438; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64439; CHECK-NEXT: [[MS1:%.*]] = ptrtoint ptr [[MS]] to i64440; CHECK-NEXT: [[GEP_START:%.*]] = getelementptr i8, ptr [[START]], i64 3441; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[START2]], 3442; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[MS1]], i64 [[TMP0]])443; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[UMAX]], -3444; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[START2]]445; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 1)446; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], [[UMIN]]447; CHECK-NEXT: [[TMP4:%.*]] = udiv i64 [[TMP3]], 3448; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[UMIN]], [[TMP4]]449; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 1450; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP6]], 4451; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]452; CHECK: [[VECTOR_PH]]:453; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP6]], 4454; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP6]], [[N_MOD_VF]]455; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[N_VEC]], 3456; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[GEP_START]], i64 [[TMP7]]457; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[N_VEC]], 3458; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP9]]459; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]460; CHECK: [[VECTOR_BODY]]:461; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]462; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3463; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 3464; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 6465; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 9466; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]]467; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP11]]468; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]]469; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP13]]470; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[NEXT_GEP]], align 1, !tbaa [[LONG_LONG_TBAA12:![0-9]+]]471; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[NEXT_GEP3]], align 1, !tbaa [[LONG_LONG_TBAA12]]472; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[NEXT_GEP4]], align 1, !tbaa [[LONG_LONG_TBAA12]]473; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[NEXT_GEP5]], align 1, !tbaa [[LONG_LONG_TBAA12]]474; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]475; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP15]]476; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP16]]477; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP17]]478; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 4, !tbaa [[INT_TBAA17:![0-9]+]]479; CHECK-NEXT: store i32 0, ptr [[TMP19]], align 4, !tbaa [[INT_TBAA17]]480; CHECK-NEXT: store i32 0, ptr [[TMP20]], align 4, !tbaa [[INT_TBAA17]]481; CHECK-NEXT: store i32 0, ptr [[TMP21]], align 4, !tbaa [[INT_TBAA17]]482; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4483; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]484; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]485; CHECK: [[MIDDLE_BLOCK]]:486; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP6]], [[N_VEC]]487; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]488; CHECK: [[SCALAR_PH]]:489;490entry:491 %gep.start = getelementptr i8, ptr %start, i64 3492 br label %loop493 494loop:495 %ptr.iv = phi ptr [ %gep.start, %entry ], [ %ptr.iv.next, %loop ]496 %recur = phi ptr [ %start, %entry ], [ %ptr.iv, %loop ]497 %l = load i64, ptr %recur, align 1, !tbaa !0498 %gep.src = getelementptr i8, ptr %src, i64 %l499 store i32 0, ptr %gep.src, align 4, !tbaa !5500 %ptr.iv.next = getelementptr nusw i8, ptr %ptr.iv, i64 3501 %ec = icmp ult ptr %ptr.iv, %ms502 br i1 %ec, label %loop, label %exit503 504exit:505 ret void506}507 508define void @cost_scalar_load_of_address(ptr noalias %src, ptr %dst) {509; CHECK-LABEL: define void @cost_scalar_load_of_address(510; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr [[DST:%.*]]) {511; CHECK-NEXT: [[ENTRY:.*]]:512; CHECK-NEXT: br label %[[LOOP:.*]]513; CHECK: [[LOOP]]:514; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]515; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]516; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4517; CHECK-NEXT: [[L_EXT:%.*]] = sext i32 [[L]] to i64518; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[L_EXT]]519; CHECK-NEXT: store i32 0, ptr [[GEP_DST]], align 4520; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1521; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 8522; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]523; CHECK: [[EXIT]]:524; CHECK-NEXT: ret void525;526entry:527 br label %loop528 529loop:530 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]531 %gep.src = getelementptr i32, ptr %src, i64 %iv532 %l = load i32, ptr %gep.src, align 4533 %l.ext = sext i32 %l to i64534 %gep.dst = getelementptr i32, ptr %dst, i64 %l.ext535 store i32 0, ptr %gep.dst, align 4536 %iv.next = add i64 %iv, 1537 %ec = icmp eq i64 %iv, 8538 br i1 %ec, label %exit, label %loop539 540exit:541 ret void542}543 544%t = type { [3 x double] }545%t.2 = type { [ 64 x double ] }546 547define double @test_scalarization_cost_for_load_of_address(ptr %src.0, ptr %src.1, ptr %src.2) {548; CHECK-LABEL: define double @test_scalarization_cost_for_load_of_address(549; CHECK-SAME: ptr [[SRC_0:%.*]], ptr [[SRC_1:%.*]], ptr [[SRC_2:%.*]]) {550; CHECK-NEXT: [[ENTRY:.*:]]551; CHECK-NEXT: br label %[[VECTOR_PH:.*]]552; CHECK: [[VECTOR_PH]]:553; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]554; CHECK: [[VECTOR_BODY]]:555; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]556; CHECK-NEXT: [[VEC_PHI:%.*]] = phi double [ 3.000000e+00, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]557; CHECK-NEXT: [[IV:%.*]] = add i64 [[INDEX]], 0558; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1559; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr [[T:%.*]], ptr [[SRC_0]], i64 [[IV]]560; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <6 x double>, ptr [[GEP_0]], align 8561; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 0, i32 3>562; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 1, i32 4>563; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <6 x double> [[WIDE_VEC]], <6 x double> poison, <2 x i32> <i32 2, i32 5>564; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[STRIDED_VEC]], splat (double 3.000000e+00)565; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x double> [[STRIDED_VEC1]], splat (double 3.000000e+00)566; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[STRIDED_VEC2]], splat (double 3.000000e+00)567; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[TMP3]], [[TMP4]]568; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]]569; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC_1]], i64 [[IV]]570; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[GEP_SRC]], align 8571; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x double> [[TMP7]], [[WIDE_LOAD]]572; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr [[T_2:%.*]], ptr [[SRC_2]], i64 [[IV]]573; CHECK-NEXT: [[TMP11:%.*]] = getelementptr [[T_2]], ptr [[SRC_2]], i64 [[TMP1]]574; CHECK-NEXT: [[GEP_72:%.*]] = getelementptr i8, ptr [[GEP_SRC_2]], i64 72575; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP11]], i64 72576; CHECK-NEXT: [[L_P_2:%.*]] = load ptr, ptr [[GEP_72]], align 8577; CHECK-NEXT: [[TMP15:%.*]] = load ptr, ptr [[TMP13]], align 8578; CHECK-NEXT: [[LV:%.*]] = load double, ptr [[L_P_2]], align 8579; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP15]], align 8580; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x double> poison, double [[LV]], i32 0581; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x double> [[TMP18]], double [[TMP17]], i32 1582; CHECK-NEXT: [[TMP20:%.*]] = fmul <2 x double> [[TMP9]], [[TMP19]]583; CHECK-NEXT: [[TMP21]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> [[TMP20]])584; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2585; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]586; CHECK: [[MIDDLE_BLOCK]]:587; CHECK-NEXT: br label %[[EXIT:.*]]588; CHECK: [[EXIT]]:589; CHECK-NEXT: ret double [[TMP21]]590;591entry:592 br label %loop593 594loop:595 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]596 %red = phi double [ 3.000000e+00, %entry ], [ %red.next, %loop ]597 %gep.0 = getelementptr %t, ptr %src.0, i64 %iv598 %l.0 = load double, ptr %gep.0, align 8599 %gep.8 = getelementptr i8, ptr %gep.0, i64 8600 %l.1 = load double, ptr %gep.8, align 8601 %gep.16 = getelementptr i8, ptr %gep.0, i64 16602 %l.2 = load double, ptr %gep.16, align 8603 %mul.0 = fmul double %l.0, 3.000000e+00604 %mul.1 = fmul double %l.1, 3.000000e+00605 %mul.2 = fmul double %l.2, 3.000000e+00606 %add.0 = fadd double %mul.0, %mul.1607 %add.1 = fadd double %add.0, %mul.2608 %gep.src = getelementptr double, ptr %src.1, i64 %iv609 %l = load double, ptr %gep.src, align 8610 %mul256.us = fmul double %add.1, %l611 %gep.src.2 = getelementptr %t.2, ptr %src.2, i64 %iv612 %gep.72 = getelementptr i8, ptr %gep.src.2, i64 72613 %l.p.2 = load ptr, ptr %gep.72, align 8614 %lv = load double, ptr %l.p.2, align 8615 %red.next = tail call double @llvm.fmuladd.f64(double %mul256.us, double %lv, double %red)616 %iv.next = add i64 %iv, 1617 %ec = icmp eq i64 %iv, 1618 br i1 %ec, label %exit, label %loop619 620exit:621 ret double %red.next622}623 624define i32 @test_ptr_iv_load_used_by_other_load(ptr %start, ptr %end) {625; CHECK-LABEL: define i32 @test_ptr_iv_load_used_by_other_load(626; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {627; CHECK-NEXT: [[ENTRY:.*]]:628; CHECK-NEXT: br label %[[LOOP:.*]]629; CHECK: [[LOOP]]:630; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ null, %[[ENTRY]] ]631; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[RED_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]632; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[IV]], align 8633; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[TMP0]], align 8634; CHECK-NEXT: [[C:%.*]] = icmp ne i8 [[TMP1]], 0635; CHECK-NEXT: [[C_EXT:%.*]] = zext i1 [[C]] to i32636; CHECK-NEXT: [[RED_NEXT]] = or i32 [[RED]], [[C_EXT]]637; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 32638; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[IV]], [[END]]639; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]640; CHECK: [[EXIT]]:641; CHECK-NEXT: [[RED_LCSSA:%.*]] = phi i32 [ [[RED]], %[[LOOP]] ]642; CHECK-NEXT: ret i32 [[RED_LCSSA]]643;644entry:645 br label %loop646 647loop: ; preds = %loop, %entry648 %iv = phi ptr [ %iv.next, %loop ], [ null, %entry ]649 %red = phi i32 [ %red.next, %loop ], [ 0, %entry ]650 %0 = load ptr, ptr %iv, align 8651 %1 = load i8, ptr %0, align 8652 %c = icmp ne i8 %1, 0653 %c.ext = zext i1 %c to i32654 %red.next = or i32 %red, %c.ext655 %iv.next = getelementptr nusw i8, ptr %iv, i64 32656 %ec = icmp eq ptr %iv, %end657 br i1 %ec, label %exit, label %loop658 659exit:660 ret i32 %red661}662 663 664define i32 @test_or_reduction_with_stride_2(i32 %scale, ptr %src) {665; CHECK-LABEL: define i32 @test_or_reduction_with_stride_2(666; CHECK-SAME: i32 [[SCALE:%.*]], ptr [[SRC:%.*]]) {667; CHECK-NEXT: [[ENTRY:.*:]]668; CHECK-NEXT: br label %[[VECTOR_PH:.*]]669; CHECK: [[VECTOR_PH]]:670; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[SCALE]], i64 0671; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer672; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]673; CHECK: [[VECTOR_BODY]]:674; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]675; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP66:%.*]], %[[VECTOR_BODY]] ]676; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2677; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0678; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2679; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4680; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6681; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 8682; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 10683; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 12684; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 14685; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 16686; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 18687; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 20688; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 22689; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 24690; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 26691; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 28692; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 30693; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP0]]694; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP1]]695; CHECK-NEXT: [[TMP18:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP2]]696; CHECK-NEXT: [[TMP19:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP3]]697; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP4]]698; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP5]]699; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP6]]700; CHECK-NEXT: [[TMP23:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP7]]701; CHECK-NEXT: [[TMP24:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP8]]702; CHECK-NEXT: [[TMP25:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP9]]703; CHECK-NEXT: [[TMP26:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP10]]704; CHECK-NEXT: [[TMP27:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP11]]705; CHECK-NEXT: [[TMP28:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP12]]706; CHECK-NEXT: [[TMP29:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP13]]707; CHECK-NEXT: [[TMP30:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP14]]708; CHECK-NEXT: [[TMP31:%.*]] = getelementptr [32 x i8], ptr [[SRC]], i64 [[TMP15]]709; CHECK-NEXT: [[TMP32:%.*]] = load i8, ptr [[TMP16]], align 1710; CHECK-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP17]], align 1711; CHECK-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP18]], align 1712; CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP19]], align 1713; CHECK-NEXT: [[TMP36:%.*]] = load i8, ptr [[TMP20]], align 1714; CHECK-NEXT: [[TMP37:%.*]] = load i8, ptr [[TMP21]], align 1715; CHECK-NEXT: [[TMP38:%.*]] = load i8, ptr [[TMP22]], align 1716; CHECK-NEXT: [[TMP39:%.*]] = load i8, ptr [[TMP23]], align 1717; CHECK-NEXT: [[TMP40:%.*]] = load i8, ptr [[TMP24]], align 1718; CHECK-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP25]], align 1719; CHECK-NEXT: [[TMP42:%.*]] = load i8, ptr [[TMP26]], align 1720; CHECK-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP27]], align 1721; CHECK-NEXT: [[TMP44:%.*]] = load i8, ptr [[TMP28]], align 1722; CHECK-NEXT: [[TMP45:%.*]] = load i8, ptr [[TMP29]], align 1723; CHECK-NEXT: [[TMP46:%.*]] = load i8, ptr [[TMP30]], align 1724; CHECK-NEXT: [[TMP47:%.*]] = load i8, ptr [[TMP31]], align 1725; CHECK-NEXT: [[TMP48:%.*]] = insertelement <16 x i8> poison, i8 [[TMP32]], i32 0726; CHECK-NEXT: [[TMP49:%.*]] = insertelement <16 x i8> [[TMP48]], i8 [[TMP33]], i32 1727; CHECK-NEXT: [[TMP50:%.*]] = insertelement <16 x i8> [[TMP49]], i8 [[TMP34]], i32 2728; CHECK-NEXT: [[TMP51:%.*]] = insertelement <16 x i8> [[TMP50]], i8 [[TMP35]], i32 3729; CHECK-NEXT: [[TMP52:%.*]] = insertelement <16 x i8> [[TMP51]], i8 [[TMP36]], i32 4730; CHECK-NEXT: [[TMP53:%.*]] = insertelement <16 x i8> [[TMP52]], i8 [[TMP37]], i32 5731; CHECK-NEXT: [[TMP54:%.*]] = insertelement <16 x i8> [[TMP53]], i8 [[TMP38]], i32 6732; CHECK-NEXT: [[TMP55:%.*]] = insertelement <16 x i8> [[TMP54]], i8 [[TMP39]], i32 7733; CHECK-NEXT: [[TMP56:%.*]] = insertelement <16 x i8> [[TMP55]], i8 [[TMP40]], i32 8734; CHECK-NEXT: [[TMP57:%.*]] = insertelement <16 x i8> [[TMP56]], i8 [[TMP41]], i32 9735; CHECK-NEXT: [[TMP58:%.*]] = insertelement <16 x i8> [[TMP57]], i8 [[TMP42]], i32 10736; CHECK-NEXT: [[TMP59:%.*]] = insertelement <16 x i8> [[TMP58]], i8 [[TMP43]], i32 11737; CHECK-NEXT: [[TMP60:%.*]] = insertelement <16 x i8> [[TMP59]], i8 [[TMP44]], i32 12738; CHECK-NEXT: [[TMP61:%.*]] = insertelement <16 x i8> [[TMP60]], i8 [[TMP45]], i32 13739; CHECK-NEXT: [[TMP62:%.*]] = insertelement <16 x i8> [[TMP61]], i8 [[TMP46]], i32 14740; CHECK-NEXT: [[TMP63:%.*]] = insertelement <16 x i8> [[TMP62]], i8 [[TMP47]], i32 15741; CHECK-NEXT: [[TMP64:%.*]] = sext <16 x i8> [[TMP63]] to <16 x i32>742; CHECK-NEXT: [[TMP65:%.*]] = mul <16 x i32> [[BROADCAST_SPLAT]], [[TMP64]]743; CHECK-NEXT: [[TMP66]] = or <16 x i32> [[TMP65]], [[VEC_PHI]]744; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16745; CHECK-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48746; CHECK-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]747; CHECK: [[MIDDLE_BLOCK]]:748; CHECK-NEXT: [[TMP68:%.*]] = call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> [[TMP66]])749; CHECK-NEXT: br label %[[SCALAR_PH:.*]]750; CHECK: [[SCALAR_PH]]:751;752entry:753 br label %loop754 755loop:756 %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]757 %reduction = phi i32 [ %reduction.next, %loop ], [ 0, %entry ]758 %gep = getelementptr [32 x i8], ptr %src, i64 %iv759 %load = load i8, ptr %gep, align 1760 %sext = sext i8 %load to i32761 %mul = mul i32 %scale, %sext762 %reduction.next = or i32 %mul, %reduction763 %iv.next = add i64 %iv, 2764 %cmp = icmp eq i64 %iv.next, 100765 br i1 %cmp, label %exit, label %loop766 767exit:768 ret i32 %reduction.next769}770 771attributes #0 = { "target-cpu"="neoverse-512tvb" }772 773!0 = !{!1, !2, i64 0}774!1 = !{!"", !2, i64 0}775!2 = !{!"long long", !3, i64 0}776!3 = !{!"omnipotent char", !4, i64 0}777!4 = !{!"Simple C/C++ TBAA"}778!5 = !{!6, !6, i64 0}779!6 = !{!"int", !3, i64 0}780