624 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -p loop-vectorize -S %s | FileCheck %s3 4target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"5target triple = "x86_64-apple-macosx10.15.0"6 7define void @test_free_instructions_feeding_geps_for_interleave_groups(ptr noalias %p.invar, ptr noalias %dst.1, ptr noalias %dst.2) {8; CHECK-LABEL: define void @test_free_instructions_feeding_geps_for_interleave_groups(9; CHECK-SAME: ptr noalias [[P_INVAR:%.*]], ptr noalias [[DST_1:%.*]], ptr noalias [[DST_2:%.*]]) {10; CHECK-NEXT: [[ENTRY:.*:]]11; CHECK-NEXT: br label %[[VECTOR_PH:.*]]12; CHECK: [[VECTOR_PH]]:13; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]14; CHECK: [[VECTOR_BODY]]:15; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]16; CHECK-NEXT: [[TMP40:%.*]] = load float, ptr [[P_INVAR]], align 417; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP40]], i64 018; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer19; CHECK-NEXT: [[TMP41:%.*]] = shl i64 [[INDEX]], 220; CHECK-NEXT: [[TMP44:%.*]] = getelementptr float, ptr [[DST_1]], i64 [[TMP41]]21; CHECK-NEXT: [[TMP42:%.*]] = load float, ptr [[P_INVAR]], align 422; CHECK-NEXT: [[BROADCAST_SPLATINSERT27:%.*]] = insertelement <2 x float> poison, float [[TMP42]], i64 023; CHECK-NEXT: [[BROADCAST_SPLAT28:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT27]], <2 x float> poison, <2 x i32> zeroinitializer24; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLAT]], <2 x float> [[BROADCAST_SPLAT28]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>25; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <4 x float> [[TMP46]], <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>26; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP47]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>27; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP44]], align 428; CHECK-NEXT: [[TMP48:%.*]] = load float, ptr [[P_INVAR]], align 429; CHECK-NEXT: [[BROADCAST_SPLATINSERT29:%.*]] = insertelement <2 x float> poison, float [[TMP48]], i64 030; CHECK-NEXT: [[BROADCAST_SPLAT30:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT29]], <2 x float> poison, <2 x i32> zeroinitializer31; CHECK-NEXT: [[TMP49:%.*]] = getelementptr float, ptr [[DST_2]], i64 [[TMP41]]32; CHECK-NEXT: [[BROADCAST_SPLAT36:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLAT30]], <2 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>33; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLAT36]], <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>34; CHECK-NEXT: [[INTERLEAVED_VEC31:%.*]] = shufflevector <8 x float> [[TMP51]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>35; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC31]], ptr [[TMP49]], align 436; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 237; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 102438; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]39; CHECK: [[MIDDLE_BLOCK]]:40; CHECK-NEXT: br label %[[EXIT:.*]]41; CHECK: [[EXIT]]:42; CHECK-NEXT: ret void43;44entry:45 br label %loop46 47loop:48 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]49 %l.0 = load float, ptr %p.invar, align 450 %iv.mul = shl i64 %iv, 251 %gep.dst.19 = getelementptr float, ptr %dst.1, i64 %iv.mul52 store float %l.0, ptr %gep.dst.19, align 453 %l.1 = load float, ptr %p.invar, align 454 %add.1 = or disjoint i64 %iv.mul, 155 %gep.dst.119 = getelementptr float, ptr %dst.1, i64 %add.156 store float %l.1, ptr %gep.dst.119, align 457 %add.2 = or disjoint i64 %iv.mul, 258 %gep.dst.129 = getelementptr float, ptr %dst.1, i64 %add.259 store float 0.000000e+00, ptr %gep.dst.129, align 460 %add.3 = or disjoint i64 %iv.mul, 361 %gep.dst.140 = getelementptr float, ptr %dst.1, i64 %add.362 store float 0.000000e+00, ptr %gep.dst.140, align 463 %l.2 = load float, ptr %p.invar, align 464 %gep.dst.247 = getelementptr float, ptr %dst.2, i64 %iv.mul65 store float %l.2, ptr %gep.dst.247, align 466 %gep.dst.255 = getelementptr float, ptr %dst.2, i64 %add.167 store float 0.000000e+00, ptr %gep.dst.255, align 468 %gep.dst.265 = getelementptr float, ptr %dst.2, i64 %add.269 store float 0.000000e+00, ptr %gep.dst.265, align 470 %gep.dst.276 = getelementptr float, ptr %dst.2, i64 %add.371 store float 0.000000e+00, ptr %gep.dst.276, align 472 %iv.next = add i64 %iv, 173 %ec = icmp eq i64 %iv.next, 102474 br i1 %ec, label %exit, label %loop75 76exit:77 ret void78}79 80define void @geps_feeding_interleave_groups_with_reuse(ptr %arg, i64 %arg1, ptr %arg2) #0 {81; CHECK-LABEL: define void @geps_feeding_interleave_groups_with_reuse(82; CHECK-SAME: ptr [[ARG:%.*]], i64 [[ARG1:%.*]], ptr [[ARG2:%.*]]) #[[ATTR0:[0-9]+]] {83; CHECK-NEXT: [[ENTRY:.*]]:84; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[ARG1]], 185; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 1886; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]87; CHECK: [[VECTOR_SCEVCHECK]]:88; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ARG]], i64 1689; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[ARG1]])90; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 091; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 192; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]93; CHECK-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[SCEVGEP]]94; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]95; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]96; CHECK: [[VECTOR_MEMCHECK]]:97; CHECK-NEXT: [[TMP20:%.*]] = shl i64 [[ARG1]], 498; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP20]], 1699; CHECK-NEXT: [[SCEVGEP12:%.*]] = getelementptr i8, ptr [[ARG2]], i64 [[TMP21]]100; CHECK-NEXT: [[TMP22:%.*]] = shl i64 [[ARG1]], 5101; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], 32102; CHECK-NEXT: [[SCEVGEP13:%.*]] = getelementptr i8, ptr [[ARG]], i64 [[TMP23]]103; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ARG2]], [[SCEVGEP13]]104; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[ARG]], [[SCEVGEP12]]105; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]106; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]107; CHECK: [[VECTOR_PH]]:108; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2109; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]110; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]111; CHECK: [[VECTOR_BODY]]:112; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]113; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 5114; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 [[TMP9]]115; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[INDEX]], 4116; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i64 [[TMP11]]117; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x float>, ptr [[TMP26]], align 4, !alias.scope [[META3:![0-9]+]]118; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 0, i32 8>119; CHECK-NEXT: [[STRIDED_VEC14:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 1, i32 9>120; CHECK-NEXT: [[STRIDED_VEC15:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 2, i32 10>121; CHECK-NEXT: [[STRIDED_VEC16:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 3, i32 11>122; CHECK-NEXT: [[STRIDED_VEC17:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 4, i32 12>123; CHECK-NEXT: [[STRIDED_VEC18:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 5, i32 13>124; CHECK-NEXT: [[STRIDED_VEC19:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 6, i32 14>125; CHECK-NEXT: [[STRIDED_VEC20:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 7, i32 15>126; CHECK-NEXT: [[TMP30:%.*]] = fadd <2 x float> [[STRIDED_VEC]], [[STRIDED_VEC17]]127; CHECK-NEXT: [[TMP31:%.*]] = fmul <2 x float> [[TMP30]], zeroinitializer128; CHECK-NEXT: [[TMP32:%.*]] = fadd <2 x float> [[STRIDED_VEC14]], [[STRIDED_VEC18]]129; CHECK-NEXT: [[TMP33:%.*]] = fmul <2 x float> [[TMP32]], zeroinitializer130; CHECK-NEXT: [[TMP34:%.*]] = fadd <2 x float> [[STRIDED_VEC15]], [[STRIDED_VEC19]]131; CHECK-NEXT: [[TMP35:%.*]] = fmul <2 x float> [[TMP34]], zeroinitializer132; CHECK-NEXT: [[TMP36:%.*]] = fadd <2 x float> [[STRIDED_VEC16]], [[STRIDED_VEC20]]133; CHECK-NEXT: [[TMP37:%.*]] = fmul <2 x float> [[TMP36]], zeroinitializer134; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <2 x float> [[TMP31]], <2 x float> [[TMP33]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>135; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <2 x float> [[TMP35]], <2 x float> [[TMP37]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>136; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <4 x float> [[TMP40]], <4 x float> [[TMP41]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>137; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP42]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>138; CHECK-NEXT: store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP28]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META3]]139; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2140; CHECK-NEXT: [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]141; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]142; CHECK: [[MIDDLE_BLOCK]]:143; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]144; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]145; CHECK: [[SCALAR_PH]]:146; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]147; CHECK-NEXT: br label %[[LOOP:.*]]148; CHECK: [[LOOP]]:149; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]150; CHECK-NEXT: [[SHL_IV_5:%.*]] = shl i64 [[IV]], 5151; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 [[SHL_IV_5]]152; CHECK-NEXT: [[ADD_5:%.*]] = or disjoint i64 [[SHL_IV_5]], 16153; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr i8, ptr [[ARG]], i64 [[ADD_5]]154; CHECK-NEXT: [[SHL_IV_4:%.*]] = shl i64 [[IV]], 4155; CHECK-NEXT: [[GEP_3:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i64 [[SHL_IV_4]]156; CHECK-NEXT: [[L_1:%.*]] = load float, ptr [[GEP_1]], align 4157; CHECK-NEXT: [[L_2:%.*]] = load float, ptr [[GEP_2]], align 4158; CHECK-NEXT: [[ADD_1:%.*]] = fadd float [[L_1]], [[L_2]]159; CHECK-NEXT: [[MUL_1:%.*]] = fmul float [[ADD_1]], 0.000000e+00160; CHECK-NEXT: store float [[MUL_1]], ptr [[GEP_3]], align 4161; CHECK-NEXT: [[GEP_4:%.*]] = getelementptr inbounds i8, ptr [[GEP_1]], i64 4162; CHECK-NEXT: [[L_3:%.*]] = load float, ptr [[GEP_4]], align 4163; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr inbounds i8, ptr [[GEP_2]], i64 4164; CHECK-NEXT: [[L_4:%.*]] = load float, ptr [[GEP_5]], align 4165; CHECK-NEXT: [[ADD_2:%.*]] = fadd float [[L_3]], [[L_4]]166; CHECK-NEXT: [[MUL_2:%.*]] = fmul float [[ADD_2]], 0.000000e+00167; CHECK-NEXT: [[GEP_6:%.*]] = getelementptr inbounds i8, ptr [[GEP_3]], i64 4168; CHECK-NEXT: store float [[MUL_2]], ptr [[GEP_6]], align 4169; CHECK-NEXT: [[GEP_7:%.*]] = getelementptr inbounds i8, ptr [[GEP_1]], i64 8170; CHECK-NEXT: [[L_5:%.*]] = load float, ptr [[GEP_7]], align 4171; CHECK-NEXT: [[GEP_8:%.*]] = getelementptr inbounds i8, ptr [[GEP_2]], i64 8172; CHECK-NEXT: [[L_6:%.*]] = load float, ptr [[GEP_8]], align 4173; CHECK-NEXT: [[ADD_3:%.*]] = fadd float [[L_5]], [[L_6]]174; CHECK-NEXT: [[MUL_3:%.*]] = fmul float [[ADD_3]], 0.000000e+00175; CHECK-NEXT: [[GEP_9:%.*]] = getelementptr inbounds i8, ptr [[GEP_3]], i64 8176; CHECK-NEXT: store float [[MUL_3]], ptr [[GEP_9]], align 4177; CHECK-NEXT: [[I27:%.*]] = getelementptr inbounds i8, ptr [[GEP_1]], i64 12178; CHECK-NEXT: [[L_7:%.*]] = load float, ptr [[I27]], align 4179; CHECK-NEXT: [[GEP_10:%.*]] = getelementptr inbounds i8, ptr [[GEP_2]], i64 12180; CHECK-NEXT: [[L_8:%.*]] = load float, ptr [[GEP_10]], align 4181; CHECK-NEXT: [[ADD_4:%.*]] = fadd float [[L_7]], [[L_8]]182; CHECK-NEXT: [[MUL_4:%.*]] = fmul float [[ADD_4]], 0.000000e+00183; CHECK-NEXT: [[GEP_11:%.*]] = getelementptr inbounds i8, ptr [[GEP_3]], i64 12184; CHECK-NEXT: store float [[MUL_4]], ptr [[GEP_11]], align 4185; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1186; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[ARG1]]187; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]188; CHECK: [[EXIT]]:189; CHECK-NEXT: ret void190;191entry:192 br label %loop193 194loop:195 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]196 %shl.iv.5 = shl i64 %iv, 5197 %gep.1 = getelementptr inbounds i8, ptr %arg, i64 %shl.iv.5198 %add.5 = or disjoint i64 %shl.iv.5, 16199 %gep.2 = getelementptr i8, ptr %arg, i64 %add.5200 %shl.iv.4 = shl i64 %iv, 4201 %gep.3 = getelementptr inbounds i8, ptr %arg2, i64 %shl.iv.4202 %l.1 = load float, ptr %gep.1, align 4203 %l.2 = load float, ptr %gep.2, align 4204 %add.1 = fadd float %l.1, %l.2205 %mul.1 = fmul float %add.1, 0.000000e+00206 store float %mul.1, ptr %gep.3, align 4207 %gep.4 = getelementptr inbounds i8, ptr %gep.1, i64 4208 %l.3 = load float, ptr %gep.4, align 4209 %gep.5 = getelementptr inbounds i8, ptr %gep.2, i64 4210 %l.4 = load float, ptr %gep.5, align 4211 %add.2 = fadd float %l.3, %l.4212 %mul.2 = fmul float %add.2, 0.000000e+00213 %gep.6 = getelementptr inbounds i8, ptr %gep.3, i64 4214 store float %mul.2, ptr %gep.6, align 4215 %gep.7 = getelementptr inbounds i8, ptr %gep.1, i64 8216 %l.5 = load float, ptr %gep.7, align 4217 %gep.8 = getelementptr inbounds i8, ptr %gep.2, i64 8218 %l.6 = load float, ptr %gep.8, align 4219 %add.3 = fadd float %l.5, %l.6220 %mul.3 = fmul float %add.3, 0.000000e+00221 %gep.9 = getelementptr inbounds i8, ptr %gep.3, i64 8222 store float %mul.3, ptr %gep.9, align 4223 %i27 = getelementptr inbounds i8, ptr %gep.1, i64 12224 %l.7 = load float, ptr %i27, align 4225 %gep.10 = getelementptr inbounds i8, ptr %gep.2, i64 12226 %l.8 = load float, ptr %gep.10, align 4227 %add.4 = fadd float %l.7, %l.8228 %mul.4 = fmul float %add.4, 0.000000e+00229 %gep.11 = getelementptr inbounds i8, ptr %gep.3, i64 12230 store float %mul.4, ptr %gep.11, align 4231 %iv.next = add i64 %iv, 1232 %ec = icmp eq i64 %iv, %arg1233 br i1 %ec, label %exit, label %loop234 235exit:236 ret void237}238 239define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N) #1 {240; CHECK-LABEL: define void @geps_feeding_interleave_groups_with_reuse2(241; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {242; CHECK-NEXT: [[ENTRY:.*]]:243; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[N]], 3244; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1245; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 56246; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]247; CHECK: [[VECTOR_SCEVCHECK]]:248; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[N]], 3249; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])250; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0251; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1252; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[A]], i64 [[MUL_RESULT]]253; CHECK-NEXT: [[TMP41:%.*]] = icmp ult ptr [[TMP32]], [[A]]254; CHECK-NEXT: [[TMP44:%.*]] = or i1 [[TMP41]], [[MUL_OVERFLOW]]255; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 4256; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])257; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0258; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1259; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT2]]260; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[TMP4]], [[SCEVGEP]]261; CHECK-NEXT: [[TMP57:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW3]]262; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 8263; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])264; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0265; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1266; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT3]]267; CHECK-NEXT: [[TMP9:%.*]] = icmp ult ptr [[TMP8]], [[SCEVGEP1]]268; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[MUL_OVERFLOW4]]269; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[A]], i64 12270; CHECK-NEXT: [[MUL6:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])271; CHECK-NEXT: [[MUL_RESULT7:%.*]] = extractvalue { i64, i1 } [[MUL6]], 0272; CHECK-NEXT: [[MUL_OVERFLOW8:%.*]] = extractvalue { i64, i1 } [[MUL6]], 1273; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP5]], i64 [[MUL_RESULT7]]274; CHECK-NEXT: [[TMP13:%.*]] = icmp ult ptr [[TMP12]], [[SCEVGEP5]]275; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[MUL_OVERFLOW8]]276; CHECK-NEXT: [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[A]], i64 16277; CHECK-NEXT: [[MUL10:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])278; CHECK-NEXT: [[MUL_RESULT11:%.*]] = extractvalue { i64, i1 } [[MUL10]], 0279; CHECK-NEXT: [[MUL_OVERFLOW12:%.*]] = extractvalue { i64, i1 } [[MUL10]], 1280; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[SCEVGEP9]], i64 [[MUL_RESULT11]]281; CHECK-NEXT: [[TMP17:%.*]] = icmp ult ptr [[TMP16]], [[SCEVGEP9]]282; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP17]], [[MUL_OVERFLOW12]]283; CHECK-NEXT: [[SCEVGEP13:%.*]] = getelementptr i8, ptr [[A]], i64 20284; CHECK-NEXT: [[MUL14:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])285; CHECK-NEXT: [[MUL_RESULT15:%.*]] = extractvalue { i64, i1 } [[MUL14]], 0286; CHECK-NEXT: [[MUL_OVERFLOW16:%.*]] = extractvalue { i64, i1 } [[MUL14]], 1287; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[SCEVGEP13]], i64 [[MUL_RESULT15]]288; CHECK-NEXT: [[TMP21:%.*]] = icmp ult ptr [[TMP20]], [[SCEVGEP13]]289; CHECK-NEXT: [[TMP22:%.*]] = or i1 [[TMP21]], [[MUL_OVERFLOW16]]290; CHECK-NEXT: [[SCEVGEP17:%.*]] = getelementptr i8, ptr [[A]], i64 24291; CHECK-NEXT: [[MUL18:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])292; CHECK-NEXT: [[MUL_RESULT19:%.*]] = extractvalue { i64, i1 } [[MUL18]], 0293; CHECK-NEXT: [[MUL_OVERFLOW20:%.*]] = extractvalue { i64, i1 } [[MUL18]], 1294; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SCEVGEP17]], i64 [[MUL_RESULT19]]295; CHECK-NEXT: [[TMP25:%.*]] = icmp ult ptr [[TMP24]], [[SCEVGEP17]]296; CHECK-NEXT: [[TMP26:%.*]] = or i1 [[TMP25]], [[MUL_OVERFLOW20]]297; CHECK-NEXT: [[SCEVGEP21:%.*]] = getelementptr i8, ptr [[A]], i64 28298; CHECK-NEXT: [[MUL22:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])299; CHECK-NEXT: [[MUL_RESULT23:%.*]] = extractvalue { i64, i1 } [[MUL22]], 0300; CHECK-NEXT: [[MUL_OVERFLOW24:%.*]] = extractvalue { i64, i1 } [[MUL22]], 1301; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[SCEVGEP21]], i64 [[MUL_RESULT23]]302; CHECK-NEXT: [[TMP29:%.*]] = icmp ult ptr [[TMP28]], [[SCEVGEP21]]303; CHECK-NEXT: [[TMP30:%.*]] = or i1 [[TMP29]], [[MUL_OVERFLOW24]]304; CHECK-NEXT: [[SCEVGEP31:%.*]] = getelementptr i8, ptr [[B]], i64 4305; CHECK-NEXT: [[MUL29:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]])306; CHECK-NEXT: [[MUL_RESULT30:%.*]] = extractvalue { i64, i1 } [[MUL29]], 0307; CHECK-NEXT: [[MUL_OVERFLOW31:%.*]] = extractvalue { i64, i1 } [[MUL29]], 1308; CHECK-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[SCEVGEP31]], i64 [[MUL_RESULT30]]309; CHECK-NEXT: [[TMP69:%.*]] = icmp ult ptr [[TMP68]], [[SCEVGEP31]]310; CHECK-NEXT: [[TMP70:%.*]] = or i1 [[TMP69]], [[MUL_OVERFLOW31]]311; CHECK-NEXT: [[MUL25:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])312; CHECK-NEXT: [[MUL_RESULT26:%.*]] = extractvalue { i64, i1 } [[MUL25]], 0313; CHECK-NEXT: [[MUL_OVERFLOW27:%.*]] = extractvalue { i64, i1 } [[MUL25]], 1314; CHECK-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT26]]315; CHECK-NEXT: [[TMP33:%.*]] = icmp ult ptr [[TMP71]], [[B]]316; CHECK-NEXT: [[TMP34:%.*]] = or i1 [[TMP33]], [[MUL_OVERFLOW27]]317; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP44]], [[TMP57]]318; CHECK-NEXT: [[TMP35:%.*]] = or i1 [[TMP6]], [[TMP10]]319; CHECK-NEXT: [[TMP36:%.*]] = or i1 [[TMP35]], [[TMP14]]320; CHECK-NEXT: [[TMP37:%.*]] = or i1 [[TMP36]], [[TMP18]]321; CHECK-NEXT: [[TMP38:%.*]] = or i1 [[TMP37]], [[TMP22]]322; CHECK-NEXT: [[TMP39:%.*]] = or i1 [[TMP38]], [[TMP26]]323; CHECK-NEXT: [[TMP40:%.*]] = or i1 [[TMP39]], [[TMP30]]324; CHECK-NEXT: [[TMP72:%.*]] = or i1 [[TMP40]], [[TMP70]]325; CHECK-NEXT: [[TMP73:%.*]] = or i1 [[TMP72]], [[TMP34]]326; CHECK-NEXT: br i1 [[TMP73]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]327; CHECK: [[VECTOR_MEMCHECK]]:328; CHECK-NEXT: [[TMP42:%.*]] = lshr i64 [[N]], 3329; CHECK-NEXT: [[TMP43:%.*]] = shl i64 [[TMP42]], 5330; CHECK-NEXT: [[TMP45:%.*]] = add nuw nsw i64 [[TMP43]], 4331; CHECK-NEXT: [[SCEVGEP29:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP45]]332; CHECK-NEXT: [[TMP53:%.*]] = add i64 [[TMP43]], 32333; CHECK-NEXT: [[SCEVGEP28:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP53]]334; CHECK-NEXT: [[TMP46:%.*]] = shl i64 [[TMP42]], 4335; CHECK-NEXT: [[TMP47:%.*]] = add nuw nsw i64 [[TMP46]], 8336; CHECK-NEXT: [[SCEVGEP30:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP47]]337; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP28]]338; CHECK-NEXT: [[BOUND2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP29]]339; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND1]], [[BOUND2]]340; CHECK-NEXT: [[BOUND132:%.*]] = icmp ult ptr [[B]], [[SCEVGEP28]]341; CHECK-NEXT: [[BOUND133:%.*]] = icmp ult ptr [[A]], [[SCEVGEP30]]342; CHECK-NEXT: [[FOUND_CONFLICT33:%.*]] = and i1 [[BOUND132]], [[BOUND133]]343; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT33]]344; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]345; CHECK: [[VECTOR_PH]]:346; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4347; CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[N_MOD_VF]], 0348; CHECK-NEXT: [[TMP49:%.*]] = select i1 [[TMP48]], i64 4, i64 [[N_MOD_VF]]349; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[TMP49]]350; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 8351; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]352; CHECK: [[VECTOR_BODY]]:353; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]354; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 8, i64 16, i64 24>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]355; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8356; CHECK-NEXT: [[TMP51:%.*]] = lshr exact i64 [[OFFSET_IDX]], 1357; CHECK-NEXT: [[TMP52:%.*]] = getelementptr nusw i32, ptr [[B]], i64 [[TMP51]]358; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP52]], align 4, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]359; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>360; CHECK-NEXT: [[STRIDED_VEC34:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>361; CHECK-NEXT: [[TMP56:%.*]] = getelementptr i32, ptr [[A]], i64 [[OFFSET_IDX]]362; CHECK-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[B]], <4 x i64> [[VEC_IND]]363; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP54]], <4 x i1> splat (i1 true), <4 x i32> poison), !alias.scope [[META15:![0-9]+]], !noalias [[META13]]364; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>365; CHECK-NEXT: [[TMP59:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC34]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>366; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_GATHER]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>367; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <8 x i32> [[TMP58]], <8 x i32> [[TMP59]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>368; CHECK-NEXT: [[TMP62:%.*]] = shufflevector <8 x i32> [[TMP60]], <8 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>369; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP61]], <16 x i32> [[TMP62]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>370; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[TMP63]], <32 x i32> poison, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>371; CHECK-NEXT: store <32 x i32> [[INTERLEAVED_VEC]], ptr [[TMP56]], align 4, !alias.scope [[META13]]372; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4373; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 32)374; CHECK-NEXT: [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]375; CHECK-NEXT: br i1 [[TMP64]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]376; CHECK: [[MIDDLE_BLOCK]]:377; CHECK-NEXT: br label %[[SCALAR_PH]]378; CHECK: [[SCALAR_PH]]:379; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]380; CHECK-NEXT: br label %[[LOOP:.*]]381; CHECK: [[LOOP]]:382; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT_7:%.*]], %[[LOOP]] ]383; CHECK-NEXT: [[SHR_1:%.*]] = lshr exact i64 [[IV]], 1384; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr nusw i32, ptr [[B]], i64 [[SHR_1]]385; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_B]], align 4386; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]387; CHECK-NEXT: store i32 [[L]], ptr [[GEP_A]], align 4388; CHECK-NEXT: [[IV_NEXT:%.*]] = or disjoint i64 [[IV]], 1389; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT]]390; CHECK-NEXT: store i32 0, ptr [[GEP_A_1]], align 4391; CHECK-NEXT: [[IV_NEXT_1:%.*]] = or disjoint i64 [[IV]], 2392; CHECK-NEXT: [[SHR_2:%.*]] = lshr exact i64 [[IV_NEXT_1]], 1393; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr i32, ptr [[B]], i64 [[SHR_2]]394; CHECK-NEXT: [[TMP65:%.*]] = load i32, ptr [[GEP_B_2]], align 4395; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_1]]396; CHECK-NEXT: store i32 [[TMP65]], ptr [[GEP_A_2]], align 4397; CHECK-NEXT: [[IV_NEXT_2:%.*]] = or disjoint i64 [[IV]], 3398; CHECK-NEXT: [[GEP_A_3:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_2]]399; CHECK-NEXT: store i32 0, ptr [[GEP_A_3]], align 4400; CHECK-NEXT: [[IV_NEXT_3:%.*]] = or disjoint i64 [[IV]], 4401; CHECK-NEXT: [[GEP_B_4:%.*]] = getelementptr i32, ptr [[B]], i64 [[IV]]402; CHECK-NEXT: [[TMP66:%.*]] = load i32, ptr [[GEP_B_4]], align 4403; CHECK-NEXT: [[GEP_A_4:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_3]]404; CHECK-NEXT: store i32 [[TMP66]], ptr [[GEP_A_4]], align 4405; CHECK-NEXT: [[IV_NEXT_4:%.*]] = or disjoint i64 [[IV]], 5406; CHECK-NEXT: [[GEP_A_5:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_4]]407; CHECK-NEXT: store i32 0, ptr [[GEP_A_5]], align 4408; CHECK-NEXT: [[IV_NEXT_5:%.*]] = or disjoint i64 [[IV]], 6409; CHECK-NEXT: [[GEP_A_6:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_5]]410; CHECK-NEXT: store i32 0, ptr [[GEP_A_6]], align 4411; CHECK-NEXT: [[IV_NEXT_6:%.*]] = or disjoint i64 [[IV]], 7412; CHECK-NEXT: [[GEP_A_7:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT_6]]413; CHECK-NEXT: store i32 0, ptr [[GEP_A_7]], align 4414; CHECK-NEXT: [[IV_NEXT_7]] = add nuw nsw i64 [[IV]], 8415; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]416; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]417; CHECK: [[EXIT]]:418; CHECK-NEXT: ret void419;420entry:421 br label %loop422 423loop:424 %iv = phi i64 [ 0, %entry ], [ %iv.next.7, %loop ]425 %shr.1 = lshr exact i64 %iv, 1426 %gep.B = getelementptr nusw i32, ptr %B, i64 %shr.1427 %l = load i32, ptr %gep.B, align 4428 %gep.A = getelementptr i32, ptr %A, i64 %iv429 store i32 %l, ptr %gep.A, align 4430 %iv.next = or disjoint i64 %iv, 1431 %gep.A.1 = getelementptr i32, ptr %A, i64 %iv.next432 store i32 0, ptr %gep.A.1, align 4433 %iv.next.1 = or disjoint i64 %iv, 2434 %shr.2 = lshr exact i64 %iv.next.1, 1435 %gep.B.2 = getelementptr i32, ptr %B, i64 %shr.2436 %1 = load i32, ptr %gep.B.2, align 4437 %gep.A.2 = getelementptr i32, ptr %A, i64 %iv.next.1438 store i32 %1, ptr %gep.A.2, align 4439 %iv.next.2 = or disjoint i64 %iv, 3440 %gep.A.3 = getelementptr i32, ptr %A, i64 %iv.next.2441 store i32 0, ptr %gep.A.3, align 4442 %iv.next.3 = or disjoint i64 %iv, 4443 %gep.B.4 = getelementptr i32, ptr %B, i64 %iv444 %2 = load i32, ptr %gep.B.4, align 4445 %gep.A.4 = getelementptr i32, ptr %A, i64 %iv.next.3446 store i32 %2, ptr %gep.A.4, align 4447 %iv.next.4 = or disjoint i64 %iv, 5448 %gep.A.5 = getelementptr i32, ptr %A, i64 %iv.next.4449 store i32 0, ptr %gep.A.5, align 4450 %iv.next.5 = or disjoint i64 %iv, 6451 %gep.A.6 = getelementptr i32, ptr %A, i64 %iv.next.5452 store i32 0, ptr %gep.A.6, align 4453 %iv.next.6 = or disjoint i64 %iv, 7454 %gep.A.7 = getelementptr i32, ptr %A, i64 %iv.next.6455 store i32 0, ptr %gep.A.7, align 4456 %iv.next.7 = add nuw nsw i64 %iv, 8457 %ec = icmp eq i64 %iv, %N458 br i1 %ec, label %exit, label %loop459 460exit:461 ret void462}463 464; Test case for https://github.com/llvm/llvm-project/issues/112922.465define void @interleave_store_double_i64(ptr %dst) {466; CHECK-LABEL: define void @interleave_store_double_i64(467; CHECK-SAME: ptr [[DST:%.*]]) {468; CHECK-NEXT: [[ENTRY:.*:]]469; CHECK-NEXT: br label %[[VECTOR_PH:.*]]470; CHECK: [[VECTOR_PH]]:471; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]472; CHECK: [[VECTOR_BODY]]:473; CHECK-NEXT: store <4 x double> <double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 4.940660e-324>, ptr [[DST]], align 8474; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]475; CHECK: [[MIDDLE_BLOCK]]:476; CHECK-NEXT: br label %[[EXIT:.*]]477; CHECK: [[EXIT]]:478; CHECK-NEXT: ret void479;480entry:481 br label %loop482 483loop:484 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]485 %gep.1 = getelementptr { double, i64 }, ptr %dst, i64 %iv, i32 1486 store i64 %iv, ptr %gep.1, align 8487 %gep.0 = getelementptr { double, i64 }, ptr %dst, i64 %iv488 store double 0.000000e+00, ptr %gep.0, align 8489 %iv.next = add i64 %iv, 1490 %ec = icmp eq i64 %iv, 1491 br i1 %ec, label %exit, label %loop492 493exit:494 ret void495}496 497define void @interleave_store_i64_double(ptr %dst) {498; CHECK-LABEL: define void @interleave_store_i64_double(499; CHECK-SAME: ptr [[DST:%.*]]) {500; CHECK-NEXT: [[ENTRY:.*]]:501; CHECK-NEXT: br label %[[LOOP:.*]]502; CHECK: [[LOOP]]:503; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]504; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr { double, i64 }, ptr [[DST]], i64 [[IV]]505; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_0]], align 8506; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr { double, i64 }, ptr [[DST]], i64 [[IV]], i32 1507; CHECK-NEXT: store i64 [[IV]], ptr [[GEP_1]], align 8508; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1509; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1510; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]511; CHECK: [[EXIT]]:512; CHECK-NEXT: ret void513;514entry:515 br label %loop516 517loop:518 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]519 %gep.0 = getelementptr { double, i64 }, ptr %dst, i64 %iv520 store double 0.000000e+00, ptr %gep.0, align 8521 %gep.1 = getelementptr { double, i64 }, ptr %dst, i64 %iv, i32 1522 store i64 %iv, ptr %gep.1, align 8523 %iv.next = add i64 %iv, 1524 %ec = icmp eq i64 %iv, 1525 br i1 %ec, label %exit, label %loop526 527exit:528 ret void529}530 531; TODO: The interleave group should likely have the same cost as @interleave_store_double_i64.532define void @interleave_store_double_i64_2(ptr %dst) {533; CHECK-LABEL: define void @interleave_store_double_i64_2(534; CHECK-SAME: ptr [[DST:%.*]]) {535; CHECK-NEXT: [[ENTRY:.*]]:536; CHECK-NEXT: br label %[[LOOP:.*]]537; CHECK: [[LOOP]]:538; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]539; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr { i64, double }, ptr [[DST]], i64 [[IV]], i32 1540; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_1]], align 8541; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr { i64, double }, ptr [[DST]], i64 [[IV]]542; CHECK-NEXT: store i64 [[IV]], ptr [[GEP_0]], align 8543; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1544; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1545; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]546; CHECK: [[EXIT]]:547; CHECK-NEXT: ret void548;549entry:550 br label %loop551 552loop:553 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]554 %gep.1 = getelementptr { i64, double }, ptr %dst, i64 %iv, i32 1555 store double 0.000000e+00, ptr %gep.1, align 8556 %gep.0 = getelementptr { i64, double }, ptr %dst, i64 %iv557 store i64 %iv, ptr %gep.0, align 8558 %iv.next = add i64 %iv, 1559 %ec = icmp eq i64 %iv, 1560 br i1 %ec, label %exit, label %loop561 562exit:563 ret void564}565 566define void @interleave_store_i64_double_2(ptr %dst) {567; CHECK-LABEL: define void @interleave_store_i64_double_2(568; CHECK-SAME: ptr [[DST:%.*]]) {569; CHECK-NEXT: [[ENTRY:.*:]]570; CHECK-NEXT: br label %[[VECTOR_PH:.*]]571; CHECK: [[VECTOR_PH]]:572; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]573; CHECK: [[VECTOR_BODY]]:574; CHECK-NEXT: store <4 x double> <double 0.000000e+00, double 0.000000e+00, double 4.940660e-324, double 0.000000e+00>, ptr [[DST]], align 8575; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]576; CHECK: [[MIDDLE_BLOCK]]:577; CHECK-NEXT: br label %[[EXIT:.*]]578; CHECK: [[EXIT]]:579; CHECK-NEXT: ret void580;581entry:582 br label %loop583 584loop:585 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]586 %gep.0 = getelementptr { i64, double }, ptr %dst, i64 %iv587 store i64 %iv, ptr %gep.0, align 8588 %gep.1 = getelementptr { i64, double }, ptr %dst, i64 %iv, i32 1589 store double 0.000000e+00, ptr %gep.1, align 8590 %iv.next = add i64 %iv, 1591 %ec = icmp eq i64 %iv, 1592 br i1 %ec, label %exit, label %loop593 594exit:595 ret void596}597 598 599 600attributes #0 = { "target-features"="+sse4.2" }601attributes #1 = { "min-legal-vector-width"="0" "target-cpu"="cascadelake" }602 603;.604; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}605; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}606; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}607; CHECK: [[META3]] = !{[[META4:![0-9]+]]}608; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]]}609; CHECK: [[META5]] = distinct !{[[META5]], !"LVerDomain"}610; CHECK: [[META6]] = !{[[META7:![0-9]+]]}611; CHECK: [[META7]] = distinct !{[[META7]], [[META5]]}612; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}613; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}614; CHECK: [[META10]] = !{[[META11:![0-9]+]]}615; CHECK: [[META11]] = distinct !{[[META11]], [[META12:![0-9]+]]}616; CHECK: [[META12]] = distinct !{[[META12]], !"LVerDomain"}617; CHECK: [[META13]] = !{[[META14:![0-9]+]]}618; CHECK: [[META14]] = distinct !{[[META14]], [[META12]]}619; CHECK: [[META15]] = !{[[META16:![0-9]+]]}620; CHECK: [[META16]] = distinct !{[[META16]], [[META12]]}621; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]]}622; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]]}623;.624