1549 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S -passes=loop-vectorize,instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -runtime-memory-check-threshold=24 < %s | FileCheck %s3 4target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"5 6; Check vectorization on an interleaved load group of factor 2 and an interleaved7; store group of factor 2.8 9; int AB[1024];10; int CD[1024];11; void test_array_load2_store2(int C, int D) {12; for (int i = 0; i < 1024; i+=2) {13; int A = AB[i];14; int B = AB[i+1];15; CD[i] = A + C;16; CD[i+1] = B * D;17; }18; }19 20 21@AB = common global [1024 x i32] zeroinitializer, align 422@CD = common global [1024 x i32] zeroinitializer, align 423 24define void @test_array_load2_store2(i32 %C, i32 %D) {25; CHECK-LABEL: @test_array_load2_store2(26; CHECK-NEXT: entry:27; CHECK-NEXT: br label [[VECTOR_PH:%.*]]28; CHECK: vector.ph:29; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i64 030; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer31; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[D:%.*]], i64 032; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer33; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]34; CHECK: vector.body:35; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]36; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 137; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr @AB, i64 [[OFFSET_IDX]]38; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 439; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>40; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>41; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]42; CHECK-NEXT: [[TMP3:%.*]] = mul nsw <4 x i32> [[STRIDED_VEC1]], [[BROADCAST_SPLAT3]]43; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr @CD, i64 [[OFFSET_IDX]]44; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>45; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 446; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 447; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 51248; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]49; CHECK: middle.block:50; CHECK-NEXT: br label [[FOR_BODY:%.*]]51; CHECK: for.end:52; CHECK-NEXT: ret void53;54entry:55 br label %for.body56 57for.body: ; preds = %for.body, %entry58 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]59 %arrayidx0 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv60 %tmp = load i32, ptr %arrayidx0, align 461 %tmp1 = or disjoint i64 %indvars.iv, 162 %arrayidx1 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %tmp163 %tmp2 = load i32, ptr %arrayidx1, align 464 %add = add nsw i32 %tmp, %C65 %mul = mul nsw i32 %tmp2, %D66 %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv67 store i32 %add, ptr %arrayidx2, align 468 %arrayidx3 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %tmp169 store i32 %mul, ptr %arrayidx3, align 470 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 271 %cmp = icmp slt i64 %indvars.iv.next, 102472 br i1 %cmp, label %for.body, label %for.end73 74for.end: ; preds = %for.body75 ret void76}77 78; int A[3072];79; struct ST S[1024];80; void test_struct_st3() {81; int *ptr = A;82; for (int i = 0; i < 1024; i++) {83; int X1 = *ptr++;84; int X2 = *ptr++;85; int X3 = *ptr++;86; T[i].x = X1 + 1;87; T[i].y = X2 + 2;88; T[i].z = X3 + 3;89; }90; }91 92 93%struct.ST3 = type { i32, i32, i32 }94@A = common global [3072 x i32] zeroinitializer, align 495@S = common global [1024 x %struct.ST3] zeroinitializer, align 496 97define void @test_struct_array_load3_store3() {98; CHECK-LABEL: @test_struct_array_load3_store3(99; CHECK-NEXT: entry:100; CHECK-NEXT: br label [[VECTOR_PH:%.*]]101; CHECK: vector.ph:102; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]103; CHECK: vector.body:104; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]105; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 12106; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr @A, i64 [[OFFSET_IDX]]107; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, ptr [[NEXT_GEP]], align 4108; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>109; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>110; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>111; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)112; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_ST3:%.*]], ptr @S, i64 [[INDEX]]113; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i32> [[STRIDED_VEC2]], splat (i32 2)114; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[STRIDED_VEC3]], splat (i32 3)115; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>116; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>117; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP5]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>118; CHECK-NEXT: store <12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 4119; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4120; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024121; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]122; CHECK: middle.block:123; CHECK-NEXT: br label [[FOR_BODY:%.*]]124; CHECK: for.end:125; CHECK-NEXT: ret void126;127entry:128 br label %for.body129 130for.body: ; preds = %for.body, %entry131 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]132 %ptr.016 = phi ptr [ @A, %entry ], [ %incdec.ptr2, %for.body ]133 %incdec.ptr = getelementptr inbounds i32, ptr %ptr.016, i64 1134 %tmp = load i32, ptr %ptr.016, align 4135 %incdec.ptr1 = getelementptr inbounds i32, ptr %ptr.016, i64 2136 %tmp1 = load i32, ptr %incdec.ptr, align 4137 %incdec.ptr2 = getelementptr inbounds i32, ptr %ptr.016, i64 3138 %tmp2 = load i32, ptr %incdec.ptr1, align 4139 %add = add nsw i32 %tmp, 1140 %x = getelementptr inbounds [1024 x %struct.ST3], ptr @S, i64 0, i64 %indvars.iv, i32 0141 store i32 %add, ptr %x, align 4142 %add3 = add nsw i32 %tmp1, 2143 %y = getelementptr inbounds [1024 x %struct.ST3], ptr @S, i64 0, i64 %indvars.iv, i32 1144 store i32 %add3, ptr %y, align 4145 %add6 = add nsw i32 %tmp2, 3146 %z = getelementptr inbounds [1024 x %struct.ST3], ptr @S, i64 0, i64 %indvars.iv, i32 2147 store i32 %add6, ptr %z, align 4148 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1149 %exitcond = icmp eq i64 %indvars.iv.next, 1024150 br i1 %exitcond, label %for.end, label %for.body151 152for.end: ; preds = %for.body153 ret void154}155 156; Check vectorization on an interleaved load group of factor 4.157 158; struct ST4{159; int x;160; int y;161; int z;162; int w;163; };164; int test_struct_load4(struct ST4 *S) {165; int r = 0;166; for (int i = 0; i < 1024; i++) {167; r += S[i].x;168; r -= S[i].y;169; r += S[i].z;170; r -= S[i].w;171; }172; return r;173; }174 175%struct.ST4 = type { i32, i32, i32, i32 }176 177define i32 @test_struct_load4(ptr nocapture readonly %S) {178;179; CHECK-LABEL: @test_struct_load4(180; CHECK-NEXT: entry:181; CHECK-NEXT: br label [[VECTOR_PH:%.*]]182; CHECK: vector.ph:183; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]184; CHECK: vector.body:185; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]186; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]187; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_ST4:%.*]], ptr [[S:%.*]], i64 [[INDEX]]188; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP0]], align 4189; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>190; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>191; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>192; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>193; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_PHI]]194; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[STRIDED_VEC2]]195; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC3]]196; CHECK-NEXT: [[TMP4]] = sub <4 x i32> [[TMP2]], [[TMP3]]197; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4198; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024199; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]200; CHECK: middle.block:201; CHECK-NEXT: br label [[FOR_BODY:%.*]]202; CHECK: for.end:203; CHECK-NEXT: [[SUB8_LCSSA:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])204; CHECK-NEXT: ret i32 [[SUB8_LCSSA]]205;206entry:207 br label %for.body208 209for.body: ; preds = %for.body, %entry210 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]211 %r.022 = phi i32 [ 0, %entry ], [ %sub8, %for.body ]212 %x = getelementptr inbounds %struct.ST4, ptr %S, i64 %indvars.iv, i32 0213 %tmp = load i32, ptr %x, align 4214 %add = add nsw i32 %tmp, %r.022215 %y = getelementptr inbounds %struct.ST4, ptr %S, i64 %indvars.iv, i32 1216 %tmp1 = load i32, ptr %y, align 4217 %sub = sub i32 %add, %tmp1218 %z = getelementptr inbounds %struct.ST4, ptr %S, i64 %indvars.iv, i32 2219 %tmp2 = load i32, ptr %z, align 4220 %add5 = add nsw i32 %sub, %tmp2221 %w = getelementptr inbounds %struct.ST4, ptr %S, i64 %indvars.iv, i32 3222 %tmp3 = load i32, ptr %w, align 4223 %sub8 = sub i32 %add5, %tmp3224 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1225 %exitcond = icmp eq i64 %indvars.iv.next, 1024226 br i1 %exitcond, label %for.end, label %for.body227 228for.end: ; preds = %for.body229 ret i32 %sub8230}231 232; Check vectorization on an interleaved store group of factor 4.233 234; void test_struct_store4(int *A, struct ST4 *B) {235; int *ptr = A;236; for (int i = 0; i < 1024; i++) {237; int X = *ptr++;238; B[i].x = X + 1;239; B[i].y = X * 2;240; B[i].z = X + 3;241; B[i].w = X + 4;242; }243; }244 245 246define void @test_struct_store4(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) {247; CHECK-LABEL: @test_struct_store4(248; CHECK-NEXT: entry:249; CHECK-NEXT: br label [[VECTOR_PH:%.*]]250; CHECK: vector.ph:251; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]252; CHECK: vector.body:253; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]254; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2255; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[OFFSET_IDX]]256; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4257; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)258; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_ST4:%.*]], ptr [[B:%.*]], i64 [[INDEX]]259; CHECK-NEXT: [[TMP1:%.*]] = shl nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)260; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 3)261; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 4)262; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>263; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>264; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP6]], <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>265; CHECK-NEXT: store <16 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 4266; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4267; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024268; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]269; CHECK: middle.block:270; CHECK-NEXT: br label [[FOR_BODY:%.*]]271; CHECK: for.cond.cleanup:272; CHECK-NEXT: ret void273;274entry:275 br label %for.body276 277for.cond.cleanup: ; preds = %for.body278 ret void279 280for.body: ; preds = %for.body, %entry281 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]282 %ptr.024 = phi ptr [ %A, %entry ], [ %incdec.ptr, %for.body ]283 %incdec.ptr = getelementptr inbounds i32, ptr %ptr.024, i64 1284 %tmp = load i32, ptr %ptr.024, align 4285 %add = add nsw i32 %tmp, 1286 %x = getelementptr inbounds %struct.ST4, ptr %B, i64 %indvars.iv, i32 0287 store i32 %add, ptr %x, align 4288 %mul = shl nsw i32 %tmp, 1289 %y = getelementptr inbounds %struct.ST4, ptr %B, i64 %indvars.iv, i32 1290 store i32 %mul, ptr %y, align 4291 %add3 = add nsw i32 %tmp, 3292 %z = getelementptr inbounds %struct.ST4, ptr %B, i64 %indvars.iv, i32 2293 store i32 %add3, ptr %z, align 4294 %add6 = add nsw i32 %tmp, 4295 %w = getelementptr inbounds %struct.ST4, ptr %B, i64 %indvars.iv, i32 3296 store i32 %add6, ptr %w, align 4297 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1298 %exitcond = icmp eq i64 %indvars.iv.next, 1024299 br i1 %exitcond, label %for.cond.cleanup, label %for.body300}301 302; Check vectorization on a reverse interleaved load group of factor 2 and303; a reverse interleaved store group of factor 2.304 305; struct ST2 {306; int x;307; int y;308; };309;310; void test_reversed_load2_store2(struct ST2 *A, struct ST2 *B) {311; for (int i = 1023; i >= 0; i--) {312; int a = A[i].x + i; // interleaved load of index 0313; int b = A[i].y - i; // interleaved load of index 1314; B[i].x = a; // interleaved store of index 0315; B[i].y = b; // interleaved store of index 1316; }317; }318 319 320%struct.ST2 = type { i32, i32 }321 322define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) {323; CHECK-LABEL: @test_reversed_load2_store2(324; CHECK-NEXT: entry:325; CHECK-NEXT: br label [[VECTOR_PH:%.*]]326; CHECK: vector.ph:327; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]328; CHECK: vector.body:329; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]330; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1023, i32 1022, i32 1021, i32 1020>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]331; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]332; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_ST2:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]]333; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 -24334; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4335; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>336; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>337; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>338; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>339; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND]]340; CHECK-NEXT: [[TMP3:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND]]341; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_ST2]], ptr [[B:%.*]], i64 [[OFFSET_IDX]]342; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 -24343; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>344; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>345; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>346; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 4347; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4348; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)349; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024350; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]351; CHECK: middle.block:352; CHECK-NEXT: br label [[FOR_BODY:%.*]]353; CHECK: for.cond.cleanup:354; CHECK-NEXT: ret void355;356entry:357 br label %for.body358 359for.cond.cleanup: ; preds = %for.body360 ret void361 362for.body: ; preds = %for.body, %entry363 %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]364 %x = getelementptr inbounds %struct.ST2, ptr %A, i64 %indvars.iv, i32 0365 %tmp = load i32, ptr %x, align 4366 %tmp1 = trunc i64 %indvars.iv to i32367 %add = add nsw i32 %tmp, %tmp1368 %y = getelementptr inbounds %struct.ST2, ptr %A, i64 %indvars.iv, i32 1369 %tmp2 = load i32, ptr %y, align 4370 %sub = sub nsw i32 %tmp2, %tmp1371 %x5 = getelementptr inbounds %struct.ST2, ptr %B, i64 %indvars.iv, i32 0372 store i32 %add, ptr %x5, align 4373 %y8 = getelementptr inbounds %struct.ST2, ptr %B, i64 %indvars.iv, i32 1374 store i32 %sub, ptr %y8, align 4375 %indvars.iv.next = add nsw i64 %indvars.iv, -1376 %cmp = icmp sgt i64 %indvars.iv, 0377 br i1 %cmp, label %for.body, label %for.cond.cleanup378}379 380; Check vectorization on an interleaved load group of factor 2 with 1 gap381; (missing the load of odd elements). Because the vectorized loop would382; speculatively access memory out-of-bounds, we must execute at least one383; iteration of the scalar loop.384 385; void even_load_static_tc(int *A, int *B) {386; for (unsigned i = 0; i < 1024; i+=2)387; B[i/2] = A[i] * 2;388; }389 390 391define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) {392; CHECK-LABEL: @even_load_static_tc(393; CHECK-NEXT: entry:394; CHECK-NEXT: br label [[VECTOR_PH:%.*]]395; CHECK: vector.ph:396; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]397; CHECK: vector.body:398; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]399; CHECK-NEXT: [[DOTIDX:%.*]] = shl i64 [[INDEX]], 3400; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[DOTIDX]]401; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4402; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>403; CHECK-NEXT: [[TMP1:%.*]] = shl nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)404; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[INDEX]], 9223372036854775804405; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP2]]406; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP3]], align 4407; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4408; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 508409; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]410; CHECK: middle.block:411; CHECK-NEXT: br label [[SCALAR_PH:%.*]]412; CHECK: scalar.ph:413; CHECK-NEXT: br label [[FOR_BODY:%.*]]414; CHECK: for.cond.cleanup:415; CHECK-NEXT: ret void416; CHECK: for.body:417; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 1016, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]418; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]419; CHECK-NEXT: [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4420; CHECK-NEXT: [[MUL:%.*]] = shl nsw i32 [[TMP]], 1421; CHECK-NEXT: [[TMP1:%.*]] = lshr exact i64 [[INDVARS_IV]], 1422; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP1]]423; CHECK-NEXT: store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4424; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2425; CHECK-NEXT: [[CMP:%.*]] = icmp samesign ult i64 [[INDVARS_IV]], 1022426; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP8:![0-9]+]]427;428entry:429 br label %for.body430 431for.cond.cleanup: ; preds = %for.body432 ret void433 434for.body: ; preds = %for.body, %entry435 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]436 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv437 %tmp = load i32, ptr %arrayidx, align 4438 %mul = shl nsw i32 %tmp, 1439 %tmp1 = lshr exact i64 %indvars.iv, 1440 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %tmp1441 store i32 %mul, ptr %arrayidx2, align 4442 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2443 %cmp = icmp ult i64 %indvars.iv.next, 1024444 br i1 %cmp, label %for.body, label %for.cond.cleanup445}446 447; Check vectorization on an interleaved load group of factor 2 with 1 gap448; (missing the load of odd elements). Because the vectorized loop would449; speculatively access memory out-of-bounds, we must execute at least one450; iteration of the scalar loop.451 452; void even_load_dynamic_tc(int *A, int *B, unsigned N) {453; for (unsigned i = 0; i < N; i+=2)454; B[i/2] = A[i] * 2;455; }456 457 458define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i64 %N) {459; CHECK-LABEL: @even_load_dynamic_tc(460; CHECK-NEXT: entry:461; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 9462; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]463; CHECK: vector.ph:464; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1465; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1466; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1467; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 3468; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0469; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 4, i64 [[N_MOD_VF]]470; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP4]]471; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1472; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]473; CHECK: vector.body:474; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]475; CHECK-NEXT: [[DOTIDX:%.*]] = shl i64 [[INDEX]], 3476; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[DOTIDX]]477; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4478; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>479; CHECK-NEXT: [[TMP6:%.*]] = shl nsw <4 x i32> [[STRIDED_VEC]], splat (i32 1)480; CHECK-NEXT: [[TMP7:%.*]] = and i64 [[INDEX]], 9223372036854775804481; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP7]]482; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4483; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4484; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]485; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]486; CHECK: middle.block:487; CHECK-NEXT: br label [[SCALAR_PH]]488; CHECK: scalar.ph:489; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]490; CHECK-NEXT: br label [[FOR_BODY:%.*]]491; CHECK: for.cond.cleanup:492; CHECK-NEXT: ret void493; CHECK: for.body:494; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]495; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]496; CHECK-NEXT: [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4497; CHECK-NEXT: [[MUL:%.*]] = shl nsw i32 [[TMP]], 1498; CHECK-NEXT: [[TMP1:%.*]] = lshr exact i64 [[INDVARS_IV]], 1499; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP1]]500; CHECK-NEXT: store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4501; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 2502; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], [[N]]503; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP10:![0-9]+]]504;505entry:506 br label %for.body507 508for.cond.cleanup: ; preds = %for.body509 ret void510 511for.body: ; preds = %for.body, %entry512 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]513 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv514 %tmp = load i32, ptr %arrayidx, align 4515 %mul = shl nsw i32 %tmp, 1516 %tmp1 = lshr exact i64 %indvars.iv, 1517 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %tmp1518 store i32 %mul, ptr %arrayidx2, align 4519 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2520 %cmp = icmp ult i64 %indvars.iv.next, %N521 br i1 %cmp, label %for.body, label %for.cond.cleanup522}523 524; Check vectorization on a reverse interleaved load group of factor 2 with 1525; gap and a reverse interleaved store group of factor 2. The interleaved load526; group should be removed since it has a gap and is reverse.527 528; struct pair {529; int x;530; int y;531; };532;533; void load_gap_reverse(struct pair *P1, struct pair *P2, int X) {534; for (int i = 1023; i >= 0; i--) {535; int a = X + i;536; int b = B[i].y - i;537; A[i].x = a;538; B[i].y = b;539; }540; }541 542 543%pair = type { i64, i64 }544define void @load_gap_reverse(ptr noalias nocapture %P1, ptr noalias nocapture %P2, i64 %X) {545; CHECK-LABEL: @load_gap_reverse(546; CHECK-NEXT: entry:547; CHECK-NEXT: br label [[VECTOR_PH:%.*]]548; CHECK: vector.ph:549; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X:%.*]], i64 0550; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer551; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]552; CHECK: vector.body:553; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]554; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1023, i64 1022, i64 1021, i64 1020>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]555; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]556; CHECK-NEXT: [[TMP0:%.*]] = sub i64 1022, [[INDEX]]557; CHECK-NEXT: [[TMP1:%.*]] = sub i64 1021, [[INDEX]]558; CHECK-NEXT: [[TMP2:%.*]] = sub i64 1020, [[INDEX]]559; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT]], [[VEC_IND]]560; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0561; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1562; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2563; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3564; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P1:%.*]], i64 [[OFFSET_IDX]]565; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P1]], i64 [[TMP0]]566; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P1]], i64 [[TMP1]]567; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P1]], i64 [[TMP2]]568; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P2:%.*]], i64 [[OFFSET_IDX]]569; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 8570; CHECK-NEXT: [[DOTSPLIT1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P2]], i64 [[TMP0]]571; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT1]], i64 8572; CHECK-NEXT: [[DOTSPLIT2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P2]], i64 [[TMP1]]573; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT2]], i64 8574; CHECK-NEXT: [[DOTSPLIT3:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P2]], i64 [[TMP2]]575; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT3]], i64 8576; CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8577; CHECK-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8578; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8579; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8580; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i64 0581; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i64 1582; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i64 2583; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i64 3584; CHECK-NEXT: [[TMP20:%.*]] = sub nsw <4 x i64> [[TMP19]], [[VEC_IND]]585; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0586; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1587; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2588; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3589; CHECK-NEXT: store i64 [[TMP21]], ptr [[TMP4]], align 8590; CHECK-NEXT: store i64 [[TMP22]], ptr [[TMP5]], align 8591; CHECK-NEXT: store i64 [[TMP23]], ptr [[TMP6]], align 8592; CHECK-NEXT: store i64 [[TMP24]], ptr [[TMP7]], align 8593; CHECK-NEXT: store i64 [[TMP25]], ptr [[TMP8]], align 8594; CHECK-NEXT: store i64 [[TMP26]], ptr [[TMP9]], align 8595; CHECK-NEXT: store i64 [[TMP27]], ptr [[TMP10]], align 8596; CHECK-NEXT: store i64 [[TMP28]], ptr [[TMP11]], align 8597; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4598; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)599; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024600; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]601; CHECK: middle.block:602; CHECK-NEXT: br label [[FOR_BODY:%.*]]603; CHECK: for.exit:604; CHECK-NEXT: ret void605;606entry:607 br label %for.body608 609for.body:610 %i = phi i64 [ 1023, %entry ], [ %i.next, %for.body ]611 %0 = add nsw i64 %X, %i612 %1 = getelementptr inbounds %pair, ptr %P1, i64 %i, i32 0613 %2 = getelementptr inbounds %pair, ptr %P2, i64 %i, i32 1614 %3 = load i64, ptr %2, align 8615 %4 = sub nsw i64 %3, %i616 store i64 %0, ptr %1, align 8617 store i64 %4, ptr %2, align 8618 %i.next = add nsw i64 %i, -1619 %cond = icmp sgt i64 %i, 0620 br i1 %cond, label %for.body, label %for.exit621 622for.exit:623 ret void624}625 626; Check vectorization on interleaved access groups identified from mixed627; loads/stores.628; void mixed_load2_store2(int *A, int *B) {629; for (unsigned i = 0; i < 1024; i+=2) {630; B[i] = A[i] * A[i+1];631; B[i+1] = A[i] + A[i+1];632; }633; }634 635 636define void @mixed_load2_store2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) {637; CHECK-LABEL: @mixed_load2_store2(638; CHECK-NEXT: entry:639; CHECK-NEXT: br label [[VECTOR_PH:%.*]]640; CHECK: vector.ph:641; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]642; CHECK: vector.body:643; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]644; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1645; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]646; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4647; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>648; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>649; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]650; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[OFFSET_IDX]]651; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>652; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>653; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC3]]654; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>655; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 4656; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4657; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512658; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]659; CHECK: middle.block:660; CHECK-NEXT: br label [[FOR_BODY:%.*]]661; CHECK: for.cond.cleanup:662; CHECK-NEXT: ret void663;664entry:665 br label %for.body666 667for.cond.cleanup: ; preds = %for.body668 ret void669 670for.body: ; preds = %for.body, %entry671 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]672 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv673 %tmp = load i32, ptr %arrayidx, align 4674 %tmp1 = or disjoint i64 %indvars.iv, 1675 %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %tmp1676 %tmp2 = load i32, ptr %arrayidx2, align 4677 %mul = mul nsw i32 %tmp2, %tmp678 %arrayidx4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv679 store i32 %mul, ptr %arrayidx4, align 4680 %tmp3 = load i32, ptr %arrayidx, align 4681 %tmp4 = load i32, ptr %arrayidx2, align 4682 %add10 = add nsw i32 %tmp4, %tmp3683 %arrayidx13 = getelementptr inbounds i32, ptr %B, i64 %tmp1684 store i32 %add10, ptr %arrayidx13, align 4685 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2686 %cmp = icmp ult i64 %indvars.iv.next, 1024687 br i1 %cmp, label %for.body, label %for.cond.cleanup688}689 690; Check vectorization on interleaved access groups identified from mixed691; loads/stores.692; void mixed_load3_store3(int *A) {693; for (unsigned i = 0; i < 1024; i++) {694; *A++ += i;695; *A++ += i;696; *A++ += i;697; }698; }699 700 701define void @mixed_load3_store3(ptr nocapture %A) {702; CHECK-LABEL: @mixed_load3_store3(703; CHECK-NEXT: entry:704; CHECK-NEXT: br label [[VECTOR_PH:%.*]]705; CHECK: vector.ph:706; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]707; CHECK: vector.body:708; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]709; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]710; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 12711; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[OFFSET_IDX]]712; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, ptr [[NEXT_GEP]], align 4713; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>714; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>715; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>716; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_IND]]717; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_IND]]718; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[STRIDED_VEC3]], [[VEC_IND]]719; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>720; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>721; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>722; CHECK-NEXT: store <12 x i32> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 4723; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4724; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)725; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024726; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]727; CHECK: middle.block:728; CHECK-NEXT: br label [[FOR_BODY:%.*]]729; CHECK: for.cond.cleanup:730; CHECK-NEXT: ret void731;732entry:733 br label %for.body734 735for.cond.cleanup: ; preds = %for.body736 ret void737 738for.body: ; preds = %for.body, %entry739 %i.013 = phi i32 [ 0, %entry ], [ %inc, %for.body ]740 %A.addr.012 = phi ptr [ %A, %entry ], [ %incdec.ptr3, %for.body ]741 %incdec.ptr = getelementptr inbounds i32, ptr %A.addr.012, i64 1742 %tmp = load i32, ptr %A.addr.012, align 4743 %add = add i32 %tmp, %i.013744 store i32 %add, ptr %A.addr.012, align 4745 %incdec.ptr1 = getelementptr inbounds i32, ptr %A.addr.012, i64 2746 %tmp1 = load i32, ptr %incdec.ptr, align 4747 %add2 = add i32 %tmp1, %i.013748 store i32 %add2, ptr %incdec.ptr, align 4749 %incdec.ptr3 = getelementptr inbounds i32, ptr %A.addr.012, i64 3750 %tmp2 = load i32, ptr %incdec.ptr1, align 4751 %add4 = add i32 %tmp2, %i.013752 store i32 %add4, ptr %incdec.ptr1, align 4753 %inc = add nuw nsw i32 %i.013, 1754 %exitcond = icmp eq i32 %inc, 1024755 br i1 %exitcond, label %for.cond.cleanup, label %for.body756}757 758; Check vectorization on interleaved access groups with members having different759; kinds of type.760 761; struct IntFloat {762; int a;763; float b;764; };765;766; int SA;767; float SB;768;769; void int_float_struct(struct IntFloat *A) {770; int SumA;771; float SumB;772; for (unsigned i = 0; i < 1024; i++) {773; SumA += A[i].a;774; SumB += A[i].b;775; }776; SA = SumA;777; SB = SumB;778; }779 780 781%struct.IntFloat = type { i32, float }782 783@SA = common global i32 0, align 4784@SB = common global float 0.000000e+00, align 4785 786define void @int_float_struct(ptr nocapture readonly %A) {787; CHECK-LABEL: @int_float_struct(788; CHECK-NEXT: entry:789; CHECK-NEXT: br label [[VECTOR_PH:%.*]]790; CHECK: vector.ph:791; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]792; CHECK: vector.body:793; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]794; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]795; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]796; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_INTFLOAT:%.*]], ptr [[A:%.*]], i64 [[INDEX]]797; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4798; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>799; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>800; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[STRIDED_VEC2]] to <4 x float>801; CHECK-NEXT: [[TMP2]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_PHI1]]802; CHECK-NEXT: [[TMP3]] = fadd fast <4 x float> [[VEC_PHI]], [[TMP1]]803; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4804; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024805; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]806; CHECK: middle.block:807; CHECK-NEXT: br label [[FOR_BODY:%.*]]808; CHECK: for.cond.cleanup:809; CHECK-NEXT: [[ADD_LCSSA:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])810; CHECK-NEXT: [[ADD3_LCSSA:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP3]])811; CHECK-NEXT: store i32 [[ADD_LCSSA]], ptr @SA, align 4812; CHECK-NEXT: store float [[ADD3_LCSSA]], ptr @SB, align 4813; CHECK-NEXT: ret void814;815entry:816 br label %for.body817 818for.cond.cleanup: ; preds = %for.body819 store i32 %add, ptr @SA, align 4820 store float %add3, ptr @SB, align 4821 ret void822 823for.body: ; preds = %for.body, %entry824 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]825 %SumB.014 = phi float [ 0.0e+00, %entry ], [ %add3, %for.body ]826 %SumA.013 = phi i32 [ 0, %entry ], [ %add, %for.body ]827 %a = getelementptr inbounds %struct.IntFloat, ptr %A, i64 %indvars.iv, i32 0828 %tmp = load i32, ptr %a, align 4829 %add = add nsw i32 %tmp, %SumA.013830 %b = getelementptr inbounds %struct.IntFloat, ptr %A, i64 %indvars.iv, i32 1831 %tmp1 = load float, ptr %b, align 4832 %add3 = fadd fast float %SumB.014, %tmp1833 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1834 %exitcond = icmp eq i64 %indvars.iv.next, 1024835 br i1 %exitcond, label %for.cond.cleanup, label %for.body836}837 838; Check vectorization of interleaved access groups in the presence of839; dependences (PR27626). The following tests check that we don't reorder840; dependent loads and stores when generating code for interleaved access841; groups. Stores should be scalarized because the required code motion would842; break dependences, and the remaining interleaved load groups should have843; gaps.844 845; PR27626_0: Ensure a strided store is not moved after a dependent (zero846; distance) strided load.847 848; void PR27626_0(struct pair *p, int z, int n) {849; for (int i = 0; i < n; i++) {850; p[i].x = z;851; p[i].y = p[i].x;852; }853; }854 855 856%pair.i32 = type { i32, i32 }857define void @PR27626_0(ptr %p, i32 %z, i64 %n) {858; CHECK-LABEL: @PR27626_0(859; CHECK-NEXT: entry:860; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 5861; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]862; CHECK: vector.ph:863; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3864; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0865; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]]866; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]]867; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]868; CHECK: vector.body:869; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]870; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 1871; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 2872; CHECK-NEXT: [[TMP4:%.*]] = or disjoint i64 [[INDEX]], 3873; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]]874; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP2]]875; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP3]]876; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP4]]877; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[INDEX]]878; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 4879; CHECK-NEXT: [[DOTSPLIT1:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP2]]880; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT1]], i64 4881; CHECK-NEXT: [[DOTSPLIT2:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP3]]882; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT2]], i64 4883; CHECK-NEXT: [[DOTSPLIT3:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP4]]884; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT3]], i64 4885; CHECK-NEXT: store i32 [[Z:%.*]], ptr [[TMP5]], align 4886; CHECK-NEXT: store i32 [[Z]], ptr [[TMP6]], align 4887; CHECK-NEXT: store i32 [[Z]], ptr [[TMP7]], align 4888; CHECK-NEXT: store i32 [[Z]], ptr [[TMP8]], align 4889; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4890; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 0891; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 2892; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 4893; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 6894; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP9]], align 4895; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP10]], align 4896; CHECK-NEXT: store i32 [[TMP15]], ptr [[TMP11]], align 4897; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP12]], align 4898; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4899; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]900; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]901; CHECK: middle.block:902; CHECK-NEXT: br label [[SCALAR_PH]]903; CHECK: scalar.ph:904; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]905; CHECK-NEXT: br label [[FOR_BODY:%.*]]906; CHECK: for.body:907; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]908; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]]909; CHECK-NEXT: [[P_I_Y_SPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]]910; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds nuw i8, ptr [[P_I_Y_SPLIT]], i64 4911; CHECK-NEXT: store i32 [[Z]], ptr [[P_I_X]], align 4912; CHECK-NEXT: store i32 [[Z]], ptr [[P_I_Y]], align 4913; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1914; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]915; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP16:![0-9]+]]916; CHECK: for.end:917; CHECK-NEXT: ret void918;919entry:920 br label %for.body921 922for.body:923 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]924 %p_i.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 0925 %p_i.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 1926 store i32 %z, ptr %p_i.x, align 4927 %0 = load i32, ptr %p_i.x, align 4928 store i32 %0, ptr %p_i.y, align 4929 %i.next = add nuw nsw i64 %i, 1930 %cond = icmp slt i64 %i.next, %n931 br i1 %cond, label %for.body, label %for.end932 933for.end:934 ret void935}936 937; PR27626_1: Ensure a strided load is not moved before a dependent (zero938; distance) strided store.939 940; void PR27626_1(struct pair *p, int n) {941; int s = 0;942; for (int i = 0; i < n; i++) {943; p[i].y = p[i].x;944; s += p[i].y945; }946; }947 948 949define i32 @PR27626_1(ptr %p, i64 %n) {950; CHECK-LABEL: @PR27626_1(951; CHECK-NEXT: entry:952; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 5953; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]954; CHECK: vector.ph:955; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3956; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0957; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]]958; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]]959; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]960; CHECK: vector.body:961; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]962; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]963; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]]964; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[INDEX]]965; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 4966; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [[PAIR_I32]], ptr [[P]], i64 [[INDEX]]967; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP4]], i64 12968; CHECK-NEXT: [[TMP19:%.*]] = getelementptr [[PAIR_I32]], ptr [[P]], i64 [[INDEX]]969; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP19]], i64 20970; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [[PAIR_I32]], ptr [[P]], i64 [[INDEX]]971; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP20]], i64 28972; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4973; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 0974; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 2975; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 4976; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 6977; CHECK-NEXT: store i32 [[TMP10]], ptr [[TMP6]], align 4978; CHECK-NEXT: store i32 [[TMP11]], ptr [[TMP7]], align 4979; CHECK-NEXT: store i32 [[TMP12]], ptr [[TMP8]], align 4980; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP9]], align 4981; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i32>, ptr [[TMP6]], align 4982; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>983; CHECK-NEXT: [[TMP14]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_PHI]]984; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4985; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]986; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]987; CHECK: middle.block:988; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP14]])989; CHECK-NEXT: br label [[SCALAR_PH]]990; CHECK: scalar.ph:991; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]992; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]993; CHECK-NEXT: br label [[FOR_BODY:%.*]]994; CHECK: for.body:995; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]996; CHECK-NEXT: [[S:%.*]] = phi i32 [ [[TMP18:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]997; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]]998; CHECK-NEXT: [[P_I_Y_SPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]]999; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds nuw i8, ptr [[P_I_Y_SPLIT]], i64 41000; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[P_I_X]], align 41001; CHECK-NEXT: store i32 [[TMP17]], ptr [[P_I_Y]], align 41002; CHECK-NEXT: [[TMP18]] = add nsw i32 [[TMP17]], [[S]]1003; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 11004; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]1005; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP18:![0-9]+]]1006; CHECK: for.end:1007; CHECK-NEXT: ret i32 [[TMP18]]1008;1009entry:1010 br label %for.body1011 1012for.body:1013 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]1014 %s = phi i32 [ %2, %for.body ], [ 0, %entry ]1015 %p_i.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 01016 %p_i.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 11017 %0 = load i32, ptr %p_i.x, align 41018 store i32 %0, ptr %p_i.y, align 41019 %1 = load i32, ptr %p_i.y, align 41020 %2 = add nsw i32 %1, %s1021 %i.next = add nuw nsw i64 %i, 11022 %cond = icmp slt i64 %i.next, %n1023 br i1 %cond, label %for.body, label %for.end1024 1025for.end:1026 %3 = phi i32 [ %2, %for.body ]1027 ret i32 %31028}1029 1030; PR27626_2: Ensure a strided store is not moved after a dependent (negative1031; distance) strided load.1032 1033; void PR27626_2(struct pair *p, int z, int n) {1034; for (int i = 0; i < n; i++) {1035; p[i].x = z;1036; p[i].y = p[i - 1].x;1037; }1038; }1039 1040 1041define void @PR27626_2(ptr %p, i64 %n, i32 %z) {1042; CHECK-LABEL: @PR27626_2(1043; CHECK-NEXT: entry:1044; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 51045; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1046; CHECK: vector.ph:1047; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31048; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 01049; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]]1050; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]]1051; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1052; CHECK: vector.body:1053; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1054; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 11055; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 21056; CHECK-NEXT: [[TMP4:%.*]] = or disjoint i64 [[INDEX]], 31057; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]]1058; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP2]]1059; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP3]]1060; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP4]]1061; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 -81062; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[INDEX]]1063; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 41064; CHECK-NEXT: [[DOTSPLIT1:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP2]]1065; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT1]], i64 41066; CHECK-NEXT: [[DOTSPLIT2:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP3]]1067; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT2]], i64 41068; CHECK-NEXT: [[DOTSPLIT3:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP4]]1069; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT3]], i64 41070; CHECK-NEXT: store i32 [[Z:%.*]], ptr [[TMP5]], align 41071; CHECK-NEXT: store i32 [[Z]], ptr [[TMP6]], align 41072; CHECK-NEXT: store i32 [[Z]], ptr [[TMP7]], align 41073; CHECK-NEXT: store i32 [[Z]], ptr [[TMP8]], align 41074; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP9]], align 41075; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 01076; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 21077; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 41078; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 61079; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP10]], align 41080; CHECK-NEXT: store i32 [[TMP15]], ptr [[TMP11]], align 41081; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP12]], align 41082; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP13]], align 41083; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41084; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1085; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]1086; CHECK: middle.block:1087; CHECK-NEXT: br label [[SCALAR_PH]]1088; CHECK: scalar.ph:1089; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]1090; CHECK-NEXT: br label [[FOR_BODY:%.*]]1091; CHECK: for.body:1092; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1093; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]]1094; CHECK-NEXT: [[P_I_MINUS_1_X:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 -81095; CHECK-NEXT: [[P_I_Y_SPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]]1096; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds nuw i8, ptr [[P_I_Y_SPLIT]], i64 41097; CHECK-NEXT: store i32 [[Z]], ptr [[P_I_X]], align 41098; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[P_I_MINUS_1_X]], align 41099; CHECK-NEXT: store i32 [[TMP19]], ptr [[P_I_Y]], align 41100; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 11101; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]1102; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP20:![0-9]+]]1103; CHECK: for.end:1104; CHECK-NEXT: ret void1105;1106entry:1107 br label %for.body1108 1109for.body:1110 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]1111 %i_minus_1 = add nuw nsw i64 %i, -11112 %p_i.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 01113 %p_i_minus_1.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i_minus_1, i32 01114 %p_i.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 11115 store i32 %z, ptr %p_i.x, align 41116 %0 = load i32, ptr %p_i_minus_1.x, align 41117 store i32 %0, ptr %p_i.y, align 41118 %i.next = add nuw nsw i64 %i, 11119 %cond = icmp slt i64 %i.next, %n1120 br i1 %cond, label %for.body, label %for.end1121 1122for.end:1123 ret void1124}1125 1126; PR27626_3: Ensure a strided load is not moved before a dependent (negative1127; distance) strided store.1128 1129; void PR27626_3(struct pair *p, int z, int n) {1130; for (int i = 0; i < n; i++) {1131; p[i + 1].y = p[i].x;1132; s += p[i].y;1133; }1134; }1135 1136 1137define i32 @PR27626_3(ptr %p, i64 %n, i32 %z) {1138; CHECK-LABEL: @PR27626_3(1139; CHECK-NEXT: entry:1140; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 51141; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1142; CHECK: vector.ph:1143; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 31144; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 01145; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]]1146; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]]1147; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1148; CHECK: vector.body:1149; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1150; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1151; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]1152; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)1153; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i64 01154; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i64 11155; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i64 21156; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP2]], i64 31157; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]]1158; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[INDEX]]1159; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 41160; CHECK-NEXT: [[DOTSPLIT3:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[TMP5]]1161; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT3]], i64 41162; CHECK-NEXT: [[DOTSPLIT4:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[TMP7]]1163; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT4]], i64 41164; CHECK-NEXT: [[DOTSPLIT5:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[TMP9]]1165; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT5]], i64 41166; CHECK-NEXT: [[DOTSPLIT6:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[TMP11]]1167; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT6]], i64 41168; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP3]], align 41169; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 01170; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 21171; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 41172; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 61173; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP6]], align 41174; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 41175; CHECK-NEXT: store i32 [[TMP15]], ptr [[TMP10]], align 41176; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP12]], align 41177; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i32>, ptr [[TMP4]], align 41178; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>1179; CHECK-NEXT: [[TMP17]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_PHI]]1180; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41181; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)1182; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1183; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]1184; CHECK: middle.block:1185; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP17]])1186; CHECK-NEXT: br label [[SCALAR_PH]]1187; CHECK: scalar.ph:1188; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]1189; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]1190; CHECK-NEXT: br label [[FOR_BODY:%.*]]1191; CHECK: for.body:1192; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1193; CHECK-NEXT: [[S:%.*]] = phi i32 [ [[TMP22:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]1194; CHECK-NEXT: [[P_I_X:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]]1195; CHECK-NEXT: [[P_I_Y_SPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]]1196; CHECK-NEXT: [[P_I_Y:%.*]] = getelementptr inbounds nuw i8, ptr [[P_I_Y_SPLIT]], i64 41197; CHECK-NEXT: [[TMP23:%.*]] = getelementptr [[PAIR_I32]], ptr [[P]], i64 [[I]]1198; CHECK-NEXT: [[P_I_PLUS_1_Y:%.*]] = getelementptr i8, ptr [[TMP23]], i64 121199; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[P_I_X]], align 41200; CHECK-NEXT: store i32 [[TMP20]], ptr [[P_I_PLUS_1_Y]], align 41201; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[P_I_Y]], align 41202; CHECK-NEXT: [[TMP22]] = add nsw i32 [[TMP21]], [[S]]1203; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 11204; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]1205; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP22:![0-9]+]]1206; CHECK: for.end:1207; CHECK-NEXT: ret i32 [[TMP22]]1208;1209entry:1210 br label %for.body1211 1212for.body:1213 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]1214 %s = phi i32 [ %2, %for.body ], [ 0, %entry ]1215 %i_plus_1 = add nuw nsw i64 %i, 11216 %p_i.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 01217 %p_i.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 11218 %p_i_plus_1.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i_plus_1, i32 11219 %0 = load i32, ptr %p_i.x, align 41220 store i32 %0, ptr %p_i_plus_1.y, align 41221 %1 = load i32, ptr %p_i.y, align 41222 %2 = add nsw i32 %1, %s1223 %i.next = add nuw nsw i64 %i, 11224 %cond = icmp slt i64 %i.next, %n1225 br i1 %cond, label %for.body, label %for.end1226 1227for.end:1228 %3 = phi i32 [ %2, %for.body ]1229 ret i32 %31230}1231 1232; PR27626_4: Ensure we form an interleaved group for strided stores in the1233; presence of a write-after-write dependence. We create a group for1234; (2) and (3) while excluding (1).1235 1236; void PR27626_4(int *a, int x, int y, int z, int n) {1237; for (int i = 0; i < n; i += 2) {1238; a[i] = x; // (1)1239; a[i] = y; // (2)1240; a[i + 1] = z; // (3)1241; }1242; }1243 1244 1245define void @PR27626_4(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) {1246; CHECK-LABEL: @PR27626_4(1247; CHECK-NEXT: entry:1248; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 2)1249; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -11250; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 11251; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 11252; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 71253; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1254; CHECK: vector.ph:1255; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 92233720368547758041256; CHECK-NEXT: [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 11257; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i64 01258; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer1259; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[Z:%.*]], i64 01260; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer1261; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1262; CHECK: vector.body:1263; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1264; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 11265; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]1266; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[A]], i64 [[OFFSET_IDX]]1267; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP5]], i64 81268; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[A]], i64 [[OFFSET_IDX]]1269; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP7]], i64 161270; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[A]], i64 [[OFFSET_IDX]]1271; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP11]], i64 241272; CHECK-NEXT: store i32 [[X:%.*]], ptr [[TMP14]], align 41273; CHECK-NEXT: store i32 [[X]], ptr [[TMP8]], align 41274; CHECK-NEXT: store i32 [[X]], ptr [[TMP9]], align 41275; CHECK-NEXT: store i32 [[X]], ptr [[TMP10]], align 41276; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>1277; CHECK-NEXT: store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP14]], align 41278; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41279; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1280; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]1281; CHECK: middle.block:1282; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]1283; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]1284; CHECK: scalar.ph:1285; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]1286; CHECK-NEXT: br label [[FOR_BODY:%.*]]1287; CHECK: for.body:1288; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1289; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]1290; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]1291; CHECK-NEXT: [[A_I_PLUS_1:%.*]] = getelementptr i8, ptr [[TMP15]], i64 41292; CHECK-NEXT: store i32 [[Y]], ptr [[TMP12]], align 41293; CHECK-NEXT: store i32 [[Z]], ptr [[A_I_PLUS_1]], align 41294; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 21295; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]1296; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP24:![0-9]+]]1297; CHECK: for.end:1298; CHECK-NEXT: ret void1299;1300entry:1301 br label %for.body1302 1303for.body:1304 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]1305 %i_plus_1 = add i64 %i, 11306 %a_i = getelementptr inbounds i32, ptr %a, i64 %i1307 %a_i_plus_1 = getelementptr inbounds i32, ptr %a, i64 %i_plus_11308 store i32 %x, ptr %a_i, align 41309 store i32 %y, ptr %a_i, align 41310 store i32 %z, ptr %a_i_plus_1, align 41311 %i.next = add nuw nsw i64 %i, 21312 %cond = icmp slt i64 %i.next, %n1313 br i1 %cond, label %for.body, label %for.end1314 1315for.end:1316 ret void1317}1318 1319; PR27626_5: Ensure we do not form an interleaved group for strided stores in1320; the presence of a write-after-write dependence.1321 1322; void PR27626_5(int *a, int x, int y, int z, int n) {1323; for (int i = 3; i < n; i += 2) {1324; a[i - 1] = x;1325; a[i - 3] = y;1326; a[i] = z;1327; }1328; }1329 1330 1331define void @PR27626_5(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) {1332; CHECK-LABEL: @PR27626_5(1333; CHECK-NEXT: entry:1334; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 5)1335; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -41336; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 11337; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 11338; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 101339; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1340; CHECK: vector.ph:1341; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 92233720368547758041342; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[N_VEC]], 11343; CHECK-NEXT: [[IND_END:%.*]] = or disjoint i64 [[TMP3]], 31344; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1345; CHECK: vector.body:1346; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1347; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 5, i64 7, i64 9>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1348; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[INDEX]], 11349; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -1)1350; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[TMP7]], i64 01351; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP7]], i64 11352; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP7]], i64 21353; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP7]], i64 31354; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -3)1355; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP8]], i64 01356; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP8]], i64 11357; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[TMP8]], i64 21358; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i64> [[TMP8]], i64 31359; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[TMP4]]1360; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP12]], i64 121361; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP4]]1362; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP33]], i64 201363; CHECK-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP4]]1364; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP34]], i64 281365; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP4]]1366; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP35]], i64 361367; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]]1368; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP16]]1369; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP18]]1370; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP20]]1371; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP22]]1372; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP24]]1373; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP26]]1374; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP28]]1375; CHECK-NEXT: store i32 [[X:%.*]], ptr [[TMP15]], align 41376; CHECK-NEXT: store i32 [[X]], ptr [[TMP17]], align 41377; CHECK-NEXT: store i32 [[X]], ptr [[TMP19]], align 41378; CHECK-NEXT: store i32 [[X]], ptr [[TMP21]], align 41379; CHECK-NEXT: store i32 [[Y:%.*]], ptr [[TMP23]], align 41380; CHECK-NEXT: store i32 [[Y]], ptr [[TMP25]], align 41381; CHECK-NEXT: store i32 [[Y]], ptr [[TMP27]], align 41382; CHECK-NEXT: store i32 [[Y]], ptr [[TMP29]], align 41383; CHECK-NEXT: store i32 [[Z:%.*]], ptr [[TMP9]], align 41384; CHECK-NEXT: store i32 [[Z]], ptr [[TMP10]], align 41385; CHECK-NEXT: store i32 [[Z]], ptr [[TMP11]], align 41386; CHECK-NEXT: store i32 [[Z]], ptr [[TMP13]], align 41387; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41388; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 8)1389; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1390; CHECK-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]1391; CHECK: middle.block:1392; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]1393; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]1394; CHECK: scalar.ph:1395; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]1396; CHECK-NEXT: br label [[FOR_BODY:%.*]]1397; CHECK: for.body:1398; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1399; CHECK-NEXT: [[A_I:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]1400; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]1401; CHECK-NEXT: [[A_I_MINUS_1:%.*]] = getelementptr i8, ptr [[TMP31]], i64 -41402; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]1403; CHECK-NEXT: [[A_I_MINUS_3:%.*]] = getelementptr i8, ptr [[TMP32]], i64 -121404; CHECK-NEXT: store i32 [[X]], ptr [[A_I_MINUS_1]], align 41405; CHECK-NEXT: store i32 [[Y]], ptr [[A_I_MINUS_3]], align 41406; CHECK-NEXT: store i32 [[Z]], ptr [[A_I]], align 41407; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 21408; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]1409; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP26:![0-9]+]]1410; CHECK: for.end:1411; CHECK-NEXT: ret void1412;1413entry:1414 br label %for.body1415 1416for.body:1417 %i = phi i64 [ %i.next, %for.body ], [ 3, %entry ]1418 %i_minus_1 = sub i64 %i, 11419 %i_minus_3 = sub i64 %i_minus_1, 21420 %a_i = getelementptr inbounds i32, ptr %a, i64 %i1421 %a_i_minus_1 = getelementptr inbounds i32, ptr %a, i64 %i_minus_11422 %a_i_minus_3 = getelementptr inbounds i32, ptr %a, i64 %i_minus_31423 store i32 %x, ptr %a_i_minus_1, align 41424 store i32 %y, ptr %a_i_minus_3, align 41425 store i32 %z, ptr %a_i, align 41426 %i.next = add nuw nsw i64 %i, 21427 %cond = icmp slt i64 %i.next, %n1428 br i1 %cond, label %for.body, label %for.end1429 1430for.end:1431 ret void1432}1433 1434; PR34743: Ensure that a cast which needs to sink after a load that belongs to1435; an interleaved group, indeeded gets sunk.1436 1437; void PR34743(short *a, int *b, int n) {1438; for (int i = 0, iv = 0; iv < n; i++, iv += 2) {1439; b[i] = a[iv] * a[iv+1] * a[iv+2];1440; }1441; }1442 1443 1444define void @PR34743(ptr %a, ptr %b, i64 %n) {1445; CHECK-LABEL: @PR34743(1446; CHECK-NEXT: entry:1447; CHECK-NEXT: [[DOTPRE:%.*]] = load i16, ptr [[A:%.*]], align 21448; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[N:%.*]], 11449; CHECK-NEXT: [[TMP1:%.*]] = add nuw i64 [[TMP0]], 11450; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 61451; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]1452; CHECK: vector.memcheck:1453; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N]], 11454; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], -41455; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP3]]1456; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP4]], i64 41457; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 21458; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]1459; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[TMP5]], i64 61460; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]1461; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]1462; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]1463; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]1464; CHECK: vector.ph:1465; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], -41466; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 11467; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i16> poison, i16 [[DOTPRE]], i64 31468; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1469; CHECK: vector.body:1470; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1471; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[STRIDED_VEC4:%.*]], [[VECTOR_BODY]] ]1472; CHECK-NEXT: [[DOTIDX:%.*]] = shl i64 [[INDEX]], 21473; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX]]1474; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP17]], i64 21475; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i16>, ptr [[TMP7]], align 4, !alias.scope [[META27:![0-9]+]]1476; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>1477; CHECK-NEXT: [[STRIDED_VEC4]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>1478; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[STRIDED_VEC4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>1479; CHECK-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32>1480; CHECK-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[STRIDED_VEC]] to <4 x i32>1481; CHECK-NEXT: [[TMP11:%.*]] = sext <4 x i16> [[STRIDED_VEC4]] to <4 x i32>1482; CHECK-NEXT: [[TMP12:%.*]] = mul nsw <4 x i32> [[TMP9]], [[TMP10]]1483; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <4 x i32> [[TMP12]], [[TMP11]]1484; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]1485; CHECK-NEXT: store <4 x i32> [[TMP13]], ptr [[TMP14]], align 4, !alias.scope [[META30:![0-9]+]], !noalias [[META27]]1486; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41487; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1488; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]1489; CHECK: middle.block:1490; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i16> [[WIDE_VEC]], i64 71491; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]1492; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]1493; CHECK: scalar.ph:1494; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ]1495; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]1496; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]1497; CHECK-NEXT: br label [[LOOP:%.*]]1498; CHECK: loop:1499; CHECK-NEXT: [[TMP16:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[LOAD2:%.*]], [[LOOP]] ]1500; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV2:%.*]], [[LOOP]] ]1501; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ], [ [[I1:%.*]], [[LOOP]] ]1502; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP16]] to i321503; CHECK-NEXT: [[I1]] = add nuw nsw i64 [[I]], 11504; CHECK-NEXT: [[IV2]] = add nuw nsw i64 [[IV]], 21505; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i16, ptr [[A]], i64 [[IV]]1506; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[TMP18]], i64 21507; CHECK-NEXT: [[LOAD1:%.*]] = load i16, ptr [[GEP1]], align 41508; CHECK-NEXT: [[CONV1:%.*]] = sext i16 [[LOAD1]] to i321509; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV2]]1510; CHECK-NEXT: [[LOAD2]] = load i16, ptr [[GEP2]], align 41511; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[LOAD2]] to i321512; CHECK-NEXT: [[MUL01:%.*]] = mul nsw i32 [[CONV]], [[CONV1]]1513; CHECK-NEXT: [[MUL012:%.*]] = mul nsw i32 [[MUL01]], [[CONV2]]1514; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]1515; CHECK-NEXT: store i32 [[MUL012]], ptr [[ARRAYIDX5]], align 41516; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], [[N]]1517; CHECK-NEXT: br i1 [[EXITCOND]], label [[END]], label [[LOOP]], !llvm.loop [[LOOP33:![0-9]+]]1518; CHECK: end:1519; CHECK-NEXT: ret void1520;1521entry:1522 %.pre = load i16, ptr %a1523 br label %loop1524 1525loop:1526 %0 = phi i16 [ %.pre, %entry ], [ %load2, %loop ]1527 %iv = phi i64 [ 0, %entry ], [ %iv2, %loop ]1528 %i = phi i64 [ 0, %entry ], [ %i1, %loop ]1529 %conv = sext i16 %0 to i321530 %i1 = add nuw nsw i64 %i, 11531 %iv1 = add nuw nsw i64 %iv, 11532 %iv2 = add nuw nsw i64 %iv, 21533 %gep1 = getelementptr inbounds i16, ptr %a, i64 %iv11534 %load1 = load i16, ptr %gep1, align 41535 %conv1 = sext i16 %load1 to i321536 %gep2 = getelementptr inbounds i16, ptr %a, i64 %iv21537 %load2 = load i16, ptr %gep2, align 41538 %conv2 = sext i16 %load2 to i321539 %mul01 = mul nsw i32 %conv, %conv11540 %mul012 = mul nsw i32 %mul01, %conv21541 %arrayidx5 = getelementptr inbounds i32, ptr %b, i64 %i1542 store i32 %mul012, ptr %arrayidx51543 %exitcond = icmp eq i64 %iv, %n1544 br i1 %exitcond, label %end, label %loop1545 1546end:1547 ret void1548}1549