brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"2; RUN: opt -mtriple=aarch64-none-linux-gnu -S -passes=loop-vectorize,instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -mattr=+sve -scalable-vectorization=on -runtime-memory-check-threshold=24 < %s | FileCheck %s3 4target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"5 6; Check vectorization on an interleaved load group of factor 2 and an interleaved7; store group of factor 2.8 9; int AB[1024];10; int CD[1024];11;  void test_array_load2_store2(int C, int D) {12;   for (int i = 0; i < 1024; i+=2) {13;     int A = AB[i];14;     int B = AB[i+1];15;     CD[i] = A + C;16;     CD[i+1] = B * D;17;   }18; }19 20 21@AB = common global [1024 x i32] zeroinitializer, align 422@CD = common global [1024 x i32] zeroinitializer, align 423 24define void @test_array_load2_store2(i32 %C, i32 %D) #1 {25; CHECK-LABEL: @test_array_load2_store2(26; CHECK-NEXT:  entry:27; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]28; CHECK:       vector.ph:29; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()30; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 231; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C:%.*]], i64 032; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer33; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[D:%.*]], i64 034; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer35; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]36; CHECK:       vector.body:37; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]38; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 139; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr @AB, i64 [[OFFSET_IDX]]40; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP2]], align 441; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])42; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 043; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 144; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <vscale x 4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]45; CHECK-NEXT:    [[TMP7:%.*]] = mul nsw <vscale x 4 x i32> [[TMP4]], [[BROADCAST_SPLAT2]]46; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr @CD, i64 [[OFFSET_IDX]]47; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]])48; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 449; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]50; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 51251; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]52; CHECK:       middle.block:53; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH:%.*]]54; CHECK:       scalar.ph:55;56entry:57  br label %for.body58 59for.body:                                         ; preds = %for.body, %entry60  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]61  %arrayidx0 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv62  %load1 = load i32, ptr %arrayidx0, align 463  %or = or disjoint i64 %indvars.iv, 164  %arrayidx1 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %or65  %load2 = load i32, ptr %arrayidx1, align 466  %add = add nsw i32 %load1, %C67  %mul = mul nsw i32 %load2, %D68  %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv69  store i32 %add, ptr %arrayidx2, align 470  %arrayidx3 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %or71  store i32 %mul, ptr %arrayidx3, align 472  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 273  %cmp = icmp slt i64 %indvars.iv.next, 102474  br i1 %cmp, label %for.body, label %for.end75 76for.end:                                          ; preds = %for.body77  ret void78}79 80; Check vectorization on an interleaved load group of factor 2 with narrower types  and an interleaved81; store group of factor 2.82 83; short AB[1024];84; int CD[1024];85;  void test_array_load2_store2(int C, int D) {86;   for (int i = 0; i < 1024; i+=2) {87;     short A = AB[i];88;     short B = AB[i+1];89;     CD[i] = A + C;90;     CD[i+1] = B * D;91;   }92; }93 94 95@AB_i16 = common global [1024 x i16] zeroinitializer, align 496 97define void @test_array_load2_i16_store2(i32 %C, i32 %D) #1 {98; CHECK-LABEL: @test_array_load2_i16_store2(99; CHECK-NEXT:  entry:100; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]101; CHECK:       vector.ph:102; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()103; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2104; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C:%.*]], i64 0105; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer106; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[D:%.*]], i64 0107; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer108; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()109; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw <vscale x 4 x i64> [[TMP2]], splat (i64 1)110; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP0]], 3111; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0112; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer113; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]114; CHECK:       vector.body:115; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]116; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]117; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr @AB_i16, <vscale x 4 x i64> [[VEC_IND]]118; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> align 2 [[TMP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> poison)119; CHECK-NEXT:    [[TMP7:%.*]] = or disjoint <vscale x 4 x i64> [[VEC_IND]], splat (i64 1)120; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr @AB_i16, <vscale x 4 x i64> [[TMP7]]121; CHECK-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> align 2 [[TMP8]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> poison)122; CHECK-NEXT:    [[TMP9:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER]] to <vscale x 4 x i32>123; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[TMP9]]124; CHECK-NEXT:    [[DOTIDX:%.*]] = shl i64 [[INDEX]], 3125; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr @CD, i64 [[DOTIDX]]126; CHECK-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER1]] to <vscale x 4 x i32>127; CHECK-NEXT:    [[TMP12:%.*]] = mul nsw <vscale x 4 x i32> [[BROADCAST_SPLAT3]], [[TMP11]]128; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP12]])129; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP14]], align 4130; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]131; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]132; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512133; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]134; CHECK:       middle.block:135; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH:%.*]]136; CHECK:       scalar.ph:137;138entry:139  br label %for.body140 141for.body:                                         ; preds = %entry, %for.body142  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]143  %arrayidx = getelementptr inbounds [1024 x i16], ptr @AB_i16, i64 0, i64 %indvars.iv144  %0 = load i16, ptr %arrayidx, align 2145  %1 = or disjoint i64 %indvars.iv, 1146  %arrayidx2 = getelementptr inbounds [1024 x i16], ptr @AB_i16, i64 0, i64 %1147  %2 = load i16, ptr %arrayidx2, align 2148  %conv = sext i16 %0 to i32149  %add3 = add nsw i32 %conv, %C150  %arrayidx5 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv151  store i32 %add3, ptr %arrayidx5, align 4152  %conv6 = sext i16 %2 to i32153  %mul = mul nsw i32 %conv6, %D154  %arrayidx9 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %1155  store i32 %mul, ptr %arrayidx9, align 4156  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2157  %cmp = icmp ult i64 %indvars.iv, 1022158  br i1 %cmp, label %for.body, label %for.end159 160for.end:                                 ; preds = %for.body161  ret void162}163 164; Check vectorization on an interleaved load group of factor 2 and an interleaved165; store group of factor 2 with narrower types.166 167; int AB[1024];168; short CD[1024];169;  void test_array_load2_store2(int C, int D) {170;   for (int i = 0; i < 1024; i+=2) {171;     short A = AB[i];172;     short B = AB[i+1];173;     CD[i] = A + C;174;     CD[i+1] = B * D;175;   }176; }177 178 179@CD_i16 = dso_local local_unnamed_addr global [1024 x i16] zeroinitializer, align 2180 181define void @test_array_load2_store2_i16(i32 noundef %C, i32 noundef %D) #1 {182; CHECK-LABEL: @test_array_load2_store2_i16(183; CHECK-NEXT:  entry:184; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]185; CHECK:       vector.ph:186; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()187; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2188; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[C:%.*]], i64 0189; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer190; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[D:%.*]], i64 0191; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer192; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()193; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw <vscale x 4 x i64> [[TMP2]], splat (i64 1)194; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP0]], 3195; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0196; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer197; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]198; CHECK:       vector.body:199; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]200; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]201; CHECK-NEXT:    [[DOTIDX:%.*]] = shl i64 [[INDEX]], 3202; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr @AB, i64 [[DOTIDX]]203; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP6]], align 4204; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])205; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0206; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1207; CHECK-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 4 x i64> [[VEC_IND]], splat (i64 1)208; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <vscale x 4 x i32> [[TMP7]], [[BROADCAST_SPLAT]]209; CHECK-NEXT:    [[TMP11:%.*]] = trunc <vscale x 4 x i32> [[TMP10]] to <vscale x 4 x i16>210; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, ptr @CD_i16, <vscale x 4 x i64> [[VEC_IND]]211; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> [[TMP11]], <vscale x 4 x ptr> align 2 [[TMP12]], <vscale x 4 x i1> splat (i1 true))212; CHECK-NEXT:    [[TMP13:%.*]] = mul nsw <vscale x 4 x i32> [[TMP8]], [[BROADCAST_SPLAT2]]213; CHECK-NEXT:    [[TMP14:%.*]] = trunc <vscale x 4 x i32> [[TMP13]] to <vscale x 4 x i16>214; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i16, ptr @CD_i16, <vscale x 4 x i64> [[TMP9]]215; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> [[TMP14]], <vscale x 4 x ptr> align 2 [[TMP15]], <vscale x 4 x i1> splat (i1 true))216; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]217; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]218; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512219; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]220; CHECK:       middle.block:221; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH:%.*]]222; CHECK:       scalar.ph:223;224entry:225  br label %for.body226 227for.body:                                         ; preds = %entry, %for.body228  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]229  %arrayidx = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv230  %0 = load i32, ptr %arrayidx, align 4231  %1 = or disjoint i64 %indvars.iv, 1232  %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %1233  %2 = load i32, ptr %arrayidx2, align 4234  %add3 = add nsw i32 %0, %C235  %conv = trunc i32 %add3 to i16236  %arrayidx5 = getelementptr inbounds [1024 x i16], ptr @CD_i16, i64 0, i64 %indvars.iv237  store i16 %conv, ptr %arrayidx5, align 2238  %mul = mul nsw i32 %2, %D239  %conv6 = trunc i32 %mul to i16240  %arrayidx9 = getelementptr inbounds [1024 x i16], ptr @CD_i16, i64 0, i64 %1241  store i16 %conv6, ptr %arrayidx9, align 2242  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2243  %cmp = icmp ult i64 %indvars.iv, 1022244  br i1 %cmp, label %for.body, label %for.end245 246for.end:                                 ; preds = %for.body247  ret void248}249 250; Check vectorization on an interleaved load group of factor 6.251; There is no dedicated ldN/stN so use gather instead252 253%struct.ST6 = type { i32, i32, i32, i32, i32, i32 }254 255define i32 @test_struct_load6(ptr %S) #1 {256; CHECK-LABEL: @test_struct_load6(257; CHECK-NEXT:  entry:258; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]259; CHECK:       vector.ph:260; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()261; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2262; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()263; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP1]], i64 0264; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer265; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]266; CHECK:       vector.body:267; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]268; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]269; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]270; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ST6:%.*]], ptr [[S:%.*]], <vscale x 4 x i64> [[VEC_IND]]271; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)272; CHECK-NEXT:    [[DOTSPLIT:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]]273; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT]], i64 4274; CHECK-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)275; CHECK-NEXT:    [[DOTSPLIT6:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]]276; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT6]], i64 8277; CHECK-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)278; CHECK-NEXT:    [[DOTSPLIT7:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]]279; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT7]], i64 12280; CHECK-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)281; CHECK-NEXT:    [[DOTSPLIT8:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]]282; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT8]], i64 16283; CHECK-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)284; CHECK-NEXT:    [[DOTSPLIT9:%.*]] = getelementptr inbounds [[STRUCT_ST6]], ptr [[S]], <vscale x 4 x i64> [[VEC_IND]]285; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT9]], i64 20286; CHECK-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)287; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]288; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER2]]289; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], [[WIDE_MASKED_GATHER3]]290; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 4 x i32> [[TMP13]], [[WIDE_MASKED_GATHER4]]291; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_GATHER5]]292; CHECK-NEXT:    [[TMP16]] = sub <vscale x 4 x i32> [[TMP12]], [[TMP15]]293; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]294; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]295; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024296; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]297; CHECK:       middle.block:298; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH:%.*]]299; CHECK:       scalar.ph:300;301entry:302  br label %for.body303 304for.body:                                         ; preds = %entry, %for.body305  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]306  %r.041 = phi i32 [ 0, %entry ], [ %sub14, %for.body ]307  %x = getelementptr inbounds %struct.ST6, ptr %S, i64 %indvars.iv, i32 0308  %0 = load i32, ptr %x, align 4309  %y = getelementptr inbounds %struct.ST6, ptr %S, i64 %indvars.iv, i32 1310  %1 = load i32, ptr %y, align 4311  %z = getelementptr inbounds %struct.ST6, ptr %S, i64 %indvars.iv, i32 2312  %2 = load i32, ptr %z, align 4313  %w = getelementptr inbounds %struct.ST6, ptr %S, i64 %indvars.iv, i32 3314  %3 = load i32, ptr %w, align 4315  %a = getelementptr inbounds %struct.ST6, ptr %S, i64 %indvars.iv, i32 4316  %4 = load i32, ptr %a, align 4317  %b = getelementptr inbounds %struct.ST6, ptr %S, i64 %indvars.iv, i32 5318  %5 = load i32, ptr %b, align 4319  %.neg36 = add i32 %0, %r.041320  %6 = add i32 %.neg36, %2321  %7 = add i32 %1, %3322  %8 = add i32 %7, %4323  %9 = add i32 %8, %5324  %sub14 = sub i32 %6, %9325  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1326  %exitcond.not = icmp eq i64 %indvars.iv.next, 1024327  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body328 329for.cond.cleanup:                                 ; preds = %for.body330  %sub14.lcssa = phi i32 [ %sub14, %for.body ]331  ret i32 %sub14.lcssa332}333 334 335; Check vectorization on a reverse interleaved load group of factor 2 and336; a reverse interleaved store group of factor 2.337 338; struct ST2 {339;  int x;340;  int y;341; };342;343; void test_reversed_load2_store2(struct ST2 *A, struct ST2 *B) {344;   for (int i = 1023; i >= 0; i--) {345;     int a = A[i].x + i;  // interleaved load of index 0346;     int b = A[i].y - i;  // interleaved load of index 1347;     B[i].x = a;          // interleaved store of index 0348;     B[i].y = b;          // interleaved store of index 1349;   }350; }351 352 353%struct.ST2 = type { i32, i32 }354 355define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) #1 {356; CHECK-LABEL: @test_reversed_load2_store2(357; CHECK-NEXT:  entry:358; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]359; CHECK:       vector.ph:360; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()361; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2362; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()363; CHECK-NEXT:    [[INDUCTION:%.*]] = sub <vscale x 4 x i32> splat (i32 1023), [[TMP2]]364; CHECK-NEXT:    [[TMP3:%.*]] = trunc nuw nsw i64 [[TMP1]] to i32365; CHECK-NEXT:    [[DOTNEG:%.*]] = sub nsw i32 0, [[TMP3]]366; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTNEG]], i64 0367; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer368; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]369; CHECK:       vector.body:370; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]371; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]372; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]373; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_ST2:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]]374; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP0]], 3375; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw i64 2, [[TMP6]]376; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 [[TMP8]]377; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP9]], align 4378; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])379; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0380; CHECK-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP10]])381; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1382; CHECK-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP11]])383; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]384; CHECK-NEXT:    [[TMP13:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE1]], [[VEC_IND]]385; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_ST2]], ptr [[B:%.*]], i64 [[OFFSET_IDX]]386; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw nsw i64 [[TMP0]], 3387; CHECK-NEXT:    [[TMP18:%.*]] = sub nsw i64 2, [[TMP15]]388; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i64 [[TMP18]]389; CHECK-NEXT:    [[REVERSE2:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP12]])390; CHECK-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP13]])391; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[REVERSE2]], <vscale x 4 x i32> [[REVERSE3]])392; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP19]], align 4393; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]394; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]395; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024396; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]397; CHECK:       middle.block:398; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH:%.*]]399; CHECK:       scalar.ph:400;401entry:402  br label %for.body403 404for.cond.cleanup:                                 ; preds = %for.body405  ret void406 407for.body:                                         ; preds = %for.body, %entry408  %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]409  %x = getelementptr inbounds %struct.ST2, ptr %A, i64 %indvars.iv, i32 0410  %load1 = load i32, ptr %x, align 4411  %trunc = trunc i64 %indvars.iv to i32412  %add = add nsw i32 %load1, %trunc413  %y = getelementptr inbounds %struct.ST2, ptr %A, i64 %indvars.iv, i32 1414  %load2 = load i32, ptr %y, align 4415  %sub = sub nsw i32 %load2, %trunc416  %x5 = getelementptr inbounds %struct.ST2, ptr %B, i64 %indvars.iv, i32 0417  store i32 %add, ptr %x5, align 4418  %y8 = getelementptr inbounds %struct.ST2, ptr %B, i64 %indvars.iv, i32 1419  store i32 %sub, ptr %y8, align 4420  %indvars.iv.next = add nsw i64 %indvars.iv, -1421  %cmp = icmp sgt i64 %indvars.iv, 0422  br i1 %cmp, label %for.body, label %for.cond.cleanup423}424 425; Check vectorization on an interleaved load group of factor 2 with 1 gap426; (missing the load of odd elements). Because the vectorized loop would427; speculatively access memory out-of-bounds, we must execute at least one428; iteration of the scalar loop.429 430; void even_load_static_tc(int *A, int *B) {431;  for (unsigned i = 0; i < 1024; i+=2)432;     B[i/2] = A[i] * 2;433; }434 435 436define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) #1 {437; CHECK-LABEL: @even_load_static_tc(438; CHECK-NEXT:  entry:439; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]440; CHECK:       vector.ph:441; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()442; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2443; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 512, [[TMP1]]444; CHECK-NEXT:    [[IND_END:%.*]] = shl nuw nsw i64 [[N_VEC]], 1445; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]446; CHECK:       vector.body:447; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]448; CHECK-NEXT:    [[DOTIDX:%.*]] = shl i64 [[INDEX]], 3449; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[DOTIDX]]450; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP4]], align 4451; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])452; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0453; CHECK-NEXT:    [[TMP6:%.*]] = shl nsw <vscale x 4 x i32> [[TMP5]], splat (i32 1)454; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[INDEX]], 9223372036854775804455; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP7]]456; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP6]], ptr [[TMP8]], align 4457; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]458; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]459; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]460; CHECK:       middle.block:461; CHECK-NEXT:    br label [[SCALAR_PH:%.*]]462; CHECK:       scalar.ph:463;464entry:465  br label %for.body466 467for.cond.cleanup:                                 ; preds = %for.body468  ret void469 470for.body:                                         ; preds = %for.body, %entry471  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]472  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv473  %load = load i32, ptr %arrayidx, align 4474  %mul = shl nsw i32 %load, 1475  %lshr = lshr exact i64 %indvars.iv, 1476  %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %lshr477  store i32 %mul, ptr %arrayidx2, align 4478  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2479  %cmp = icmp ult i64 %indvars.iv.next, 1024480  br i1 %cmp, label %for.body, label %for.cond.cleanup481}482 483; Check vectorization on an interleaved load group of factor 2 with 1 gap484; (missing the load of odd elements). Because the vectorized loop would485; speculatively access memory out-of-bounds, we must execute at least one486; iteration of the scalar loop.487 488; void even_load_dynamic_tc(int *A, int *B, unsigned N) {489;  for (unsigned i = 0; i < N; i+=2)490;     B[i/2] = A[i] * 2;491; }492 493 494define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias nocapture %B, i64 %N) #1 {495; CHECK-LABEL: @even_load_dynamic_tc(496; CHECK-NEXT:  entry:497; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 2)498; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[UMAX]], -1499; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1500; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()501; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2502; CHECK-NEXT:    [[MIN_ITERS_CHECK_NOT_NOT:%.*]] = icmp samesign ult i64 [[TMP1]], [[TMP3]]503; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_NOT_NOT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]504; CHECK:       vector.ph:505; CHECK-NEXT:    [[TMP4:%.*]] = add nuw i64 [[TMP1]], 1506; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()507; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 2508; CHECK-NEXT:    [[TMP7:%.*]] = add nsw i64 [[TMP6]], -1509; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[TMP4]], [[TMP7]]510; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[N_MOD_VF]], 0511; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP6]], i64 [[N_MOD_VF]]512; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP9]]513; CHECK-NEXT:    [[IND_END:%.*]] = shl i64 [[N_VEC]], 1514; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]515; CHECK:       vector.body:516; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]517; CHECK-NEXT:    [[DOTIDX:%.*]] = shl i64 [[INDEX]], 3518; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[DOTIDX]]519; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP12]], align 4520; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])521; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0522; CHECK-NEXT:    [[TMP14:%.*]] = shl nsw <vscale x 4 x i32> [[TMP13]], splat (i32 1)523; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[INDEX]], 9223372036854775804524; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP15]]525; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP14]], ptr [[TMP16]], align 4526; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]527; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]528; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]529; CHECK:       middle.block:530; CHECK-NEXT:    br label [[SCALAR_PH]]531; CHECK:       scalar.ph:532;533entry:534  br label %for.body535 536for.cond.cleanup:                                 ; preds = %for.body537  ret void538 539for.body:                                         ; preds = %for.body, %entry540  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]541  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv542  %load = load i32, ptr %arrayidx, align 4543  %mul = shl nsw i32 %load, 1544  %lshr = lshr exact i64 %indvars.iv, 1545  %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %lshr546  store i32 %mul, ptr %arrayidx2, align 4547  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2548  %cmp = icmp ult i64 %indvars.iv.next, %N549  br i1 %cmp, label %for.body, label %for.cond.cleanup550}551 552; Check vectorization on a reverse interleaved load group of factor 2 with 1553; gap and a reverse interleaved store group of factor 2. The interleaved load554; group should be removed since it has a gap and is reverse.555 556; struct pair {557;  int x;558;  int y;559; };560;561; void load_gap_reverse(struct pair *P1, struct pair *P2, int X) {562;   for (int i = 1023; i >= 0; i--) {563;     int a = X + i;564;     int b = A[i].y - i;565;     B[i].x = a;566;     B[i].y = b;567;   }568; }569 570%pair = type { i64, i64 }571define void @load_gap_reverse(ptr noalias nocapture readonly %P1, ptr noalias nocapture readonly %P2, i64 %X) #1 {572; CHECK-LABEL: @load_gap_reverse(573; CHECK-NEXT:  entry:574; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]575; CHECK:       vector.ph:576; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()577; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2578; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[X:%.*]], i64 0579; CHECK-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer580; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()581; CHECK-NEXT:    [[INDUCTION:%.*]] = sub <vscale x 4 x i64> splat (i64 1023), [[TMP2]]582; CHECK-NEXT:    [[DOTNEG:%.*]] = sub nsw i64 0, [[TMP1]]583; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[DOTNEG]], i64 0584; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer585; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]586; CHECK:       vector.body:587; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]588; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]589; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <vscale x 4 x i64> [[BROADCAST_SPLAT1]], [[VEC_IND]]590; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P1:%.*]], <vscale x 4 x i64> [[VEC_IND]]591; CHECK-NEXT:    [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P2:%.*]], <vscale x 4 x i64> [[VEC_IND]]592; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT]], i64 8593; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[TMP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)594; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], [[VEC_IND]]595; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i64.nxv4p0(<vscale x 4 x i64> [[TMP4]], <vscale x 4 x ptr> align 8 [[TMP5]], <vscale x 4 x i1> splat (i1 true))596; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i64.nxv4p0(<vscale x 4 x i64> [[TMP7]], <vscale x 4 x ptr> align 8 [[TMP6]], <vscale x 4 x i1> splat (i1 true))597; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]598; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]599; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024600; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]601; CHECK:       middle.block:602; CHECK-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH:%.*]]603; CHECK:       scalar.ph:604;605entry:606  br label %for.body607 608for.body:609  %i = phi i64 [ 1023, %entry ], [ %i.next, %for.body ]610  %0 = add nsw i64 %X, %i611  %1 = getelementptr inbounds %pair, ptr %P1, i64 %i, i32 0612  %2 = getelementptr inbounds %pair, ptr %P2, i64 %i, i32 1613  %3 = load i64, ptr %2, align 8614  %4 = sub nsw i64 %3, %i615  store i64 %0, ptr %1, align 8616  store i64 %4, ptr %2, align 8617  %i.next = add nsw i64 %i, -1618  %cond = icmp sgt i64 %i, 0619  br i1 %cond, label %for.body, label %for.exit620 621for.exit:622  ret void623}624 625; Check vectorization on interleaved access groups identified from mixed626; loads/stores.627; void mixed_load2_store2(int *A, int *B) {628;   for (unsigned i = 0; i < 1024; i+=2)  {629;     B[i] = A[i] * A[i+1];630;     B[i+1] = A[i] + A[i+1];631;   }632; }633 634 635define void @mixed_load2_store2(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) #1 {636; CHECK-LABEL: @mixed_load2_store2(637; CHECK-NEXT:  entry:638; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]639; CHECK:       vector.ph:640; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()641; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2642; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]643; CHECK:       vector.body:644; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]645; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1646; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]647; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP2]], align 4648; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])649; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0650; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1651; CHECK-NEXT:    [[TMP6:%.*]] = mul nsw <vscale x 4 x i32> [[TMP4]], [[TMP3]]652; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[OFFSET_IDX]]653; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])654; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0655; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1656; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[TMP8]], [[TMP7]]657; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP9]])658; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4659; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]660; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512661; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]662; CHECK:       middle.block:663; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH:%.*]]664; CHECK:       scalar.ph:665;666entry:667  br label %for.body668 669for.cond.cleanup:                                 ; preds = %for.body670  ret void671 672for.body:                                         ; preds = %for.body, %entry673  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]674  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv675  %load1 = load i32, ptr %arrayidx, align 4676  %or = or disjoint i64 %indvars.iv, 1677  %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %or678  %load2 = load i32, ptr %arrayidx2, align 4679  %mul = mul nsw i32 %load2, %load1680  %arrayidx4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv681  store i32 %mul, ptr %arrayidx4, align 4682  %load3 = load i32, ptr %arrayidx, align 4683  %load4 = load i32, ptr %arrayidx2, align 4684  %add10 = add nsw i32 %load4, %load3685  %arrayidx13 = getelementptr inbounds i32, ptr %B, i64 %or686  store i32 %add10, ptr %arrayidx13, align 4687  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2688  %cmp = icmp ult i64 %indvars.iv.next, 1024689  br i1 %cmp, label %for.body, label %for.cond.cleanup690}691 692 693; Check vectorization on interleaved access groups with members having different694; kinds of type.695 696; struct IntFloat {697;   int a;698;   float b;699; };700;701; int SA;702; float SB;703;704; void int_float_struct(struct IntFloat *A) {705;   int SumA;706;   float SumB;707;   for (unsigned i = 0; i < 1024; i++)  {708;     SumA += A[i].a;709;     SumB += A[i].b;710;   }711;   SA = SumA;712;   SB = SumB;713; }714 715 716%struct.IntFloat = type { i32, float }717 718@SA = common global i32 0, align 4719@SB = common global float 0.000000e+00, align 4720 721define void @int_float_struct(ptr nocapture readonly %p) #0 {722; CHECK-LABEL: @int_float_struct(723; CHECK-NEXT:  entry:724; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]725; CHECK:       vector.ph:726; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()727; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2728; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]729; CHECK:       vector.body:730; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]731; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]732; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]733; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_INTFLOAT:%.*]], ptr [[P:%.*]], i64 [[INDEX]]734; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP2]], align 4735; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])736; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0737; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1738; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <vscale x 4 x i32> [[TMP4]] to <vscale x 4 x float>739; CHECK-NEXT:    [[TMP6]] = add <vscale x 4 x i32> [[TMP3]], [[VEC_PHI1]]740; CHECK-NEXT:    [[TMP7]] = fadd fast <vscale x 4 x float> [[VEC_PHI]], [[TMP5]]741; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]742; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024743; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]744; CHECK:       middle.block:745; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH:%.*]]746; CHECK:       scalar.ph:747;748entry:749  br label %for.body750 751for.cond.cleanup:                                 ; preds = %for.body752  store i32 %add, ptr @SA, align 4753  store float %add3, ptr @SB, align 4754  ret void755 756for.body:                                         ; preds = %for.body, %entry757  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]758  %SumB.014 = phi float [ 0.0e+00, %entry ], [ %add3, %for.body ]759  %SumA.013 = phi i32 [ 0, %entry ], [ %add, %for.body ]760  %a = getelementptr inbounds %struct.IntFloat, ptr %p, i64 %indvars.iv, i32 0761  %load1 = load i32, ptr %a, align 4762  %add = add nsw i32 %load1, %SumA.013763  %b = getelementptr inbounds %struct.IntFloat, ptr %p, i64 %indvars.iv, i32 1764  %load2 = load float, ptr %b, align 4765  %add3 = fadd fast float %SumB.014, %load2766  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1767  %exitcond = icmp eq i64 %indvars.iv.next, 1024768  br i1 %exitcond, label %for.cond.cleanup, label %for.body769}770 771; Check vectorization of interleaved access groups in the presence of772; dependences (PR27626). The following tests check that we don't reorder773; dependent loads and stores when generating code for interleaved access774; groups. Stores should be scalarized because the required code motion would775; break dependences, and the remaining interleaved load groups should have776; gaps.777 778; PR27626_0: Ensure a strided store is not moved after a dependent (zero779;            distance) strided load.780 781; void PR27626_0(struct pair *p, int z, int n) {782;   for (int i = 0; i < n; i++) {783;     p[i].x = z;784;     p[i].y = p[i].x;785;   }786; }787 788%pair.i32 = type { i32, i32 }789define void @PR27626_0(ptr %p, i32 %z, i64 %n) #1 {790; CHECK-LABEL: @PR27626_0(791; CHECK-NEXT:  entry:792; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)793; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()794; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2795; CHECK-NEXT:    [[MIN_ITERS_CHECK_NOT:%.*]] = icmp samesign ugt i64 [[SMAX]], [[TMP1]]796; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_NOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH:%.*]]797; CHECK:       vector.ph:798; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()799; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2800; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], -1801; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], [[TMP4]]802; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0803; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]804; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]805; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0806; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer807; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()808; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0809; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer810; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]811; CHECK:       vector.body:812; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]813; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]814; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], <vscale x 4 x i64> [[VEC_IND]]815; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 4 x ptr> [[TMP12]], i64 0816; CHECK-NEXT:    [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]817; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT]], i64 4818; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true))819; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP14]], align 4820; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])821; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0822; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP15]], <vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true))823; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]824; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]825; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]826; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]827; CHECK:       middle.block:828; CHECK-NEXT:    br label [[SCALAR_PH]]829; CHECK:       scalar.ph:830;831entry:832  br label %for.body833 834for.body:835  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]836  %p_i.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 0837  %p_i.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 1838  store i32 %z, ptr %p_i.x, align 4839  %0 = load i32, ptr %p_i.x, align 4840  store i32 %0, ptr %p_i.y, align 4841  %i.next = add nuw nsw i64 %i, 1842  %cond = icmp slt i64 %i.next, %n843  br i1 %cond, label %for.body, label %for.end844 845for.end:846  ret void847}848 849; PR27626_1: Ensure a strided load is not moved before a dependent (zero850;            distance) strided store.851 852; void PR27626_1(struct pair *p, int n) {853;   int s = 0;854;   for (int i = 0; i < n; i++) {855;     p[i].y = p[i].x;856;     s += p[i].y857;   }858; }859 860define i32 @PR27626_1(ptr %p, i64 %n) #1 {861; CHECK-LABEL: @PR27626_1(862; CHECK-NEXT:  entry:863; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)864; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()865; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2866; CHECK-NEXT:    [[MIN_ITERS_CHECK_NOT:%.*]] = icmp samesign ugt i64 [[SMAX]], [[TMP1]]867; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_NOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH:%.*]]868; CHECK:       vector.ph:869; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()870; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2871; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], -1872; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], [[TMP4]]873; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0874; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]875; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]876; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()877; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0878; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer879; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]880; CHECK:       vector.body:881; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]882; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]883; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]884; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]]885; CHECK-NEXT:    [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]886; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT]], i64 4887; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x ptr> [[TMP13]], i64 0888; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP12]], align 4889; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])890; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0891; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP14]], <vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true))892; CHECK-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x i32>, ptr [[TMP15]], align 4893; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC1]])894; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0895; CHECK-NEXT:    [[TMP17]] = add <vscale x 4 x i32> [[TMP16]], [[VEC_PHI]]896; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]897; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]898; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]899; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]900; CHECK:       middle.block:901; CHECK-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP17]])902; CHECK-NEXT:    br label [[SCALAR_PH]]903; CHECK:       scalar.ph:904;905entry:906  br label %for.body907 908for.body:909  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]910  %s = phi i32 [ %2, %for.body ], [ 0, %entry ]911  %p_i.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 0912  %p_i.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 1913  %0 = load i32, ptr %p_i.x, align 4914  store i32 %0, ptr %p_i.y, align 4915  %1 = load i32, ptr %p_i.y, align 4916  %2 = add nsw i32 %1, %s917  %i.next = add nuw nsw i64 %i, 1918  %cond = icmp slt i64 %i.next, %n919  br i1 %cond, label %for.body, label %for.end920 921for.end:922  %3 = phi i32 [ %2, %for.body ]923  ret i32 %3924}925 926; PR27626_2: Ensure a strided store is not moved after a dependent (negative927;            distance) strided load.928 929; void PR27626_2(struct pair *p, int z, int n) {930;   for (int i = 0; i < n; i++) {931;     p[i].x = z;932;     p[i].y = p[i - 1].x;933;   }934; }935 936define void @PR27626_2(ptr %p, i64 %n, i32 %z) #1 {937; CHECK-LABEL: @PR27626_2(938; CHECK-NEXT:  entry:939; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)940; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()941; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2942; CHECK-NEXT:    [[MIN_ITERS_CHECK_NOT:%.*]] = icmp samesign ugt i64 [[SMAX]], [[TMP1]]943; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_NOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH:%.*]]944; CHECK:       vector.ph:945; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()946; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2947; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], -1948; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], [[TMP4]]949; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0950; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]951; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]952; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0953; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer954; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()955; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0956; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer957; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]958; CHECK:       vector.body:959; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]960; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]961; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], <vscale x 4 x i64> [[VEC_IND]]962; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 -8963; CHECK-NEXT:    [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]964; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT]], i64 4965; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true))966; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP13]], align 4967; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])968; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0969; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP15]], <vscale x 4 x ptr> align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true))970; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]971; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]972; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]973; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]974; CHECK:       middle.block:975; CHECK-NEXT:    br label [[SCALAR_PH]]976; CHECK:       scalar.ph:977;978entry:979  br label %for.body980 981for.body:982  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]983  %i_minus_1 = add nuw nsw i64 %i, -1984  %p_i.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 0985  %p_i_minus_1.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i_minus_1, i32 0986  %p_i.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 1987  store i32 %z, ptr %p_i.x, align 4988  %0 = load i32, ptr %p_i_minus_1.x, align 4989  store i32 %0, ptr %p_i.y, align 4990  %i.next = add nuw nsw i64 %i, 1991  %cond = icmp slt i64 %i.next, %n992  br i1 %cond, label %for.body, label %for.end993 994for.end:995  ret void996}997 998; PR27626_3: Ensure a strided load is not moved before a dependent (negative999;            distance) strided store.1000 1001; void PR27626_3(struct pair *p, int z, int n) {1002;   for (int i = 0; i < n; i++) {1003;     p[i + 1].y = p[i].x;1004;     s += p[i].y;1005;   }1006; }1007 1008define i32 @PR27626_3(ptr %p, i64 %n, i32 %z) #1 {1009; CHECK-LABEL: @PR27626_3(1010; CHECK-NEXT:  entry:1011; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)1012; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()1013; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 21014; CHECK-NEXT:    [[MIN_ITERS_CHECK_NOT:%.*]] = icmp samesign ugt i64 [[SMAX]], [[TMP1]]1015; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK_NOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH:%.*]]1016; CHECK:       vector.ph:1017; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()1018; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 21019; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], -11020; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], [[TMP4]]1021; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 01022; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]1023; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]1024; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()1025; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 01026; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer1027; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]1028; CHECK:       vector.body:1029; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1030; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1031; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]1032; CHECK-NEXT:    [[TMP12:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 1)1033; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]]1034; CHECK-NEXT:    [[DOTSPLIT:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[INDEX]]1035; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 41036; CHECK-NEXT:    [[DOTSPLIT3:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], <vscale x 4 x i64> [[TMP12]]1037; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, <vscale x 4 x ptr> [[DOTSPLIT3]], i64 41038; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP13]], align 41039; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])1040; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 01041; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true))1042; CHECK-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x i32>, ptr [[TMP14]], align 41043; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC1]])1044; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 01045; CHECK-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[TMP17]], [[VEC_PHI]]1046; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]1047; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]1048; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1049; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]1050; CHECK:       middle.block:1051; CHECK-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP18]])1052; CHECK-NEXT:    br label [[SCALAR_PH]]1053; CHECK:       scalar.ph:1054;1055entry:1056  br label %for.body1057 1058for.body:1059  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]1060  %s = phi i32 [ %2, %for.body ], [ 0, %entry ]1061  %i_plus_1 = add nuw nsw i64 %i, 11062  %p_i.x = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 01063  %p_i.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 11064  %p_i_plus_1.y = getelementptr inbounds %pair.i32, ptr %p, i64 %i_plus_1, i32 11065  %0 = load i32, ptr %p_i.x, align 41066  store i32 %0, ptr %p_i_plus_1.y, align 41067  %1 = load i32, ptr %p_i.y, align 41068  %2 = add nsw i32 %1, %s1069  %i.next = add nuw nsw i64 %i, 11070  %cond = icmp slt i64 %i.next, %n1071  br i1 %cond, label %for.body, label %for.end1072 1073for.end:1074  %3 = phi i32 [ %2, %for.body ]1075  ret i32 %31076}1077 1078; PR27626_4: Ensure we form an interleaved group for strided stores in the1079;            presence of a write-after-write dependence. We create a group for1080;            (2) and (3) while excluding (1).1081 1082; void PR27626_4(int *a, int x, int y, int z, int n) {1083;   for (int i = 0; i < n; i += 2) {1084;     a[i] = x;      // (1)1085;     a[i] = y;      // (2)1086;     a[i + 1] = z;  // (3)1087;   }1088; }1089 1090define void @PR27626_4(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) #1 {1091; CHECK-LABEL: @PR27626_4(1092; CHECK-NEXT:  entry:1093; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 2)1094; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -11095; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 11096; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 11097; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()1098; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 21099; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i64 [[TMP2]], [[TMP4]]1100; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1101; CHECK:       vector.ph:1102; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()1103; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 21104; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP7]]1105; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNOT]]1106; CHECK-NEXT:    [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 11107; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 01108; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer1109; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Y:%.*]], i64 01110; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer1111; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 01112; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer1113; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()1114; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw <vscale x 4 x i64> [[TMP10]], splat (i64 1)1115; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP6]], 31116; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 01117; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer1118; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]1119; CHECK:       vector.body:1120; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1121; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1122; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]1123; CHECK-NEXT:    [[P:%.*]] = extractelement <vscale x 4 x ptr> [[TMP13]], i64 01124; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true))1125; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[BROADCAST_SPLAT2]], <vscale x 4 x i32> [[BROADCAST_SPLAT4]])1126; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[P]], align 41127; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]1128; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]1129; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1130; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]1131; CHECK:       middle.block:1132; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]1133; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]1134; CHECK:       scalar.ph:1135;1136entry:1137  br label %for.body1138 1139for.body:1140  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]1141  %i_plus_1 = add i64 %i, 11142  %a_i = getelementptr inbounds i32, ptr %a, i64 %i1143  %a_i_plus_1 = getelementptr inbounds i32, ptr %a, i64 %i_plus_11144  store i32 %x, ptr %a_i, align 41145  store i32 %y, ptr %a_i, align 41146  store i32 %z, ptr %a_i_plus_1, align 41147  %i.next = add nuw nsw i64 %i, 21148  %cond = icmp slt i64 %i.next, %n1149  br i1 %cond, label %for.body, label %for.end1150 1151for.end:1152  ret void1153}1154 1155; PR27626_5: Ensure we do not form an interleaved group for strided stores in1156;            the presence of a write-after-write dependence.1157 1158; void PR27626_5(int *a, int x, int y, int z, int n) {1159;   for (int i = 3; i < n; i += 2) {1160;     a[i - 1] = x;1161;     a[i - 3] = y;1162;     a[i] = z;1163;   }1164; }1165 1166define void @PR27626_5(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) #1 {1167; CHECK-LABEL: @PR27626_5(1168; CHECK-NEXT:  entry:1169; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 5)1170; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -41171; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 11172; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 11173; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()1174; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 21175; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i64 [[TMP2]], [[TMP4]]1176; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1177; CHECK:       vector.ph:1178; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()1179; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 21180; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP8]]1181; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNOT]]1182; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[N_VEC]], 11183; CHECK-NEXT:    [[IND_END:%.*]] = or disjoint i64 [[TMP11]], 31184; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 01185; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer1186; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Y:%.*]], i64 01187; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer1188; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 01189; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer1190; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()1191; CHECK-NEXT:    [[TMP19:%.*]] = shl nuw nsw <vscale x 4 x i64> [[TMP10]], splat (i64 1)1192; CHECK-NEXT:    [[INDUCTION:%.*]] = add nuw nsw <vscale x 4 x i64> [[TMP19]], splat (i64 3)1193; CHECK-NEXT:    [[TMP12:%.*]] = shl nuw nsw i64 [[TMP7]], 31194; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 01195; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer1196; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]1197; CHECK:       vector.body:1198; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1199; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1200; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i64> [[VEC_IND]], splat (i64 -1)1201; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 4 x i64> [[VEC_IND]], splat (i64 -3)1202; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]1203; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], <vscale x 4 x i64> [[TMP13]]1204; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], <vscale x 4 x i64> [[TMP14]]1205; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x ptr> align 4 [[TMP16]], <vscale x 4 x i1> splat (i1 true))1206; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT2]], <vscale x 4 x ptr> align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true))1207; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT4]], <vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true))1208; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]1209; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]1210; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1211; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]1212; CHECK:       middle.block:1213; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]1214; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]1215; CHECK:       scalar.ph:1216;1217entry:1218  br label %for.body1219 1220for.body:1221  %i = phi i64 [ %i.next, %for.body ], [ 3, %entry ]1222  %i_minus_1 = sub i64 %i, 11223  %i_minus_3 = sub i64 %i_minus_1, 21224  %a_i = getelementptr inbounds i32, ptr %a, i64 %i1225  %a_i_minus_1 = getelementptr inbounds i32, ptr %a, i64 %i_minus_11226  %a_i_minus_3 = getelementptr inbounds i32, ptr %a, i64 %i_minus_31227  store i32 %x, ptr %a_i_minus_1, align 41228  store i32 %y, ptr %a_i_minus_3, align 41229  store i32 %z, ptr %a_i, align 41230  %i.next = add nuw nsw i64 %i, 21231  %cond = icmp slt i64 %i.next, %n1232  br i1 %cond, label %for.body, label %for.end1233 1234for.end:1235  ret void1236}1237 1238; PR34743: Ensure that a cast which needs to sink after a load that belongs to1239; an interleaved group, indeeded gets sunk.1240 1241; void PR34743(short *a, int *b, int n) {1242;   for (int i = 0, iv = 0; iv < n; i++, iv += 2) {1243;     b[i] = a[iv] * a[iv+1] * a[iv+2];1244;   }1245; }1246 1247define void @PR34743(ptr %a, ptr %b, i64 %n) #1 {1248; CHECK-LABEL: @PR34743(1249; CHECK-NEXT:  entry:1250; CHECK-NEXT:    [[DOTPRE:%.*]] = load i16, ptr [[A:%.*]], align 21251; CHECK-NEXT:    [[TMP0:%.*]] = lshr i64 [[N:%.*]], 11252; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i64 [[TMP0]], 11253; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()1254; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 21255; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]1256; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]1257; CHECK:       vector.memcheck:1258; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[N]], 11259; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -41260; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP5]]1261; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP6]], i64 41262; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 21263; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]1264; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[TMP7]], i64 61265; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]1266; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]1267; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]1268; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]1269; CHECK:       vector.ph:1270; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()1271; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw nsw i64 [[TMP9]], 21272; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP10]]1273; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP1]], [[DOTNOT]]1274; CHECK-NEXT:    [[IND_END:%.*]] = shl i64 [[N_VEC]], 11275; CHECK-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()1276; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw nsw <vscale x 4 x i64> [[TMP14]], splat (i64 1)1277; CHECK-NEXT:    [[TMP17:%.*]] = shl nuw nsw i64 [[TMP9]], 31278; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP17]], i64 01279; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer1280; CHECK-NEXT:    [[TMP33:%.*]] = call i32 @llvm.vscale.i32()1281; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw nsw i32 [[TMP33]], 21282; CHECK-NEXT:    [[TMP34:%.*]] = add nsw i32 [[TMP16]], -11283; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[DOTPRE]], i32 [[TMP34]]1284; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]1285; CHECK:       vector.body:1286; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1287; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_MASKED_GATHER4:%.*]], [[VECTOR_BODY]] ]1288; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP15]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1289; CHECK-NEXT:    [[TMP18:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 1)1290; CHECK-NEXT:    [[TMP19:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 2)1291; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i16, ptr [[A]], <vscale x 4 x i64> [[TMP18]]1292; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> poison), !alias.scope [[META34:![0-9]+]]1293; CHECK-NEXT:    [[TMP21:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER]] to <vscale x 4 x i32>1294; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i16, ptr [[A]], <vscale x 4 x i64> [[TMP19]]1295; CHECK-NEXT:    [[WIDE_MASKED_GATHER4]] = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP22]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> poison), !alias.scope [[META34]]1296; CHECK-NEXT:    [[TMP23:%.*]] = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> [[VECTOR_RECUR]], <vscale x 4 x i16> [[WIDE_MASKED_GATHER4]], i32 -1)1297; CHECK-NEXT:    [[TMP24:%.*]] = sext <vscale x 4 x i16> [[TMP23]] to <vscale x 4 x i32>1298; CHECK-NEXT:    [[TMP25:%.*]] = sext <vscale x 4 x i16> [[WIDE_MASKED_GATHER4]] to <vscale x 4 x i32>1299; CHECK-NEXT:    [[TMP26:%.*]] = mul nsw <vscale x 4 x i32> [[TMP24]], [[TMP21]]1300; CHECK-NEXT:    [[TMP27:%.*]] = mul nsw <vscale x 4 x i32> [[TMP26]], [[TMP25]]1301; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]1302; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP27]], ptr [[TMP28]], align 4, !alias.scope [[META37:![0-9]+]], !noalias [[META34]]1303; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]1304; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]1305; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1306; CHECK-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]1307; CHECK:       middle.block:1308; CHECK-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vscale.i32()1309; CHECK-NEXT:    [[TMP31:%.*]] = shl nuw nsw i32 [[TMP30]], 21310; CHECK-NEXT:    [[TMP32:%.*]] = add nsw i32 [[TMP31]], -11311; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_MASKED_GATHER4]], i32 [[TMP32]]1312; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]1313; CHECK-NEXT:    br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]1314; CHECK:       scalar.ph:1315;1316entry:1317  %.pre = load i16, ptr %a1318  br label %loop1319 1320loop:1321  %0 = phi i16 [ %.pre, %entry ], [ %load2, %loop ]1322  %iv = phi i64 [ 0, %entry ], [ %iv2, %loop ]1323  %i = phi i64 [ 0, %entry ], [ %i1, %loop ]1324  %conv = sext i16 %0 to i321325  %i1 = add nuw nsw i64 %i, 11326  %iv1 = add nuw nsw i64 %iv, 11327  %iv2 = add nuw nsw i64 %iv, 21328  %gep1 = getelementptr inbounds i16, ptr %a, i64 %iv11329  %load1 = load i16, ptr %gep1, align 41330  %conv1 = sext i16 %load1 to i321331  %gep2 = getelementptr inbounds i16, ptr %a, i64 %iv21332  %load2 = load i16, ptr %gep2, align 41333  %conv2 = sext i16 %load2 to i321334  %mul01 = mul nsw i32 %conv, %conv11335  %mul012 = mul nsw i32 %mul01, %conv21336  %arrayidx5 = getelementptr inbounds i32, ptr %b, i64 %i1337  store i32 %mul012, ptr %arrayidx51338  %exitcond = icmp eq i64 %iv, %n1339  br i1 %exitcond, label %end, label %loop1340 1341end:1342  ret void1343}1344 1345; Check vectorization on an interleaved load/store groups of factor 31346 1347; for (int i = 0; i < 1024; ++i) {1348;   dst[i].x = a[i].x + b[i].x;1349;   dst[i].y = a[i].y - b[i].y;1350;   dst[i].z = a[i].z << b[i].z;1351; }1352;1353; TODO: Support scalable interleave groups once we can also codegen1354; @llvm.[de]interleave31355%struct.xyz = type { i32, i32, i32 }1356 1357define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {1358; CHECK-LABEL: @interleave_deinterleave_factor3(1359; CHECK-NEXT:  entry:1360; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()1361; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 2561362; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1363; CHECK:       vector.ph:1364; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()1365; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 21366; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]1367; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 1024, [[N_MOD_VF]]1368; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()1369; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 01370; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer1371; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]1372; CHECK:       vector.body:1373; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1374; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1375; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]1376; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP19]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)1377; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], <vscale x 4 x i64> [[VEC_IND]]1378; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)1379; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <vscale x 4 x i32> [[TMP11]], [[TMP7]]1380; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]1381; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP14]], <vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true))1382; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP19]], i64 41383; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)1384; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP20]], i64 41385; CHECK-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP22]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)1386; CHECK-NEXT:    [[TMP16:%.*]] = sub nsw <vscale x 4 x i32> [[TMP8]], [[TMP12]]1387; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP10]], i64 41388; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x ptr> align 4 [[TMP23]], <vscale x 4 x i1> splat (i1 true))1389; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP19]], i64 81390; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)1391; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP20]], i64 81392; CHECK-NEXT:    [[TMP13:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP24]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)1393; CHECK-NEXT:    [[TMP17:%.*]] = shl <vscale x 4 x i32> [[TMP9]], [[TMP13]]1394; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP10]], i64 81395; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP17]], <vscale x 4 x ptr> align 4 [[TMP25]], <vscale x 4 x i1> splat (i1 true))1396; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]1397; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]1398; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1399; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]1400; CHECK:       middle.block:1401; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 01402; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]1403; CHECK:       scalar.ph:1404;1405entry:1406  br label %for.body1407 1408for.body:1409  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]1410  %arrayidx = getelementptr inbounds %struct.xyz, ptr %a, i64 %indvars.iv1411  %0 = load i32, ptr %arrayidx, align 41412  %arrayidx2 = getelementptr inbounds %struct.xyz, ptr %b, i64 %indvars.iv1413  %1 = load i32, ptr %arrayidx2, align 41414  %add = add nsw i32 %1, %01415  %arrayidx5 = getelementptr inbounds %struct.xyz, ptr %dst, i64 %indvars.iv1416  store i32 %add, ptr %arrayidx5, align 41417  %y = getelementptr inbounds nuw i8, ptr %arrayidx, i64 41418  %2 = load i32, ptr %y, align 41419  %y11 = getelementptr inbounds nuw i8, ptr %arrayidx2, i64 41420  %3 = load i32, ptr %y11, align 41421  %sub = sub nsw i32 %2, %31422  %y14 = getelementptr inbounds nuw i8, ptr %arrayidx5, i64 41423  store i32 %sub, ptr %y14, align 41424  %z = getelementptr inbounds nuw i8, ptr %arrayidx, i64 81425  %4 = load i32, ptr %z, align 41426  %z19 = getelementptr inbounds nuw i8, ptr %arrayidx2, i64 81427  %5 = load i32, ptr %z19, align 41428  %shl = shl i32 %4, %51429  %z22 = getelementptr inbounds nuw i8, ptr %arrayidx5, i64 81430  store i32 %shl, ptr %z22, align 41431  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11432  %exitcond.not = icmp eq i64 %indvars.iv.next, 10241433  br i1 %exitcond.not, label %for.end, label %for.body1434 1435for.end:1436  ret void1437}1438 1439; Check vectorization on an interleaved load/store groups of factor 41440 1441; for (int i = 0; i < 1024; ++i) {1442;   dst[i].x = a[i].x + b[i].x;1443;   dst[i].y = a[i].y - b[i].y;1444;   dst[i].z = a[i].z << b[i].z;1445;   dst[i].t = a[i].t >> b[i].t;1446; }1447%struct.xyzt = type { i32, i32, i32, i32 }1448 1449define void @interleave_deinterleave(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {1450; CHECK-LABEL: @interleave_deinterleave(1451; CHECK-NEXT:  entry:1452; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()1453; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 2561454; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1455; CHECK:       vector.ph:1456; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()1457; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 21458; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]1459; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 1024, [[N_MOD_VF]]1460; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]1461; CHECK:       vector.body:1462; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1463; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_XYZT:%.*]], ptr [[A:%.*]], i64 [[INDEX]]1464; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP6]], align 41465; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])1466; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 01467; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 11468; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 21469; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 31470; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B:%.*]], i64 [[INDEX]]1471; CHECK-NEXT:    [[WIDE_VEC8:%.*]] = load <vscale x 16 x i32>, ptr [[TMP13]], align 41472; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC8]])1473; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 01474; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 11475; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 21476; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 31477; CHECK-NEXT:    [[TMP20:%.*]] = add nsw <vscale x 4 x i32> [[TMP16]], [[TMP9]]1478; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[DST:%.*]], i64 [[INDEX]]1479; CHECK-NEXT:    [[TMP22:%.*]] = sub nsw <vscale x 4 x i32> [[TMP10]], [[TMP17]]1480; CHECK-NEXT:    [[TMP23:%.*]] = shl <vscale x 4 x i32> [[TMP11]], [[TMP18]]1481; CHECK-NEXT:    [[TMP24:%.*]] = ashr <vscale x 4 x i32> [[TMP12]], [[TMP19]]1482; CHECK-NEXT:    [[INTERLEAVED_VEC13:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[TMP23]], <vscale x 4 x i32> [[TMP24]])1483; CHECK-NEXT:    store <vscale x 16 x i32> [[INTERLEAVED_VEC13]], ptr [[TMP21]], align 41484; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]1485; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1486; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]1487; CHECK:       middle.block:1488; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 01489; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]1490; CHECK:       scalar.ph:1491;1492entry:1493  br label %for.body1494 1495for.body:1496  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]1497  %arrayidx = getelementptr inbounds %struct.xyzt, ptr %a, i64 %indvars.iv1498  %0 = load i32, ptr %arrayidx, align 41499  %arrayidx2 = getelementptr inbounds %struct.xyzt, ptr %b, i64 %indvars.iv1500  %1 = load i32, ptr %arrayidx2, align 41501  %add = add nsw i32 %1, %01502  %arrayidx5 = getelementptr inbounds %struct.xyzt, ptr %dst, i64 %indvars.iv1503  store i32 %add, ptr %arrayidx5, align 41504  %y = getelementptr inbounds nuw i8, ptr %arrayidx, i64 41505  %2 = load i32, ptr %y, align 41506  %y11 = getelementptr inbounds nuw i8, ptr %arrayidx2, i64 41507  %3 = load i32, ptr %y11, align 41508  %sub = sub nsw i32 %2, %31509  %y14 = getelementptr inbounds nuw i8, ptr %arrayidx5, i64 41510  store i32 %sub, ptr %y14, align 41511  %z = getelementptr inbounds nuw i8, ptr %arrayidx, i64 81512  %4 = load i32, ptr %z, align 41513  %z19 = getelementptr inbounds nuw i8, ptr %arrayidx2, i64 81514  %5 = load i32, ptr %z19, align 41515  %shl = shl i32 %4, %51516  %z22 = getelementptr inbounds nuw i8, ptr %arrayidx5, i64 81517  store i32 %shl, ptr %z22, align 41518  %t = getelementptr inbounds nuw i8, ptr %arrayidx, i64 121519  %6 = load i32, ptr %t, align 41520  %t27 = getelementptr inbounds nuw i8, ptr %arrayidx2, i64 121521  %7 = load i32, ptr %t27, align 41522  %shr = ashr i32 %6, %71523  %t30 = getelementptr inbounds nuw i8, ptr %arrayidx5, i64 121524  store i32 %shr, ptr %t30, align 41525  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11526  %exitcond.not = icmp eq i64 %indvars.iv.next, 10241527  br i1 %exitcond.not, label %for.end, label %for.body1528 1529for.end:1530  ret void1531}1532 1533; Check vectorization on a reverse interleaved load/store groups of factor 41534 1535; for (int i = 1023; i >= 0; i--) {1536;   int a = A[i].x + i;1537;   int b = A[i].y - i;1538;   int c = A[i].z * i;1539;   int d = A[i].t << i;1540;   B[i].x = a;1541;   B[i].y = b;1542;   B[i].z = c;1543;   B[i].t = d;1544; }1545 1546define void @interleave_deinterleave_reverse(ptr noalias nocapture readonly %A, ptr noalias nocapture %B) #1{1547; CHECK-LABEL: @interleave_deinterleave_reverse(1548; CHECK-NEXT:  entry:1549; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]1550; CHECK:       vector.ph:1551; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()1552; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 21553; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()1554; CHECK-NEXT:    [[INDUCTION:%.*]] = sub <vscale x 4 x i32> splat (i32 1023), [[TMP2]]1555; CHECK-NEXT:    [[TMP3:%.*]] = trunc nuw nsw i64 [[TMP1]] to i321556; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw i32 0, [[TMP3]]1557; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP4]], i64 01558; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer1559; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]1560; CHECK:       vector.body:1561; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1562; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1563; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]1564; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_XYZT:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]]1565; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP0]], 41566; CHECK-NEXT:    [[TMP9:%.*]] = sub nsw i64 4, [[TMP6]]1567; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP9]]1568; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP10]], align 41569; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])1570; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 01571; CHECK-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP13]])1572; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 11573; CHECK-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP14]])1574; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 21575; CHECK-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP15]])1576; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 31577; CHECK-NEXT:    [[REVERSE5:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP16]])1578; CHECK-NEXT:    [[TMP17:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]1579; CHECK-NEXT:    [[TMP18:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE3]], [[VEC_IND]]1580; CHECK-NEXT:    [[TMP19:%.*]] = mul nsw <vscale x 4 x i32> [[REVERSE4]], [[VEC_IND]]1581; CHECK-NEXT:    [[TMP20:%.*]] = shl nuw nsw <vscale x 4 x i32> [[REVERSE5]], [[VEC_IND]]1582; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B:%.*]], i64 [[OFFSET_IDX]]1583; CHECK-NEXT:    [[TMP22:%.*]] = shl nuw nsw i64 [[TMP0]], 41584; CHECK-NEXT:    [[TMP25:%.*]] = sub nsw i64 4, [[TMP22]]1585; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP25]]1586; CHECK-NEXT:    [[REVERSE6:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP17]])1587; CHECK-NEXT:    [[REVERSE7:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP18]])1588; CHECK-NEXT:    [[REVERSE8:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP19]])1589; CHECK-NEXT:    [[REVERSE9:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP20]])1590; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[REVERSE6]], <vscale x 4 x i32> [[REVERSE7]], <vscale x 4 x i32> [[REVERSE8]], <vscale x 4 x i32> [[REVERSE9]])1591; CHECK-NEXT:    store <vscale x 16 x i32> [[INTERLEAVED_VEC11]], ptr [[TMP26]], align 41592; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]1593; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]1594; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10241595; CHECK-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]1596; CHECK:       middle.block:1597; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH:%.*]]1598; CHECK:       scalar.ph:1599;1600entry:1601  br label %for.body1602for.cond.cleanup:                                 ; preds = %for.body1603  ret void1604for.body:                                         ; preds = %for.body, %entry1605  %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]1606  %x = getelementptr inbounds %struct.xyzt, ptr %A, i64 %indvars.iv, i32 01607  %load1 = load i32, ptr %x, align 41608  %trunc = trunc i64 %indvars.iv to i321609  %add = add nsw i32 %load1, %trunc1610  %y = getelementptr inbounds %struct.xyzt, ptr %A, i64 %indvars.iv, i32 11611  %load2 = load i32, ptr %y, align 41612  %sub = sub nsw i32 %load2, %trunc1613  %z = getelementptr inbounds %struct.xyzt, ptr %A, i64 %indvars.iv, i32 21614  %load3 = load i32, ptr %z, align 41615  %mul = mul nsw i32 %load3, %trunc1616  %t = getelementptr inbounds %struct.xyzt, ptr %A, i64 %indvars.iv, i32 31617  %load4 = load i32, ptr %t, align 41618  %shl = shl nuw nsw i32 %load4, %trunc1619  %x5 = getelementptr inbounds %struct.xyzt, ptr %B, i64 %indvars.iv, i32 01620  store i32 %add, ptr %x5, align 41621  %y8 = getelementptr inbounds %struct.xyzt, ptr %B, i64 %indvars.iv, i32 11622  store i32 %sub, ptr %y8, align 41623  %z5 = getelementptr inbounds %struct.xyzt, ptr %B, i64 %indvars.iv, i32 21624  store i32 %mul, ptr %z5, align 41625  %t8 = getelementptr inbounds %struct.xyzt, ptr %B, i64 %indvars.iv, i32 31626  store i32 %shl, ptr %t8, align 41627  %indvars.iv.next = add nsw i64 %indvars.iv, -11628  %cmp = icmp sgt i64 %indvars.iv, 01629  br i1 %cmp, label %for.body, label %for.cond.cleanup1630 1631}1632attributes #1 = { "target-features"="+sve" vscale_range(1, 16) }1633attributes #0 = { "target-features"="+sve" vscale_range(1, 16) }1634