730 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -p loop-vectorize -mcpu=skylake -S %s | FileCheck %s3 4target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"5target triple = "x86_64-unknown-linux"6 7; Test cases for https://github.com/llvm/llvm-project/issues/82936.8define void @test_4xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {9; CHECK-LABEL: define void @test_4xi64(10; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] {11; CHECK-NEXT: [[ENTRY:.*]]:12; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 413; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]14; CHECK: [[VECTOR_PH]]:15; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 416; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]17; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]18; CHECK: [[VECTOR_BODY]]:19; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]20; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]21; CHECK-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX]], align 822; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 023; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer24; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV]], i32 025; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 826; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]27; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP3]], align 828; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 129; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]30; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]31; CHECK: [[MIDDLE_BLOCK]]:32; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]33; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]34; CHECK: [[SCALAR_PH]]:35; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]36; CHECK-NEXT: br label %[[LOOP:.*]]37; CHECK: [[LOOP]]:38; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]39; CHECK-NEXT: [[DATA_2:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]]40; CHECK-NEXT: [[L_2:%.*]] = load i64, ptr [[DATA_2]], align 841; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV1]], i32 042; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 843; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_2]], [[L_0]]44; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_0]], align 845; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV1]], i32 146; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 847; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_2]], [[L_1]]48; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 849; CHECK-NEXT: [[DATA_4:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV1]], i32 250; CHECK-NEXT: [[L_4:%.*]] = load i64, ptr [[DATA_4]], align 851; CHECK-NEXT: [[MUL_2:%.*]] = mul i64 [[L_2]], [[L_4]]52; CHECK-NEXT: store i64 [[MUL_2]], ptr [[DATA_4]], align 853; CHECK-NEXT: [[DATA_3:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV1]], i32 354; CHECK-NEXT: [[L_3:%.*]] = load i64, ptr [[DATA_3]], align 855; CHECK-NEXT: [[MUL_3:%.*]] = mul i64 [[L_2]], [[L_3]]56; CHECK-NEXT: store i64 [[MUL_3]], ptr [[DATA_3]], align 857; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 158; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]59; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]60; CHECK: [[EXIT]]:61; CHECK-NEXT: ret void62;63entry:64 br label %loop65 66loop:67 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]68 %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv69 %l.factor = load i64, ptr %arrayidx, align 870 %data.0 = getelementptr inbounds { i64 , i64, i64, i64 }, ptr %data, i64 %iv, i32 071 %l.0 = load i64, ptr %data.0, align 872 %mul.0 = mul i64 %l.factor, %l.073 store i64 %mul.0, ptr %data.0, align 874 %data.1 = getelementptr inbounds { i64 , i64, i64, i64 }, ptr %data, i64 %iv, i32 175 %l.1 = load i64, ptr %data.1, align 876 %mul.1 = mul i64 %l.factor, %l.177 store i64 %mul.1, ptr %data.1, align 878 %data.2 = getelementptr inbounds { i64 , i64, i64, i64 }, ptr %data, i64 %iv, i32 279 %l.2 = load i64, ptr %data.2, align 880 %mul.2 = mul i64 %l.factor, %l.281 store i64 %mul.2, ptr %data.2, align 882 %data.3 = getelementptr inbounds { i64 , i64, i64, i64 }, ptr %data, i64 %iv, i32 383 %l.3 = load i64, ptr %data.3, align 884 %mul.3 = mul i64 %l.factor, %l.385 store i64 %mul.3, ptr %data.3, align 886 %iv.next = add nuw nsw i64 %iv, 187 %ec = icmp eq i64 %iv.next, %n88 br i1 %ec, label %exit, label %loop89 90exit:91 ret void92}93 94define void @test_2xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {95; CHECK-LABEL: define void @test_2xi64(96; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {97; CHECK-NEXT: [[ENTRY:.*]]:98; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 499; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]100; CHECK: [[VECTOR_PH]]:101; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4102; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]103; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]104; CHECK: [[VECTOR_BODY]]:105; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]106; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]107; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[ARRAYIDX]], align 8108; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1109; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]110; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8111; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>112; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>113; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]114; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]115; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>116; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>117; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8118; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4119; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]120; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]121; CHECK: [[MIDDLE_BLOCK]]:122; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]123; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]124; CHECK: [[SCALAR_PH]]:125; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]126; CHECK-NEXT: br label %[[LOOP:.*]]127; CHECK: [[LOOP]]:128; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]129; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]]130; CHECK-NEXT: [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8131; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[IV1]], 1132; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]133; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8134; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR]], [[L_0]]135; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_0]], align 8136; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[TMP0]], 1137; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]138; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8139; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR]], [[L_1]]140; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8141; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1142; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]143; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]144; CHECK: [[EXIT]]:145; CHECK-NEXT: ret void146;147entry:148 br label %loop149 150loop:151 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]152 %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv153 %l.factor = load i64, ptr %arrayidx, align 8154 %1 = shl nsw i64 %iv, 1155 %data.0 = getelementptr inbounds i64, ptr %data, i64 %1156 %l.0 = load i64, ptr %data.0, align 8157 %mul.0 = mul i64 %l.factor, %l.0158 store i64 %mul.0, ptr %data.0, align 8159 %3 = or disjoint i64 %1, 1160 %data.1 = getelementptr inbounds i64, ptr %data, i64 %3161 %l.1 = load i64, ptr %data.1, align 8162 %mul.1 = mul i64 %l.factor, %l.1163 store i64 %mul.1, ptr %data.1, align 8164 %iv.next = add nuw nsw i64 %iv, 1165 %ec = icmp eq i64 %iv.next, %n166 br i1 %ec, label %exit, label %loop167 168exit:169 ret void170}171 172define void @test_2xi64_interleave_loads_order_flipped(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {173; CHECK-LABEL: define void @test_2xi64_interleave_loads_order_flipped(174; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {175; CHECK-NEXT: [[ENTRY:.*]]:176; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4177; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]178; CHECK: [[VECTOR_PH]]:179; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4180; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]181; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]182; CHECK: [[VECTOR_BODY]]:183; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]184; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]185; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[ARRAYIDX]], align 8186; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1187; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]188; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8189; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>190; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>191; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]192; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]193; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>194; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>195; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8196; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4197; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]198; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]199; CHECK: [[MIDDLE_BLOCK]]:200; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]201; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]202; CHECK: [[SCALAR_PH]]:203; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]204; CHECK-NEXT: br label %[[LOOP:.*]]205; CHECK: [[LOOP]]:206; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]207; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]]208; CHECK-NEXT: [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8209; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[IV1]], 1210; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]211; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8212; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[TMP0]], 1213; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]214; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8215; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR]], [[L_1]]216; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_0]], align 8217; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR]], [[L_0]]218; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8219; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1220; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]221; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]222; CHECK: [[EXIT]]:223; CHECK-NEXT: ret void224;225entry:226 br label %loop227 228loop:229 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]230 %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv231 %l.factor = load i64, ptr %arrayidx, align 8232 %1 = shl nsw i64 %iv, 1233 %data.0 = getelementptr inbounds i64, ptr %data, i64 %1234 %l.0 = load i64, ptr %data.0, align 8235 %3 = or disjoint i64 %1, 1236 %data.1 = getelementptr inbounds i64, ptr %data, i64 %3237 %l.1 = load i64, ptr %data.1, align 8238 %mul.0 = mul i64 %l.factor, %l.1239 store i64 %mul.0, ptr %data.0, align 8240 %mul.1 = mul i64 %l.factor, %l.0241 store i64 %mul.1, ptr %data.1, align 8242 %iv.next = add nuw nsw i64 %iv, 1243 %ec = icmp eq i64 %iv.next, %n244 br i1 %ec, label %exit, label %loop245 246exit:247 ret void248}249 250define void @test_2xi64_store_order_flipped_1(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {251; CHECK-LABEL: define void @test_2xi64_store_order_flipped_1(252; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {253; CHECK-NEXT: [[ENTRY:.*]]:254; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4255; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]256; CHECK: [[VECTOR_PH]]:257; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4258; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]259; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]260; CHECK: [[VECTOR_BODY]]:261; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]262; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]263; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[ARRAYIDX]], align 8264; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1265; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]266; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8267; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>268; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>269; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]270; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]271; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>272; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>273; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8274; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4275; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]276; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]277; CHECK: [[MIDDLE_BLOCK]]:278; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]279; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]280; CHECK: [[SCALAR_PH]]:281; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]282; CHECK-NEXT: br label %[[LOOP:.*]]283; CHECK: [[LOOP]]:284; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]285; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]]286; CHECK-NEXT: [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8287; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[IV1]], 1288; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]289; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8290; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR]], [[L_0]]291; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[TMP0]], 1292; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]293; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8294; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR]], [[L_1]]295; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_0]], align 8296; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_1]], align 8297; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1298; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]299; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]300; CHECK: [[EXIT]]:301; CHECK-NEXT: ret void302;303entry:304 br label %loop305 306loop:307 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]308 %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv309 %l.factor = load i64, ptr %arrayidx, align 8310 %1 = shl nsw i64 %iv, 1311 %data.0 = getelementptr inbounds i64, ptr %data, i64 %1312 %l.0 = load i64, ptr %data.0, align 8313 %mul.0 = mul i64 %l.factor, %l.0314 %3 = or disjoint i64 %1, 1315 %data.1 = getelementptr inbounds i64, ptr %data, i64 %3316 %l.1 = load i64, ptr %data.1, align 8317 %mul.1 = mul i64 %l.factor, %l.1318 store i64 %mul.1, ptr %data.0, align 8319 store i64 %mul.0, ptr %data.1, align 8320 %iv.next = add nuw nsw i64 %iv, 1321 %ec = icmp eq i64 %iv.next, %n322 br i1 %ec, label %exit, label %loop323 324exit:325 ret void326}327 328define void @test_2xi64_store_order_flipped_2(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {329; CHECK-LABEL: define void @test_2xi64_store_order_flipped_2(330; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {331; CHECK-NEXT: [[ENTRY:.*]]:332; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4333; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]334; CHECK: [[VECTOR_PH]]:335; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4336; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]337; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]338; CHECK: [[VECTOR_BODY]]:339; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]340; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]341; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[ARRAYIDX]], align 8342; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[IV]], 1343; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]344; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8345; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>346; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>347; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]348; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]349; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>350; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP8]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>351; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 8352; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4353; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]354; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]355; CHECK: [[MIDDLE_BLOCK]]:356; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]357; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]358; CHECK: [[SCALAR_PH]]:359; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]360; CHECK-NEXT: br label %[[LOOP:.*]]361; CHECK: [[LOOP]]:362; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]363; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]]364; CHECK-NEXT: [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8365; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[IV1]], 1366; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]367; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8368; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR]], [[L_0]]369; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[TMP0]], 1370; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]371; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8372; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR]], [[L_1]]373; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_1]], align 8374; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_0]], align 8375; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1376; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]377; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]378; CHECK: [[EXIT]]:379; CHECK-NEXT: ret void380;381entry:382 br label %loop383 384loop:385 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]386 %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv387 %l.factor = load i64, ptr %arrayidx, align 8388 %1 = shl nsw i64 %iv, 1389 %data.0 = getelementptr inbounds i64, ptr %data, i64 %1390 %l.0 = load i64, ptr %data.0, align 8391 %mul.0 = mul i64 %l.factor, %l.0392 %3 = or disjoint i64 %1, 1393 %data.1 = getelementptr inbounds i64, ptr %data, i64 %3394 %l.1 = load i64, ptr %data.1, align 8395 %mul.1 = mul i64 %l.factor, %l.1396 store i64 %mul.0, ptr %data.1, align 8397 store i64 %mul.1, ptr %data.0, align 8398 %iv.next = add nuw nsw i64 %iv, 1399 %ec = icmp eq i64 %iv.next, %n400 br i1 %ec, label %exit, label %loop401 402exit:403 ret void404}405 406define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noalias %src.0, ptr noalias %src.1, i64 noundef %n) {407; CHECK-LABEL: define void @test_2xi64_different_loads_feeding_fmul(408; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[SRC_0:%.*]], ptr noalias [[SRC_1:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {409; CHECK-NEXT: [[ENTRY:.*]]:410; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 4411; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]412; CHECK: [[VECTOR_PH]]:413; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4414; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0415; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]416; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP3]]417; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]418; CHECK: [[VECTOR_BODY]]:419; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]420; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i64, ptr [[SRC_0]], i64 [[IV]]421; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[GEP_SRC_0]], align 8422; CHECK-NEXT: [[TMP5:%.*]] = shl nsw i64 [[IV]], 1423; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP5]]424; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[GEP_SRC_0]], align 8425; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]426; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[TMP5]], 1427; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP7]]428; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP8]], align 8429; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>430; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[SRC_1]], i64 [[IV]]431; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP10]], align 8432; CHECK-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[WIDE_LOAD2]], [[STRIDED_VEC]]433; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>434; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP14]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>435; CHECK-NEXT: store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP16]], align 8436; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4437; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]438; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]439; CHECK: [[MIDDLE_BLOCK]]:440; CHECK-NEXT: br label %[[SCALAR_PH]]441; CHECK: [[SCALAR_PH]]:442; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]443; CHECK-NEXT: br label %[[LOOP:.*]]444; CHECK: [[LOOP]]:445; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]446; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i64, ptr [[SRC_0]], i64 [[IV1]]447; CHECK-NEXT: [[L_SRC_0:%.*]] = load i64, ptr [[GEP_SRC_2]], align 8448; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[IV1]], 1449; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]450; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[GEP_SRC_2]], align 8451; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_SRC_0]], [[L_0]]452; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_0]], align 8453; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[TMP0]], 1454; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]455; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8456; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds i64, ptr [[SRC_1]], i64 [[IV1]]457; CHECK-NEXT: [[L_SRC_1:%.*]] = load i64, ptr [[GEP_SRC_1]], align 8458; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_SRC_1]], [[L_1]]459; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8460; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1461; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]462; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]463; CHECK: [[EXIT]]:464; CHECK-NEXT: ret void465;466entry:467 br label %loop468 469loop:470 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]471 %gep.src.0 = getelementptr inbounds i64, ptr %src.0, i64 %iv472 %l.src.0 = load i64, ptr %gep.src.0, align 8473 %1 = shl nsw i64 %iv, 1474 %data.0 = getelementptr inbounds i64, ptr %data, i64 %1475 %l.0 = load i64, ptr %gep.src.0, align 8476 %mul.0 = mul i64 %l.src.0, %l.0477 store i64 %mul.0, ptr %data.0, align 8478 %3 = or disjoint i64 %1, 1479 %data.1 = getelementptr inbounds i64, ptr %data, i64 %3480 %l.1 = load i64, ptr %data.1, align 8481 %gep.src.1 = getelementptr inbounds i64, ptr %src.1, i64 %iv482 %l.src.1 = load i64, ptr %gep.src.1, align 8483 %mul.1 = mul i64 %l.src.1, %l.1484 store i64 %mul.1, ptr %data.1, align 8485 %iv.next = add nuw nsw i64 %iv, 1486 %ec = icmp eq i64 %iv.next, %n487 br i1 %ec, label %exit, label %loop488 489exit:490 ret void491}492 493define void @test_3xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {494; CHECK-LABEL: define void @test_3xi64(495; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {496; CHECK-NEXT: [[ENTRY:.*]]:497; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4498; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]499; CHECK: [[VECTOR_PH]]:500; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4501; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]502; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]503; CHECK: [[VECTOR_BODY]]:504; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]505; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]506; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[ARRAYIDX]], align 8507; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds { i64, i64, i64 }, ptr [[DATA]], i64 [[IV]], i32 0508; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP3]], align 8509; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>510; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>511; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>512; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]513; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]514; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[STRIDED_VEC2]]515; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>516; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>517; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> [[TMP11]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>518; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP12]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>519; CHECK-NEXT: store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 8520; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4521; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]522; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]523; CHECK: [[MIDDLE_BLOCK]]:524; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]525; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]526; CHECK: [[SCALAR_PH]]:527; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]528; CHECK-NEXT: br label %[[LOOP:.*]]529; CHECK: [[LOOP]]:530; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]531; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]]532; CHECK-NEXT: [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8533; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds { i64, i64, i64 }, ptr [[DATA]], i64 [[IV1]], i32 0534; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8535; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR]], [[L_0]]536; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_0]], align 8537; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds { i64, i64, i64 }, ptr [[DATA]], i64 [[IV1]], i32 1538; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8539; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR]], [[L_1]]540; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8541; CHECK-NEXT: [[DATA_2:%.*]] = getelementptr inbounds { i64, i64, i64 }, ptr [[DATA]], i64 [[IV1]], i32 2542; CHECK-NEXT: [[L_2:%.*]] = load i64, ptr [[DATA_2]], align 8543; CHECK-NEXT: [[MUL_2:%.*]] = mul i64 [[L_FACTOR]], [[L_2]]544; CHECK-NEXT: store i64 [[MUL_2]], ptr [[DATA_2]], align 8545; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1546; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]547; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]548; CHECK: [[EXIT]]:549; CHECK-NEXT: ret void550;551entry:552 br label %loop553 554loop:555 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]556 %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv557 %l.factor = load i64, ptr %arrayidx, align 8558 %data.0 = getelementptr inbounds { i64 , i64, i64 }, ptr %data, i64 %iv, i32 0559 %l.0 = load i64, ptr %data.0, align 8560 %mul.0 = mul i64 %l.factor, %l.0561 store i64 %mul.0, ptr %data.0, align 8562 %data.1 = getelementptr inbounds { i64 , i64, i64 }, ptr %data, i64 %iv, i32 1563 %l.1 = load i64, ptr %data.1, align 8564 %mul.1 = mul i64 %l.factor, %l.1565 store i64 %mul.1, ptr %data.1, align 8566 %data.2 = getelementptr inbounds { i64 , i64, i64 }, ptr %data, i64 %iv, i32 2567 %l.2 = load i64, ptr %data.2, align 8568 %mul.2 = mul i64 %l.factor, %l.2569 store i64 %mul.2, ptr %data.2, align 8570 %iv.next = add nuw nsw i64 %iv, 1571 %ec = icmp eq i64 %iv.next, %n572 br i1 %ec, label %exit, label %loop573 574exit:575 ret void576}577 578define void @test_2xi32(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {579; CHECK-LABEL: define void @test_2xi32(580; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {581; CHECK-NEXT: [[ENTRY:.*]]:582; CHECK-NEXT: br label %[[LOOP:.*]]583; CHECK: [[LOOP]]:584; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]585; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]586; CHECK-NEXT: [[L_FACTOR:%.*]] = load i32, ptr [[ARRAYIDX]], align 8587; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[IV]], i32 0588; CHECK-NEXT: [[L_0:%.*]] = load i32, ptr [[DATA_0]], align 8589; CHECK-NEXT: [[MUL_0:%.*]] = mul i32 [[L_FACTOR]], [[L_0]]590; CHECK-NEXT: store i32 [[MUL_0]], ptr [[DATA_0]], align 8591; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[IV]], i32 1592; CHECK-NEXT: [[L_1:%.*]] = load i32, ptr [[DATA_1]], align 8593; CHECK-NEXT: [[MUL_1:%.*]] = mul i32 [[L_FACTOR]], [[L_1]]594; CHECK-NEXT: store i32 [[MUL_1]], ptr [[DATA_1]], align 8595; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1596; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]597; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]598; CHECK: [[EXIT]]:599; CHECK-NEXT: ret void600;601entry:602 br label %loop603 604loop:605 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]606 %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv607 %l.factor = load i32 , ptr %arrayidx, align 8608 %data.0 = getelementptr inbounds { i32, i32, i32 }, ptr %data, i64 %iv, i32 0609 %l.0 = load i32, ptr %data.0, align 8610 %mul.0 = mul i32 %l.factor, %l.0611 store i32 %mul.0, ptr %data.0, align 8612 %data.1 = getelementptr inbounds { i32, i32, i32 }, ptr %data, i64 %iv, i32 1613 %l.1 = load i32, ptr %data.1, align 8614 %mul.1 = mul i32 %l.factor, %l.1615 store i32%mul.1, ptr %data.1, align 8616 %iv.next = add nuw nsw i64 %iv, 1617 %ec = icmp eq i64 %iv.next, %n618 br i1 %ec, label %exit, label %loop619 620exit:621 ret void622}623 624define void @test_3xi32(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {625; CHECK-LABEL: define void @test_3xi32(626; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {627; CHECK-NEXT: [[ENTRY:.*]]:628; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 8629; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]630; CHECK: [[VECTOR_PH]]:631; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8632; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0633; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 8, i64 [[N_MOD_VF]]634; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP1]]635; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]636; CHECK: [[VECTOR_BODY]]:637; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]638; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]639; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[ARRAYIDX]], align 8640; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>641; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[IV]], i32 0642; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <24 x i32>, ptr [[TMP5]], align 8643; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC1]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>644; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <24 x i32> [[WIDE_VEC1]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>645; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <24 x i32> [[WIDE_VEC1]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>646; CHECK-NEXT: [[TMP7:%.*]] = mul <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC2]]647; CHECK-NEXT: [[TMP8:%.*]] = mul <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC3]]648; CHECK-NEXT: [[TMP10:%.*]] = mul <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC4]]649; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> [[TMP8]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>650; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>651; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x i32> [[TMP12]], <16 x i32> [[TMP13]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>652; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP14]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>653; CHECK-NEXT: store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 8654; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8655; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]656; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]657; CHECK: [[MIDDLE_BLOCK]]:658; CHECK-NEXT: br label %[[SCALAR_PH]]659; CHECK: [[SCALAR_PH]]:660; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]661; CHECK-NEXT: br label %[[LOOP:.*]]662; CHECK: [[LOOP]]:663; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]664; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV1]]665; CHECK-NEXT: [[L_FACTOR:%.*]] = load i32, ptr [[ARRAYIDX1]], align 8666; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[IV1]], i32 0667; CHECK-NEXT: [[L_0:%.*]] = load i32, ptr [[DATA_0]], align 8668; CHECK-NEXT: [[MUL_0:%.*]] = mul i32 [[L_FACTOR]], [[L_0]]669; CHECK-NEXT: store i32 [[MUL_0]], ptr [[DATA_0]], align 8670; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[IV1]], i32 1671; CHECK-NEXT: [[L_1:%.*]] = load i32, ptr [[DATA_1]], align 8672; CHECK-NEXT: [[MUL_1:%.*]] = mul i32 [[L_FACTOR]], [[L_1]]673; CHECK-NEXT: store i32 [[MUL_1]], ptr [[DATA_1]], align 8674; CHECK-NEXT: [[DATA_2:%.*]] = getelementptr inbounds { i32, i32, i32 }, ptr [[DATA]], i64 [[IV1]], i32 2675; CHECK-NEXT: [[L_2:%.*]] = load i32, ptr [[DATA_2]], align 8676; CHECK-NEXT: [[MUL_2:%.*]] = mul i32 [[L_FACTOR]], [[L_2]]677; CHECK-NEXT: store i32 [[MUL_2]], ptr [[DATA_2]], align 8678; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1679; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]680; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]681; CHECK: [[EXIT]]:682; CHECK-NEXT: ret void683;684entry:685 br label %loop686 687loop:688 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]689 %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv690 %l.factor = load i32 , ptr %arrayidx, align 8691 %data.0 = getelementptr inbounds { i32, i32, i32 }, ptr %data, i64 %iv, i32 0692 %l.0 = load i32, ptr %data.0, align 8693 %mul.0 = mul i32 %l.factor, %l.0694 store i32 %mul.0, ptr %data.0, align 8695 %data.1 = getelementptr inbounds { i32, i32, i32 }, ptr %data, i64 %iv, i32 1696 %l.1 = load i32, ptr %data.1, align 8697 %mul.1 = mul i32 %l.factor, %l.1698 store i32%mul.1, ptr %data.1, align 8699 %data.2 = getelementptr inbounds { i32, i32, i32 }, ptr %data, i64 %iv, i32 2700 %l.2 = load i32, ptr %data.2, align 8701 %mul.2 = mul i32 %l.factor, %l.2702 store i32 %mul.2, ptr %data.2, align 8703 %iv.next = add nuw nsw i64 %iv, 1704 %ec = icmp eq i64 %iv.next, %n705 br i1 %ec, label %exit, label %loop706 707exit:708 ret void709}710;.711; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}712; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}713; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}714; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}715; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}716; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}717; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}718; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}719; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}720; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}721; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}722; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}723; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}724; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}725; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}726; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}727; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}728; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}729;.730