361 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 52; RUN: opt -p loop-vectorize -force-vector-interleave=1 -S -mcpu=neoverse-512tvb %s | FileCheck --check-prefixes=CHECK %s3 4target triple = "aarch64-unknown-linux"5 6define void @load_store_interleave_group(ptr noalias %data) {7; CHECK-LABEL: define void @load_store_interleave_group(8; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] {9; CHECK-NEXT: [[ENTRY:.*:]]10; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()11; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 112; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP5]]13; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]14; CHECK: [[VECTOR_PH]]:15; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()16; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 217; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]18; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]19; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]20; CHECK: [[VECTOR_BODY]]:21; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]22; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 123; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]24; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP1]], align 825; CHECK-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 826; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]27; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]28; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]29; CHECK: [[MIDDLE_BLOCK]]:30; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]31; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]32; CHECK: [[SCALAR_PH]]:33;34entry:35 br label %loop36 37loop:38 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]39 %mul.2 = shl nsw i64 %iv, 140 %data.0 = getelementptr inbounds i64, ptr %data, i64 %mul.241 %l.0 = load i64, ptr %data.0, align 842 store i64 %l.0, ptr %data.0, align 843 %add.1 = or disjoint i64 %mul.2, 144 %data.1 = getelementptr inbounds i64, ptr %data, i64 %add.145 %l.1 = load i64, ptr %data.1, align 846 store i64 %l.1, ptr %data.1, align 847 %iv.next = add nuw nsw i64 %iv, 148 %ec = icmp eq i64 %iv.next, 10049 br i1 %ec, label %exit, label %loop50 51exit:52 ret void53}54 55define void @test_2xi64_unary_op_load_interleave_group(ptr noalias %data, ptr noalias %factor) {56; CHECK-LABEL: define void @test_2xi64_unary_op_load_interleave_group(57; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]]) #[[ATTR0]] {58; CHECK-NEXT: [[ENTRY:.*:]]59; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()60; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 161; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1111, [[TMP5]]62; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]63; CHECK: [[VECTOR_PH]]:64; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()65; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 266; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1111, [[TMP3]]67; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1111, [[N_MOD_VF]]68; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]69; CHECK: [[VECTOR_BODY]]:70; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]71; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 172; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]73; CHECK-NEXT: [[TMP7:%.*]] = load <vscale x 2 x double>, ptr [[TMP1]], align 874; CHECK-NEXT: [[TMP9:%.*]] = fneg <vscale x 2 x double> [[TMP7]]75; CHECK-NEXT: store <vscale x 2 x double> [[TMP9]], ptr [[TMP1]], align 876; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]77; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]78; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]79; CHECK: [[MIDDLE_BLOCK]]:80; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1111, [[N_VEC]]81; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]82; CHECK: [[SCALAR_PH]]:83;84entry:85 br label %loop86 87loop:88 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]89 %1 = shl nsw i64 %iv, 190 %data.0 = getelementptr inbounds double, ptr %data, i64 %191 %l.0 = load double, ptr %data.0, align 892 %neg.0 = fneg double %l.093 store double %neg.0, ptr %data.0, align 894 %3 = or disjoint i64 %1, 195 %data.1 = getelementptr inbounds double, ptr %data, i64 %396 %l.1 = load double, ptr %data.1, align 897 %neg.1 = fneg double %l.198 store double %neg.1, ptr %data.1, align 899 %iv.next = add nuw nsw i64 %iv, 1100 %ec = icmp eq i64 %iv.next, 1111101 br i1 %ec, label %exit, label %loop102 103exit:104 ret void105}106 107define void @test_masked_interleave_group(i32 %N, ptr %mask, ptr %src, ptr %dst) {108; IC1-LABEL: define void @test_masked_interleave_group(109; IC1-SAME: i32 [[N:%.*]], ptr [[MASK:%.*]], ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {110; IC1-NEXT: [[ENTRY:.*:]]111; IC1-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64112; IC1-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1113; IC1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()114; IC1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2115; IC1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP3]], i64 8)116; IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]117; IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]118; IC1: [[VECTOR_MEMCHECK]]:119; IC1-NEXT: [[TMP4:%.*]] = zext i32 [[N]] to i64120; IC1-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 4121; IC1-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 16122; IC1-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]123; IC1-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP4]], 1124; IC1-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[TMP7]]125; IC1-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]126; IC1-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]127; IC1-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[MASK]], [[SCEVGEP]]128; IC1-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]129; IC1-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]130; IC1-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]131; IC1-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]132; IC1-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]133; IC1-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]134; IC1: [[VECTOR_PH]]:135; IC1-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()136; IC1-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4137; IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP9]]138; IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]139; IC1-NEXT: [[TMP10:%.*]] = trunc i64 [[N_VEC]] to i32140; IC1-NEXT: [[TMP11:%.*]] = mul i64 [[N_VEC]], 16141; IC1-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]142; IC1-NEXT: [[TMP13:%.*]] = mul i64 [[N_VEC]], 16143; IC1-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]144; IC1-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[N_VEC]]145; IC1-NEXT: br label %[[VECTOR_BODY:.*]]146; IC1: [[VECTOR_BODY]]:147; IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]148; IC1-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16149; IC1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX]]150; IC1-NEXT: [[OFFSET_IDX6:%.*]] = mul i64 [[INDEX]], 16151; IC1-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX6]]152; IC1-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[INDEX]]153; IC1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[NEXT_GEP8]], align 1, !alias.scope [[META6:![0-9]+]]154; IC1-NEXT: [[TMP16:%.*]] = icmp eq <vscale x 4 x i8> [[WIDE_LOAD]], zeroinitializer155; IC1-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]])156; IC1-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x float> @llvm.masked.load.nxv16f32.p0(ptr align 4 [[NEXT_GEP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x float> poison), !alias.scope [[META9:![0-9]+]]157; IC1-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave4.nxv16f32(<vscale x 16 x float> [[WIDE_MASKED_VEC]])158; IC1-NEXT: [[TMP17:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0159; IC1-NEXT: [[TMP18:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1160; IC1-NEXT: [[TMP19:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 2161; IC1-NEXT: [[TMP20:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 3162; IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> [[TMP17]], <vscale x 4 x float> [[TMP18]], <vscale x 4 x float> [[TMP19]], <vscale x 4 x float> [[TMP20]])163; IC1-NEXT: [[INTERLEAVED_MASK9:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]])164; IC1-NEXT: call void @llvm.masked.store.nxv16f32.p0(<vscale x 16 x float> [[INTERLEAVED_VEC]], ptr align 4 [[NEXT_GEP]], <vscale x 16 x i1> [[INTERLEAVED_MASK9]]), !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]165; IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]166; IC1-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]167; IC1-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]168; IC1: [[MIDDLE_BLOCK]]:169; IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]170; IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]171; IC1: [[SCALAR_PH]]:172;173; CHECK-LABEL: define void @test_masked_interleave_group(174; CHECK-SAME: i32 [[N:%.*]], ptr [[MASK:%.*]], ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {175; CHECK-NEXT: [[ENTRY:.*]]:176; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64177; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1178; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()179; CHECK-NEXT: [[UMAX:%.*]] = shl nuw i64 [[TMP2]], 3180; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]181; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]182; CHECK: [[VECTOR_MEMCHECK]]:183; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[N]] to i64184; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 4185; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 16186; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]187; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP4]], 1188; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[TMP7]]189; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]190; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]191; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[MASK]], [[SCEVGEP]]192; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]193; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]194; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]195; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]196; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]197; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]198; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:199; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()200; CHECK-NEXT: [[TMP20:%.*]] = shl nuw i64 [[TMP8]], 4201; CHECK-NEXT: [[MIN_ITERS_CHECK6:%.*]] = icmp ult i64 [[TMP1]], [[TMP20]]202; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK6]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]203; CHECK: [[VECTOR_PH]]:204; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64()205; CHECK-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP21]], 16206; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP9]]207; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]208; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]209; CHECK: [[VECTOR_BODY]]:210; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]211; CHECK-NEXT: [[TMP25:%.*]] = mul i64 [[INDEX1]], 16212; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP25]]213; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[INDEX1]], 16214; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP26]]215; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[INDEX1]]216; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[NEXT_GEP10]], align 1, !alias.scope [[META6:![0-9]+]]217; CHECK-NEXT: [[TMP27:%.*]] = icmp eq <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer218; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP27]], <vscale x 16 x i1> [[TMP27]], <vscale x 16 x i1> [[TMP27]], <vscale x 16 x i1> [[TMP27]])219; CHECK-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x float> @llvm.masked.load.nxv64f32.p0(ptr align 4 [[NEXT_GEP9]], <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x float> poison), !alias.scope [[META9:![0-9]+]]220; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float> } @llvm.vector.deinterleave4.nxv64f32(<vscale x 64 x float> [[WIDE_MASKED_VEC]])221; CHECK-NEXT: [[TMP28:%.*]] = extractvalue { <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float> } [[STRIDED_VEC]], 0222; CHECK-NEXT: [[TMP16:%.*]] = extractvalue { <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float> } [[STRIDED_VEC]], 1223; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float> } [[STRIDED_VEC]], 2224; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float>, <vscale x 16 x float> } [[STRIDED_VEC]], 3225; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x float> @llvm.vector.interleave4.nxv64f32(<vscale x 16 x float> [[TMP28]], <vscale x 16 x float> [[TMP16]], <vscale x 16 x float> [[TMP17]], <vscale x 16 x float> [[TMP18]])226; CHECK-NEXT: [[INTERLEAVED_MASK9:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP27]], <vscale x 16 x i1> [[TMP27]], <vscale x 16 x i1> [[TMP27]], <vscale x 16 x i1> [[TMP27]])227; CHECK-NEXT: call void @llvm.masked.store.nxv64f32.p0(<vscale x 64 x float> [[INTERLEAVED_VEC]], ptr align 4 [[NEXT_GEP1]], <vscale x 64 x i1> [[INTERLEAVED_MASK9]]), !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]228; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], [[TMP9]]229; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]230; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]231; CHECK: [[MIDDLE_BLOCK]]:232; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]233; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]234; CHECK: [[VEC_EPILOG_ITER_CHECK]]:235; CHECK-NEXT: [[TMP10:%.*]] = trunc i64 [[N_VEC]] to i32236; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[N_VEC]], 16237; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]238; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[N_VEC]], 16239; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]240; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[N_VEC]]241; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[UMAX]]242; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF15:![0-9]+]]243; CHECK: [[VEC_EPILOG_PH]]:244; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]245; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64()246; CHECK-NEXT: [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 8247; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i64 [[TMP1]], [[TMP23]]248; CHECK-NEXT: [[INDEX:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF10]]249; CHECK-NEXT: [[TMP24:%.*]] = trunc i64 [[INDEX]] to i32250; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16251; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX]]252; CHECK-NEXT: [[OFFSET_IDX6:%.*]] = mul i64 [[INDEX]], 16253; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX6]]254; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[INDEX]]255; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]256; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:257; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT23:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]258; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = mul i64 [[INDEX12]], 16259; CHECK-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX1]]260; CHECK-NEXT: [[OFFSET_IDX14:%.*]] = mul i64 [[INDEX12]], 16261; CHECK-NEXT: [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX14]]262; CHECK-NEXT: [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[INDEX12]]263; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <vscale x 8 x i8>, ptr [[NEXT_GEP16]], align 1, !alias.scope [[META6]]264; CHECK-NEXT: [[TMP30:%.*]] = icmp eq <vscale x 8 x i8> [[WIDE_LOAD17]], zeroinitializer265; CHECK-NEXT: [[INTERLEAVED_MASK18:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> [[TMP30]], <vscale x 8 x i1> [[TMP30]], <vscale x 8 x i1> [[TMP30]], <vscale x 8 x i1> [[TMP30]])266; CHECK-NEXT: [[WIDE_MASKED_VEC19:%.*]] = call <vscale x 32 x float> @llvm.masked.load.nxv32f32.p0(ptr align 4 [[NEXT_GEP15]], <vscale x 32 x i1> [[INTERLEAVED_MASK18]], <vscale x 32 x float> poison), !alias.scope [[META9]]267; CHECK-NEXT: [[STRIDED_VEC20:%.*]] = call { <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float> } @llvm.vector.deinterleave4.nxv32f32(<vscale x 32 x float> [[WIDE_MASKED_VEC19]])268; CHECK-NEXT: [[TMP31:%.*]] = extractvalue { <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float> } [[STRIDED_VEC20]], 0269; CHECK-NEXT: [[TMP32:%.*]] = extractvalue { <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float> } [[STRIDED_VEC20]], 1270; CHECK-NEXT: [[TMP33:%.*]] = extractvalue { <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float> } [[STRIDED_VEC20]], 2271; CHECK-NEXT: [[TMP34:%.*]] = extractvalue { <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float>, <vscale x 8 x float> } [[STRIDED_VEC20]], 3272; CHECK-NEXT: [[INTERLEAVED_VEC21:%.*]] = call <vscale x 32 x float> @llvm.vector.interleave4.nxv32f32(<vscale x 8 x float> [[TMP31]], <vscale x 8 x float> [[TMP32]], <vscale x 8 x float> [[TMP33]], <vscale x 8 x float> [[TMP34]])273; CHECK-NEXT: [[INTERLEAVED_MASK22:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> [[TMP30]], <vscale x 8 x i1> [[TMP30]], <vscale x 8 x i1> [[TMP30]], <vscale x 8 x i1> [[TMP30]])274; CHECK-NEXT: call void @llvm.masked.store.nxv32f32.p0(<vscale x 32 x float> [[INTERLEAVED_VEC21]], ptr align 4 [[NEXT_GEP13]], <vscale x 32 x i1> [[INTERLEAVED_MASK22]]), !alias.scope [[META11]], !noalias [[META13]]275; CHECK-NEXT: [[INDEX_NEXT23]] = add nuw i64 [[INDEX12]], [[TMP23]]276; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT23]], [[INDEX]]277; CHECK-NEXT: br i1 [[TMP35]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]278; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:279; CHECK-NEXT: [[CMP_N24:%.*]] = icmp eq i64 [[TMP1]], [[INDEX]]280; CHECK-NEXT: br i1 [[CMP_N24]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]281; CHECK: [[VEC_EPILOG_SCALAR_PH]]:282; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP24]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ENTRY]] ]283; CHECK-NEXT: [[BC_RESUME_VAL25:%.*]] = phi ptr [ [[NEXT_GEP]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[DST]], %[[VECTOR_MEMCHECK]] ], [ [[DST]], %[[ENTRY]] ]284; CHECK-NEXT: [[BC_RESUME_VAL26:%.*]] = phi ptr [ [[NEXT_GEP7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[SRC]], %[[VECTOR_MEMCHECK]] ], [ [[SRC]], %[[ENTRY]] ]285; CHECK-NEXT: [[BC_RESUME_VAL27:%.*]] = phi ptr [ [[NEXT_GEP8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[MASK]], %[[VECTOR_MEMCHECK]] ], [ [[MASK]], %[[ENTRY]] ]286; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]287; CHECK: [[LOOP_HEADER]]:288; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]289; CHECK-NEXT: [[DST_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL25]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[DST_IV_NEXT:%.*]], %[[LOOP_LATCH]] ]290; CHECK-NEXT: [[SRC_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL26]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SRC_IV_NEXT:%.*]], %[[LOOP_LATCH]] ]291; CHECK-NEXT: [[MASK_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL27]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[MASK_IV_NEXT:%.*]], %[[LOOP_LATCH]] ]292; CHECK-NEXT: [[MASK_IV_NEXT]] = getelementptr i8, ptr [[MASK_IV]], i64 1293; CHECK-NEXT: [[MASK_VAL:%.*]] = load i8, ptr [[MASK_IV]], align 1294; CHECK-NEXT: [[SHOULD_COPY:%.*]] = icmp eq i8 [[MASK_VAL]], 0295; CHECK-NEXT: br i1 [[SHOULD_COPY]], label %[[THEN:.*]], label %[[LOOP_LATCH]]296; CHECK: [[THEN]]:297; CHECK-NEXT: [[ELEM0:%.*]] = load float, ptr [[SRC_IV]], align 4298; CHECK-NEXT: store float [[ELEM0]], ptr [[DST_IV]], align 4299; CHECK-NEXT: [[SRC_1_PTR:%.*]] = getelementptr i8, ptr [[SRC_IV]], i64 4300; CHECK-NEXT: [[S1:%.*]] = load float, ptr [[SRC_1_PTR]], align 4301; CHECK-NEXT: [[DST_1_PTR:%.*]] = getelementptr i8, ptr [[DST_IV]], i64 4302; CHECK-NEXT: store float [[S1]], ptr [[DST_1_PTR]], align 4303; CHECK-NEXT: [[SRC_2_PTR:%.*]] = getelementptr i8, ptr [[SRC_IV]], i64 8304; CHECK-NEXT: [[S2:%.*]] = load float, ptr [[SRC_2_PTR]], align 4305; CHECK-NEXT: [[DST_2_PTR:%.*]] = getelementptr i8, ptr [[DST_IV]], i64 8306; CHECK-NEXT: store float [[S2]], ptr [[DST_2_PTR]], align 4307; CHECK-NEXT: [[SRC_3_PTR:%.*]] = getelementptr i8, ptr [[SRC_IV]], i64 12308; CHECK-NEXT: [[S3:%.*]] = load float, ptr [[SRC_3_PTR]], align 4309; CHECK-NEXT: [[DST_3_PTR:%.*]] = getelementptr i8, ptr [[DST_IV]], i64 12310; CHECK-NEXT: store float [[S3]], ptr [[DST_3_PTR]], align 4311; CHECK-NEXT: br label %[[LOOP_LATCH]]312; CHECK: [[LOOP_LATCH]]:313; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1314; CHECK-NEXT: [[SRC_IV_NEXT]] = getelementptr i8, ptr [[SRC_IV]], i64 16315; CHECK-NEXT: [[DST_IV_NEXT]] = getelementptr i8, ptr [[DST_IV]], i64 16316; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], [[N]]317; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP17:![0-9]+]]318; CHECK: [[EXIT]]:319; CHECK-NEXT: ret void320;321entry:322 br label %loop.header323 324loop.header:325 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]326 %dst.iv = phi ptr [ %dst, %entry ], [ %dst.iv.next, %loop.latch ]327 %src.iv = phi ptr [ %src, %entry ], [ %src.iv.next, %loop.latch ]328 %mask.iv = phi ptr [ %mask, %entry ], [ %mask.iv.next, %loop.latch ]329 %mask.iv.next = getelementptr i8, ptr %mask.iv, i64 1330 %mask.val = load i8, ptr %mask.iv, align 1331 %should.copy = icmp eq i8 %mask.val, 0332 br i1 %should.copy, label %then, label %loop.latch333 334then:335 %elem0 = load float, ptr %src.iv, align 4336 store float %elem0, ptr %dst.iv, align 4337 %src.1.ptr = getelementptr i8, ptr %src.iv, i64 4338 %s1 = load float, ptr %src.1.ptr, align 4339 %dst.1.ptr = getelementptr i8, ptr %dst.iv, i64 4340 store float %s1, ptr %dst.1.ptr, align 4341 %src.2.ptr = getelementptr i8, ptr %src.iv, i64 8342 %s2 = load float, ptr %src.2.ptr, align 4343 %dst.2.ptr = getelementptr i8, ptr %dst.iv, i64 8344 store float %s2, ptr %dst.2.ptr, align 4345 %src.3.ptr = getelementptr i8, ptr %src.iv, i64 12346 %s3 = load float, ptr %src.3.ptr, align 4347 %dst.3.ptr = getelementptr i8, ptr %dst.iv, i64 12348 store float %s3, ptr %dst.3.ptr, align 4349 br label %loop.latch350 351loop.latch:352 %iv.next = add i32 %iv, 1353 %src.iv.next = getelementptr i8, ptr %src.iv, i64 16354 %dst.iv.next = getelementptr i8, ptr %dst.iv, i64 16355 %ec = icmp eq i32 %iv, %N356 br i1 %ec, label %exit, label %loop.header357 358exit:359 ret void360}361