385 lines · plain
1; REQUIRES: asserts2; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -passes=loop-vectorize -S < %s 2>&1 | FileCheck %s3; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -passes=loop-vectorize -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize -S < %s 2>&1 | FileCheck --check-prefix=CHECK-DBG %s4; RUN: opt -mtriple=aarch64-none-linux-gnu -passes=loop-vectorize -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize -S < %s 2>%t | FileCheck --check-prefix=CHECK-NO-SVE %s5; RUN: cat %t | FileCheck %s -check-prefix=CHECK-NO-SVE-REMARKS6 7target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"8 9; These tests validate the behaviour of scalable vectorization factor hints,10; where the following applies:11;12; * If the backend does not support scalable vectors, ignore the hint and let13; the vectorizer pick a VF.14; * If there are no dependencies and assuming the VF is a power of 2 the VF15; should be accepted. This applies to both fixed and scalable VFs.16; * If the dependency is too small to use scalable vectors, change the VF to17; fixed, where existing behavior applies (clamping).18; * If scalable vectorization is feasible given the dependency and the VF is19; valid, accept it. Otherwise, clamp to the max scalable VF.20 21; test122;23; Scalable vectorization unfeasible, clamp VF from (4, scalable) -> (4, fixed).24;25; The pragma applied to this loop implies a scalable vector <vscale x 4 x i32>26; be used for vectorization. For fixed vectors the MaxVF=8, otherwise there27; would be a dependence between vector lanes for vectors greater than 256 bits.28;29; void test1(int *a, int *b, int N) {30; #pragma clang loop vectorize(enable) vectorize_width(4, scalable)31; for (int i=0; i<N; ++i) {32; a[i + 8] = a[i] + b[i];33; }34; }35;36; For scalable vectorization 'vscale' has to be considered, for this example37; unless max(vscale)=2 it's unsafe to vectorize. For SVE max(vscale)=16, check38; fixed-width vectorization is used instead.39 40; CHECK-DBG: LV: Checking a loop in 'test1'41; CHECK-DBG: LV: Scalable vectorization is available42; CHECK-DBG: LV: Max legal vector width too small, scalable vectorization unfeasible.43; CHECK-DBG: remark: <unknown>:0:0: Max legal vector width too small, scalable vectorization unfeasible.44; CHECK-DBG: LV: The max safe fixed VF is: 8.45; CHECK-DBG: LV: Selecting VF: 4.46; CHECK-LABEL: @test147; CHECK: <4 x i32>48define void @test1(ptr %a, ptr %b) #0 {49entry:50 br label %loop51 52loop:53 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]54 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv55 %0 = load i32, ptr %arrayidx, align 456 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv57 %1 = load i32, ptr %arrayidx2, align 458 %add = add nsw i32 %1, %059 %2 = add nuw nsw i64 %iv, 860 %arrayidx5 = getelementptr inbounds i32, ptr %a, i64 %261 store i32 %add, ptr %arrayidx5, align 462 %iv.next = add nuw nsw i64 %iv, 163 %exitcond.not = icmp eq i64 %iv.next, 102464 br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !065 66exit:67 ret void68}69 70!0 = !{!0, !1, !2}71!1 = !{!"llvm.loop.vectorize.width", i32 4}72!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}73 74; test275;76; Scalable vectorization unfeasible, clamp VF from (8, scalable) -> (4, fixed).77;78; void test2(int *a, int *b, int N) {79; #pragma clang loop vectorize(enable) vectorize_width(8, scalable)80; for (int i=0; i<N; ++i) {81; a[i + 4] = a[i] + b[i];82; }83; }84 85; CHECK-DBG: LV: Checking a loop in 'test2'86; CHECK-DBG: LV: Scalable vectorization is available87; CHECK-DBG: LV: Max legal vector width too small, scalable vectorization unfeasible.88; CHECK-DBG: LV: The max safe fixed VF is: 4.89; CHECK-DBG: LV: User VF=vscale x 8 is unsafe. Ignoring scalable UserVF.90; CHECK-DBG: LV: Selecting VF: 4.91; CHECK-LABEL: @test292; CHECK: <4 x i32>93define void @test2(ptr %a, ptr %b) #0 {94entry:95 br label %loop96 97loop:98 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]99 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv100 %0 = load i32, ptr %arrayidx, align 4101 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv102 %1 = load i32, ptr %arrayidx2, align 4103 %add = add nsw i32 %1, %0104 %2 = add nuw nsw i64 %iv, 4105 %arrayidx5 = getelementptr inbounds i32, ptr %a, i64 %2106 store i32 %add, ptr %arrayidx5, align 4107 %iv.next = add nuw nsw i64 %iv, 1108 %exitcond.not = icmp eq i64 %iv.next, 1024109 br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !3110 111exit:112 ret void113}114 115!3 = !{!3, !4, !5}116!4 = !{!"llvm.loop.vectorize.width", i32 8}117!5 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}118 119; test3120;121; Scalable vectorization feasible and the VF is valid.122;123; Specifies a vector of <vscale x 2 x i32>, i.e. maximum of 32 x i32 with 2124; words per 128-bits (unpacked).125;126; void test3(int *a, int *b, int N) {127; #pragma clang loop vectorize(enable) vectorize_width(2, scalable)128; for (int i=0; i<N; ++i) {129; a[i + 32] = a[i] + b[i];130; }131; }132;133; Max fixed VF=32, Max scalable VF=2, safe to vectorize.134 135; CHECK-DBG-LABEL: LV: Checking a loop in 'test3'136; CHECK-DBG: LV: Scalable vectorization is available137; CHECK-DBG: LV: The max safe scalable VF is: vscale x 2.138; CHECK-DBG: LV: Using user VF vscale x 2.139; CHECK-LABEL: @test3140; CHECK: <vscale x 2 x i32>141define void @test3(ptr %a, ptr %b) #0 {142entry:143 br label %loop144 145loop:146 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]147 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv148 %0 = load i32, ptr %arrayidx, align 4149 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv150 %1 = load i32, ptr %arrayidx2, align 4151 %add = add nsw i32 %1, %0152 %2 = add nuw nsw i64 %iv, 32153 %arrayidx5 = getelementptr inbounds i32, ptr %a, i64 %2154 store i32 %add, ptr %arrayidx5, align 4155 %iv.next = add nuw nsw i64 %iv, 1156 %exitcond.not = icmp eq i64 %iv.next, 1024157 br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !6158 159exit:160 ret void161}162 163!6 = !{!6, !7, !8}164!7 = !{!"llvm.loop.vectorize.width", i32 2}165!8 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}166 167; test4168;169; Scalable vectorization feasible, but the given VF is unsafe. Should ignore170; the hint and leave it to the vectorizer to pick a more suitable VF.171;172; Specifies a vector of <vscale x 4 x i32>, i.e. maximum of 64 x i32 with 4173; words per 128-bits (packed).174;175; void test4(int *a, int *b, int N) {176; #pragma clang loop vectorize(enable) vectorize_width(4, scalable)177; for (int i=0; i<N; ++i) {178; a[i + 32] = a[i] + b[i];179; }180; }181;182; Max fixed VF=32, Max scalable VF=2, unsafe to vectorize.183 184; CHECK-DBG-LABEL: LV: Checking a loop in 'test4'185; CHECK-DBG: LV: Scalable vectorization is available186; CHECK-DBG: LV: The max safe scalable VF is: vscale x 2.187; CHECK-DBG: LV: User VF=vscale x 4 is unsafe. Ignoring scalable UserVF.188; CHECK-DBG: remark: <unknown>:0:0: User-specified vectorization factor vscale x 4 is unsafe. Ignoring the hint to let the compiler pick a more suitable value.189; CHECK-DBG: Found feasible scalable VF = vscale x 2190; CHECK-DBG: LV: Selecting VF: 4.191; CHECK-LABEL: @test4192; CHECK-NOT: <vscale x 4 x i32>193define void @test4(ptr %a, ptr %b) #0 {194entry:195 br label %loop196 197loop:198 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]199 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv200 %0 = load i32, ptr %arrayidx, align 4201 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv202 %1 = load i32, ptr %arrayidx2, align 4203 %add = add nsw i32 %1, %0204 %2 = add nuw nsw i64 %iv, 32205 %arrayidx5 = getelementptr inbounds i32, ptr %a, i64 %2206 store i32 %add, ptr %arrayidx5, align 4207 %iv.next = add nuw nsw i64 %iv, 1208 %exitcond.not = icmp eq i64 %iv.next, 1024209 br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !9210 211exit:212 ret void213}214 215!9 = !{!9, !10, !11}216!10 = !{!"llvm.loop.vectorize.width", i32 4}217!11 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}218 219; test5220;221; Scalable vectorization feasible and the VF is valid.222;223; Specifies a vector of <vscale x 4 x i32>, i.e. maximum of 64 x i32 with 4224; words per 128-bits (packed).225;226; void test5(int *a, int *b, int N) {227; #pragma clang loop vectorize(enable) vectorize_width(4, scalable)228; for (int i=0; i<N; ++i) {229; a[i + 128] = a[i] + b[i];230; }231; }232;233; Max fixed VF=128, Max scalable VF=8, safe to vectorize.234 235; CHECK-DBG-LABEL: LV: Checking a loop in 'test5'236; CHECK-DBG: LV: Scalable vectorization is available237; CHECK-DBG: LV: The max safe scalable VF is: vscale x 8.238; CHECK-DBG: LV: Using user VF vscale x 4239; CHECK-LABEL: @test5240; CHECK: <vscale x 4 x i32>241define void @test5(ptr %a, ptr %b) #0 {242entry:243 br label %loop244 245loop:246 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]247 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv248 %0 = load i32, ptr %arrayidx, align 4249 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv250 %1 = load i32, ptr %arrayidx2, align 4251 %add = add nsw i32 %1, %0252 %2 = add nuw nsw i64 %iv, 128253 %arrayidx5 = getelementptr inbounds i32, ptr %a, i64 %2254 store i32 %add, ptr %arrayidx5, align 4255 %iv.next = add nuw nsw i64 %iv, 1256 %exitcond.not = icmp eq i64 %iv.next, 1024257 br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !12258 259exit:260 ret void261}262 263!12 = !{!12, !13, !14}264!13 = !{!"llvm.loop.vectorize.width", i32 4}265!14 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}266 267; test6268;269; Scalable vectorization feasible, but the VF is unsafe. Should ignore270; the hint and leave it to the vectorizer to pick a more suitable VF.271;272; Specifies a vector of <vscale x 16 x i32>, i.e. maximum of 256 x i32.273;274; void test6(int *a, int *b, int N) {275; #pragma clang loop vectorize(enable) vectorize_width(16, scalable)276; for (int i=0; i<N; ++i) {277; a[i + 128] = a[i] + b[i];278; }279; }280;281; Max fixed VF=128, Max scalable VF=8, unsafe to vectorize.282 283; CHECK-DBG-LABEL: LV: Checking a loop in 'test6'284; CHECK-DBG: LV: Scalable vectorization is available285; CHECK-DBG: LV: The max safe scalable VF is: vscale x 8.286; CHECK-DBG: LV: User VF=vscale x 16 is unsafe. Ignoring scalable UserVF.287; CHECK-DBG: remark: <unknown>:0:0: User-specified vectorization factor vscale x 16 is unsafe. Ignoring the hint to let the compiler pick a more suitable value.288; CHECK-DBG: LV: Found feasible scalable VF = vscale x 4289; CHECK-DBG: Selecting VF: vscale x 4.290; CHECK-LABEL: @test6291; CHECK: <vscale x 4 x i32>292define void @test6(ptr %a, ptr %b) #0 {293entry:294 br label %loop295 296loop:297 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]298 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv299 %0 = load i32, ptr %arrayidx, align 4300 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv301 %1 = load i32, ptr %arrayidx2, align 4302 %add = add nsw i32 %1, %0303 %2 = add nuw nsw i64 %iv, 128304 %arrayidx5 = getelementptr inbounds i32, ptr %a, i64 %2305 store i32 %add, ptr %arrayidx5, align 4306 %iv.next = add nuw nsw i64 %iv, 1307 %exitcond.not = icmp eq i64 %iv.next, 1024308 br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !15309 310exit:311 ret void312}313 314!15 = !{!15, !16, !17}315!16 = !{!"llvm.loop.vectorize.width", i32 16}316!17 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}317 318; CHECK-NO-SVE-REMARKS-LABEL: LV: Checking a loop in 'test_no_sve'319; CHECK-NO-SVE-REMARKS: LV: User VF=vscale x 4 is ignored because scalable vectors are not available.320; CHECK-NO-SVE-REMARKS: remark: <unknown>:0:0: User-specified vectorization factor vscale x 4 is ignored because the target does not support scalable vectors. The compiler will pick a more suitable value.321; CHECK-NO-SVE-REMARKS: LV: Selecting VF: 4.322; CHECK-NO-SVE-LABEL: @test_no_sve323; CHECK-NO-SVE: <4 x i32>324; CHECK-NO-SVE-NOT: <vscale x 4 x i32>325define void @test_no_sve(ptr %a, ptr %b) #0 {326entry:327 br label %loop328 329loop:330 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]331 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv332 %0 = load i32, ptr %arrayidx, align 4333 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv334 %1 = load i32, ptr %arrayidx2, align 4335 %add = add nsw i32 %1, %0336 store i32 %add, ptr %arrayidx, align 4337 %iv.next = add nuw nsw i64 %iv, 1338 %exitcond.not = icmp eq i64 %iv.next, 1024339 br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !18340 341exit:342 ret void343}344 345!18 = !{!18, !19, !20}346!19 = !{!"llvm.loop.vectorize.width", i32 4}347!20 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}348 349; Test the LV falls back to fixed-width vectorization if scalable vectors are350; supported but max vscale is undefined.351;352; CHECK-DBG-LABEL: LV: Checking a loop in 'test_no_max_vscale'353; CHECK-DBG: LV: Scalable vectorization is available354; CHECK-DBG: The max safe fixed VF is: 4.355; CHECK-DBG: LV: User VF=vscale x 4 is unsafe. Ignoring scalable UserVF.356; CHECK-DBG: LV: Selecting VF: 4.357; CHECK-LABEL: @test_no_max_vscale358; CHECK: <4 x i32>359define void @test_no_max_vscale(ptr %a, ptr %b) #0 {360entry:361 br label %loop362 363loop:364 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]365 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv366 %0 = load i32, ptr %arrayidx, align 4367 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv368 %1 = load i32, ptr %arrayidx2, align 4369 %add = add nsw i32 %1, %0370 %2 = add nuw nsw i64 %iv, 4371 %arrayidx5 = getelementptr inbounds i32, ptr %a, i64 %2372 store i32 %add, ptr %arrayidx5, align 4373 %iv.next = add nuw nsw i64 %iv, 1374 %exitcond.not = icmp eq i64 %iv.next, 1024375 br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !21376 377exit:378 ret void379}380 381attributes #0 = { vscale_range(1, 16) }382!21 = !{!21, !22, !23}383!22 = !{!"llvm.loop.vectorize.width", i32 4}384!23 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}385