244 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=on -mtriple riscv64-linux-gnu -mattr=+v,+f -S 2>%t | FileCheck %s3 4target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"5target triple = "riscv64"6 7; Dependence distance between read and write is greater than the trip8; count of the loop. Thus, values written are never read for any9; valid vectorization of the loop.10define void @test(ptr %p) {11; CHECK-LABEL: @test(12; CHECK-NEXT: entry:13; CHECK-NEXT: br label [[VECTOR_PH:%.*]]14; CHECK: vector.ph:15; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]16; CHECK: vector.body:17; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]18; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 200, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]19; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)20; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[INDEX]]21; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 32 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])22; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 20023; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]24; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP_LOAD]], ptr align 32 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])25; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP8]] to i6426; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]27; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]28; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 029; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]30; CHECK: middle.block:31; CHECK-NEXT: br label [[LOOP:%.*]]32; CHECK: exit:33; CHECK-NEXT: ret void34;35entry:36 br label %loop37 38loop:39 %iv = phi i64 [0, %entry], [%iv.next, %loop]40 %a1 = getelementptr i64, ptr %p, i64 %iv41 %v = load i64, ptr %a1, align 3242 %offset = add i64 %iv, 20043 %a2 = getelementptr i64, ptr %p, i64 %offset44 store i64 %v, ptr %a2, align 3245 %iv.next = add i64 %iv, 146 %cmp = icmp ne i64 %iv, 19947 br i1 %cmp, label %loop, label %exit48 49exit:50 ret void51}52 53; Dependence distance is less than trip count, thus we must prove that54; chosen VF guaranteed to be less than dependence distance.55define void @test_may_clobber(ptr %p) {56; CHECK-LABEL: @test_may_clobber(57; CHECK-NEXT: entry:58; CHECK-NEXT: br label [[VECTOR_PH:%.*]]59; CHECK: vector.ph:60; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]61; CHECK: vector.body:62; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]63; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[INDEX]]64; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 3265; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 10066; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]67; CHECK-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP4]], align 3268; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 469; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20070; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]71; CHECK: middle.block:72; CHECK-NEXT: br label [[LOOP:%.*]]73; CHECK: exit:74; CHECK-NEXT: ret void75;76entry:77 br label %loop78 79loop:80 %iv = phi i64 [0, %entry], [%iv.next, %loop]81 %a1 = getelementptr i64, ptr %p, i64 %iv82 %v = load i64, ptr %a1, align 3283 %offset = add i64 %iv, 10084 %a2 = getelementptr i64, ptr %p, i64 %offset85 store i64 %v, ptr %a2, align 3286 %iv.next = add i64 %iv, 187 %cmp = icmp ne i64 %iv, 19988 br i1 %cmp, label %loop, label %exit89 90exit:91 ret void92}93 94; Trviailly no overlap due to maximum possible value of VLEN and LMUL95define void @trivial_due_max_vscale(ptr %p) {96; CHECK-LABEL: @trivial_due_max_vscale(97; CHECK-NEXT: entry:98; CHECK-NEXT: br label [[VECTOR_PH:%.*]]99; CHECK: vector.ph:100; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]101; CHECK: vector.body:102; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]103; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 200, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]104; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)105; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[INDEX]]106; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 32 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])107; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 8192108; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]109; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP_LOAD]], ptr align 32 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])110; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP8]] to i64111; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]112; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]113; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0114; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]115; CHECK: middle.block:116; CHECK-NEXT: br label [[LOOP:%.*]]117; CHECK: exit:118; CHECK-NEXT: ret void119;120entry:121 br label %loop122 123loop:124 %iv = phi i64 [0, %entry], [%iv.next, %loop]125 %a1 = getelementptr i64, ptr %p, i64 %iv126 %v = load i64, ptr %a1, align 32127 %offset = add i64 %iv, 8192128 %a2 = getelementptr i64, ptr %p, i64 %offset129 store i64 %v, ptr %a2, align 32130 %iv.next = add i64 %iv, 1131 %cmp = icmp ne i64 %iv, 199132 br i1 %cmp, label %loop, label %exit133 134exit:135 ret void136}137 138; Dependence distance could be violated via LMUL>=2 or interleaving139define void @no_high_lmul_or_interleave(ptr %p) {140; CHECK-LABEL: @no_high_lmul_or_interleave(141; CHECK-NEXT: entry:142; CHECK-NEXT: br label [[VECTOR_PH:%.*]]143; CHECK: vector.ph:144; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]145; CHECK: vector.body:146; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]147; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 200, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]148; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)149; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[INDEX]]150; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 32 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])151; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1024152; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]153; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP_LOAD]], ptr align 32 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])154; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP8]] to i64155; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]156; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]157; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0158; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]159; CHECK: middle.block:160; CHECK-NEXT: br label [[LOOP:%.*]]161; CHECK: exit:162; CHECK-NEXT: ret void163;164entry:165 br label %loop166 167loop:168 %iv = phi i64 [0, %entry], [%iv.next, %loop]169 %a1 = getelementptr i64, ptr %p, i64 %iv170 %v = load i64, ptr %a1, align 32171 %offset = add i64 %iv, 1024172 %a2 = getelementptr i64, ptr %p, i64 %offset173 store i64 %v, ptr %a2, align 32174 %iv.next = add i64 %iv, 1175 %cmp = icmp ne i64 %iv, 199176 br i1 %cmp, label %loop, label %exit177 178exit:179 ret void180}181 182@a = external global [10 x [12 x i16]]183 184; Test case for https://github.com/llvm/llvm-project/issues/134696.185define void @safe_load_store_distance_not_pow_of_2(i64 %N) {186; CHECK-LABEL: @safe_load_store_distance_not_pow_of_2(187; CHECK-NEXT: entry:188; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N:%.*]], i64 1)189; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[N]], [[UMIN]]190; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 3191; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[UMIN]], [[TMP1]]192; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 1193; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], 8194; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]195; CHECK: vector.ph:196; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 8197; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0198; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 8, i64 [[N_MOD_VF]]199; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP5]]200; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[N_VEC]], 3201; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]202; CHECK: vector.body:203; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]204; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 3, i64 6, i64 9, i64 12, i64 15, i64 18, i64 21>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]205; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [10 x [12 x i16]], ptr @a, i64 0, i64 8, <8 x i64> [[VEC_IND]]206; CHECK-NEXT: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> zeroinitializer, <8 x ptr> align 2 [[TMP7]], <8 x i1> splat (i1 true))207; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8208; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <8 x i64> [[VEC_IND]], splat (i64 24)209; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]210; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]211; CHECK: middle.block:212; CHECK-NEXT: br label [[SCALAR_PH]]213; CHECK: scalar.ph:214; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]215; CHECK-NEXT: br label [[LOOP:%.*]]216; CHECK: loop:217; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]218; CHECK-NEXT: [[GEP:%.*]] = getelementptr [10 x [12 x i16]], ptr @a, i64 0, i64 0, i64 [[IV]]219; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[GEP]], align 2220; CHECK-NEXT: [[GEP_OFF:%.*]] = getelementptr [10 x [12 x i16]], ptr @a, i64 0, i64 8, i64 [[IV]]221; CHECK-NEXT: store i16 0, ptr [[GEP_OFF]], align 2222; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 3223; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IV]], [[N]]224; CHECK-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]]225; CHECK: exit:226; CHECK-NEXT: ret void227;228entry:229 br label %loop230 231loop:232 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]233 %gep = getelementptr [10 x [12 x i16]], ptr @a, i64 0, i64 0, i64 %iv234 %1 = load i16, ptr %gep, align 2235 %gep.off = getelementptr [10 x [12 x i16]], ptr @a, i64 0, i64 8, i64 %iv236 store i16 0, ptr %gep.off, align 2237 %iv.next = add nsw i64 %iv, 3238 %cmp = icmp ult i64 %iv, %N239 br i1 %cmp, label %loop, label %exit240 241exit:242 ret void243}244