253 lines · plain
1; RUN: opt -S -passes=loop-vectorize,instcombine -force-vector-interleave=1 -mattr=+sve -mtriple aarch64-unknown-linux-gnu \2; RUN: -prefer-predicate-over-epilogue=scalar-epilogue -pass-remarks-missed=loop-vectorize < %s 2>%t | FileCheck %s3; RUN: cat %t | FileCheck %s --check-prefix=CHECK-REMARKS4; RUN: opt -S -passes=loop-vectorize,instcombine -force-vector-interleave=1 -force-target-instruction-cost=1 -mattr=+sve -mtriple aarch64-unknown-linux-gnu \5; RUN: -prefer-predicate-over-epilogue=scalar-epilogue -pass-remarks-missed=loop-vectorize < %s 2>%t | FileCheck %s6; RUN: cat %t | FileCheck %s --check-prefix=CHECK-REMARKS7 8define void @vec_load(i64 %N, ptr nocapture %a, ptr nocapture readonly %b) {9; CHECK-LABEL: @vec_load10; CHECK: vector.body:11; CHECK: %[[LOAD:.*]] = load <vscale x 2 x double>, ptr12; CHECK: call <vscale x 2 x double> @foo_vec(<vscale x 2 x double> %[[LOAD]])13entry:14 %cmp7 = icmp sgt i64 %N, 015 br i1 %cmp7, label %for.body, label %for.end16 17for.body: ; preds = %for.body.preheader, %for.body18 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]19 %arrayidx = getelementptr inbounds double, ptr %b, i64 %iv20 %0 = load double, ptr %arrayidx, align 821 %1 = call double @foo(double %0) #022 %add = fadd double %1, 1.000000e+0023 %arrayidx2 = getelementptr inbounds double, ptr %a, i64 %iv24 store double %add, ptr %arrayidx2, align 825 %iv.next = add nuw nsw i64 %iv, 126 %exitcond.not = icmp eq i64 %iv.next, %N27 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !128 29for.end: ; preds = %for.body, %entry30 ret void31}32 33define void @vec_scalar(i64 %N, ptr nocapture %a) {34; CHECK-LABEL: @vec_scalar35; CHECK: vector.body:36; CHECK: call <vscale x 2 x double> @foo_vec(<vscale x 2 x double> splat (double 1.000000e+01))37entry:38 %cmp7 = icmp sgt i64 %N, 039 br i1 %cmp7, label %for.body, label %for.end40 41for.body: ; preds = %for.body.preheader, %for.body42 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]43 %0 = call double @foo(double 10.0) #044 %sub = fsub double %0, 1.000000e+0045 %arrayidx = getelementptr inbounds double, ptr %a, i64 %iv46 store double %sub, ptr %arrayidx, align 847 %iv.next = add nuw nsw i64 %iv, 148 %exitcond.not = icmp eq i64 %iv.next, %N49 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !150 51for.end: ; preds = %for.body, %entry52 ret void53}54 55define void @vec_ptr(i64 %N, ptr noalias %a, ptr readnone %b) {56; CHECK-LABEL: @vec_ptr57; CHECK: for.body:58; CHECK: %[[LOAD:.*]] = load ptr, ptr59; CHECK: call i64 @bar(ptr %[[LOAD]])60entry:61 %cmp7 = icmp sgt i64 %N, 062 br i1 %cmp7, label %for.body, label %for.end63 64for.body:65 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]66 %gep = getelementptr ptr, ptr %b, i64 %iv67 %load = load ptr, ptr %gep68 %call = call i64 @bar(ptr %load) #169 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv70 store i64 %call, ptr %arrayidx71 %iv.next = add nuw nsw i64 %iv, 172 %exitcond = icmp eq i64 %iv.next, 102473 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !174 75for.end:76 ret void77}78 79define void @vec_intrinsic(i64 %N, ptr nocapture readonly %a) {80; CHECK-LABEL: @vec_intrinsic81; CHECK: vector.body:82; CHECK: %[[LOAD:.*]] = load <vscale x 2 x double>, ptr83; CHECK: call fast <vscale x 2 x double> @sin_vec_nxv2f64(<vscale x 2 x double> %[[LOAD]])84entry:85 %cmp7 = icmp sgt i64 %N, 086 br i1 %cmp7, label %for.body, label %for.end87 88for.body:89 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]90 %arrayidx = getelementptr inbounds double, ptr %a, i64 %iv91 %0 = load double, ptr %arrayidx, align 892 %1 = call fast double @llvm.sin.f64(double %0) #293 %add = fadd fast double %1, 1.000000e+0094 store double %add, ptr %arrayidx, align 895 %iv.next = add nuw nsw i64 %iv, 196 %exitcond = icmp eq i64 %iv.next, %N97 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !198 99for.end:100 ret void101}102 103; CHECK-REMARKS: UserVF ignored because of invalid costs.104; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load105; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin106; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store107define void @vec_sin_no_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {108; CHECK: @vec_sin_no_mapping109; CHECK: call fast <2 x float> @llvm.sin.v2f32110; CHECK-NOT: <vscale x111entry:112 br label %for.body113 114for.body: ; preds = %entry, %for.body115 %i.07 = phi i64 [ %inc, %for.body ], [ 0, %entry ]116 %arrayidx = getelementptr inbounds float, ptr %src, i64 %i.07117 %0 = load float, ptr %arrayidx, align 4, !dbg !11118 %1 = tail call fast float @llvm.sin.f32(float %0), !dbg !12119 %arrayidx1 = getelementptr inbounds float, ptr %dst, i64 %i.07120 store float %1, ptr %arrayidx1, align 4, !dbg !13121 %inc = add nuw nsw i64 %i.07, 1122 %exitcond.not = icmp eq i64 %inc, %n123 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1124 125for.cond.cleanup: ; preds = %for.body126 ret void127}128 129; CHECK-REMARKS: UserVF ignored because of invalid costs.130; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load131; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): fadd132; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin133; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin134; CHECK-REMARKS-NEXT: t.c:3:40: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store135define void @vec_sin_no_mapping_ite(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {136; CHECK: @vec_sin_no_mapping_ite137; CHECK-NOT: <vscale x138; CHECK: ret139entry:140 br label %for.body141 142for.body: ; preds = %entry, %if.end143 %i.07 = phi i64 [ %inc, %if.end ], [ 0, %entry ]144 %arrayidx = getelementptr inbounds float, ptr %src, i64 %i.07145 %0 = load float, ptr %arrayidx, align 4, !dbg !11146 %cmp = fcmp ugt float %0, 0.0000147 br i1 %cmp, label %if.then, label %if.else148if.then:149 %1 = tail call fast float @llvm.sin.f32(float %0), !dbg !12150 br label %if.end151if.else:152 %add = fadd float %0, 12.0, !dbg !13153 %2 = tail call fast float @llvm.sin.f32(float %add), !dbg !13154 br label %if.end155if.end:156 %3 = phi float [%1, %if.then], [%2, %if.else]157 %arrayidx1 = getelementptr inbounds float, ptr %dst, i64 %i.07158 store float %3, ptr %arrayidx1, align 4, !dbg !14159 %inc = add nuw nsw i64 %i.07, 1160 %exitcond.not = icmp eq i64 %inc, %n161 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1162 163for.cond.cleanup: ; preds = %for.body164 ret void165}166 167; CHECK-REMARKS: UserVF ignored because of invalid costs.168; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load169; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin170; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store171define void @vec_sin_fixed_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {172; CHECK: @vec_sin_fixed_mapping173; CHECK: call fast <2 x float> @llvm.sin.v2f32174; CHECK-NOT: <vscale x175entry:176 br label %for.body177 178for.body: ; preds = %entry, %for.body179 %i.07 = phi i64 [ %inc, %for.body ], [ 0, %entry ]180 %arrayidx = getelementptr inbounds float, ptr %src, i64 %i.07181 %0 = load float, ptr %arrayidx, align 4, !dbg !11182 %1 = tail call fast float @llvm.sin.f32(float %0) #3, !dbg !12183 %arrayidx1 = getelementptr inbounds float, ptr %dst, i64 %i.07184 store float %1, ptr %arrayidx1, align 4, !dbg !13185 %inc = add nuw nsw i64 %i.07, 1186 %exitcond.not = icmp eq i64 %inc, %n187 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1188 189for.cond.cleanup: ; preds = %for.body190 ret void191}192 193; Even though there are no function mappings attached to the call194; in the loop below we can still vectorize the loop because SVE has195; hardware support in the form of the 'fqsrt' instruction.196define void @vec_sqrt_no_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) {197; CHECK: @vec_sqrt_no_mapping198; CHECK: call fast <vscale x 2 x float> @llvm.sqrt.nxv2f32199entry:200 br label %for.body201 202for.body: ; preds = %entry, %for.body203 %i.07 = phi i64 [ %inc, %for.body ], [ 0, %entry ]204 %arrayidx = getelementptr inbounds float, ptr %src, i64 %i.07205 %0 = load float, ptr %arrayidx, align 4206 %1 = tail call fast float @llvm.sqrt.f32(float %0)207 %arrayidx1 = getelementptr inbounds float, ptr %dst, i64 %i.07208 store float %1, ptr %arrayidx1, align 4209 %inc = add nuw nsw i64 %i.07, 1210 %exitcond.not = icmp eq i64 %inc, %n211 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1212 213for.cond.cleanup: ; preds = %for.body214 ret void215}216 217 218declare double @foo(double)219declare i64 @bar(ptr)220declare double @llvm.sin.f64(double)221declare float @llvm.sin.f32(float)222declare float @llvm.sqrt.f32(float)223 224declare <vscale x 2 x double> @foo_vec(<vscale x 2 x double>)225declare <vscale x 2 x i64> @bar_vec(<vscale x 2 x ptr>)226declare <vscale x 2 x double> @sin_vec_nxv2f64(<vscale x 2 x double>)227declare <2 x double> @sin_vec_v2f64(<2 x double>)228 229attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }230attributes #1 = { "vector-function-abi-variant"="_ZGVsNxv_bar(bar_vec)" }231attributes #2 = { "vector-function-abi-variant"="_ZGVsNxv_llvm.sin.f64(sin_vec_nxv2f64)" }232attributes #3 = { "vector-function-abi-variant"="_ZGV_LLVM_N2v_llvm.sin.f64(sin_vec_v2f64)" }233 234!1 = distinct !{!1, !2, !3}235!2 = !{!"llvm.loop.vectorize.width", i32 2}236!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}237 238!llvm.dbg.cu = !{!4}239!llvm.module.flags = !{!7}240!llvm.ident = !{!8}241 242!4 = distinct !DICompileUnit(language: DW_LANG_C99, file: !5, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !6, splitDebugInlining: false, nameTableKind: None)243!5 = !DIFile(filename: "t.c", directory: "somedir")244!6 = !{}245!7 = !{i32 2, !"Debug Info Version", i32 3}246!8 = !{!"clang"}247!9 = distinct !DISubprogram(name: "foo", scope: !5, file: !5, line: 2, type: !10, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !4, retainedNodes: !6)248!10 = !DISubroutineType(types: !6)249!11 = !DILocation(line: 3, column: 10, scope: !9)250!12 = !DILocation(line: 3, column: 20, scope: !9)251!13 = !DILocation(line: 3, column: 30, scope: !9)252!14 = !DILocation(line: 3, column: 40, scope: !9)253