119 lines · plain
1; RUN: opt %loadNPMPolly '-passes=polly-custom<opt-isl>' -polly-pattern-matching-based-opts=true -debug -polly-tc-opt=true -disable-output < %s 2>&1 | FileCheck %s2; RUN: opt %loadNPMPolly '-passes=polly-custom<opt-isl;ast>' -polly-print-ast -polly-pattern-matching-based-opts=true -polly-target-throughput-vector-fma=1 -polly-target-latency-vector-fma=8 -polly-target-1st-cache-level-size=32768 -polly-target-vector-register-bitwidth=256 -polly-target-2nd-cache-level-size=262144 -polly-tc-opt=true -disable-output < %s | FileCheck %s --check-prefix=PATTERN-MATCHING-OPTS3; REQUIRES: asserts4;5; C := A * B + C6; Check that the pattern matching optimizations can detect different7; permutations of GEMM loop and produce the correct ISL AST. In this case,8; dimensions of band nodes can be implicitly permuted by the algorithm9; applied during the schedule generation. It should be taken into the10; account during the pattern matching optimizations.11; for (i = 0; i < _PB_NI; i++)12; for (k = 0; k < _PB_NK; ++k)13; for (j = 0; j < _PB_NJ; j++)14; C[i][j] += A[i][k] * B[k][j];15;16; CHECK: The tensor contraction pattern was detected17; CHECK: The matrix multiplication pattern was detected18;19; PATTERN-MATCHING-OPTS: // 1st level tiling - Tiles20; PATTERN-MATCHING-OPTS-NEXT: for (int c1 = 0; c1 <= 3; c1 += 1) {21; PATTERN-MATCHING-OPTS-NEXT: for (int c3 = 256 * c1; c3 <= 256 * c1 + 255; c3 += 1)22; PATTERN-MATCHING-OPTS-NEXT: for (int c4 = 0; c4 <= 1023; c4 += 1)23; PATTERN-MATCHING-OPTS-NEXT: CopyStmt_0(0, c3, c4);24; PATTERN-MATCHING-OPTS-NEXT: for (int c2 = 0; c2 <= 10; c2 += 1) {25; PATTERN-MATCHING-OPTS-NEXT: for (int c6 = 96 * c2; c6 <= min(1023, 96 * c2 + 95); c6 += 1)26; PATTERN-MATCHING-OPTS-NEXT: for (int c7 = 256 * c1; c7 <= 256 * c1 + 255; c7 += 1)27; PATTERN-MATCHING-OPTS-NEXT: CopyStmt_1(0, c1, c2, c6, c7);28; PATTERN-MATCHING-OPTS-NEXT: // 1st level tiling - Points29; PATTERN-MATCHING-OPTS-NEXT: // Register tiling - Tiles30; PATTERN-MATCHING-OPTS-NEXT: for (int c3 = 0; c3 <= 127; c3 += 1)31; PATTERN-MATCHING-OPTS-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + 255); c4 += 1)32; PATTERN-MATCHING-OPTS-NEXT: for (int c5 = 0; c5 <= 255; c5 += 1) {33; PATTERN-MATCHING-OPTS-NEXT: // Loop Vectorizer Disabled34; PATTERN-MATCHING-OPTS-NEXT: // Register tiling - Points35; PATTERN-MATCHING-OPTS-NEXT: {36; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3);37; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 1);38; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 2);39; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 3);40; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 4);41; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 5);42; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 6);43; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 7);44; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3);45; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 1);46; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 2);47; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 3);48; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 4);49; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 5);50; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 6);51; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 7);52; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3);53; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 1);54; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 2);55; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 3);56; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 4);57; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 5);58; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 6);59; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 7);60; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3);61; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 1);62; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 2);63; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 3);64; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 4);65; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 5);66; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 6);67; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 7);68; PATTERN-MATCHING-OPTS-NEXT: }69; PATTERN-MATCHING-OPTS-NEXT: }70; PATTERN-MATCHING-OPTS-NEXT: }71; PATTERN-MATCHING-OPTS-NEXT: }72;73target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"74target triple = "x86_64-unknown-unknown"75 76define internal void @kernel_gemm(i32 %ni, i32 %nj, i32 %nk, double %alpha, double %beta, ptr %C, ptr %A, ptr %B) {77entry:78 br label %entry.split79 80entry.split: ; preds = %entry81 br label %for.cond1.preheader82 83for.cond1.preheader: ; preds = %for.inc20, %entry.split84 %indvars.iv41 = phi i64 [ 0, %entry.split ], [ %indvars.iv.next42, %for.inc20 ]85 br label %for.cond4.preheader86 87for.cond4.preheader: ; preds = %for.inc17, %for.cond1.preheader88 %indvars.iv38 = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next39, %for.inc17 ]89 br label %for.body690 91for.body6: ; preds = %for.body6, %for.cond4.preheader92 %indvars.iv = phi i64 [ 0, %for.cond4.preheader ], [ %indvars.iv.next, %for.body6 ]93 %arrayidx8 = getelementptr inbounds [1024 x double], ptr %A, i64 %indvars.iv41, i64 %indvars.iv3894 %tmp = load double, ptr %arrayidx8, align 895 %arrayidx12 = getelementptr inbounds [1024 x double], ptr %B, i64 %indvars.iv38, i64 %indvars.iv96 %tmp1 = load double, ptr %arrayidx12, align 897 %mul = fmul double %tmp, %tmp198 %arrayidx16 = getelementptr inbounds [1024 x double], ptr %C, i64 %indvars.iv41, i64 %indvars.iv99 %tmp2 = load double, ptr %arrayidx16, align 8100 %add = fadd double %tmp2, %mul101 store double %add, ptr %arrayidx16, align 8102 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1103 %exitcond = icmp ne i64 %indvars.iv.next, 1024104 br i1 %exitcond, label %for.body6, label %for.inc17105 106for.inc17: ; preds = %for.body6107 %indvars.iv.next39 = add nuw nsw i64 %indvars.iv38, 1108 %exitcond40 = icmp ne i64 %indvars.iv.next39, 1024109 br i1 %exitcond40, label %for.cond4.preheader, label %for.inc20110 111for.inc20: ; preds = %for.inc17112 %indvars.iv.next42 = add nuw nsw i64 %indvars.iv41, 1113 %exitcond43 = icmp ne i64 %indvars.iv.next42, 1024114 br i1 %exitcond43, label %for.cond1.preheader, label %for.end22115 116for.end22: ; preds = %for.inc20117 ret void118}119