178 lines · plain
1; RUN: opt %loadNPMPolly -polly-pattern-matching-based-opts=true -polly-target-throughput-vector-fma=1 -polly-target-latency-vector-fma=8 -polly-target-1st-cache-level-size=0 -polly-target-vector-register-bitwidth=256 '-passes=polly-custom<opt-isl;ast>' -polly-print-ast -disable-output < %s 2>&1 | FileCheck %s2 3; RUN: opt %loadNPMPolly -polly-pattern-matching-based-opts=true -polly-target-throughput-vector-fma=1 -polly-target-latency-vector-fma=8 -polly-target-1st-cache-level-associativity=8 -polly-target-2nd-cache-level-associativity=8 -polly-target-1st-cache-level-size=32768 -polly-target-vector-register-bitwidth=256 -polly-target-2nd-cache-level-size=262144 '-passes=polly-custom<opt-isl;ast>' -polly-print-ast -disable-output < %s 2>&1 | FileCheck %s --check-prefix=EXTRACTION-OF-MACRO-KERNEL4;5; /* C := alpha*A*B + beta*C */6; for (i = 0; i < _PB_NI; i++)7; for (j = 0; j < _PB_NJ; j++)8; {9; C[i][j] *= beta;10; for (k = 0; k < _PB_NK; ++k)11; C[i][j] += alpha * A[i][k] * B[k][j];12; }13;14; CHECK-LABEL: :: isl ast :: kernel_gemm :: %bb8---%bb32 15; CHECK: {16; CHECK-NEXT: // 1st level tiling - Tiles17; CHECK-NEXT: for (int c0 = 0; c0 <= 32; c0 += 1)18; CHECK-NEXT: for (int c1 = 0; c1 <= 32; c1 += 1) {19; CHECK-NEXT: // 1st level tiling - Points20; CHECK-NEXT: for (int c2 = 0; c2 <= 31; c2 += 1)21; CHECK-NEXT: for (int c3 = 0; c3 <= 31; c3 += 1)22; CHECK-NEXT: Stmt_bb9(32 * c0 + c2, 32 * c1 + c3);23; CHECK-NEXT: }24; CHECK-NEXT: // Register tiling - Tiles25; CHECK-NEXT: for (int c0 = 0; c0 <= 131; c0 += 1)26; CHECK-NEXT: for (int c1 = 0; c1 <= 263; c1 += 1)27; CHECK-NEXT: for (int c2 = 0; c2 <= 1023; c2 += 1) {28; CHECK-NEXT: // Register tiling - Points29; CHECK-NEXT: {30; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0, c2);31; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 1, c2);32; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 2, c2);33; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 3, c2);34; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 4, c2);35; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 5, c2);36; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 6, c2);37; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 7, c2);38; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0, c2);39; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 1, c2);40; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 2, c2);41; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 3, c2);42; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 4, c2);43; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 5, c2);44; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 6, c2);45; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 7, c2);46; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0, c2);47; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 1, c2);48; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 2, c2);49; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 3, c2);50; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 4, c2);51; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 5, c2);52; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 6, c2);53; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 7, c2);54; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0, c2);55; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 1, c2);56; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 2, c2);57; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 3, c2);58; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 4, c2);59; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 5, c2);60; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 6, c2);61; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 7, c2);62; CHECK-NEXT: }63; CHECK-NEXT: }64; CHECK-NEXT: }65;66; EXTRACTION-OF-MACRO-KERNEL-LABEL: :: isl ast :: kernel_gemm :: %bb8---%bb32 67; EXTRACTION-OF-MACRO-KERNEL: {68; EXTRACTION-OF-MACRO-KERNEL-NEXT: // 1st level tiling - Tiles69; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c0 = 0; c0 <= 32; c0 += 1)70; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c1 = 0; c1 <= 32; c1 += 1) {71; EXTRACTION-OF-MACRO-KERNEL-NEXT: // 1st level tiling - Points72; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c2 = 0; c2 <= 31; c2 += 1)73; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c3 = 0; c3 <= 31; c3 += 1)74; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_bb9(32 * c0 + c2, 32 * c1 + c3);75; EXTRACTION-OF-MACRO-KERNEL-NEXT: }76; EXTRACTION-OF-MACRO-KERNEL-NEXT: // 1st level tiling - Tiles77; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c1 = 0; c1 <= 3; c1 += 1) {78; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c3 = 0; c3 <= 1055; c3 += 1)79; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c4 = 256 * c1; c4 <= 256 * c1 + 255; c4 += 1)80; EXTRACTION-OF-MACRO-KERNEL-NEXT: CopyStmt_0(0, c3, c4);81; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c2 = 0; c2 <= 10; c2 += 1) {82; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c6 = 96 * c2; c6 <= 96 * c2 + 95; c6 += 1)83; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c7 = 256 * c1; c7 <= 256 * c1 + 255; c7 += 1)84; EXTRACTION-OF-MACRO-KERNEL-NEXT: CopyStmt_1(0, c1, c2, c6, c7);85; EXTRACTION-OF-MACRO-KERNEL-NEXT: // 1st level tiling - Points86; EXTRACTION-OF-MACRO-KERNEL-NEXT: // Register tiling - Tiles87; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c3 = 0; c3 <= 131; c3 += 1)88; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c4 = 0; c4 <= 23; c4 += 1)89; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c5 = 0; c5 <= 255; c5 += 1) {90; EXTRACTION-OF-MACRO-KERNEL-NEXT: // Loop Vectorizer Disabled91; EXTRACTION-OF-MACRO-KERNEL-NEXT: // Register tiling - Points92; EXTRACTION-OF-MACRO-KERNEL-NEXT: {93; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3, 256 * c1 + c5);94; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 1, 256 * c1 + c5);95; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 2, 256 * c1 + c5);96; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 3, 256 * c1 + c5);97; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 4, 256 * c1 + c5);98; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 5, 256 * c1 + c5);99; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 6, 256 * c1 + c5);100; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 7, 256 * c1 + c5);101; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3, 256 * c1 + c5);102; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 1, 256 * c1 + c5);103; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 2, 256 * c1 + c5);104; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 3, 256 * c1 + c5);105; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 4, 256 * c1 + c5);106; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 5, 256 * c1 + c5);107; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 6, 256 * c1 + c5);108; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 7, 256 * c1 + c5);109; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3, 256 * c1 + c5);110; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 1, 256 * c1 + c5);111; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 2, 256 * c1 + c5);112; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 3, 256 * c1 + c5);113; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 4, 256 * c1 + c5);114; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 5, 256 * c1 + c5);115; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 6, 256 * c1 + c5);116; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 7, 256 * c1 + c5);117; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3, 256 * c1 + c5);118; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 1, 256 * c1 + c5);119; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 2, 256 * c1 + c5);120; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 3, 256 * c1 + c5);121; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 4, 256 * c1 + c5);122; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 5, 256 * c1 + c5);123; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 6, 256 * c1 + c5);124; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 7, 256 * c1 + c5);125; EXTRACTION-OF-MACRO-KERNEL-NEXT: }126; EXTRACTION-OF-MACRO-KERNEL-NEXT: }127; EXTRACTION-OF-MACRO-KERNEL-NEXT: }128; EXTRACTION-OF-MACRO-KERNEL-NEXT: }129; EXTRACTION-OF-MACRO-KERNEL-NEXT: }130;131target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"132target triple = "x86_64-unknown-unknown"133 134define internal void @kernel_gemm(i32 %arg, i32 %arg1, i32 %arg2, double %arg3, double %arg4, ptr %arg5, ptr %arg6, ptr %arg7) #0 {135bb:136 br label %bb8137 138bb8: ; preds = %bb29, %bb139 %tmp = phi i64 [ 0, %bb ], [ %tmp30, %bb29 ]140 br label %bb9141 142bb9: ; preds = %bb26, %bb8143 %tmp10 = phi i64 [ 0, %bb8 ], [ %tmp27, %bb26 ]144 %tmp11 = getelementptr inbounds [1056 x double], ptr %arg5, i64 %tmp, i64 %tmp10145 %tmp12 = load double, ptr %tmp11, align 8146 %tmp13 = fmul double %tmp12, %arg4147 store double %tmp13, ptr %tmp11, align 8148 br label %Copy_0149 150Copy_0: ; preds = %Copy_0, %bb9151 %tmp15 = phi i64 [ 0, %bb9 ], [ %tmp24, %Copy_0 ]152 %tmp16 = getelementptr inbounds [1024 x double], ptr %arg6, i64 %tmp, i64 %tmp15153 %tmp17 = load double, ptr %tmp16, align 8154 %tmp18 = fmul double %tmp17, %arg3155 %tmp19 = getelementptr inbounds [1056 x double], ptr %arg7, i64 %tmp15, i64 %tmp10156 %tmp20 = load double, ptr %tmp19, align 8157 %tmp21 = fmul double %tmp18, %tmp20158 %tmp22 = load double, ptr %tmp11, align 8159 %tmp23 = fadd double %tmp22, %tmp21160 store double %tmp23, ptr %tmp11, align 8161 %tmp24 = add nuw nsw i64 %tmp15, 1162 %tmp25 = icmp ne i64 %tmp24, 1024163 br i1 %tmp25, label %Copy_0, label %bb26164 165bb26: ; preds = %Copy_0166 %tmp27 = add nuw nsw i64 %tmp10, 1167 %tmp28 = icmp ne i64 %tmp27, 1056168 br i1 %tmp28, label %bb9, label %bb29169 170bb29: ; preds = %bb26171 %tmp30 = add nuw nsw i64 %tmp, 1172 %tmp31 = icmp ne i64 %tmp30, 1056173 br i1 %tmp31, label %bb8, label %bb32174 175bb32: ; preds = %bb29176 ret void177}178