496 lines · plain
1; RUN: opt %loadNPMPolly -polly-pattern-matching-based-opts=true -polly-target-throughput-vector-fma=1 -polly-target-latency-vector-fma=8 -polly-target-1st-cache-level-associativity=8 -polly-target-2nd-cache-level-associativity=8 -polly-target-1st-cache-level-size=32768 -polly-target-vector-register-bitwidth=256 -polly-target-2nd-cache-level-size=262144 '-passes=polly-custom<opt-isl;ast>' -polly-print-ast -disable-output < %s | FileCheck %s2;3; opt %loadNPMPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \4; -polly-target-throughput-vector-fma=1 \5; -polly-target-latency-vector-fma=8 \6; -passes=polly-codegen -polly-target-1st-cache-level-associativity=8 \7; -polly-target-2nd-cache-level-associativity=8 \8; -polly-target-1st-cache-level-size=32768 \9; -polly-target-vector-register-bitwidth=256 \10; -polly-target-2nd-cache-level-size=262144 -gvn -licm -slp-vectorizer \11; -mcpu=corei7 -stats -S < %s 2>&1 | FileCheck %s --check-prefix=AUTO-VECTORIZATION12;13;14; /* We isolate a set of partial tile prefixes, which contains only partial15; tile prefixes that have exactly Mr x Nr iterations of the two innermost16; loops produced by the optimization of the matrix multiplication. Mr and17; Nr are parameters of the micro-kernel (see getMicroKernelParams and18; getMacroKernelParams from lib/Transform/ScheduleOptimizer.cpp for19; details). This test check that in case of parametric bounds it helps to20; get rid of the conditional expressions of the unrolled innermost loops,21; which prevents stores and loads of the unrolled loops from being sunk22; and hoisted. Otherwise, it causes a run-time regression in comparison23; to the vectorized code with sunk and hoisted memory accesses. */24;25; /* C := A * B + C */26; for (i = 0; i < _PB_NI; i++)27; for (j = 0; j < _PB_NJ; j++)28; for (k = 0; k < _PB_NK; ++k)29; C[i][j] += A[i][k] * B[k][j];30;31; CHECK: if (ni >= 1) {32; CHECK-NEXT: // 1st level tiling - Tiles33; CHECK-NEXT: for (int c0 = 0; c0 <= floord(nj - 1, 2048); c0 += 1)34; CHECK-NEXT: for (int c1 = 0; c1 <= floord(nk - 1, 256); c1 += 1) {35; CHECK-NEXT: for (int c3 = 2048 * c0; c3 <= min(nj - 1, 2048 * c0 + 2047); c3 += 1)36; CHECK-NEXT: for (int c4 = 256 * c1; c4 <= min(nk - 1, 256 * c1 + 255); c4 += 1)37; CHECK-NEXT: CopyStmt_0(0, c3, c4);38; CHECK-NEXT: for (int c2 = 0; c2 <= floord(ni - 1, 96); c2 += 1) {39; CHECK-NEXT: for (int c6 = 96 * c2; c6 <= min(ni - 1, 96 * c2 + 95); c6 += 1)40; CHECK-NEXT: for (int c7 = 256 * c1; c7 <= min(nk - 1, 256 * c1 + 255); c7 += 1)41; CHECK-NEXT: CopyStmt_1(c0, c1, c2, c6, c7);42; CHECK-NEXT: // 1st level tiling - Points43; CHECK-NEXT: // Register tiling - Tiles44; CHECK-NEXT: {45; CHECK-NEXT: if (ni >= 96 * c2 + 4)46; CHECK-NEXT: for (int c3 = 0; c3 <= min(255, -256 * c0 + nj / 8 - 1); c3 += 1) {47; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + ni / 4 - 1); c4 += 1)48; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {49; CHECK-NEXT: // Loop Vectorizer Disabled50; CHECK-NEXT: // Register tiling - Points51; CHECK-NEXT: {52; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3, 256 * c1 + c5);53; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);54; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);55; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);56; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);57; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);58; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);59; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);60; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);61; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);62; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);63; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);64; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);65; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);66; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);67; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);68; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3, 256 * c1 + c5);69; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);70; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);71; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);72; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);73; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);74; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);75; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);76; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3, 256 * c1 + c5);77; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);78; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);79; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);80; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);81; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);82; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);83; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);84; CHECK-NEXT: }85; CHECK-NEXT: }86; CHECK-NEXT: if ((ni >= 96 * c2 + 5 && 96 * c2 + 7 >= ni && c3 >= 0) || (ni >= 96 * c2 + 8 && 96 * c2 + 95 >= ni && ni % 4 >= 1)) {87; CHECK-NEXT: if (96 * c2 + 7 >= ni) {88; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {89; CHECK-NEXT: // Loop Vectorizer Disabled90; CHECK-NEXT: // Register tiling - Points91; CHECK-NEXT: {92; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3, 256 * c1 + c5);93; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);94; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);95; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);96; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);97; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);98; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);99; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);100; CHECK-NEXT: if (ni >= 96 * c2 + 6) {101; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3, 256 * c1 + c5);102; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);103; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);104; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);105; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);106; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);107; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);108; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);109; CHECK-NEXT: if (96 * c2 + 7 == ni) {110; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);111; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);112; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);113; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);114; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);115; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);116; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);117; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);118; CHECK-NEXT: }119; CHECK-NEXT: }120; CHECK-NEXT: }121; CHECK-NEXT: }122; CHECK-NEXT: } else {123; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {124; CHECK-NEXT: // Loop Vectorizer Disabled125; CHECK-NEXT: // Register tiling - Points126; CHECK-NEXT: {127; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);128; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);129; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);130; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);131; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);132; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);133; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);134; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);135; CHECK-NEXT: if (ni % 4 >= 2) {136; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3, 256 * c1 + c5);137; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);138; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);139; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);140; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);141; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);142; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);143; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);144; CHECK-NEXT: if ((ni + 1) % 4 == 0) {145; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);146; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);147; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);148; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);149; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);150; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);151; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);152; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);153; CHECK-NEXT: }154; CHECK-NEXT: }155; CHECK-NEXT: }156; CHECK-NEXT: }157; CHECK-NEXT: }158; CHECK-NEXT: }159; CHECK-NEXT: }160; CHECK-NEXT: if (96 * c2 + 3 >= ni || 2048 * c0 + 7 >= nj || (2048 * c0 + 2047 >= nj && nj % 8 >= 1)) {161; CHECK-NEXT: if (2048 * c0 + 7 >= nj) {162; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + (ni - 1) / 4); c4 += 1)163; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {164; CHECK-NEXT: // Loop Vectorizer Disabled165; CHECK-NEXT: // Register tiling - Points166; CHECK-NEXT: {167; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0, 256 * c1 + c5);168; CHECK-NEXT: if (nj >= 2048 * c0 + 2) {169; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 1, 256 * c1 + c5);170; CHECK-NEXT: if (nj >= 2048 * c0 + 3) {171; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 2, 256 * c1 + c5);172; CHECK-NEXT: if (nj >= 2048 * c0 + 4) {173; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 3, 256 * c1 + c5);174; CHECK-NEXT: if (nj >= 2048 * c0 + 5) {175; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 4, 256 * c1 + c5);176; CHECK-NEXT: if (nj >= 2048 * c0 + 6) {177; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 5, 256 * c1 + c5);178; CHECK-NEXT: if (2048 * c0 + 7 == nj)179; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, nj - 1, 256 * c1 + c5);180; CHECK-NEXT: }181; CHECK-NEXT: }182; CHECK-NEXT: }183; CHECK-NEXT: }184; CHECK-NEXT: }185; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 2) {186; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0, 256 * c1 + c5);187; CHECK-NEXT: if (nj >= 2048 * c0 + 2) {188; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 1, 256 * c1 + c5);189; CHECK-NEXT: if (nj >= 2048 * c0 + 3) {190; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 2, 256 * c1 + c5);191; CHECK-NEXT: if (nj >= 2048 * c0 + 4) {192; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 3, 256 * c1 + c5);193; CHECK-NEXT: if (nj >= 2048 * c0 + 5) {194; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 4, 256 * c1 + c5);195; CHECK-NEXT: if (nj >= 2048 * c0 + 6) {196; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 5, 256 * c1 + c5);197; CHECK-NEXT: if (2048 * c0 + 7 == nj)198; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, nj - 1, 256 * c1 + c5);199; CHECK-NEXT: }200; CHECK-NEXT: }201; CHECK-NEXT: }202; CHECK-NEXT: }203; CHECK-NEXT: }204; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 3) {205; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0, 256 * c1 + c5);206; CHECK-NEXT: if (nj >= 2048 * c0 + 2) {207; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 1, 256 * c1 + c5);208; CHECK-NEXT: if (nj >= 2048 * c0 + 3) {209; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 2, 256 * c1 + c5);210; CHECK-NEXT: if (nj >= 2048 * c0 + 4) {211; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 3, 256 * c1 + c5);212; CHECK-NEXT: if (nj >= 2048 * c0 + 5) {213; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 4, 256 * c1 + c5);214; CHECK-NEXT: if (nj >= 2048 * c0 + 6) {215; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 5, 256 * c1 + c5);216; CHECK-NEXT: if (2048 * c0 + 7 == nj)217; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, nj - 1, 256 * c1 + c5);218; CHECK-NEXT: }219; CHECK-NEXT: }220; CHECK-NEXT: }221; CHECK-NEXT: }222; CHECK-NEXT: }223; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 4) {224; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0, 256 * c1 + c5);225; CHECK-NEXT: if (nj >= 2048 * c0 + 2) {226; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 1, 256 * c1 + c5);227; CHECK-NEXT: if (nj >= 2048 * c0 + 3) {228; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 2, 256 * c1 + c5);229; CHECK-NEXT: if (nj >= 2048 * c0 + 4) {230; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 3, 256 * c1 + c5);231; CHECK-NEXT: if (nj >= 2048 * c0 + 5) {232; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 4, 256 * c1 + c5);233; CHECK-NEXT: if (nj >= 2048 * c0 + 6) {234; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 5, 256 * c1 + c5);235; CHECK-NEXT: if (2048 * c0 + 7 == nj)236; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, nj - 1, 256 * c1 + c5);237; CHECK-NEXT: }238; CHECK-NEXT: }239; CHECK-NEXT: }240; CHECK-NEXT: }241; CHECK-NEXT: }242; CHECK-NEXT: }243; CHECK-NEXT: }244; CHECK-NEXT: }245; CHECK-NEXT: }246; CHECK-NEXT: }247; CHECK-NEXT: } else if (96 * c2 + 3 >= ni) {248; CHECK-NEXT: for (int c3 = 0; c3 <= min(255, -256 * c0 + (nj - 1) / 8); c3 += 1)249; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {250; CHECK-NEXT: // Loop Vectorizer Disabled251; CHECK-NEXT: // Register tiling - Points252; CHECK-NEXT: {253; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3, 256 * c1 + c5);254; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) {255; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);256; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) {257; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);258; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) {259; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);260; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) {261; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);262; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) {263; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);264; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) {265; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);266; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8)267; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);268; CHECK-NEXT: }269; CHECK-NEXT: }270; CHECK-NEXT: }271; CHECK-NEXT: }272; CHECK-NEXT: }273; CHECK-NEXT: }274; CHECK-NEXT: if (ni >= 96 * c2 + 2) {275; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);276; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) {277; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);278; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) {279; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);280; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) {281; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);282; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) {283; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);284; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) {285; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);286; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) {287; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);288; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8)289; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);290; CHECK-NEXT: }291; CHECK-NEXT: }292; CHECK-NEXT: }293; CHECK-NEXT: }294; CHECK-NEXT: }295; CHECK-NEXT: }296; CHECK-NEXT: if (96 * c2 + 3 == ni) {297; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);298; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) {299; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);300; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) {301; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);302; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) {303; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);304; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) {305; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);306; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) {307; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);308; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) {309; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);310; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8)311; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);312; CHECK-NEXT: }313; CHECK-NEXT: }314; CHECK-NEXT: }315; CHECK-NEXT: }316; CHECK-NEXT: }317; CHECK-NEXT: }318; CHECK-NEXT: }319; CHECK-NEXT: }320; CHECK-NEXT: }321; CHECK-NEXT: }322; CHECK-NEXT: } else {323; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + (ni - 1) / 4); c4 += 1)324; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {325; CHECK-NEXT: // Loop Vectorizer Disabled326; CHECK-NEXT: // Register tiling - Points327; CHECK-NEXT: {328; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5);329; CHECK-NEXT: if (nj % 8 >= 2) {330; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj, 256 * c1 + c5);331; CHECK-NEXT: if (nj % 8 >= 3) {332; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5);333; CHECK-NEXT: if (nj % 8 >= 4) {334; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5);335; CHECK-NEXT: if (nj % 8 >= 5) {336; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5);337; CHECK-NEXT: if (nj % 8 >= 6) {338; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5);339; CHECK-NEXT: if ((nj + 1) % 8 == 0)340; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, nj - 1, 256 * c1 + c5);341; CHECK-NEXT: }342; CHECK-NEXT: }343; CHECK-NEXT: }344; CHECK-NEXT: }345; CHECK-NEXT: }346; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 2) {347; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5);348; CHECK-NEXT: if (nj % 8 >= 2) {349; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj, 256 * c1 + c5);350; CHECK-NEXT: if (nj % 8 >= 3) {351; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5);352; CHECK-NEXT: if (nj % 8 >= 4) {353; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5);354; CHECK-NEXT: if (nj % 8 >= 5) {355; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5);356; CHECK-NEXT: if (nj % 8 >= 6) {357; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5);358; CHECK-NEXT: if ((nj + 1) % 8 == 0)359; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, nj - 1, 256 * c1 + c5);360; CHECK-NEXT: }361; CHECK-NEXT: }362; CHECK-NEXT: }363; CHECK-NEXT: }364; CHECK-NEXT: }365; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 3) {366; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5);367; CHECK-NEXT: if (nj % 8 >= 2) {368; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj, 256 * c1 + c5);369; CHECK-NEXT: if (nj % 8 >= 3) {370; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5);371; CHECK-NEXT: if (nj % 8 >= 4) {372; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5);373; CHECK-NEXT: if (nj % 8 >= 5) {374; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5);375; CHECK-NEXT: if (nj % 8 >= 6) {376; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5);377; CHECK-NEXT: if ((nj + 1) % 8 == 0)378; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, nj - 1, 256 * c1 + c5);379; CHECK-NEXT: }380; CHECK-NEXT: }381; CHECK-NEXT: }382; CHECK-NEXT: }383; CHECK-NEXT: }384; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 4) {385; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5);386; CHECK-NEXT: if (nj % 8 >= 2) {387; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj, 256 * c1 + c5);388; CHECK-NEXT: if (nj % 8 >= 3) {389; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5);390; CHECK-NEXT: if (nj % 8 >= 4) {391; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5);392; CHECK-NEXT: if (nj % 8 >= 5) {393; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5);394; CHECK-NEXT: if (nj % 8 >= 6) {395; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5);396; CHECK-NEXT: if ((nj + 1) % 8 == 0)397; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, nj - 1, 256 * c1 + c5);398; CHECK-NEXT: }399; CHECK-NEXT: }400; CHECK-NEXT: }401; CHECK-NEXT: }402; CHECK-NEXT: }403; CHECK-NEXT: }404; CHECK-NEXT: }405; CHECK-NEXT: }406; CHECK-NEXT: }407; CHECK-NEXT: }408; CHECK-NEXT: }409; CHECK-NEXT: }410; CHECK-NEXT: }411; CHECK-NEXT: }412; CHECK-NEXT: }413; CHECK-NEXT: }414;415 416; AUTO-VECTORIZATION: fmul <4 x double>417; AUTO-VECTORIZATION: fadd <4 x double>418 419; AUTO-VECTORIZATION: 36 SLP - Number of vector instructions generated420; AUTO-VECTORIZATION: 453 licm - Number of instructions hoisted out of loop421; AUTO-VECTORIZATION: 2 licm - Number of load insts hoisted or sunk422; AUTO-VECTORIZATION: 32 licm - Number of memory locations promoted to registers423;424target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"425target triple = "x86_64-unknown-unknown"426 427define internal void @kernel_gemm(i32 %ni, i32 %nj, i32 %nk, double %alpha, double %beta, ptr %C, ptr %A, ptr %B) #0 {428entry:429 br label %entry.split430 431entry.split: ; preds = %entry432 %cmp39 = icmp sgt i32 %ni, 0433 br i1 %cmp39, label %for.cond1.preheader.lr.ph, label %for.end22434 435for.cond1.preheader.lr.ph: ; preds = %entry.split436 br label %for.cond1.preheader437 438for.cond1.preheader: ; preds = %for.inc20, %for.cond1.preheader.lr.ph439 %indvars.iv45 = phi i64 [ 0, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next46, %for.inc20 ]440 %cmp237 = icmp sgt i32 %nj, 0441 br i1 %cmp237, label %for.cond4.preheader.lr.ph, label %for.inc20442 443for.cond4.preheader.lr.ph: ; preds = %for.cond1.preheader444 br label %for.cond4.preheader445 446for.cond4.preheader: ; preds = %for.inc17, %for.cond4.preheader.lr.ph447 %indvars.iv41 = phi i64 [ 0, %for.cond4.preheader.lr.ph ], [ %indvars.iv.next42, %for.inc17 ]448 %cmp535 = icmp sgt i32 %nk, 0449 br i1 %cmp535, label %for.body6.lr.ph, label %for.inc17450 451for.body6.lr.ph: ; preds = %for.cond4.preheader452 br label %for.body6453 454for.body6: ; preds = %for.body6, %for.body6.lr.ph455 %indvars.iv = phi i64 [ 0, %for.body6.lr.ph ], [ %indvars.iv.next, %for.body6 ]456 %arrayidx8 = getelementptr inbounds [1024 x double], ptr %A, i64 %indvars.iv45, i64 %indvars.iv457 %tmp = load double, ptr %arrayidx8, align 8458 %arrayidx12 = getelementptr inbounds [1024 x double], ptr %B, i64 %indvars.iv, i64 %indvars.iv41459 %tmp1 = load double, ptr %arrayidx12, align 8460 %mul = fmul double %tmp, %tmp1461 %arrayidx16 = getelementptr inbounds [1024 x double], ptr %C, i64 %indvars.iv45, i64 %indvars.iv41462 %tmp2 = load double, ptr %arrayidx16, align 8463 %add = fadd double %tmp2, %mul464 store double %add, ptr %arrayidx16, align 8465 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1466 %wide.trip.count = zext i32 %nk to i64467 %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count468 br i1 %exitcond, label %for.body6, label %for.cond4.for.inc17_crit_edge469 470for.cond4.for.inc17_crit_edge: ; preds = %for.body6471 br label %for.inc17472 473for.inc17: ; preds = %for.cond4.for.inc17_crit_edge, %for.cond4.preheader474 %indvars.iv.next42 = add nuw nsw i64 %indvars.iv41, 1475 %wide.trip.count43 = zext i32 %nj to i64476 %exitcond44 = icmp ne i64 %indvars.iv.next42, %wide.trip.count43477 br i1 %exitcond44, label %for.cond4.preheader, label %for.cond1.for.inc20_crit_edge478 479for.cond1.for.inc20_crit_edge: ; preds = %for.inc17480 br label %for.inc20481 482for.inc20: ; preds = %for.cond1.for.inc20_crit_edge, %for.cond1.preheader483 %indvars.iv.next46 = add nuw nsw i64 %indvars.iv45, 1484 %wide.trip.count47 = zext i32 %ni to i64485 %exitcond48 = icmp ne i64 %indvars.iv.next46, %wide.trip.count47486 br i1 %exitcond48, label %for.cond1.preheader, label %for.cond.for.end22_crit_edge487 488for.cond.for.end22_crit_edge: ; preds = %for.inc20489 br label %for.end22490 491for.end22: ; preds = %for.cond.for.end22_crit_edge, %entry.split492 ret void493}494 495attributes #0 = { nounwind uwtable "target-cpu"="x86-64" "target-features"="+aes,+avx,+cmov,+cx16,+fxsr,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" }496