381 lines · cpp
1//===- RemarkTest.cpp - Remark unit tests -------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "mlir/IR/Diagnostics.h"10#include "mlir/IR/MLIRContext.h"11#include "mlir/IR/Remarks.h"12#include "mlir/Remark/RemarkStreamer.h"13#include "mlir/Support/TypeID.h"14#include "llvm/ADT/StringRef.h"15#include "llvm/IR/LLVMRemarkStreamer.h"16#include "llvm/Remarks/RemarkFormat.h"17#include "llvm/Support/FileSystem.h"18#include "llvm/Support/LogicalResult.h"19#include "llvm/Support/YAMLParser.h"20#include "gmock/gmock.h"21#include "gtest/gtest.h"22#include <optional>23 24using namespace llvm;25using namespace mlir;26using namespace testing;27namespace {28 29TEST(Remark, TestOutputOptimizationRemark) {30 std::string categoryVectorizer("Vectorizer");31 std::string categoryRegister("Register");32 std::string categoryUnroll("Unroll");33 std::string categoryInliner("Inliner");34 std::string categoryReroller("Reroller");35 std::string myPassname1("myPass1");36 SmallString<64> tmpPathStorage;37 sys::fs::createUniquePath("remarks-%%%%%%.yaml", tmpPathStorage,38 /*MakeAbsolute=*/true);39 std::string yamlFile =40 std::string(tmpPathStorage.data(), tmpPathStorage.size());41 ASSERT_FALSE(yamlFile.empty());42 43 {44 MLIRContext context;45 Location loc = UnknownLoc::get(&context);46 47 context.printOpOnDiagnostic(true);48 context.printStackTraceOnDiagnostic(true);49 50 // Setup the remark engine51 mlir::remark::RemarkCategories cats{/*all=*/"",52 /*passed=*/categoryVectorizer,53 /*missed=*/categoryUnroll,54 /*analysis=*/categoryRegister,55 /*failed=*/categoryInliner};56 std::unique_ptr<remark::RemarkEmittingPolicyAll> policy =57 std::make_unique<remark::RemarkEmittingPolicyAll>();58 LogicalResult isEnabled =59 mlir::remark::enableOptimizationRemarksWithLLVMStreamer(60 context, yamlFile, llvm::remarks::Format::YAML, std::move(policy),61 cats);62 ASSERT_TRUE(succeeded(isEnabled)) << "Failed to enable remark engine";63 64 // PASS: something succeeded65 remark::passed(loc, remark::RemarkOpts::name("Pass1")66 .category(categoryVectorizer)67 .subCategory(myPassname1)68 .function("bar"))69 << "vectorized loop" << remark::metric("tripCount", 128);70 71 // ANALYSIS: neutral insight72 remark::analysis(73 loc, remark::RemarkOpts::name("Analysis1").category(categoryRegister))74 << "Kernel uses 168 registers";75 76 // MISSED: explain why + suggest a fix77 remark::missed(loc, remark::RemarkOpts::name("Miss1")78 .category(categoryUnroll)79 .subCategory(myPassname1))80 << remark::reason("not profitable at this size")81 << remark::suggest("increase unroll factor to >=4");82 83 // FAILURE: action attempted but failed84 remark::failed(loc, remark::RemarkOpts::name("Failed1")85 .category(categoryInliner)86 .subCategory(myPassname1))87 << remark::reason("failed due to unsupported pattern");88 89 // FAILURE: Won't show up90 remark::failed(loc, remark::RemarkOpts::name("Failed2")91 .category(categoryReroller)92 .subCategory(myPassname1))93 << remark::reason("failed due to rerolling pattern");94 }95 96 // Read the file97 auto bufferOrErr = MemoryBuffer::getFile(yamlFile);98 ASSERT_TRUE(static_cast<bool>(bufferOrErr)) << "Failed to open remarks file";99 std::string content = bufferOrErr.get()->getBuffer().str();100 101 EXPECT_THAT(content, HasSubstr("--- !Passed"));102 EXPECT_THAT(content, HasSubstr("Name: Pass1"));103 EXPECT_THAT(content, HasSubstr("Pass: 'Vectorizer:myPass1'"));104 EXPECT_THAT(content, HasSubstr("Function: bar"));105 EXPECT_THAT(content, HasSubstr("Remark: vectorized loop"));106 EXPECT_THAT(content, HasSubstr("tripCount: '128'"));107 108 EXPECT_THAT(content, HasSubstr("--- !Analysis"));109 EXPECT_THAT(content, HasSubstr("Pass: Register"));110 EXPECT_THAT(content, HasSubstr("Name: Analysis1"));111 EXPECT_THAT(content, HasSubstr("Function: '<unknown function>'"));112 EXPECT_THAT(content, HasSubstr("Remark: Kernel uses 168 registers"));113 114 EXPECT_THAT(content, HasSubstr("--- !Missed"));115 EXPECT_THAT(content, HasSubstr("Pass: 'Unroll:myPass1'"));116 EXPECT_THAT(content, HasSubstr("Name: Miss1"));117 EXPECT_THAT(content, HasSubstr("Function: '<unknown function>'"));118 EXPECT_THAT(content,119 HasSubstr("Reason: not profitable at this size"));120 EXPECT_THAT(content,121 HasSubstr("Suggestion: 'increase unroll factor to >=4'"));122 123 EXPECT_THAT(content, HasSubstr("--- !Failure"));124 EXPECT_THAT(content, HasSubstr("Pass: 'Inliner:myPass1'"));125 EXPECT_THAT(content, HasSubstr("Name: Failed1"));126 EXPECT_THAT(content, HasSubstr("Function: '<unknown function>'"));127 EXPECT_THAT(content,128 HasSubstr("Reason: failed due to unsupported pattern"));129 130 EXPECT_THAT(content, Not(HasSubstr("Failed2")));131 EXPECT_THAT(content, Not(HasSubstr("Reroller")));132 133 // Also verify document order to avoid false positives.134 size_t iPassed = content.find("--- !Passed");135 size_t iAnalysis = content.find("--- !Analysis");136 size_t iMissed = content.find("--- !Missed");137 size_t iFailure = content.find("--- !Failure");138 139 ASSERT_NE(iPassed, std::string::npos);140 ASSERT_NE(iAnalysis, std::string::npos);141 ASSERT_NE(iMissed, std::string::npos);142 ASSERT_NE(iFailure, std::string::npos);143 144 EXPECT_LT(iPassed, iAnalysis);145 EXPECT_LT(iAnalysis, iMissed);146 EXPECT_LT(iMissed, iFailure);147}148 149TEST(Remark, TestNoOutputOptimizationRemark) {150 const auto *pass1Msg = "My message";151 152 std::string categoryFailName("myImportantCategory");153 std::string myPassname1("myPass1");154 SmallString<64> tmpPathStorage;155 sys::fs::createUniquePath("remarks-%%%%%%.yaml", tmpPathStorage,156 /*MakeAbsolute=*/true);157 std::string yamlFile =158 std::string(tmpPathStorage.data(), tmpPathStorage.size());159 ASSERT_FALSE(yamlFile.empty());160 std::error_code ec =161 llvm::sys::fs::remove(yamlFile, /*IgnoreNonExisting=*/true);162 if (ec) {163 FAIL() << "Failed to remove file " << yamlFile << ": " << ec.message();164 }165 {166 MLIRContext context;167 Location loc = UnknownLoc::get(&context);168 remark::failed(loc, remark::RemarkOpts::name("myfail")169 .category(categoryFailName)170 .subCategory(myPassname1))171 << remark::reason(pass1Msg);172 }173 // No setup, so no output file should be created174 // check!175 bool fileExists = llvm::sys::fs::exists(yamlFile);176 EXPECT_FALSE(fileExists)177 << "Expected no YAML file to be created without setupOptimizationRemarks";178}179 180TEST(Remark, TestOutputOptimizationRemarkDiagnostic) {181 std::string categoryVectorizer("Vectorizer");182 std::string categoryRegister("Register");183 std::string categoryUnroll("Unroll");184 std::string myPassname1("myPass1");185 std::string fName("foo");186 187 llvm::SmallVector<std::string> seenMsg;188 {189 MLIRContext context;190 Location loc = UnknownLoc::get(&context);191 192 context.printOpOnDiagnostic(true);193 context.printStackTraceOnDiagnostic(true);194 195 // Register a handler that captures the diagnostic.196 ScopedDiagnosticHandler handler(&context, [&](Diagnostic &diag) {197 seenMsg.push_back(diag.str());198 return success();199 });200 201 // Setup the remark engine202 mlir::remark::RemarkCategories cats{/*all=*/"",203 /*passed=*/categoryVectorizer,204 /*missed=*/categoryUnroll,205 /*analysis=*/categoryRegister,206 /*failed=*/categoryUnroll};207 std::unique_ptr<remark::RemarkEmittingPolicyAll> policy =208 std::make_unique<remark::RemarkEmittingPolicyAll>();209 LogicalResult isEnabled = remark::enableOptimizationRemarks(210 context, nullptr, std::move(policy), cats, true);211 212 ASSERT_TRUE(succeeded(isEnabled)) << "Failed to enable remark engine";213 214 // PASS: something succeeded215 remark::passed(loc, remark::RemarkOpts::name("pass1")216 .category(categoryVectorizer)217 .function(fName)218 .subCategory(myPassname1))219 << "vectorized loop" << remark::metric("tripCount", 128);220 221 // ANALYSIS: neutral insight222 remark::analysis(loc, remark::RemarkOpts::name("Analysis1")223 .category(categoryRegister)224 .function(fName))225 << "Kernel uses 168 registers";226 227 // MISSED: explain why + suggest a fix228 int target = 128;229 int tripBad = 4;230 int threshold = 256;231 232 remark::missed(loc, {"", categoryUnroll, "unroller2", ""})233 << remark::reason("tripCount={0} < threshold={1}", tripBad, threshold);234 235 remark::missed(loc, {"", categoryUnroll, "", ""})236 << remark::reason("tripCount={0} < threshold={1}", tripBad, threshold)237 << remark::suggest("increase unroll to {0}", target);238 239 // FAILURE: action attempted but failed240 remark::failed(loc, {"", categoryUnroll, "", ""})241 << remark::reason("failed due to unsupported pattern");242 }243 // clang-format off244 unsigned long expectedSize = 5;245 ASSERT_EQ(seenMsg.size(), expectedSize);246 EXPECT_EQ(seenMsg[0], "[Passed] pass1 | Category:Vectorizer:myPass1 | Function=foo | Remark=\"vectorized loop\", tripCount=128");247 EXPECT_EQ(seenMsg[1], "[Analysis] Analysis1 | Category:Register | Function=foo | Remark=\"Kernel uses 168 registers\"");248 EXPECT_EQ(seenMsg[2], "[Missed] | Category:Unroll:unroller2 | Reason=\"tripCount=4 < threshold=256\"");249 EXPECT_EQ(seenMsg[3], "[Missed] | Category:Unroll | Reason=\"tripCount=4 < threshold=256\", Suggestion=\"increase unroll to 128\"");250 EXPECT_EQ(seenMsg[4], "[Failure] | Category:Unroll | Reason=\"failed due to unsupported pattern\"");251 // clang-format on252}253 254/// Custom remark streamer that prints remarks to stderr.255class MyCustomStreamer : public remark::detail::MLIRRemarkStreamerBase {256public:257 MyCustomStreamer() = default;258 259 void streamOptimizationRemark(const remark::detail::Remark &remark) override {260 llvm::errs() << "Custom remark: ";261 remark.print(llvm::errs(), true);262 llvm::errs() << "\n";263 }264};265 266TEST(Remark, TestCustomOptimizationRemarkDiagnostic) {267 testing::internal::CaptureStderr();268 const auto *pass1Msg = "My message";269 const auto *pass2Msg = "My another message";270 const auto *pass3Msg = "Do not show this message";271 272 std::string categoryLoopunroll("LoopUnroll");273 std::string categoryInline("Inliner");274 std::string myPassname1("myPass1");275 std::string myPassname2("myPass2");276 277 {278 MLIRContext context;279 Location loc = UnknownLoc::get(&context);280 281 // Setup the remark engine282 mlir::remark::RemarkCategories cats{/*all=*/"",283 /*passed=*/categoryLoopunroll,284 /*missed=*/std::nullopt,285 /*analysis=*/std::nullopt,286 /*failed=*/categoryLoopunroll};287 288 std::unique_ptr<remark::RemarkEmittingPolicyAll> policy =289 std::make_unique<remark::RemarkEmittingPolicyAll>();290 LogicalResult isEnabled = remark::enableOptimizationRemarks(291 context, std::make_unique<MyCustomStreamer>(), std::move(policy), cats,292 true);293 ASSERT_TRUE(succeeded(isEnabled)) << "Failed to enable remark engine";294 295 // Remark 1: pass, category LoopUnroll296 remark::passed(loc, {"", categoryLoopunroll, myPassname1, ""}) << pass1Msg;297 // Remark 2: failure, category LoopUnroll298 remark::failed(loc, {"", categoryLoopunroll, myPassname2, ""})299 << remark::reason(pass2Msg);300 // Remark 3: pass, category Inline (should not be printed)301 remark::passed(loc, {"", categoryInline, myPassname1, ""}) << pass3Msg;302 }303 304 llvm::errs().flush();305 std::string errOut = ::testing::internal::GetCapturedStderr();306 307 // Expect exactly two "Custom remark:" lines.308 auto first = errOut.find("Custom remark:");309 EXPECT_NE(first, std::string::npos);310 auto second = errOut.find("Custom remark:", first + 1);311 EXPECT_NE(second, std::string::npos);312 auto third = errOut.find("Custom remark:", second + 1);313 EXPECT_EQ(third, std::string::npos);314 315 // Containment checks for messages.316 EXPECT_NE(errOut.find(pass1Msg), std::string::npos); // printed317 EXPECT_NE(errOut.find(pass2Msg), std::string::npos); // printed318 EXPECT_EQ(errOut.find(pass3Msg), std::string::npos); // filtered out319}320 321TEST(Remark, TestRemarkFinal) {322 testing::internal::CaptureStderr();323 const auto *pass1Msg = "I failed";324 const auto *pass2Msg = "I failed too";325 const auto *pass3Msg = "I succeeded";326 const auto *pass4Msg = "I succeeded too";327 328 std::string categoryLoopunroll("LoopUnroll");329 330 {331 MLIRContext context;332 Location loc = FileLineColLoc::get(&context, "test.cpp", 1, 5);333 Location locOther = FileLineColLoc::get(&context, "test.cpp", 55, 5);334 335 // Setup the remark engine336 mlir::remark::RemarkCategories cats{/*all=*/"",337 /*passed=*/categoryLoopunroll,338 /*missed=*/categoryLoopunroll,339 /*analysis=*/categoryLoopunroll,340 /*failed=*/categoryLoopunroll};341 342 std::unique_ptr<remark::RemarkEmittingPolicyFinal> policy =343 std::make_unique<remark::RemarkEmittingPolicyFinal>();344 LogicalResult isEnabled = remark::enableOptimizationRemarks(345 context, std::make_unique<MyCustomStreamer>(), std::move(policy), cats,346 true);347 ASSERT_TRUE(succeeded(isEnabled)) << "Failed to enable remark engine";348 349 // Remark 1: failure350 remark::failed(351 loc, remark::RemarkOpts::name("Unroller").category(categoryLoopunroll))352 << pass1Msg;353 354 // Remark 2: failure355 remark::missed(356 loc, remark::RemarkOpts::name("Unroller").category(categoryLoopunroll))357 << remark::reason(pass2Msg);358 359 // Remark 3: pass360 remark::passed(361 loc, remark::RemarkOpts::name("Unroller").category(categoryLoopunroll))362 << pass3Msg;363 364 // Remark 4: pass365 remark::passed(366 locOther,367 remark::RemarkOpts::name("Unroller").category(categoryLoopunroll))368 << pass4Msg;369 }370 371 llvm::errs().flush();372 std::string errOut = ::testing::internal::GetCapturedStderr();373 374 // Containment checks for messages.375 EXPECT_EQ(errOut.find(pass1Msg), std::string::npos); // dropped376 EXPECT_EQ(errOut.find(pass2Msg), std::string::npos); // dropped377 EXPECT_NE(errOut.find(pass3Msg), std::string::npos); // shown378 EXPECT_NE(errOut.find(pass4Msg), std::string::npos); // shown379}380} // namespace381