392 lines · cpp
1//===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/Basic/Sarif.h"10#include "clang/Basic/DiagnosticOptions.h"11#include "clang/Basic/FileManager.h"12#include "clang/Basic/FileSystemOptions.h"13#include "clang/Basic/LangOptions.h"14#include "clang/Basic/SourceLocation.h"15#include "clang/Basic/SourceManager.h"16#include "llvm/ADT/StringRef.h"17#include "llvm/Support/FormatVariadic.h"18#include "llvm/Support/JSON.h"19#include "llvm/Support/MemoryBuffer.h"20#include "llvm/Support/VirtualFileSystem.h"21#include "llvm/Support/raw_ostream.h"22#include "gmock/gmock.h"23#include "gtest/gtest.h"24 25#include <algorithm>26 27using namespace clang;28 29namespace {30 31using LineCol = std::pair<unsigned int, unsigned int>;32 33static std::string serializeSarifDocument(llvm::json::Object &&Doc) {34 std::string Output;35 llvm::json::Value Value(std::move(Doc));36 llvm::raw_string_ostream OS{Output};37 OS << llvm::formatv("{0}", Value);38 return Output;39}40 41class SarifDocumentWriterTest : public ::testing::Test {42protected:43 SarifDocumentWriterTest()44 : InMemoryFileSystem(45 llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>()),46 FileMgr(FileSystemOptions(), InMemoryFileSystem),47 Diags(DiagnosticIDs::create(), DiagOpts, new IgnoringDiagConsumer()),48 SourceMgr(Diags, FileMgr) {}49 50 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;51 FileManager FileMgr;52 DiagnosticOptions DiagOpts;53 DiagnosticsEngine Diags;54 SourceManager SourceMgr;55 LangOptions LangOpts;56 57 FileID registerSource(llvm::StringRef Name, const char *SourceText,58 bool IsMainFile = false) {59 std::unique_ptr<llvm::MemoryBuffer> SourceBuf =60 llvm::MemoryBuffer::getMemBuffer(SourceText);61 FileEntryRef SourceFile =62 FileMgr.getVirtualFileRef(Name, SourceBuf->getBufferSize(), 0);63 SourceMgr.overrideFileContents(SourceFile, std::move(SourceBuf));64 FileID FID = SourceMgr.getOrCreateFileID(SourceFile, SrcMgr::C_User);65 if (IsMainFile)66 SourceMgr.setMainFileID(FID);67 return FID;68 }69 70 CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin,71 LineCol End) {72 auto BeginLoc = SourceMgr.translateLineCol(FID, Begin.first, Begin.second);73 auto EndLoc = SourceMgr.translateLineCol(FID, End.first, End.second);74 return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false};75 }76};77 78TEST_F(SarifDocumentWriterTest, canCreateEmptyDocument) {79 // GIVEN:80 SarifDocumentWriter Writer{SourceMgr};81 82 // WHEN:83 const llvm::json::Object &EmptyDoc = Writer.createDocument();84 std::vector<StringRef> Keys(EmptyDoc.size());85 std::transform(EmptyDoc.begin(), EmptyDoc.end(), Keys.begin(),86 [](auto Item) { return Item.getFirst(); });87 88 // THEN:89 ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version"));90}91 92// Test that a newly inserted run will associate correct tool names93TEST_F(SarifDocumentWriterTest, canCreateDocumentWithOneRun) {94 // GIVEN:95 SarifDocumentWriter Writer{SourceMgr};96 const char *ShortName = "sariftest";97 const char *LongName = "sarif writer test";98 99 // WHEN:100 Writer.createRun(ShortName, LongName);101 Writer.endRun();102 const llvm::json::Object &Doc = Writer.createDocument();103 const llvm::json::Array *Runs = Doc.getArray("runs");104 105 // THEN:106 // A run was created107 ASSERT_THAT(Runs, testing::NotNull());108 109 // It is the only run110 ASSERT_EQ(Runs->size(), 1UL);111 112 // The tool associated with the run was the tool113 const llvm::json::Object *Driver =114 Runs->begin()->getAsObject()->getObject("tool")->getObject("driver");115 ASSERT_THAT(Driver, testing::NotNull());116 117 ASSERT_TRUE(Driver->getString("name").has_value());118 ASSERT_TRUE(Driver->getString("fullName").has_value());119 ASSERT_TRUE(Driver->getString("language").has_value());120 121 EXPECT_EQ(*Driver->getString("name"), ShortName);122 EXPECT_EQ(*Driver->getString("fullName"), LongName);123 EXPECT_EQ(*Driver->getString("language"), "en-US");124}125 126TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) {127#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST128 GTEST_SKIP() << "This death test is only available for debug builds.";129#endif130 // GIVEN:131 SarifDocumentWriter Writer{SourceMgr};132 133 // WHEN:134 // A SarifDocumentWriter::createRun(...) was not called prior to135 // SarifDocumentWriter::appendResult(...)136 // But a rule exists137 auto RuleIdx = Writer.createRule(SarifRule::create());138 const SarifResult &EmptyResult = SarifResult::create(RuleIdx);139 140 // THEN:141 auto Matcher = ::testing::AnyOf(142 ::testing::HasSubstr("create a run first"),143 ::testing::HasSubstr("no runs associated with the document"));144 ASSERT_DEATH(Writer.appendResult(EmptyResult), Matcher);145}146 147TEST_F(SarifDocumentWriterTest, settingInvalidRankWillCrash) {148#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST149 GTEST_SKIP() << "This death test is only available for debug builds.";150#endif151 // GIVEN:152 SarifDocumentWriter Writer{SourceMgr};153 154 // WHEN:155 // A SarifReportingConfiguration is created with an invalid "rank"156 // * Ranks below 0.0 are invalid157 // * Ranks above 100.0 are invalid158 159 // THEN: The builder will crash in either case160 EXPECT_DEATH(SarifReportingConfiguration::create().setRank(-1.0),161 ::testing::HasSubstr("Rule rank cannot be smaller than 0.0"));162 EXPECT_DEATH(SarifReportingConfiguration::create().setRank(101.0),163 ::testing::HasSubstr("Rule rank cannot be larger than 100.0"));164}165 166TEST_F(SarifDocumentWriterTest, creatingResultWithDisabledRuleWillCrash) {167#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST168 GTEST_SKIP() << "This death test is only available for debug builds.";169#endif170 171 // GIVEN:172 SarifDocumentWriter Writer{SourceMgr};173 174 // WHEN:175 // A disabled Rule is created, and a result is create referencing this rule176 const auto &Config = SarifReportingConfiguration::create().disable();177 auto RuleIdx =178 Writer.createRule(SarifRule::create().setDefaultConfiguration(Config));179 const SarifResult &Result = SarifResult::create(RuleIdx);180 181 // THEN:182 // SarifResult::create(...) will produce a crash183 ASSERT_DEATH(184 Writer.appendResult(Result),185 ::testing::HasSubstr("Cannot add a result referencing a disabled Rule"));186}187 188// Test adding rule and result shows up in the final document189TEST_F(SarifDocumentWriterTest, addingResultWithValidRuleAndRunIsOk) {190 // GIVEN:191 SarifDocumentWriter Writer{SourceMgr};192 const SarifRule &Rule =193 SarifRule::create()194 .setRuleId("clang.unittest")195 .setDescription("Example rule created during unit tests")196 .setName("clang unit test");197 198 // WHEN:199 Writer.createRun("sarif test", "sarif test runner");200 unsigned RuleIdx = Writer.createRule(Rule);201 const SarifResult &Result = SarifResult::create(RuleIdx);202 203 Writer.appendResult(Result);204 const llvm::json::Object &Doc = Writer.createDocument();205 206 // THEN:207 // A document with a valid schema and version exists208 ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull());209 ASSERT_THAT(Doc.get("version"), ::testing::NotNull());210 const llvm::json::Array *Runs = Doc.getArray("runs");211 212 // A run exists on this document213 ASSERT_THAT(Runs, ::testing::NotNull());214 ASSERT_EQ(Runs->size(), 1UL);215 const llvm::json::Object *TheRun = Runs->back().getAsObject();216 217 // The run has slots for tools, results, rules and artifacts218 ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull());219 ASSERT_THAT(TheRun->get("results"), ::testing::NotNull());220 ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull());221 const llvm::json::Object *Driver =222 TheRun->getObject("tool")->getObject("driver");223 const llvm::json::Array *Results = TheRun->getArray("results");224 const llvm::json::Array *Artifacts = TheRun->getArray("artifacts");225 226 // The tool is as expected227 ASSERT_TRUE(Driver->getString("name").has_value());228 ASSERT_TRUE(Driver->getString("fullName").has_value());229 230 EXPECT_EQ(*Driver->getString("name"), "sarif test");231 EXPECT_EQ(*Driver->getString("fullName"), "sarif test runner");232 233 // The results are as expected234 EXPECT_EQ(Results->size(), 1UL);235 236 // The artifacts are as expected237 EXPECT_TRUE(Artifacts->empty());238}239 240TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithDefaultRuleConfig) {241 // GIVEN:242 const std::string ExpectedOutput =243 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"warning","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";244 245 SarifDocumentWriter Writer{SourceMgr};246 const SarifRule &Rule =247 SarifRule::create()248 .setRuleId("clang.unittest")249 .setDescription("Example rule created during unit tests")250 .setName("clang unit test");251 252 // WHEN: A run contains a result253 Writer.createRun("sarif test", "sarif test runner", "1.0.0");254 unsigned RuleIdx = Writer.createRule(Rule);255 const SarifResult &Result = SarifResult::create(RuleIdx);256 Writer.appendResult(Result);257 std::string Output = serializeSarifDocument(Writer.createDocument());258 259 // THEN:260 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));261}262 263TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithCustomRuleConfig) {264 // GIVEN:265 const std::string ExpectedOutput =266 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"error","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"error","rank":35.5},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";267 268 SarifDocumentWriter Writer{SourceMgr};269 const SarifRule &Rule =270 SarifRule::create()271 .setRuleId("clang.unittest")272 .setDescription("Example rule created during unit tests")273 .setName("clang unit test")274 .setDefaultConfiguration(SarifReportingConfiguration::create()275 .setLevel(SarifResultLevel::Error)276 .setRank(35.5));277 278 // WHEN: A run contains a result279 Writer.createRun("sarif test", "sarif test runner", "1.0.0");280 unsigned RuleIdx = Writer.createRule(Rule);281 const SarifResult &Result = SarifResult::create(RuleIdx);282 Writer.appendResult(Result);283 std::string Output = serializeSarifDocument(Writer.createDocument());284 285 // THEN:286 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));287}288 289// Check that serializing artifacts from results produces valid SARIF290TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) {291 // GIVEN:292 const std::string ExpectedOutput =293 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";294 295 SarifDocumentWriter Writer{SourceMgr};296 const SarifRule &Rule =297 SarifRule::create()298 .setRuleId("clang.unittest")299 .setDescription("Example rule created during unit tests")300 .setName("clang unit test");301 302 // WHEN: A result is added with valid source locations for its diagnostics303 Writer.createRun("sarif test", "sarif test runner", "1.0.0");304 unsigned RuleIdx = Writer.createRule(Rule);305 306 llvm::SmallVector<CharSourceRange, 1> DiagLocs;307 const char *SourceText = "int foo = 0;\n"308 "int bar = 1;\n"309 "float x = 0.0\n";310 311 FileID MainFileID =312 registerSource("/main.cpp", SourceText, /* IsMainFile = */ true);313 CharSourceRange SourceCSR =314 getFakeCharSourceRange(MainFileID, {3, 14}, {3, 14});315 316 DiagLocs.push_back(SourceCSR);317 318 const SarifResult &Result =319 SarifResult::create(RuleIdx)320 .setLocations(DiagLocs)321 .setDiagnosticMessage("expected ';' after top level declarator")322 .setDiagnosticLevel(SarifResultLevel::Error);323 Writer.appendResult(Result);324 std::string Output = serializeSarifDocument(Writer.createDocument());325 326 // THEN: Assert that the serialized SARIF is as expected327 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));328}329 330TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) {331 // GIVEN:332 const std::string ExpectedOutput =333 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]},{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1,"uri":"file:///test-header-1.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2,"uri":"file:///test-header-2.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3,"uri":"file:///test-header-3.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";334 335 const char *SourceText = "int foo = 0;\n"336 "int foo = 1;\n"337 "float x = 0.0;\n";338 FileID MainFileID =339 registerSource("/main.cpp", SourceText, /* IsMainFile = */ true);340 CharSourceRange DiagLoc{getFakeCharSourceRange(MainFileID, {2, 5}, {2, 8})};341 342 SarifDocumentWriter Writer{SourceMgr};343 const SarifRule &Rule =344 SarifRule::create()345 .setRuleId("clang.unittest")346 .setDescription("Example rule created during unit tests")347 .setName("clang unit test");348 349 constexpr unsigned int NumCases = 3;350 llvm::SmallVector<ThreadFlow, NumCases> Threadflows;351 const char *HeaderTexts[NumCases]{("#pragma once\n"352 "#include <foo>"),353 ("#ifndef FOO\n"354 "#define FOO\n"355 "#endif"),356 ("#ifdef FOO\n"357 "#undef FOO\n"358 "#endif")};359 const char *HeaderNames[NumCases]{"/test-header-1.h", "/test-header-2.h",360 "/test-header-3.h"};361 ThreadFlowImportance Importances[NumCases]{ThreadFlowImportance::Essential,362 ThreadFlowImportance::Important,363 ThreadFlowImportance::Unimportant};364 for (size_t Idx = 0; Idx != NumCases; ++Idx) {365 FileID FID = registerSource(HeaderNames[Idx], HeaderTexts[Idx]);366 CharSourceRange &&CSR = getFakeCharSourceRange(FID, {1, 1}, {2, 8});367 std::string Message = llvm::formatv("Message #{0}", Idx + 1);368 ThreadFlow Item = ThreadFlow::create()369 .setRange(CSR)370 .setImportance(Importances[Idx])371 .setMessage(Message);372 Threadflows.push_back(Item);373 }374 375 // WHEN: A result containing code flows and diagnostic locations is added376 Writer.createRun("sarif test", "sarif test runner", "1.0.0");377 unsigned RuleIdx = Writer.createRule(Rule);378 const SarifResult &Result =379 SarifResult::create(RuleIdx)380 .setLocations({DiagLoc})381 .setDiagnosticMessage("Redefinition of 'foo'")382 .setThreadFlows(Threadflows)383 .setDiagnosticLevel(SarifResultLevel::Warning);384 Writer.appendResult(Result);385 std::string Output = serializeSarifDocument(Writer.createDocument());386 387 // THEN: Assert that the serialized SARIF is as expected388 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));389}390 391} // namespace392