1658 lines · cpp
1//===-- TUSchedulerTests.cpp ------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "Annotations.h"10#include "ClangdServer.h"11#include "Compiler.h"12#include "Config.h"13#include "Diagnostics.h"14#include "GlobalCompilationDatabase.h"15#include "Matchers.h"16#include "ParsedAST.h"17#include "Preamble.h"18#include "TUScheduler.h"19#include "TestFS.h"20#include "TestIndex.h"21#include "clang-include-cleaner/Record.h"22#include "support/Cancellation.h"23#include "support/Context.h"24#include "support/Path.h"25#include "support/TestTracer.h"26#include "support/Threading.h"27#include "clang/Basic/DiagnosticDriver.h"28#include "llvm/ADT/ArrayRef.h"29#include "llvm/ADT/FunctionExtras.h"30#include "llvm/ADT/ScopeExit.h"31#include "llvm/ADT/StringExtras.h"32#include "llvm/ADT/StringMap.h"33#include "llvm/ADT/StringRef.h"34#include "gmock/gmock.h"35#include "gtest/gtest.h"36#include <atomic>37#include <chrono>38#include <condition_variable>39#include <cstdint>40#include <functional>41#include <memory>42#include <mutex>43#include <optional>44#include <string>45#include <utility>46#include <vector>47 48namespace clang {49namespace clangd {50namespace {51 52using ::testing::AllOf;53using ::testing::AnyOf;54using ::testing::Contains;55using ::testing::Each;56using ::testing::ElementsAre;57using ::testing::Eq;58using ::testing::Field;59using ::testing::IsEmpty;60using ::testing::Not;61using ::testing::Pair;62using ::testing::Pointee;63using ::testing::SizeIs;64using ::testing::UnorderedElementsAre;65 66MATCHER_P2(TUState, PreambleActivity, ASTActivity, "") {67 if (arg.PreambleActivity != PreambleActivity) {68 *result_listener << "preamblestate is "69 << static_cast<uint8_t>(arg.PreambleActivity);70 return false;71 }72 if (arg.ASTActivity.K != ASTActivity) {73 *result_listener << "aststate is " << arg.ASTActivity.K;74 return false;75 }76 return true;77}78 79// Simple ContextProvider to verify the provider is invoked & contexts are used.80static Key<std::string> BoundPath;81Context bindPath(PathRef F) {82 return Context::current().derive(BoundPath, F.str());83}84llvm::StringRef boundPath() {85 const std::string *V = Context::current().get(BoundPath);86 return V ? *V : llvm::StringRef("");87}88 89TUScheduler::Options optsForTest() {90 TUScheduler::Options Opts(ClangdServer::optsForTest());91 Opts.ContextProvider = bindPath;92 return Opts;93}94 95class TUSchedulerTests : public ::testing::Test {96protected:97 ParseInputs getInputs(PathRef File, std::string Contents) {98 ParseInputs Inputs;99 Inputs.CompileCommand = *CDB.getCompileCommand(File);100 Inputs.TFS = &FS;101 Inputs.Contents = std::move(Contents);102 Inputs.Opts = ParseOptions();103 return Inputs;104 }105 106 void updateWithCallback(TUScheduler &S, PathRef File,107 llvm::StringRef Contents, WantDiagnostics WD,108 llvm::unique_function<void()> CB) {109 updateWithCallback(S, File, getInputs(File, std::string(Contents)), WD,110 std::move(CB));111 }112 113 void updateWithCallback(TUScheduler &S, PathRef File, ParseInputs Inputs,114 WantDiagnostics WD,115 llvm::unique_function<void()> CB) {116 WithContextValue Ctx(llvm::make_scope_exit(std::move(CB)));117 S.update(File, Inputs, WD);118 }119 120 static Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>121 DiagsCallbackKey;122 123 /// A diagnostics callback that should be passed to TUScheduler when it's used124 /// in updateWithDiags.125 static std::unique_ptr<ParsingCallbacks> captureDiags() {126 class CaptureDiags : public ParsingCallbacks {127 public:128 void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {129 reportDiagnostics(File, AST.getDiagnostics(), Publish);130 }131 132 void onFailedAST(PathRef File, llvm::StringRef Version,133 std::vector<Diag> Diags, PublishFn Publish) override {134 reportDiagnostics(File, Diags, Publish);135 }136 137 private:138 void reportDiagnostics(PathRef File, llvm::ArrayRef<Diag> Diags,139 PublishFn Publish) {140 auto *D = Context::current().get(DiagsCallbackKey);141 if (!D)142 return;143 Publish([&]() {144 const_cast<llvm::unique_function<void(PathRef, std::vector<Diag>)> &>(145 *D)(File, Diags);146 });147 }148 };149 return std::make_unique<CaptureDiags>();150 }151 152 /// Schedule an update and call \p CB with the diagnostics it produces, if153 /// any. The TUScheduler should be created with captureDiags as a154 /// DiagsCallback for this to work.155 void updateWithDiags(TUScheduler &S, PathRef File, ParseInputs Inputs,156 WantDiagnostics WD,157 llvm::unique_function<void(std::vector<Diag>)> CB) {158 Path OrigFile = File.str();159 WithContextValue Ctx(DiagsCallbackKey,160 [OrigFile, CB = std::move(CB)](161 PathRef File, std::vector<Diag> Diags) mutable {162 assert(File == OrigFile);163 CB(std::move(Diags));164 });165 S.update(File, std::move(Inputs), WD);166 }167 168 void updateWithDiags(TUScheduler &S, PathRef File, llvm::StringRef Contents,169 WantDiagnostics WD,170 llvm::unique_function<void(std::vector<Diag>)> CB) {171 return updateWithDiags(S, File, getInputs(File, std::string(Contents)), WD,172 std::move(CB));173 }174 175 MockFS FS;176 MockCompilationDatabase CDB;177};178 179Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>180 TUSchedulerTests::DiagsCallbackKey;181 182TEST_F(TUSchedulerTests, MissingFiles) {183 TUScheduler S(CDB, optsForTest());184 185 auto Added = testPath("added.cpp");186 FS.Files[Added] = "x";187 188 auto Missing = testPath("missing.cpp");189 FS.Files[Missing] = "";190 191 S.update(Added, getInputs(Added, "x"), WantDiagnostics::No);192 193 // Assert each operation for missing file is an error (even if it's194 // available in VFS).195 S.runWithAST("", Missing,196 [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });197 S.runWithPreamble(198 "", Missing, TUScheduler::Stale,199 [&](Expected<InputsAndPreamble> Preamble) { EXPECT_ERROR(Preamble); });200 // remove() shouldn't crash on missing files.201 S.remove(Missing);202 203 // Assert there aren't any errors for added file.204 S.runWithAST("", Added,205 [&](Expected<InputsAndAST> AST) { EXPECT_TRUE(bool(AST)); });206 S.runWithPreamble("", Added, TUScheduler::Stale,207 [&](Expected<InputsAndPreamble> Preamble) {208 EXPECT_TRUE(bool(Preamble));209 });210 S.remove(Added);211 212 // Assert that all operations fail after removing the file.213 S.runWithAST("", Added,214 [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });215 S.runWithPreamble("", Added, TUScheduler::Stale,216 [&](Expected<InputsAndPreamble> Preamble) {217 ASSERT_FALSE(bool(Preamble));218 llvm::consumeError(Preamble.takeError());219 });220 // remove() shouldn't crash on missing files.221 S.remove(Added);222}223 224TEST_F(TUSchedulerTests, WantDiagnostics) {225 std::atomic<int> CallbackCount(0);226 {227 // To avoid a racy test, don't allow tasks to actually run on the worker228 // thread until we've scheduled them all.229 Notification Ready;230 TUScheduler S(CDB, optsForTest(), captureDiags());231 auto Path = testPath("foo.cpp");232 // Semicolons here and in the following inputs are significant. They ensure233 // preamble stays the same across runs. Otherwise we might get multiple234 // diagnostics callbacks, once with the stale preamble and another with the235 // fresh preamble.236 updateWithDiags(S, Path, ";", WantDiagnostics::Yes,237 [&](std::vector<Diag>) { Ready.wait(); });238 updateWithDiags(S, Path, ";request diags", WantDiagnostics::Yes,239 [&](std::vector<Diag>) { ++CallbackCount; });240 updateWithDiags(S, Path, ";auto (clobbered)", WantDiagnostics::Auto,241 [&](std::vector<Diag>) {242 ADD_FAILURE()243 << "auto should have been cancelled by auto";244 });245 updateWithDiags(S, Path, ";request no diags", WantDiagnostics::No,246 [&](std::vector<Diag>) {247 ADD_FAILURE() << "no diags should not be called back";248 });249 updateWithDiags(S, Path, ";auto (produces)", WantDiagnostics::Auto,250 [&](std::vector<Diag>) { ++CallbackCount; });251 Ready.notify();252 253 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));254 }255 EXPECT_EQ(2, CallbackCount);256}257 258TEST_F(TUSchedulerTests, Debounce) {259 auto Opts = optsForTest();260 Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(500));261 TUScheduler S(CDB, Opts, captureDiags());262 auto Path = testPath("foo.cpp");263 // Issue a write that's going to be debounced away.264 updateWithDiags(S, Path, "auto (debounced)", WantDiagnostics::Auto,265 [&](std::vector<Diag>) {266 ADD_FAILURE()267 << "auto should have been debounced and canceled";268 });269 // Sleep a bit to verify that it's really debounce that's holding diagnostics.270 std::this_thread::sleep_for(std::chrono::milliseconds(50));271 272 // Issue another write, this time we'll wait for its diagnostics.273 Notification N;274 updateWithDiags(S, Path, "auto (timed out)", WantDiagnostics::Auto,275 [&](std::vector<Diag>) { N.notify(); });276 EXPECT_TRUE(N.wait(timeoutSeconds(60)));277 278 // Once we start shutting down the TUScheduler, this one becomes a dead write.279 updateWithDiags(S, Path, "auto (discarded)", WantDiagnostics::Auto,280 [&](std::vector<Diag>) {281 ADD_FAILURE()282 << "auto should have been discarded (dead write)";283 });284}285 286TEST_F(TUSchedulerTests, Cancellation) {287 // We have the following update/read sequence288 // U0289 // U1(WantDiags=Yes) <-- cancelled290 // R1 <-- cancelled291 // U2(WantDiags=Yes) <-- cancelled292 // R2A <-- cancelled293 // R2B294 // U3(WantDiags=Yes)295 // R3 <-- cancelled296 std::vector<StringRef> DiagsSeen, ReadsSeen, ReadsCanceled;297 {298 Notification Proceed; // Ensure we schedule everything.299 TUScheduler S(CDB, optsForTest(), captureDiags());300 auto Path = testPath("foo.cpp");301 // Helper to schedule a named update and return a function to cancel it.302 auto Update = [&](StringRef ID) -> Canceler {303 auto T = cancelableTask();304 WithContext C(std::move(T.first));305 updateWithDiags(306 S, Path, ("//" + ID).str(), WantDiagnostics::Yes,307 [&, ID](std::vector<Diag> Diags) { DiagsSeen.push_back(ID); });308 return std::move(T.second);309 };310 // Helper to schedule a named read and return a function to cancel it.311 auto Read = [&](StringRef ID) -> Canceler {312 auto T = cancelableTask();313 WithContext C(std::move(T.first));314 S.runWithAST(ID, Path, [&, ID](llvm::Expected<InputsAndAST> E) {315 if (auto Err = E.takeError()) {316 if (Err.isA<CancelledError>()) {317 ReadsCanceled.push_back(ID);318 consumeError(std::move(Err));319 } else {320 ADD_FAILURE() << "Non-cancelled error for " << ID << ": "321 << llvm::toString(std::move(Err));322 }323 } else {324 ReadsSeen.push_back(ID);325 }326 });327 return std::move(T.second);328 };329 330 updateWithCallback(S, Path, "", WantDiagnostics::Yes,331 [&]() { Proceed.wait(); });332 // The second parens indicate cancellation, where present.333 Update("U1")();334 Read("R1")();335 Update("U2")();336 Read("R2A")();337 Read("R2B");338 Update("U3");339 Read("R3")();340 Proceed.notify();341 342 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));343 }344 EXPECT_THAT(DiagsSeen, ElementsAre("U2", "U3"))345 << "U1 and all dependent reads were cancelled. "346 "U2 has a dependent read R2A. "347 "U3 was not cancelled.";348 EXPECT_THAT(ReadsSeen, ElementsAre("R2B"))349 << "All reads other than R2B were cancelled";350 EXPECT_THAT(ReadsCanceled, ElementsAre("R1", "R2A", "R3"))351 << "All reads other than R2B were cancelled";352}353 354TEST_F(TUSchedulerTests, InvalidationNoCrash) {355 auto Path = testPath("foo.cpp");356 TUScheduler S(CDB, optsForTest(), captureDiags());357 358 Notification StartedRunning;359 Notification ScheduledChange;360 // We expect invalidation logic to not crash by trying to invalidate a running361 // request.362 S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);363 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));364 S.runWithAST(365 "invalidatable-but-running", Path,366 [&](llvm::Expected<InputsAndAST> AST) {367 StartedRunning.notify();368 ScheduledChange.wait();369 ASSERT_TRUE(bool(AST));370 },371 TUScheduler::InvalidateOnUpdate);372 StartedRunning.wait();373 S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);374 ScheduledChange.notify();375 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));376}377 378TEST_F(TUSchedulerTests, Invalidation) {379 auto Path = testPath("foo.cpp");380 TUScheduler S(CDB, optsForTest(), captureDiags());381 std::atomic<int> Builds(0), Actions(0);382 383 Notification Start;384 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {385 ++Builds;386 Start.wait();387 });388 S.runWithAST(389 "invalidatable", Path,390 [&](llvm::Expected<InputsAndAST> AST) {391 ++Actions;392 EXPECT_FALSE(bool(AST));393 llvm::Error E = AST.takeError();394 EXPECT_TRUE(E.isA<CancelledError>());395 handleAllErrors(std::move(E), [&](const CancelledError &E) {396 EXPECT_EQ(E.Reason, static_cast<int>(ErrorCode::ContentModified));397 });398 },399 TUScheduler::InvalidateOnUpdate);400 S.runWithAST(401 "not-invalidatable", Path,402 [&](llvm::Expected<InputsAndAST> AST) {403 ++Actions;404 EXPECT_TRUE(bool(AST));405 },406 TUScheduler::NoInvalidation);407 updateWithDiags(S, Path, "b", WantDiagnostics::Auto, [&](std::vector<Diag>) {408 ++Builds;409 ADD_FAILURE() << "Shouldn't build, all dependents invalidated";410 });411 S.runWithAST(412 "invalidatable", Path,413 [&](llvm::Expected<InputsAndAST> AST) {414 ++Actions;415 EXPECT_FALSE(bool(AST));416 llvm::Error E = AST.takeError();417 EXPECT_TRUE(E.isA<CancelledError>());418 consumeError(std::move(E));419 },420 TUScheduler::InvalidateOnUpdate);421 updateWithDiags(S, Path, "c", WantDiagnostics::Auto,422 [&](std::vector<Diag>) { ++Builds; });423 S.runWithAST(424 "invalidatable", Path,425 [&](llvm::Expected<InputsAndAST> AST) {426 ++Actions;427 EXPECT_TRUE(bool(AST)) << "Shouldn't be invalidated, no update follows";428 },429 TUScheduler::InvalidateOnUpdate);430 Start.notify();431 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));432 433 EXPECT_EQ(2, Builds.load()) << "Middle build should be skipped";434 EXPECT_EQ(4, Actions.load()) << "All actions should run (some with error)";435}436 437// We don't invalidate requests for updates that don't change the file content.438// These are mostly "refresh this file" events synthesized inside clangd itself.439// (Usually the AST rebuild is elided after verifying that all inputs are440// unchanged, but invalidation decisions happen earlier and so independently).441// See https://github.com/clangd/clangd/issues/620442TEST_F(TUSchedulerTests, InvalidationUnchanged) {443 auto Path = testPath("foo.cpp");444 TUScheduler S(CDB, optsForTest(), captureDiags());445 std::atomic<int> Actions(0);446 447 Notification Start;448 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {449 Start.wait();450 });451 S.runWithAST(452 "invalidatable", Path,453 [&](llvm::Expected<InputsAndAST> AST) {454 ++Actions;455 EXPECT_TRUE(bool(AST))456 << "Should not invalidate based on an update with same content: "457 << llvm::toString(AST.takeError());458 },459 TUScheduler::InvalidateOnUpdate);460 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {461 ADD_FAILURE() << "Shouldn't build, identical to previous";462 });463 Start.notify();464 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));465 466 EXPECT_EQ(1, Actions.load()) << "All actions should run";467}468 469TEST_F(TUSchedulerTests, ManyUpdates) {470 const int FilesCount = 3;471 const int UpdatesPerFile = 10;472 473 std::mutex Mut;474 int TotalASTReads = 0;475 int TotalPreambleReads = 0;476 int TotalUpdates = 0;477 llvm::StringMap<int> LatestDiagVersion;478 479 // Run TUScheduler and collect some stats.480 {481 auto Opts = optsForTest();482 Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(50));483 TUScheduler S(CDB, Opts, captureDiags());484 485 std::vector<std::string> Files;486 for (int I = 0; I < FilesCount; ++I) {487 std::string Name = "foo" + std::to_string(I) + ".cpp";488 Files.push_back(testPath(Name));489 this->FS.Files[Files.back()] = "";490 }491 492 StringRef Contents1 = R"cpp(int a;)cpp";493 StringRef Contents2 = R"cpp(int main() { return 1; })cpp";494 StringRef Contents3 = R"cpp(int a; int b; int sum() { return a + b; })cpp";495 496 StringRef AllContents[] = {Contents1, Contents2, Contents3};497 const int AllContentsSize = 3;498 499 // Scheduler may run tasks asynchronously, but should propagate the500 // context. We stash a nonce in the context, and verify it in the task.501 static Key<int> NonceKey;502 int Nonce = 0;503 504 for (int FileI = 0; FileI < FilesCount; ++FileI) {505 for (int UpdateI = 0; UpdateI < UpdatesPerFile; ++UpdateI) {506 auto Contents = AllContents[(FileI + UpdateI) % AllContentsSize];507 508 auto File = Files[FileI];509 auto Inputs = getInputs(File, Contents.str());510 {511 WithContextValue WithNonce(NonceKey, ++Nonce);512 Inputs.Version = std::to_string(UpdateI);513 updateWithDiags(514 S, File, Inputs, WantDiagnostics::Auto,515 [File, Nonce, Version(Inputs.Version), &Mut, &TotalUpdates,516 &LatestDiagVersion](std::vector<Diag>) {517 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));518 EXPECT_EQ(File, boundPath());519 520 std::lock_guard<std::mutex> Lock(Mut);521 ++TotalUpdates;522 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());523 // Make sure Diags are for a newer version.524 auto It = LatestDiagVersion.try_emplace(File, -1);525 const int PrevVersion = It.first->second;526 int CurVersion;527 ASSERT_TRUE(llvm::to_integer(Version, CurVersion, 10));528 EXPECT_LT(PrevVersion, CurVersion);529 It.first->getValue() = CurVersion;530 });531 }532 {533 WithContextValue WithNonce(NonceKey, ++Nonce);534 S.runWithAST(535 "CheckAST", File,536 [File, Inputs, Nonce, &Mut,537 &TotalASTReads](Expected<InputsAndAST> AST) {538 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));539 EXPECT_EQ(File, boundPath());540 541 ASSERT_TRUE((bool)AST);542 EXPECT_EQ(AST->Inputs.Contents, Inputs.Contents);543 EXPECT_EQ(AST->Inputs.Version, Inputs.Version);544 EXPECT_EQ(AST->AST.version(), Inputs.Version);545 546 std::lock_guard<std::mutex> Lock(Mut);547 ++TotalASTReads;548 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());549 });550 }551 552 {553 WithContextValue WithNonce(NonceKey, ++Nonce);554 S.runWithPreamble(555 "CheckPreamble", File, TUScheduler::Stale,556 [File, Inputs, Nonce, &Mut,557 &TotalPreambleReads](Expected<InputsAndPreamble> Preamble) {558 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));559 EXPECT_EQ(File, boundPath());560 561 ASSERT_TRUE((bool)Preamble);562 EXPECT_EQ(Preamble->Contents, Inputs.Contents);563 564 std::lock_guard<std::mutex> Lock(Mut);565 ++TotalPreambleReads;566 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());567 });568 }569 }570 }571 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));572 } // TUScheduler destructor waits for all operations to finish.573 574 std::lock_guard<std::mutex> Lock(Mut);575 // Updates might get coalesced in preamble thread and result in dropping576 // diagnostics for intermediate snapshots.577 EXPECT_GE(TotalUpdates, FilesCount);578 EXPECT_LE(TotalUpdates, FilesCount * UpdatesPerFile);579 // We should receive diags for last update.580 for (const auto &Entry : LatestDiagVersion)581 EXPECT_EQ(Entry.second, UpdatesPerFile - 1);582 EXPECT_EQ(TotalASTReads, FilesCount * UpdatesPerFile);583 EXPECT_EQ(TotalPreambleReads, FilesCount * UpdatesPerFile);584}585 586TEST_F(TUSchedulerTests, EvictedAST) {587 std::atomic<int> BuiltASTCounter(0);588 auto Opts = optsForTest();589 Opts.AsyncThreadsCount = 1;590 Opts.RetentionPolicy.MaxRetainedASTs = 2;591 trace::TestTracer Tracer;592 TUScheduler S(CDB, Opts);593 594 llvm::StringLiteral SourceContents = R"cpp(595 int* a;596 double* b = a;597 )cpp";598 llvm::StringLiteral OtherSourceContents = R"cpp(599 int* a;600 double* b = a + 0;601 )cpp";602 603 auto Foo = testPath("foo.cpp");604 auto Bar = testPath("bar.cpp");605 auto Baz = testPath("baz.cpp");606 607 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));608 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));609 // Build one file in advance. We will not access it later, so it will be the610 // one that the cache will evict.611 updateWithCallback(S, Foo, SourceContents, WantDiagnostics::Yes,612 [&BuiltASTCounter]() { ++BuiltASTCounter; });613 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));614 ASSERT_EQ(BuiltASTCounter.load(), 1);615 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));616 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));617 618 // Build two more files. Since we can retain only 2 ASTs, these should be619 // the ones we see in the cache later.620 updateWithCallback(S, Bar, SourceContents, WantDiagnostics::Yes,621 [&BuiltASTCounter]() { ++BuiltASTCounter; });622 updateWithCallback(S, Baz, SourceContents, WantDiagnostics::Yes,623 [&BuiltASTCounter]() { ++BuiltASTCounter; });624 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));625 ASSERT_EQ(BuiltASTCounter.load(), 3);626 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));627 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(2));628 629 // Check only the last two ASTs are retained.630 ASSERT_THAT(S.getFilesWithCachedAST(), UnorderedElementsAre(Bar, Baz));631 632 // Access the old file again.633 updateWithCallback(S, Foo, OtherSourceContents, WantDiagnostics::Yes,634 [&BuiltASTCounter]() { ++BuiltASTCounter; });635 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));636 ASSERT_EQ(BuiltASTCounter.load(), 4);637 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));638 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));639 640 // Check the AST for foo.cpp is retained now and one of the others got641 // evicted.642 EXPECT_THAT(S.getFilesWithCachedAST(),643 UnorderedElementsAre(Foo, AnyOf(Bar, Baz)));644}645 646// We send "empty" changes to TUScheduler when we think some external event647// *might* have invalidated current state (e.g. a header was edited).648// Verify that this doesn't evict our cache entries.649TEST_F(TUSchedulerTests, NoopChangesDontThrashCache) {650 auto Opts = optsForTest();651 Opts.RetentionPolicy.MaxRetainedASTs = 1;652 TUScheduler S(CDB, Opts);653 654 auto Foo = testPath("foo.cpp");655 auto FooInputs = getInputs(Foo, "int x=1;");656 auto Bar = testPath("bar.cpp");657 auto BarInputs = getInputs(Bar, "int x=2;");658 659 // After opening Foo then Bar, AST cache contains Bar.660 S.update(Foo, FooInputs, WantDiagnostics::Auto);661 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));662 S.update(Bar, BarInputs, WantDiagnostics::Auto);663 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));664 ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));665 666 // Any number of no-op updates to Foo don't dislodge Bar from the cache.667 S.update(Foo, FooInputs, WantDiagnostics::Auto);668 S.update(Foo, FooInputs, WantDiagnostics::Auto);669 S.update(Foo, FooInputs, WantDiagnostics::Auto);670 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));671 ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));672 // In fact each file has been built only once.673 ASSERT_EQ(S.fileStats().lookup(Foo).ASTBuilds, 1u);674 ASSERT_EQ(S.fileStats().lookup(Bar).ASTBuilds, 1u);675}676 677TEST_F(TUSchedulerTests, EmptyPreamble) {678 TUScheduler S(CDB, optsForTest());679 680 auto Foo = testPath("foo.cpp");681 auto Header = testPath("foo.h");682 683 FS.Files[Header] = "void foo()";684 FS.Timestamps[Header] = time_t(0);685 auto *WithPreamble = R"cpp(686 #include "foo.h"687 int main() {}688 )cpp";689 auto *WithEmptyPreamble = R"cpp(int main() {})cpp";690 S.update(Foo, getInputs(Foo, WithPreamble), WantDiagnostics::Auto);691 S.runWithPreamble(692 "getNonEmptyPreamble", Foo, TUScheduler::Stale,693 [&](Expected<InputsAndPreamble> Preamble) {694 // We expect to get a non-empty preamble.695 EXPECT_GT(696 cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,697 0u);698 });699 // Wait while the preamble is being built.700 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));701 702 // Update the file which results in an empty preamble.703 S.update(Foo, getInputs(Foo, WithEmptyPreamble), WantDiagnostics::Auto);704 // Wait while the preamble is being built.705 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));706 S.runWithPreamble(707 "getEmptyPreamble", Foo, TUScheduler::Stale,708 [&](Expected<InputsAndPreamble> Preamble) {709 // We expect to get an empty preamble.710 EXPECT_EQ(711 cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,712 0u);713 });714}715 716TEST_F(TUSchedulerTests, ASTSignalsSmokeTests) {717 TUScheduler S(CDB, optsForTest());718 auto Foo = testPath("foo.cpp");719 auto Header = testPath("foo.h");720 721 FS.Files[Header] = "namespace tar { int foo(); }";722 const char *Contents = R"cpp(723 #include "foo.h"724 namespace ns {725 int func() {726 return tar::foo());727 }728 } // namespace ns729 )cpp";730 // Update the file which results in an empty preamble.731 S.update(Foo, getInputs(Foo, Contents), WantDiagnostics::Yes);732 // Wait while the preamble is being built.733 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));734 Notification TaskRun;735 S.runWithPreamble(736 "ASTSignals", Foo, TUScheduler::Stale,737 [&](Expected<InputsAndPreamble> IP) {738 ASSERT_FALSE(!IP);739 std::vector<std::pair<StringRef, int>> NS;740 for (const auto &P : IP->Signals->RelatedNamespaces)741 NS.emplace_back(P.getKey(), P.getValue());742 EXPECT_THAT(NS,743 UnorderedElementsAre(Pair("ns::", 1), Pair("tar::", 1)));744 745 std::vector<std::pair<SymbolID, int>> Sym;746 for (const auto &P : IP->Signals->ReferencedSymbols)747 Sym.emplace_back(P.getFirst(), P.getSecond());748 EXPECT_THAT(Sym, UnorderedElementsAre(Pair(ns("tar").ID, 1),749 Pair(ns("ns").ID, 1),750 Pair(func("tar::foo").ID, 1),751 Pair(func("ns::func").ID, 1)));752 TaskRun.notify();753 });754 TaskRun.wait();755}756 757TEST_F(TUSchedulerTests, RunWaitsForPreamble) {758 // Testing strategy: we update the file and schedule a few preamble reads at759 // the same time. All reads should get the same non-null preamble.760 TUScheduler S(CDB, optsForTest());761 auto Foo = testPath("foo.cpp");762 auto *NonEmptyPreamble = R"cpp(763 #define FOO 1764 #define BAR 2765 766 int main() {}767 )cpp";768 constexpr int ReadsToSchedule = 10;769 std::mutex PreamblesMut;770 std::vector<const void *> Preambles(ReadsToSchedule, nullptr);771 S.update(Foo, getInputs(Foo, NonEmptyPreamble), WantDiagnostics::Auto);772 for (int I = 0; I < ReadsToSchedule; ++I) {773 S.runWithPreamble(774 "test", Foo, TUScheduler::Stale,775 [I, &PreamblesMut, &Preambles](Expected<InputsAndPreamble> IP) {776 std::lock_guard<std::mutex> Lock(PreamblesMut);777 Preambles[I] = cantFail(std::move(IP)).Preamble;778 });779 }780 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));781 // Check all actions got the same non-null preamble.782 std::lock_guard<std::mutex> Lock(PreamblesMut);783 ASSERT_NE(Preambles[0], nullptr);784 ASSERT_THAT(Preambles, Each(Preambles[0]));785}786 787TEST_F(TUSchedulerTests, NoopOnEmptyChanges) {788 TUScheduler S(CDB, optsForTest(), captureDiags());789 790 auto Source = testPath("foo.cpp");791 auto Header = testPath("foo.h");792 793 FS.Files[Header] = "int a;";794 FS.Timestamps[Header] = time_t(0);795 796 std::string SourceContents = R"cpp(797 #include "foo.h"798 int b = a;799 )cpp";800 801 // Return value indicates if the updated callback was received.802 auto DoUpdate = [&](std::string Contents) -> bool {803 std::atomic<bool> Updated(false);804 Updated = false;805 updateWithDiags(S, Source, Contents, WantDiagnostics::Yes,806 [&Updated](std::vector<Diag>) { Updated = true; });807 bool UpdateFinished = S.blockUntilIdle(timeoutSeconds(60));808 if (!UpdateFinished)809 ADD_FAILURE() << "Updated has not finished in one second. Threading bug?";810 return Updated;811 };812 813 // Test that subsequent updates with the same inputs do not cause rebuilds.814 ASSERT_TRUE(DoUpdate(SourceContents));815 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);816 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);817 ASSERT_FALSE(DoUpdate(SourceContents));818 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);819 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);820 821 // Update to a header should cause a rebuild, though.822 FS.Timestamps[Header] = time_t(1);823 ASSERT_TRUE(DoUpdate(SourceContents));824 ASSERT_FALSE(DoUpdate(SourceContents));825 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 2u);826 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);827 828 // Update to the contents should cause a rebuild.829 SourceContents += "\nint c = b;";830 ASSERT_TRUE(DoUpdate(SourceContents));831 ASSERT_FALSE(DoUpdate(SourceContents));832 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 3u);833 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);834 835 // Update to the compile commands should also cause a rebuild.836 CDB.ExtraClangFlags.push_back("-DSOMETHING");837 ASSERT_TRUE(DoUpdate(SourceContents));838 ASSERT_FALSE(DoUpdate(SourceContents));839 // This causes 2 AST builds always. We first build an AST with the stale840 // preamble, and build a second AST once the fresh preamble is ready.841 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 5u);842 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 3u);843}844 845// We rebuild if a completely missing header exists, but not if one is added846// on a higher-priority include path entry (for performance).847// (Previously we wouldn't automatically rebuild when files were added).848TEST_F(TUSchedulerTests, MissingHeader) {849 CDB.ExtraClangFlags.push_back("-I" + testPath("a"));850 CDB.ExtraClangFlags.push_back("-I" + testPath("b"));851 // Force both directories to exist so they don't get pruned.852 FS.Files.try_emplace("a/__unused__");853 FS.Files.try_emplace("b/__unused__");854 TUScheduler S(CDB, optsForTest(), captureDiags());855 856 auto Source = testPath("foo.cpp");857 auto HeaderA = testPath("a/foo.h");858 auto HeaderB = testPath("b/foo.h");859 860 auto *SourceContents = R"cpp(861 #include "foo.h"862 int c = b;863 )cpp";864 865 ParseInputs Inputs = getInputs(Source, SourceContents);866 std::atomic<size_t> DiagCount(0);867 868 // Update the source contents, which should trigger an initial build with869 // the header file missing.870 updateWithDiags(871 S, Source, Inputs, WantDiagnostics::Yes,872 [&DiagCount](std::vector<Diag> Diags) {873 ++DiagCount;874 EXPECT_THAT(Diags,875 ElementsAre(Field(&Diag::Message, "'foo.h' file not found"),876 Field(&Diag::Message,877 "use of undeclared identifier 'b'")));878 });879 S.blockUntilIdle(timeoutSeconds(60));880 881 FS.Files[HeaderB] = "int b;";882 FS.Timestamps[HeaderB] = time_t(1);883 884 // The addition of the missing header file triggers a rebuild, no errors.885 updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,886 [&DiagCount](std::vector<Diag> Diags) {887 ++DiagCount;888 EXPECT_THAT(Diags, IsEmpty());889 });890 891 // Ensure previous assertions are done before we touch the FS again.892 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));893 // Add the high-priority header file, which should reintroduce the error.894 FS.Files[HeaderA] = "int a;";895 FS.Timestamps[HeaderA] = time_t(1);896 897 // This isn't detected: we don't stat a/foo.h to validate the preamble.898 updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,899 [&DiagCount](std::vector<Diag> Diags) {900 ++DiagCount;901 ADD_FAILURE()902 << "Didn't expect new diagnostics when adding a/foo.h";903 });904 905 // Forcing the reload should cause a rebuild.906 Inputs.ForceRebuild = true;907 updateWithDiags(908 S, Source, Inputs, WantDiagnostics::Yes,909 [&DiagCount](std::vector<Diag> Diags) {910 ++DiagCount;911 ElementsAre(Field(&Diag::Message, "use of undeclared identifier 'b'"));912 });913 914 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));915 EXPECT_EQ(DiagCount, 3U);916}917 918TEST_F(TUSchedulerTests, NoChangeDiags) {919 trace::TestTracer Tracer;920 TUScheduler S(CDB, optsForTest(), captureDiags());921 922 auto FooCpp = testPath("foo.cpp");923 const auto *Contents = "int a; int b;";924 925 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));926 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(0));927 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));928 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));929 updateWithDiags(930 S, FooCpp, Contents, WantDiagnostics::No,931 [](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });932 S.runWithAST("touchAST", FooCpp, [](Expected<InputsAndAST> IA) {933 // Make sure the AST was actually built.934 cantFail(std::move(IA));935 });936 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));937 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));938 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(1));939 940 // Even though the inputs didn't change and AST can be reused, we need to941 // report the diagnostics, as they were not reported previously.942 std::atomic<bool> SeenDiags(false);943 updateWithDiags(S, FooCpp, Contents, WantDiagnostics::Auto,944 [&](std::vector<Diag>) { SeenDiags = true; });945 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));946 ASSERT_TRUE(SeenDiags);947 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(1));948 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));949 950 // Subsequent request does not get any diagnostics callback because the same951 // diags have previously been reported and the inputs didn't change.952 updateWithDiags(953 S, FooCpp, Contents, WantDiagnostics::Auto,954 [&](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });955 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));956}957 958TEST_F(TUSchedulerTests, Run) {959 for (bool Sync : {false, true}) {960 auto Opts = optsForTest();961 if (Sync)962 Opts.AsyncThreadsCount = 0;963 TUScheduler S(CDB, Opts);964 std::atomic<int> Counter(0);965 S.run("add 1", /*Path=*/"", [&] { ++Counter; });966 S.run("add 2", /*Path=*/"", [&] { Counter += 2; });967 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));968 EXPECT_EQ(Counter.load(), 3);969 970 Notification TaskRun;971 Key<int> TestKey;972 WithContextValue CtxWithKey(TestKey, 10);973 const char *Path = "somepath";974 S.run("props context", Path, [&] {975 EXPECT_EQ(Context::current().getExisting(TestKey), 10);976 EXPECT_EQ(Path, boundPath());977 TaskRun.notify();978 });979 TaskRun.wait();980 }981}982 983TEST_F(TUSchedulerTests, TUStatus) {984 class CaptureTUStatus : public ClangdServer::Callbacks {985 public:986 void onFileUpdated(PathRef File, const TUStatus &Status) override {987 auto ASTAction = Status.ASTActivity.K;988 auto PreambleAction = Status.PreambleActivity;989 std::lock_guard<std::mutex> Lock(Mutex);990 // Only push the action if it has changed. Since TUStatus can be published991 // from either Preamble or AST thread and when one changes the other stays992 // the same.993 // Note that this can result in missing some updates when something other994 // than action kind changes, e.g. when AST is built/reused the action kind995 // stays as Building.996 if (ASTActions.empty() || ASTActions.back() != ASTAction)997 ASTActions.push_back(ASTAction);998 if (PreambleActions.empty() || PreambleActions.back() != PreambleAction)999 PreambleActions.push_back(PreambleAction);1000 }1001 1002 std::vector<PreambleAction> preambleStatuses() {1003 std::lock_guard<std::mutex> Lock(Mutex);1004 return PreambleActions;1005 }1006 1007 std::vector<ASTAction::Kind> astStatuses() {1008 std::lock_guard<std::mutex> Lock(Mutex);1009 return ASTActions;1010 }1011 1012 private:1013 std::mutex Mutex;1014 std::vector<ASTAction::Kind> ASTActions;1015 std::vector<PreambleAction> PreambleActions;1016 } CaptureTUStatus;1017 MockFS FS;1018 MockCompilationDatabase CDB;1019 ClangdServer Server(CDB, FS, ClangdServer::optsForTest(), &CaptureTUStatus);1020 Annotations Code("int m^ain () {}");1021 1022 // We schedule the following tasks in the queue:1023 // [Update] [GoToDefinition]1024 Server.addDocument(testPath("foo.cpp"), Code.code(), "1",1025 WantDiagnostics::Auto);1026 ASSERT_TRUE(Server.blockUntilIdleForTest());1027 Server.locateSymbolAt(testPath("foo.cpp"), Code.point(),1028 [](Expected<std::vector<LocatedSymbol>> Result) {1029 ASSERT_TRUE((bool)Result);1030 });1031 ASSERT_TRUE(Server.blockUntilIdleForTest());1032 1033 EXPECT_THAT(CaptureTUStatus.preambleStatuses(),1034 ElementsAre(1035 // PreambleThread starts idle, as the update is first handled1036 // by ASTWorker.1037 PreambleAction::Idle,1038 // Then it starts building first preamble and releases that to1039 // ASTWorker.1040 PreambleAction::Building,1041 // Then goes idle and stays that way as we don't receive any1042 // more update requests.1043 PreambleAction::Idle));1044 EXPECT_THAT(CaptureTUStatus.astStatuses(),1045 ElementsAre(1046 // Starts handling the update action and blocks until the1047 // first preamble is built.1048 ASTAction::RunningAction,1049 // Afterwards it builds an AST for that preamble to publish1050 // diagnostics.1051 ASTAction::Building,1052 // Then goes idle.1053 ASTAction::Idle,1054 // Afterwards we start executing go-to-def.1055 ASTAction::RunningAction,1056 // Then go idle.1057 ASTAction::Idle));1058}1059 1060TEST_F(TUSchedulerTests, CommandLineErrors) {1061 // We should see errors from command-line parsing inside the main file.1062 CDB.ExtraClangFlags = {"-fsome-unknown-flag"};1063 1064 // (!) 'Ready' must live longer than TUScheduler.1065 Notification Ready;1066 1067 TUScheduler S(CDB, optsForTest(), captureDiags());1068 std::vector<Diag> Diagnostics;1069 updateWithDiags(S, testPath("foo.cpp"), "void test() {}",1070 WantDiagnostics::Yes, [&](std::vector<Diag> D) {1071 Diagnostics = std::move(D);1072 Ready.notify();1073 });1074 Ready.wait();1075 1076 EXPECT_THAT(1077 Diagnostics,1078 ElementsAre(AllOf(1079 Field(&Diag::ID, Eq(diag::err_drv_unknown_argument)),1080 Field(&Diag::Name, Eq("drv_unknown_argument")),1081 Field(&Diag::Message, "unknown argument: '-fsome-unknown-flag'"))));1082}1083 1084TEST_F(TUSchedulerTests, CommandLineWarnings) {1085 // We should not see warnings from command-line parsing.1086 CDB.ExtraClangFlags = {"-Wsome-unknown-warning"};1087 1088 // (!) 'Ready' must live longer than TUScheduler.1089 Notification Ready;1090 1091 TUScheduler S(CDB, optsForTest(), captureDiags());1092 std::vector<Diag> Diagnostics;1093 updateWithDiags(S, testPath("foo.cpp"), "void test() {}",1094 WantDiagnostics::Yes, [&](std::vector<Diag> D) {1095 Diagnostics = std::move(D);1096 Ready.notify();1097 });1098 Ready.wait();1099 1100 EXPECT_THAT(Diagnostics, IsEmpty());1101}1102 1103TEST(DebouncePolicy, Compute) {1104 namespace c = std::chrono;1105 DebouncePolicy::clock::duration History[] = {1106 c::seconds(0),1107 c::seconds(5),1108 c::seconds(10),1109 c::seconds(20),1110 };1111 DebouncePolicy Policy;1112 Policy.Min = c::seconds(3);1113 Policy.Max = c::seconds(25);1114 // Call Policy.compute(History) and return seconds as a float.1115 auto Compute = [&](llvm::ArrayRef<DebouncePolicy::clock::duration> History) {1116 return c::duration_cast<c::duration<float, c::seconds::period>>(1117 Policy.compute(History))1118 .count();1119 };1120 EXPECT_NEAR(10, Compute(History), 0.01) << "(upper) median = 10";1121 Policy.RebuildRatio = 1.5;1122 EXPECT_NEAR(15, Compute(History), 0.01) << "median = 10, ratio = 1.5";1123 Policy.RebuildRatio = 3;1124 EXPECT_NEAR(25, Compute(History), 0.01) << "constrained by max";1125 Policy.RebuildRatio = 0;1126 EXPECT_NEAR(3, Compute(History), 0.01) << "constrained by min";1127 EXPECT_NEAR(25, Compute({}), 0.01) << "no history -> max";1128}1129 1130TEST_F(TUSchedulerTests, AsyncPreambleThread) {1131 // Blocks preamble thread while building preamble with \p BlockVersion until1132 // \p N is notified.1133 class BlockPreambleThread : public ParsingCallbacks {1134 public:1135 BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N)1136 : BlockVersion(BlockVersion), N(N) {}1137 void onPreambleAST(1138 PathRef Path, llvm::StringRef Version, CapturedASTCtx,1139 std::shared_ptr<const include_cleaner::PragmaIncludes>) override {1140 if (Version == BlockVersion)1141 N.wait();1142 }1143 1144 private:1145 llvm::StringRef BlockVersion;1146 Notification &N;1147 };1148 1149 static constexpr llvm::StringLiteral InputsV0 = "v0";1150 static constexpr llvm::StringLiteral InputsV1 = "v1";1151 Notification Ready;1152 TUScheduler S(CDB, optsForTest(),1153 std::make_unique<BlockPreambleThread>(InputsV1, Ready));1154 1155 Path File = testPath("foo.cpp");1156 auto PI = getInputs(File, "");1157 PI.Version = InputsV0.str();1158 S.update(File, PI, WantDiagnostics::Auto);1159 S.blockUntilIdle(timeoutSeconds(60));1160 1161 // Block preamble builds.1162 PI.Version = InputsV1.str();1163 // Issue second update which will block preamble thread.1164 S.update(File, PI, WantDiagnostics::Auto);1165 1166 Notification RunASTAction;1167 // Issue an AST read, which shouldn't be blocked and see latest version of the1168 // file.1169 S.runWithAST("test", File, [&](Expected<InputsAndAST> AST) {1170 ASSERT_TRUE(bool(AST));1171 // Make sure preamble is built with stale inputs, but AST was built using1172 // new ones.1173 EXPECT_THAT(AST->AST.preambleVersion(), InputsV0);1174 EXPECT_THAT(AST->Inputs.Version, InputsV1.str());1175 RunASTAction.notify();1176 });1177 RunASTAction.wait();1178 Ready.notify();1179}1180 1181TEST_F(TUSchedulerTests, OnlyPublishWhenPreambleIsBuilt) {1182 struct PreamblePublishCounter : public ParsingCallbacks {1183 PreamblePublishCounter(int &PreamblePublishCount)1184 : PreamblePublishCount(PreamblePublishCount) {}1185 void onPreamblePublished(PathRef File) override { ++PreamblePublishCount; }1186 int &PreamblePublishCount;1187 };1188 1189 int PreamblePublishCount = 0;1190 TUScheduler S(CDB, optsForTest(),1191 std::make_unique<PreamblePublishCounter>(PreamblePublishCount));1192 1193 Path File = testPath("foo.cpp");1194 S.update(File, getInputs(File, ""), WantDiagnostics::Auto);1195 S.blockUntilIdle(timeoutSeconds(60));1196 EXPECT_EQ(PreamblePublishCount, 1);1197 // Same contents, no publish.1198 S.update(File, getInputs(File, ""), WantDiagnostics::Auto);1199 S.blockUntilIdle(timeoutSeconds(60));1200 EXPECT_EQ(PreamblePublishCount, 1);1201 // New contents, should publish.1202 S.update(File, getInputs(File, "#define FOO"), WantDiagnostics::Auto);1203 S.blockUntilIdle(timeoutSeconds(60));1204 EXPECT_EQ(PreamblePublishCount, 2);1205}1206 1207TEST_F(TUSchedulerTests, PublishWithStalePreamble) {1208 // Callbacks that blocks the preamble thread after the first preamble is1209 // built and stores preamble/main-file versions for diagnostics released.1210 class BlockPreambleThread : public ParsingCallbacks {1211 public:1212 using DiagsCB = std::function<void(ParsedAST &)>;1213 BlockPreambleThread(Notification &UnblockPreamble, DiagsCB CB)1214 : UnblockPreamble(UnblockPreamble), CB(std::move(CB)) {}1215 1216 void onPreambleAST(1217 PathRef Path, llvm::StringRef Version, CapturedASTCtx,1218 std::shared_ptr<const include_cleaner::PragmaIncludes>) override {1219 if (BuildBefore)1220 ASSERT_TRUE(UnblockPreamble.wait(timeoutSeconds(60)))1221 << "Expected notification";1222 BuildBefore = true;1223 }1224 1225 void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {1226 CB(AST);1227 }1228 1229 void onFailedAST(PathRef File, llvm::StringRef Version,1230 std::vector<Diag> Diags, PublishFn Publish) override {1231 ADD_FAILURE() << "Received failed ast for: " << File << " with version "1232 << Version << '\n';1233 }1234 1235 private:1236 bool BuildBefore = false;1237 Notification &UnblockPreamble;1238 std::function<void(ParsedAST &)> CB;1239 };1240 1241 // Helpers for issuing blocking update requests on a TUScheduler, whose1242 // onMainAST callback would call onDiagnostics.1243 class DiagCollector {1244 public:1245 void onDiagnostics(ParsedAST &AST) {1246 std::scoped_lock<std::mutex> Lock(DiagMu);1247 DiagVersions.emplace_back(1248 std::make_pair(AST.preambleVersion()->str(), AST.version().str()));1249 DiagsReceived.notify_all();1250 }1251 1252 std::pair<std::string, std::string>1253 waitForNewDiags(TUScheduler &S, PathRef File, ParseInputs PI) {1254 std::unique_lock<std::mutex> Lock(DiagMu);1255 // Perform the update under the lock to make sure it isn't handled until1256 // we're waiting for it.1257 S.update(File, std::move(PI), WantDiagnostics::Auto);1258 size_t OldSize = DiagVersions.size();1259 bool ReceivedDiags = DiagsReceived.wait_for(1260 Lock, std::chrono::seconds(5),1261 [this, OldSize] { return OldSize + 1 == DiagVersions.size(); });1262 if (!ReceivedDiags) {1263 ADD_FAILURE() << "Timed out waiting for diags";1264 return {"invalid", "version"};1265 }1266 return DiagVersions.back();1267 }1268 1269 std::vector<std::pair<std::string, std::string>> diagVersions() {1270 std::scoped_lock<std::mutex> Lock(DiagMu);1271 return DiagVersions;1272 }1273 1274 private:1275 std::condition_variable DiagsReceived;1276 std::mutex DiagMu;1277 std::vector<std::pair</*PreambleVersion*/ std::string,1278 /*MainFileVersion*/ std::string>>1279 DiagVersions;1280 };1281 1282 DiagCollector Collector;1283 Notification UnblockPreamble;1284 auto DiagCallbacks = std::make_unique<BlockPreambleThread>(1285 UnblockPreamble,1286 [&Collector](ParsedAST &AST) { Collector.onDiagnostics(AST); });1287 TUScheduler S(CDB, optsForTest(), std::move(DiagCallbacks));1288 Path File = testPath("foo.cpp");1289 auto BlockForDiags = [&](ParseInputs PI) {1290 return Collector.waitForNewDiags(S, File, std::move(PI));1291 };1292 1293 // Build first preamble.1294 auto PI = getInputs(File, "");1295 PI.Version = PI.Contents = "1";1296 ASSERT_THAT(BlockForDiags(PI), testing::Pair("1", "1"));1297 1298 // Now preamble thread is blocked, so rest of the requests sees only the1299 // stale preamble.1300 PI.Version = "2";1301 PI.Contents = "#define BAR\n" + PI.Version;1302 ASSERT_THAT(BlockForDiags(PI), testing::Pair("1", "2"));1303 1304 PI.Version = "3";1305 PI.Contents = "#define FOO\n" + PI.Version;1306 ASSERT_THAT(BlockForDiags(PI), testing::Pair("1", "3"));1307 1308 UnblockPreamble.notify();1309 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1310 1311 // Make sure that we have eventual consistency.1312 EXPECT_THAT(Collector.diagVersions().back(), Pair(PI.Version, PI.Version));1313 1314 // Check that WantDiagnostics::No doesn't emit any diags.1315 PI.Version = "4";1316 PI.Contents = "#define FOO\n" + PI.Version;1317 S.update(File, PI, WantDiagnostics::No);1318 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1319 EXPECT_THAT(Collector.diagVersions().back(), Pair("3", "3"));1320}1321 1322// If a header file is missing from the CDB (or inferred using heuristics), and1323// it's included by another open file, then we parse it using that files flags.1324TEST_F(TUSchedulerTests, IncluderCache) {1325 static std::string Main = testPath("main.cpp"), Main2 = testPath("main2.cpp"),1326 Main3 = testPath("main3.cpp"),1327 NoCmd = testPath("no_cmd.h"),1328 Unreliable = testPath("unreliable.h"),1329 OK = testPath("ok.h"),1330 NotIncluded = testPath("not_included.h");1331 struct NoHeadersCDB : public GlobalCompilationDatabase {1332 std::optional<tooling::CompileCommand>1333 getCompileCommand(PathRef File) const override {1334 if (File == NoCmd || File == NotIncluded || FailAll)1335 return std::nullopt;1336 auto Basic = getFallbackCommand(File);1337 Basic.Heuristic.clear();1338 if (File == Unreliable) {1339 Basic.Heuristic = "not reliable";1340 } else if (File == Main) {1341 Basic.CommandLine.push_back("-DMAIN");1342 } else if (File == Main2) {1343 Basic.CommandLine.push_back("-DMAIN2");1344 } else if (File == Main3) {1345 Basic.CommandLine.push_back("-DMAIN3");1346 }1347 return Basic;1348 }1349 1350 std::atomic<bool> FailAll{false};1351 } CDB;1352 TUScheduler S(CDB, optsForTest());1353 auto GetFlags = [&](PathRef Header) {1354 S.update(Header, getInputs(Header, ";"), WantDiagnostics::Yes);1355 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1356 Notification CmdDone;1357 tooling::CompileCommand Cmd;1358 S.runWithPreamble("GetFlags", Header, TUScheduler::StaleOrAbsent,1359 [&](llvm::Expected<InputsAndPreamble> Inputs) {1360 ASSERT_FALSE(!Inputs) << Inputs.takeError();1361 Cmd = std::move(Inputs->Command);1362 CmdDone.notify();1363 });1364 CmdDone.wait();1365 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1366 return Cmd.CommandLine;1367 };1368 1369 for (const auto &Path : {NoCmd, Unreliable, OK, NotIncluded})1370 FS.Files[Path] = ";";1371 1372 // Initially these files have normal commands from the CDB.1373 EXPECT_THAT(GetFlags(Main), Contains("-DMAIN")) << "sanity check";1374 EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN"))) << "no includes yet";1375 1376 // Now make Main include the others, and some should pick up its flags.1377 const char *AllIncludes = R"cpp(1378 #include "no_cmd.h"1379 #include "ok.h"1380 #include "unreliable.h"1381 )cpp";1382 S.update(Main, getInputs(Main, AllIncludes), WantDiagnostics::Yes);1383 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1384 EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN"))1385 << "Included from main file, has no own command";1386 EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))1387 << "Included from main file, own command is heuristic";1388 EXPECT_THAT(GetFlags(OK), Not(Contains("-DMAIN")))1389 << "Included from main file, but own command is used";1390 EXPECT_THAT(GetFlags(NotIncluded), Not(Contains("-DMAIN")))1391 << "Not included from main file";1392 1393 // Open another file - it won't overwrite the associations with Main.1394 std::string SomeIncludes = R"cpp(1395 #include "no_cmd.h"1396 #include "not_included.h"1397 )cpp";1398 S.update(Main2, getInputs(Main2, SomeIncludes), WantDiagnostics::Yes);1399 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1400 EXPECT_THAT(GetFlags(NoCmd),1401 AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))1402 << "mainfile association is stable";1403 EXPECT_THAT(GetFlags(NotIncluded),1404 AllOf(Contains("-DMAIN2"), Not(Contains("-DMAIN"))))1405 << "new headers are associated with new mainfile";1406 1407 // Remove includes from main - this marks the associations as invalid but1408 // doesn't actually remove them until another preamble claims them.1409 S.update(Main, getInputs(Main, ""), WantDiagnostics::Yes);1410 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1411 EXPECT_THAT(GetFlags(NoCmd),1412 AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))1413 << "mainfile association not updated yet!";1414 1415 // Open yet another file - this time it claims the associations.1416 S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);1417 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1418 EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))1419 << "association invalidated and then claimed by main3";1420 EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))1421 << "association invalidated but not reclaimed";1422 EXPECT_THAT(GetFlags(NotIncluded), Contains("-DMAIN2"))1423 << "association still valid";1424 1425 // Delete the file from CDB, it should invalidate the associations.1426 CDB.FailAll = true;1427 EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN3")))1428 << "association should've been invalidated.";1429 // Also run update for Main3 to invalidate the preeamble to make sure next1430 // update populates include cache associations.1431 S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);1432 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1433 // Re-add the file and make sure nothing crashes.1434 CDB.FailAll = false;1435 S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);1436 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1437 EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))1438 << "association invalidated and then claimed by main3";1439}1440 1441TEST_F(TUSchedulerTests, PreservesLastActiveFile) {1442 for (bool Sync : {false, true}) {1443 auto Opts = optsForTest();1444 if (Sync)1445 Opts.AsyncThreadsCount = 0;1446 TUScheduler S(CDB, Opts);1447 1448 auto CheckNoFileActionsSeesLastActiveFile =1449 [&](llvm::StringRef LastActiveFile) {1450 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1451 std::atomic<int> Counter(0);1452 // We only check for run and runQuick as runWithAST and1453 // runWithPreamble is always bound to a file.1454 S.run("run-UsesLastActiveFile", /*Path=*/"", [&] {1455 ++Counter;1456 EXPECT_EQ(LastActiveFile, boundPath());1457 });1458 S.runQuick("runQuick-UsesLastActiveFile", /*Path=*/"", [&] {1459 ++Counter;1460 EXPECT_EQ(LastActiveFile, boundPath());1461 });1462 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1463 EXPECT_EQ(2, Counter.load());1464 };1465 1466 // Check that we see no file initially1467 CheckNoFileActionsSeesLastActiveFile("");1468 1469 // Now check that every action scheduled with a particular file changes the1470 // LastActiveFile.1471 auto Path = testPath("run.cc");1472 S.run(Path, Path, [] {});1473 CheckNoFileActionsSeesLastActiveFile(Path);1474 1475 Path = testPath("runQuick.cc");1476 S.runQuick(Path, Path, [] {});1477 CheckNoFileActionsSeesLastActiveFile(Path);1478 1479 Path = testPath("runWithAST.cc");1480 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);1481 S.runWithAST(Path, Path, [](llvm::Expected<InputsAndAST> Inp) {1482 EXPECT_TRUE(bool(Inp));1483 });1484 CheckNoFileActionsSeesLastActiveFile(Path);1485 1486 Path = testPath("runWithPreamble.cc");1487 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);1488 S.runWithPreamble(1489 Path, Path, TUScheduler::Stale,1490 [](llvm::Expected<InputsAndPreamble> Inp) { EXPECT_TRUE(bool(Inp)); });1491 CheckNoFileActionsSeesLastActiveFile(Path);1492 1493 Path = testPath("update.cc");1494 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);1495 CheckNoFileActionsSeesLastActiveFile(Path);1496 1497 // An update with the same contents should not change LastActiveFile.1498 auto LastActive = Path;1499 Path = testPath("runWithAST.cc");1500 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);1501 CheckNoFileActionsSeesLastActiveFile(LastActive);1502 }1503}1504 1505TEST_F(TUSchedulerTests, PreambleThrottle) {1506 const int NumRequests = 4;1507 // Silly throttler that waits for 4 requests, and services them in reverse.1508 // Doesn't honor cancellation but records it.1509 struct : public PreambleThrottler {1510 std::mutex Mu;1511 std::vector<std::string> Acquires;1512 std::vector<RequestID> Releases;1513 llvm::DenseMap<RequestID, Callback> Callbacks;1514 // If set, the notification is signalled after acquiring the specified ID.1515 std::optional<std::pair<RequestID, Notification *>> Notify;1516 1517 RequestID acquire(llvm::StringRef Filename, Callback CB) override {1518 RequestID ID;1519 Callback Invoke;1520 {1521 std::lock_guard<std::mutex> Lock(Mu);1522 ID = Acquires.size();1523 Acquires.emplace_back(Filename);1524 // If we're full, satisfy this request immediately.1525 if (Acquires.size() == NumRequests) {1526 Invoke = std::move(CB);1527 } else {1528 Callbacks.try_emplace(ID, std::move(CB));1529 }1530 }1531 if (Invoke)1532 Invoke();1533 {1534 std::lock_guard<std::mutex> Lock(Mu);1535 if (Notify && ID == Notify->first) {1536 Notify->second->notify();1537 Notify.reset();1538 }1539 }1540 return ID;1541 }1542 1543 void release(RequestID ID) override {1544 Callback SatisfyNext;1545 {1546 std::lock_guard<std::mutex> Lock(Mu);1547 Releases.push_back(ID);1548 if (ID > 0 && Acquires.size() == NumRequests)1549 SatisfyNext = std::move(Callbacks[ID - 1]);1550 }1551 if (SatisfyNext)1552 SatisfyNext();1553 }1554 1555 void reset() {1556 Acquires.clear();1557 Releases.clear();1558 Callbacks.clear();1559 }1560 } Throttler;1561 1562 struct CaptureBuiltFilenames : public ParsingCallbacks {1563 std::vector<std::string> &Filenames;1564 CaptureBuiltFilenames(std::vector<std::string> &Filenames)1565 : Filenames(Filenames) {}1566 void onPreambleAST(1567 PathRef Path, llvm::StringRef Version, CapturedASTCtx,1568 std::shared_ptr<const include_cleaner::PragmaIncludes> PI) override {1569 // Deliberately no synchronization.1570 // The PreambleThrottler should serialize these calls, if not then tsan1571 // will find a bug here.1572 Filenames.emplace_back(Path);1573 }1574 };1575 1576 auto Opts = optsForTest();1577 Opts.AsyncThreadsCount = 2 * NumRequests; // throttler is the bottleneck1578 Opts.PreambleThrottler = &Throttler;1579 1580 std::vector<std::string> Filenames;1581 1582 {1583 std::vector<std::string> BuiltFilenames;1584 TUScheduler S(CDB, Opts,1585 std::make_unique<CaptureBuiltFilenames>(BuiltFilenames));1586 for (unsigned I = 0; I < NumRequests; ++I) {1587 auto Path = testPath(std::to_string(I) + ".cc");1588 Filenames.push_back(Path);1589 S.update(Path, getInputs(Path, ""), WantDiagnostics::Yes);1590 }1591 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(60)));1592 1593 // The throttler saw all files, and we built them.1594 EXPECT_THAT(Throttler.Acquires,1595 testing::UnorderedElementsAreArray(Filenames));1596 EXPECT_THAT(BuiltFilenames,1597 testing::UnorderedElementsAreArray(Filenames));1598 // We built the files in reverse order that the throttler saw them.1599 EXPECT_THAT(BuiltFilenames,1600 testing::ElementsAreArray(Throttler.Acquires.rbegin(),1601 Throttler.Acquires.rend()));1602 // Resources for each file were correctly released.1603 EXPECT_THAT(Throttler.Releases, ElementsAre(3, 2, 1, 0));1604 }1605 1606 Throttler.reset();1607 1608 // This time, enqueue 2 files, then cancel one of them while still waiting.1609 // Finally shut down the server. Observe that everything gets cleaned up.1610 Notification AfterAcquire2;1611 Notification AfterFinishA;1612 Throttler.Notify = {1, &AfterAcquire2};1613 std::vector<std::string> BuiltFilenames;1614 auto A = testPath("a.cc");1615 auto B = testPath("b.cc");1616 Filenames = {A, B};1617 {1618 TUScheduler S(CDB, Opts,1619 std::make_unique<CaptureBuiltFilenames>(BuiltFilenames));1620 updateWithCallback(S, A, getInputs(A, ""), WantDiagnostics::Yes,1621 [&] { AfterFinishA.notify(); });1622 S.update(B, getInputs(B, ""), WantDiagnostics::Yes);1623 AfterAcquire2.wait();1624 1625 // The throttler saw all files, but we built none.1626 EXPECT_THAT(Throttler.Acquires,1627 testing::UnorderedElementsAreArray(Filenames));1628 EXPECT_THAT(BuiltFilenames, testing::IsEmpty());1629 // We haven't released anything yet, we're still waiting.1630 EXPECT_THAT(Throttler.Releases, testing::IsEmpty());1631 1632 // FIXME: This is flaky, because the request can be destroyed after shutdown1633 // if it hasn't been dequeued yet (stop() resets NextRequest).1634#if 01635 // Now close file A, which will shut down its AST worker.1636 S.remove(A);1637 // Request is destroyed after the queue shutdown, so release() has happened.1638 AfterFinishA.wait();1639 // We still didn't build anything.1640 EXPECT_THAT(BuiltFilenames, testing::IsEmpty());1641 // But we've cancelled the request to build A (not sure which its ID is).1642 EXPECT_THAT(Throttler.Releases, ElementsAre(AnyOf(1, 0)));1643#endif1644 1645 // Now shut down the TU Scheduler.1646 }1647 // The throttler saw all files, but we built none.1648 EXPECT_THAT(Throttler.Acquires,1649 testing::UnorderedElementsAreArray(Filenames));1650 EXPECT_THAT(BuiltFilenames, testing::IsEmpty());1651 // We gave up waiting and everything got released (in some order).1652 EXPECT_THAT(Throttler.Releases, UnorderedElementsAre(1, 0));1653}1654 1655} // namespace1656} // namespace clangd1657} // namespace clang1658