501 lines · cpp
1//===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/DirectoryWatcher/DirectoryWatcher.h"10#include "llvm/Support/FileSystem.h"11#include "llvm/Support/Path.h"12#include "llvm/Support/raw_ostream.h"13#include "llvm/Testing/Support/Error.h"14#include "gtest/gtest.h"15#include <condition_variable>16#include <future>17#include <mutex>18#include <optional>19#include <thread>20 21using namespace llvm;22using namespace llvm::sys;23using namespace llvm::sys::fs;24using namespace clang;25 26namespace clang {27static bool operator==(const DirectoryWatcher::Event &lhs,28 const DirectoryWatcher::Event &rhs) {29 return lhs.Filename == rhs.Filename &&30 static_cast<int>(lhs.Kind) == static_cast<int>(rhs.Kind);31}32} // namespace clang33 34namespace {35 36typedef DirectoryWatcher::Event::EventKind EventKind;37 38// We've observed this test being significantly flaky when running on a heavily39// loaded machine (e.g. when it's being run as part of the full check-clang40// suite). Set a high timeout value to avoid this flakiness. The 60s timeout41// value was determined empirically. It's a timeout value, not a sleep value,42// and the test should require much less time in practice the vast majority of43// instances. The cases where we do come close to (or still end up hitting) the44// longer timeout are most likely to occur when other tests are also running at45// the same time (e.g. as part of the full check-clang suite), in which case the46// latency of the timeout will be masked by the latency of the other tests.47constexpr std::chrono::seconds EventualResultTimeout(60);48 49struct DirectoryWatcherTestFixture {50 std::string TestRootDir;51 std::string TestWatchedDir;52 53 DirectoryWatcherTestFixture() {54 SmallString<128> pathBuf;55#ifndef NDEBUG56 std::error_code UniqDirRes =57#endif58 createUniqueDirectory("dirwatcher", pathBuf);59 assert(!UniqDirRes);60 TestRootDir = std::string(pathBuf.str());61 path::append(pathBuf, "watch");62 TestWatchedDir = std::string(pathBuf.str());63#ifndef NDEBUG64 std::error_code CreateDirRes =65#endif66 create_directory(TestWatchedDir, false);67 assert(!CreateDirRes);68 }69 70 ~DirectoryWatcherTestFixture() { remove_directories(TestRootDir); }71 72 SmallString<128> getPathInWatched(const std::string &testFile) {73 SmallString<128> pathBuf;74 pathBuf = TestWatchedDir;75 path::append(pathBuf, testFile);76 return pathBuf;77 }78 79 void addFile(const std::string &testFile) {80 Expected<file_t> ft = openNativeFileForWrite(getPathInWatched(testFile),81 CD_CreateNew, OF_None);82 if (ft) {83 closeFile(*ft);84 } else {85 llvm::errs() << llvm::toString(ft.takeError()) << "\n";86 llvm::errs() << getPathInWatched(testFile) << "\n";87 llvm_unreachable("Couldn't create test file.");88 }89 }90 91 void deleteFile(const std::string &testFile) {92 std::error_code EC =93 remove(getPathInWatched(testFile), /*IgnoreNonExisting=*/false);94 ASSERT_FALSE(EC);95 }96};97 98std::string eventKindToString(const EventKind K) {99 switch (K) {100 case EventKind::Removed:101 return "Removed";102 case EventKind::Modified:103 return "Modified";104 case EventKind::WatchedDirRemoved:105 return "WatchedDirRemoved";106 case EventKind::WatcherGotInvalidated:107 return "WatcherGotInvalidated";108 }109 llvm_unreachable("unknown event kind");110}111 112struct VerifyingConsumer {113 std::vector<DirectoryWatcher::Event> ExpectedInitial;114 const std::vector<DirectoryWatcher::Event> ExpectedInitialCopy;115 std::vector<DirectoryWatcher::Event> ExpectedNonInitial;116 const std::vector<DirectoryWatcher::Event> ExpectedNonInitialCopy;117 std::vector<DirectoryWatcher::Event> OptionalNonInitial;118 std::vector<DirectoryWatcher::Event> UnexpectedInitial;119 std::vector<DirectoryWatcher::Event> UnexpectedNonInitial;120 std::mutex Mtx;121 std::condition_variable ResultIsReady;122 123 VerifyingConsumer(124 const std::vector<DirectoryWatcher::Event> &ExpectedInitial,125 const std::vector<DirectoryWatcher::Event> &ExpectedNonInitial,126 const std::vector<DirectoryWatcher::Event> &OptionalNonInitial = {})127 : ExpectedInitial(ExpectedInitial), ExpectedInitialCopy(ExpectedInitial),128 ExpectedNonInitial(ExpectedNonInitial), ExpectedNonInitialCopy(ExpectedNonInitial),129 OptionalNonInitial(OptionalNonInitial) {}130 131 // This method is used by DirectoryWatcher.132 void consume(DirectoryWatcher::Event E, bool IsInitial) {133 if (IsInitial)134 consumeInitial(E);135 else136 consumeNonInitial(E);137 }138 139 void consumeInitial(DirectoryWatcher::Event E) {140 std::unique_lock<std::mutex> L(Mtx);141 auto It = llvm::find(ExpectedInitial, E);142 if (It == ExpectedInitial.end()) {143 UnexpectedInitial.push_back(E);144 } else {145 ExpectedInitial.erase(It);146 }147 if (result()) {148 L.unlock();149 ResultIsReady.notify_one();150 }151 }152 153 void consumeNonInitial(DirectoryWatcher::Event E) {154 std::unique_lock<std::mutex> L(Mtx);155 auto It = llvm::find(ExpectedNonInitial, E);156 if (It == ExpectedNonInitial.end()) {157 auto OptIt = llvm::find(OptionalNonInitial, E);158 if (OptIt != OptionalNonInitial.end()) {159 OptionalNonInitial.erase(OptIt);160 } else {161 UnexpectedNonInitial.push_back(E);162 }163 } else {164 ExpectedNonInitial.erase(It);165 }166 if (result()) {167 L.unlock();168 ResultIsReady.notify_one();169 }170 }171 172 // This method is used by DirectoryWatcher.173 void consume(llvm::ArrayRef<DirectoryWatcher::Event> Es, bool IsInitial) {174 for (const auto &E : Es)175 consume(E, IsInitial);176 }177 178 // Not locking - caller has to lock Mtx.179 std::optional<bool> result() const {180 if (ExpectedInitial.empty() && ExpectedNonInitial.empty() &&181 UnexpectedInitial.empty() && UnexpectedNonInitial.empty())182 return true;183 if (!UnexpectedInitial.empty() || !UnexpectedNonInitial.empty())184 return false;185 return std::nullopt;186 }187 188 // This method is used by tests.189 // \returns true on success190 bool blockUntilResult() {191 std::unique_lock<std::mutex> L(Mtx);192 while (true) {193 if (result())194 return *result();195 196 ResultIsReady.wait(L, [this]() { return result().has_value(); });197 }198 return false; // Just to make compiler happy.199 }200 201 void printUnmetExpectations(llvm::raw_ostream &OS) {202 // If there was any issue, print the expected state203 if (204 !ExpectedInitial.empty()205 ||206 !ExpectedNonInitial.empty()207 ||208 !UnexpectedInitial.empty()209 ||210 !UnexpectedNonInitial.empty()211 ) {212 OS << "Expected initial events: \n";213 for (const auto &E : ExpectedInitialCopy) {214 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";215 }216 OS << "Expected non-initial events: \n";217 for (const auto &E : ExpectedNonInitialCopy) {218 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";219 }220 }221 222 if (!ExpectedInitial.empty()) {223 OS << "Expected but not seen initial events: \n";224 for (const auto &E : ExpectedInitial) {225 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";226 }227 }228 if (!ExpectedNonInitial.empty()) {229 OS << "Expected but not seen non-initial events: \n";230 for (const auto &E : ExpectedNonInitial) {231 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";232 }233 }234 if (!UnexpectedInitial.empty()) {235 OS << "Unexpected initial events seen: \n";236 for (const auto &E : UnexpectedInitial) {237 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";238 }239 }240 if (!UnexpectedNonInitial.empty()) {241 OS << "Unexpected non-initial events seen: \n";242 for (const auto &E : UnexpectedNonInitial) {243 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";244 }245 }246 }247};248 249void checkEventualResultWithTimeout(VerifyingConsumer &TestConsumer) {250 std::packaged_task<int(void)> task(251 [&TestConsumer]() { return TestConsumer.blockUntilResult(); });252 std::future<int> WaitForExpectedStateResult = task.get_future();253 std::thread worker(std::move(task));254 worker.detach();255 256 EXPECT_TRUE(WaitForExpectedStateResult.wait_for(EventualResultTimeout) ==257 std::future_status::ready)258 << "The expected result state wasn't reached before the time-out.";259 std::unique_lock<std::mutex> L(TestConsumer.Mtx);260 EXPECT_TRUE(TestConsumer.result().has_value());261 if (TestConsumer.result()) {262 EXPECT_TRUE(*TestConsumer.result());263 }264 if ((TestConsumer.result() && !*TestConsumer.result()) ||265 !TestConsumer.result())266 TestConsumer.printUnmetExpectations(llvm::outs());267}268} // namespace269 270TEST(DirectoryWatcherTest, InitialScanSync) {271 DirectoryWatcherTestFixture fixture;272 273 fixture.addFile("a");274 fixture.addFile("b");275 fixture.addFile("c");276 277 VerifyingConsumer TestConsumer{278 {{EventKind::Modified, "a"},279 {EventKind::Modified, "b"},280 {EventKind::Modified, "c"}},281 {},282 // We have to ignore these as it's a race between the test process283 // which is scanning the directory and kernel which is sending284 // notification.285 {{EventKind::Modified, "a"},286 {EventKind::Modified, "b"},287 {EventKind::Modified, "c"}}288 };289 290 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =291 DirectoryWatcher::create(292 fixture.TestWatchedDir,293 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,294 bool IsInitial) {295 TestConsumer.consume(Events, IsInitial);296 },297 /*waitForInitialSync=*/true);298 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());299 300 checkEventualResultWithTimeout(TestConsumer);301}302 303TEST(DirectoryWatcherTest, InitialScanAsync) {304 DirectoryWatcherTestFixture fixture;305 306 fixture.addFile("a");307 fixture.addFile("b");308 fixture.addFile("c");309 310 VerifyingConsumer TestConsumer{311 {{EventKind::Modified, "a"},312 {EventKind::Modified, "b"},313 {EventKind::Modified, "c"}},314 {},315 // We have to ignore these as it's a race between the test process316 // which is scanning the directory and kernel which is sending317 // notification.318 {{EventKind::Modified, "a"},319 {EventKind::Modified, "b"},320 {EventKind::Modified, "c"}}321 };322 323 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =324 DirectoryWatcher::create(325 fixture.TestWatchedDir,326 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,327 bool IsInitial) {328 TestConsumer.consume(Events, IsInitial);329 },330 /*waitForInitialSync=*/false);331 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());332 333 checkEventualResultWithTimeout(TestConsumer);334}335 336TEST(DirectoryWatcherTest, AddFiles) {337 DirectoryWatcherTestFixture fixture;338 339 VerifyingConsumer TestConsumer{340 {},341 {{EventKind::Modified, "a"},342 {EventKind::Modified, "b"},343 {EventKind::Modified, "c"}}};344 345 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =346 DirectoryWatcher::create(347 fixture.TestWatchedDir,348 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,349 bool IsInitial) {350 TestConsumer.consume(Events, IsInitial);351 },352 /*waitForInitialSync=*/true);353 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());354 355 fixture.addFile("a");356 fixture.addFile("b");357 fixture.addFile("c");358 359 checkEventualResultWithTimeout(TestConsumer);360}361 362TEST(DirectoryWatcherTest, ModifyFile) {363 DirectoryWatcherTestFixture fixture;364 365 fixture.addFile("a");366 367 VerifyingConsumer TestConsumer{368 {{EventKind::Modified, "a"}},369 {{EventKind::Modified, "a"}},370 {{EventKind::Modified, "a"}}};371 372 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =373 DirectoryWatcher::create(374 fixture.TestWatchedDir,375 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,376 bool IsInitial) {377 TestConsumer.consume(Events, IsInitial);378 },379 /*waitForInitialSync=*/true);380 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());381 382 // modify the file383 {384 std::error_code error;385 llvm::raw_fd_ostream bStream(fixture.getPathInWatched("a"), error,386 CD_OpenExisting);387 assert(!error);388 bStream << "foo";389 }390 391 checkEventualResultWithTimeout(TestConsumer);392}393 394TEST(DirectoryWatcherTest, DeleteFile) {395 DirectoryWatcherTestFixture fixture;396 397 fixture.addFile("a");398 399 VerifyingConsumer TestConsumer{400 {{EventKind::Modified, "a"}},401 {{EventKind::Removed, "a"}},402 {{EventKind::Modified, "a"}, {EventKind::Removed, "a"}}};403 404 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =405 DirectoryWatcher::create(406 fixture.TestWatchedDir,407 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,408 bool IsInitial) {409 TestConsumer.consume(Events, IsInitial);410 },411 /*waitForInitialSync=*/true);412 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());413 414 fixture.deleteFile("a");415 416 checkEventualResultWithTimeout(TestConsumer);417}418 419TEST(DirectoryWatcherTest, DeleteWatchedDir) {420 DirectoryWatcherTestFixture fixture;421 422 VerifyingConsumer TestConsumer{423 {},424 {{EventKind::WatchedDirRemoved, ""},425 {EventKind::WatcherGotInvalidated, ""}}};426 427 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =428 DirectoryWatcher::create(429 fixture.TestWatchedDir,430 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,431 bool IsInitial) {432 TestConsumer.consume(Events, IsInitial);433 },434 /*waitForInitialSync=*/true);435 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());436 437 remove_directories(fixture.TestWatchedDir);438 439 checkEventualResultWithTimeout(TestConsumer);440}441 442TEST(DirectoryWatcherTest, InvalidatedWatcher) {443 DirectoryWatcherTestFixture fixture;444 445 VerifyingConsumer TestConsumer{446 {}, {{EventKind::WatcherGotInvalidated, ""}}};447 448 {449 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =450 DirectoryWatcher::create(451 fixture.TestWatchedDir,452 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,453 bool IsInitial) {454 TestConsumer.consume(Events, IsInitial);455 },456 /*waitForInitialSync=*/true);457 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());458 } // DW is destructed here.459 460 checkEventualResultWithTimeout(TestConsumer);461}462 463TEST(DirectoryWatcherTest, InvalidatedWatcherAsync) {464 DirectoryWatcherTestFixture fixture;465 fixture.addFile("a");466 467 // This test is checking that we get the initial notification for 'a' before468 // the final 'invalidated'. Strictly speaking, we do not care whether 'a' is469 // processed or not, only that it is neither racing with, nor after470 // 'invalidated'. In practice, it is always processed in our implementations.471 VerifyingConsumer TestConsumer{472 {{EventKind::Modified, "a"}},473 {{EventKind::WatcherGotInvalidated, ""}},474 // We have to ignore these as it's a race between the test process475 // which is scanning the directory and kernel which is sending476 // notification.477 {{EventKind::Modified, "a"}},478 };479 480 // A counter that can help detect data races on the event receiver,481 // particularly if used with TSan. Expected count will be 2 or 3 depending on482 // whether we get the kernel event or not (see comment above).483 unsigned Count = 0;484 {485 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =486 DirectoryWatcher::create(487 fixture.TestWatchedDir,488 [&TestConsumer,489 &Count](llvm::ArrayRef<DirectoryWatcher::Event> Events,490 bool IsInitial) {491 Count += 1;492 TestConsumer.consume(Events, IsInitial);493 },494 /*waitForInitialSync=*/false);495 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());496 } // DW is destructed here.497 498 checkEventualResultWithTimeout(TestConsumer);499 ASSERT_TRUE(Count == 2u || Count == 3u);500}501