493 lines · cpp
1//===- llvm/unittest/Support/JobserverTest.cpp ----------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// Jobserver.h unit tests.11///12//===----------------------------------------------------------------------===//13 14#include "llvm/Support/Jobserver.h"15#include "llvm/Config/llvm-config.h"16#include "llvm/Support/Debug.h"17#include "llvm/Support/Parallel.h"18#include "llvm/Support/Program.h"19#include "llvm/Support/ThreadPool.h"20#include "llvm/Support/raw_ostream.h"21#include "gtest/gtest.h"22#include <future>23#include <random>24#include <stdlib.h>25 26#if defined(LLVM_ON_UNIX)27#include "llvm/ADT/SmallString.h"28#include "llvm/Support/FileSystem.h"29#include <atomic>30#include <condition_variable>31#include <fcntl.h>32#include <mutex>33#include <sys/stat.h>34#include <thread>35#include <unistd.h>36#elif defined(_WIN32)37#include <windows.h>38#endif39 40#define DEBUG_TYPE "jobserver-test"41 42using namespace llvm;43 44// Provided by the unit test main to locate the current test binary.45extern const char *TestMainArgv0;46 47namespace {48 49// RAII helper to set an environment variable for the duration of a test.50class ScopedEnvironment {51 std::string Name;52 std::string OldValue;53 bool HadOldValue;54 55public:56 ScopedEnvironment(const char *Name, const char *Value) : Name(Name) {57#if defined(_WIN32)58 char *Old = nullptr;59 size_t OldLen;60 errno_t err = _dupenv_s(&Old, &OldLen, Name);61 if (err == 0 && Old != nullptr) {62 HadOldValue = true;63 OldValue = Old;64 free(Old);65 } else {66 HadOldValue = false;67 }68 _putenv_s(Name, Value);69#else70 const char *Old = getenv(Name);71 if (Old) {72 HadOldValue = true;73 OldValue = Old;74 } else {75 HadOldValue = false;76 }77 setenv(Name, Value, 1);78#endif79 }80 81 ~ScopedEnvironment() {82#if defined(_WIN32)83 if (HadOldValue)84 _putenv_s(Name.c_str(), OldValue.c_str());85 else86 // On Windows, setting an environment variable to an empty string87 // unsets it, making getenv() return NULL.88 _putenv_s(Name.c_str(), "");89#else90 if (HadOldValue)91 setenv(Name.c_str(), OldValue.c_str(), 1);92 else93 unsetenv(Name.c_str());94#endif95 }96};97 98TEST(Jobserver, Slot) {99 // Default constructor creates an invalid slot.100 JobSlot S1;101 EXPECT_FALSE(S1.isValid());102 EXPECT_FALSE(S1.isImplicit());103 104 // Create an implicit slot.105 JobSlot S2 = JobSlot::createImplicit();106 EXPECT_TRUE(S2.isValid());107 EXPECT_TRUE(S2.isImplicit());108 109 // Create an explicit slot.110 JobSlot S3 = JobSlot::createExplicit(42);111 EXPECT_TRUE(S3.isValid());112 EXPECT_FALSE(S3.isImplicit());113 114 // Test move construction.115 JobSlot S4 = std::move(S2);116 EXPECT_TRUE(S4.isValid());117 EXPECT_TRUE(S4.isImplicit());118 EXPECT_FALSE(S2.isValid()); // S2 is now invalid.119 120 // Test move assignment.121 S1 = std::move(S3);122 EXPECT_TRUE(S1.isValid());123 EXPECT_FALSE(S1.isImplicit());124 EXPECT_FALSE(S3.isValid()); // S3 is now invalid.125}126 127// Test fixture for parsing tests to ensure the singleton state is128// reset between each test case.129class JobserverParsingTest : public ::testing::Test {130protected:131 void TearDown() override { JobserverClient::resetForTesting(); }132};133 134TEST_F(JobserverParsingTest, NoMakeflags) {135 // No MAKEFLAGS, should be null.136 ScopedEnvironment Env("MAKEFLAGS", "");137 // On Unix, setting an env var to "" makes getenv() return an empty138 // string, not NULL. We must call unsetenv() to test the case where139 // the variable is truly not present.140#if !defined(_WIN32)141 unsetenv("MAKEFLAGS");142#endif143 EXPECT_EQ(JobserverClient::getInstance(), nullptr);144}145 146TEST_F(JobserverParsingTest, EmptyMakeflags) {147 // Empty MAKEFLAGS, should be null.148 ScopedEnvironment Env("MAKEFLAGS", "");149 EXPECT_EQ(JobserverClient::getInstance(), nullptr);150}151 152TEST_F(JobserverParsingTest, DryRunFlag) {153 // Dry-run flag 'n', should be null.154 ScopedEnvironment Env("MAKEFLAGS", "n -j --jobserver-auth=fifo:/tmp/foo");155 EXPECT_EQ(JobserverClient::getInstance(), nullptr);156}157 158// Separate fixture for non-threaded client tests.159class JobserverClientTest : public JobserverParsingTest {};160 161#if defined(LLVM_ON_UNIX)162// RAII helper to create and clean up a temporary FIFO file.163class ScopedFifo {164 SmallString<128> Path;165 bool IsValid = false;166 167public:168 ScopedFifo() {169 // To get a unique, non-colliding name for a FIFO, we use the170 // createTemporaryFile function to reserve a name in the filesystem.171 std::error_code EC =172 sys::fs::createTemporaryFile("jobserver-test", "fifo", Path);173 if (EC)174 return;175 // Then we immediately remove the regular file it created, but keep the176 // unique path.177 sys::fs::remove(Path);178 // Finally, we create the FIFO at that safe, unique path.179 if (mkfifo(Path.c_str(), 0600) != 0)180 return;181 IsValid = true;182 }183 184 ~ScopedFifo() {185 if (IsValid)186 sys::fs::remove(Path);187 }188 189 const char *c_str() const { return Path.data(); }190 bool isValid() const { return IsValid; }191};192 193TEST_F(JobserverClientTest, UnixClientFifo) {194 // This test covers basic FIFO client creation and behavior with an empty195 // FIFO. No job tokens are available.196 ScopedFifo F;197 ASSERT_TRUE(F.isValid());198 199 // Intentionally inserted \t in environment string.200 std::string Makeflags = " \t -j4\t \t--jobserver-auth=fifo:";201 Makeflags += F.c_str();202 ScopedEnvironment Env("MAKEFLAGS", Makeflags.c_str());203 204 JobserverClient *Client = JobserverClient::getInstance();205 ASSERT_NE(Client, nullptr);206 207 // Get the implicit token.208 JobSlot S1 = Client->tryAcquire();209 EXPECT_TRUE(S1.isValid());210 EXPECT_TRUE(S1.isImplicit());211 212 // FIFO is empty, next acquire fails.213 JobSlot S2 = Client->tryAcquire();214 EXPECT_FALSE(S2.isValid());215 216 // Release does not write to the pipe for the implicit token.217 Client->release(std::move(S1));218 219 // Re-acquire the implicit token.220 S1 = Client->tryAcquire();221 EXPECT_TRUE(S1.isValid());222}223 224#if LLVM_ENABLE_THREADS225// Unique anchor whose address helps locate the current test binary.226static int JobserverTestAnchor = 0;227 228// Test fixture for tests that use the jobserver strategy. It creates a229// temporary FIFO, sets MAKEFLAGS, and provides a helper to pre-load the FIFO230// with job tokens, simulating `make -jN`.231class JobserverStrategyTest : public JobserverParsingTest {232protected:233 std::unique_ptr<ScopedFifo> TheFifo;234 std::thread MakeThread;235 std::atomic<bool> StopMakeThread{false};236 // Save and restore the global parallel strategy to avoid interfering with237 // other tests in the same process.238 ThreadPoolStrategy SavedStrategy;239 240 void SetUp() override {241 SavedStrategy = parallel::strategy;242 TheFifo = std::make_unique<ScopedFifo>();243 ASSERT_TRUE(TheFifo->isValid());244 245 std::string MakeFlags = "--jobserver-auth=fifo:";246 MakeFlags += TheFifo->c_str();247 setenv("MAKEFLAGS", MakeFlags.c_str(), 1);248 }249 250 void TearDown() override {251 if (MakeThread.joinable()) {252 StopMakeThread = true;253 MakeThread.join();254 }255 unsetenv("MAKEFLAGS");256 TheFifo.reset();257 // Restore the original strategy to ensure subsequent tests are unaffected.258 parallel::strategy = SavedStrategy;259 }260 261 // Starts a background thread that emulates `make`. It populates the FIFO262 // with initial tokens and then recycles tokens released by clients.263 void startMakeProxy(int NumInitialJobs) {264 MakeThread = std::thread([this, NumInitialJobs]() {265 LLVM_DEBUG(dbgs() << "[MakeProxy] Thread started.\n");266 // Open the FIFO for reading and writing. This call does not block.267 int RWFd = open(TheFifo->c_str(), O_RDWR);268 LLVM_DEBUG(dbgs() << "[MakeProxy] Opened FIFO " << TheFifo->c_str()269 << " with O_RDWR, FD=" << RWFd << "\n");270 if (RWFd == -1) {271 LLVM_DEBUG(272 dbgs()273 << "[MakeProxy] ERROR: Failed to open FIFO with O_RDWR. Errno: "274 << errno << "\n");275 return;276 }277 278 // Populate with initial jobs.279 LLVM_DEBUG(dbgs() << "[MakeProxy] Writing " << NumInitialJobs280 << " initial tokens.\n");281 for (int i = 0; i < NumInitialJobs; ++i) {282 if (write(RWFd, "+", 1) != 1) {283 LLVM_DEBUG(dbgs()284 << "[MakeProxy] ERROR: Failed to write initial token " << i285 << ".\n");286 close(RWFd);287 return;288 }289 }290 LLVM_DEBUG(dbgs() << "[MakeProxy] Finished writing initial tokens.\n");291 292 // Make the read non-blocking so we can periodically check StopMakeThread.293 int flags = fcntl(RWFd, F_GETFL, 0);294 fcntl(RWFd, F_SETFL, flags | O_NONBLOCK);295 296 while (!StopMakeThread) {297 char Token;298 ssize_t Ret = read(RWFd, &Token, 1);299 if (Ret == 1) {300 LLVM_DEBUG(dbgs() << "[MakeProxy] Read token '" << Token301 << "' to recycle.\n");302 // A client released a token, 'make' makes it available again.303 std::this_thread::sleep_for(std::chrono::microseconds(100));304 ssize_t WRet;305 do {306 WRet = write(RWFd, &Token, 1);307 } while (WRet < 0 && errno == EINTR);308 if (WRet <= 0) {309 LLVM_DEBUG(310 dbgs()311 << "[MakeProxy] ERROR: Failed to write recycled token.\n");312 break; // Error, stop the proxy.313 }314 LLVM_DEBUG(dbgs()315 << "[MakeProxy] Wrote token '" << Token << "' back.\n");316 } else if (Ret < 0 && errno != EAGAIN && errno != EWOULDBLOCK) {317 LLVM_DEBUG(dbgs() << "[MakeProxy] ERROR: Read failed with errno "318 << errno << ".\n");319 break; // Error, stop the proxy.320 }321 // Yield to prevent this thread from busy-waiting.322 std::this_thread::sleep_for(std::chrono::milliseconds(1));323 }324 LLVM_DEBUG(dbgs() << "[MakeProxy] Thread stopping.\n");325 close(RWFd);326 });327 328 // Give the proxy thread a moment to start and populate the FIFO.329 // This is a simple way to avoid a race condition where the client starts330 // before the initial tokens are in the pipe.331 std::this_thread::sleep_for(std::chrono::milliseconds(50));332 }333};334 335TEST_F(JobserverStrategyTest, ThreadPoolConcurrencyIsLimited) {336 // This test simulates `make -j3`. We will have 1 implicit job slot and337 // we will add 2 explicit job tokens to the FIFO, for a total of 3.338 const int NumExplicitJobs = 2;339 const int ConcurrencyLimit = NumExplicitJobs + 1; // +1 for the implicit slot340 const int NumTasks = 8; // More tasks than available slots.341 342 LLVM_DEBUG(dbgs() << "Calling startMakeProxy with " << NumExplicitJobs343 << " jobs.\n");344 startMakeProxy(NumExplicitJobs);345 LLVM_DEBUG(dbgs() << "MakeProxy is running.\n");346 347 // Create the thread pool. Its constructor will call jobserver_concurrency()348 // and create a client that reads from our pre-loaded FIFO.349 StdThreadPool Pool(jobserver_concurrency());350 351 std::atomic<int> ActiveTasks{0};352 std::atomic<int> MaxActiveTasks{0};353 std::atomic<int> CompletedTasks{0};354 std::mutex M;355 std::condition_variable CV;356 357 // Dispatch more tasks than there are job slots. The pool should block358 // and only run up to `ConcurrencyLimit` tasks at once.359 for (int i = 0; i < NumTasks; ++i) {360 Pool.async([&, i] {361 // Track the number of concurrently running tasks.362 int CurrentActive = ++ActiveTasks;363 LLVM_DEBUG(dbgs() << "Task " << i << ": Active tasks: " << CurrentActive364 << "\n");365 (void)i;366 int OldMax = MaxActiveTasks.load();367 while (CurrentActive > OldMax)368 MaxActiveTasks.compare_exchange_weak(OldMax, CurrentActive);369 370 std::this_thread::sleep_for(std::chrono::milliseconds(25));371 372 --ActiveTasks;373 if (++CompletedTasks == NumTasks) {374 std::lock_guard<std::mutex> Lock(M);375 CV.notify_one();376 }377 });378 }379 380 // Wait for all tasks to complete.381 std::unique_lock<std::mutex> Lock(M);382 CV.wait(Lock, [&] { return CompletedTasks == NumTasks; });383 384 LLVM_DEBUG(dbgs() << "Test finished. Max active tasks was " << MaxActiveTasks385 << ".\n");386 // The key assertion: the maximum number of concurrent tasks should387 // not have exceeded the limit imposed by the jobserver.388 EXPECT_LE(MaxActiveTasks, ConcurrencyLimit);389 EXPECT_EQ(CompletedTasks, NumTasks);390}391 392// Parent-side driver that spawns a fresh process to run the child test which393// validates that parallelFor respects the jobserver limit when it is the first394// user of the default executor in that process.395TEST_F(JobserverStrategyTest, ParallelForIsLimited_Subprocess) {396 // Mark child execution.397 setenv("LLVM_JOBSERVER_TEST_CHILD", "1", 1);398 399 // Find the current test binary and build args to run only the child test.400 std::string Executable =401 sys::fs::getMainExecutable(TestMainArgv0, &JobserverTestAnchor);402 ASSERT_FALSE(Executable.empty()) << "Failed to get main executable path";403 SmallVector<StringRef, 4> Args{Executable,404 "--gtest_filter=JobserverStrategyTest."405 "ParallelForIsLimited_SubprocessChild"};406 407 std::string Error;408 bool ExecFailed = false;409 int RC = sys::ExecuteAndWait(Executable, Args, std::nullopt, {}, 0, 0, &Error,410 &ExecFailed);411 unsetenv("LLVM_JOBSERVER_TEST_CHILD");412 ASSERT_FALSE(ExecFailed) << Error;413 ASSERT_EQ(RC, 0) << "Executable failed with exit code " << RC;414}415 416// Child-side test: create FIFO and make-proxy in this process, set the417// jobserver strategy, and then run parallelFor.418TEST_F(JobserverStrategyTest, ParallelForIsLimited_SubprocessChild) {419 if (!getenv("LLVM_JOBSERVER_TEST_CHILD"))420 GTEST_SKIP() << "Not running in child mode";421 422 // This test verifies that llvm::parallelFor respects the jobserver limit.423 const int NumExplicitJobs = 3;424 const int ConcurrencyLimit = NumExplicitJobs + 1; // +1 implicit425 const int NumTasks = 20;426 427 startMakeProxy(NumExplicitJobs);428 429 // Set the global strategy before any default executor is created.430 parallel::strategy = jobserver_concurrency();431 432 std::atomic<int> ActiveTasks{0};433 std::atomic<int> MaxActiveTasks{0};434 435 parallelFor(0, NumTasks, [&]([[maybe_unused]] int i) {436 int CurrentActive = ++ActiveTasks;437 int OldMax = MaxActiveTasks.load();438 while (CurrentActive > OldMax)439 MaxActiveTasks.compare_exchange_weak(OldMax, CurrentActive);440 std::this_thread::sleep_for(std::chrono::milliseconds(20));441 --ActiveTasks;442 });443 444 EXPECT_LE(MaxActiveTasks, ConcurrencyLimit);445}446 447// Parent-side driver for parallelSort child test.448TEST_F(JobserverStrategyTest, ParallelSortIsLimited_Subprocess) {449 setenv("LLVM_JOBSERVER_TEST_CHILD", "1", 1);450 451 std::string Executable =452 sys::fs::getMainExecutable(TestMainArgv0, &JobserverTestAnchor);453 ASSERT_FALSE(Executable.empty()) << "Failed to get main executable path";454 SmallVector<StringRef, 4> Args{Executable,455 "--gtest_filter=JobserverStrategyTest."456 "ParallelSortIsLimited_SubprocessChild"};457 458 std::string Error;459 bool ExecFailed = false;460 int RC = sys::ExecuteAndWait(Executable, Args, std::nullopt, {}, 0, 0, &Error,461 &ExecFailed);462 unsetenv("LLVM_JOBSERVER_TEST_CHILD");463 ASSERT_FALSE(ExecFailed) << Error;464 ASSERT_EQ(RC, 0) << "Executable failed with exit code " << RC;465}466 467// Child-side test: ensure parallelSort runs and completes correctly under the468// jobserver strategy when it owns default executor initialization.469TEST_F(JobserverStrategyTest, ParallelSortIsLimited_SubprocessChild) {470 if (!getenv("LLVM_JOBSERVER_TEST_CHILD"))471 GTEST_SKIP() << "Not running in child mode";472 473 const int NumExplicitJobs = 3;474 startMakeProxy(NumExplicitJobs);475 476 parallel::strategy = jobserver_concurrency();477 478 std::vector<int> V(1024);479 std::mt19937 randEngine;480 std::uniform_int_distribution<int> dist;481 for (int &i : V)482 i = dist(randEngine);483 484 parallelSort(V.begin(), V.end());485 ASSERT_TRUE(llvm::is_sorted(V));486}487 488#endif // LLVM_ENABLE_THREADS489 490#endif // defined(LLVM_ON_UNIX)491 492} // end anonymous namespace493