From 7afdfb7d91e5ab37631faefcb9712f96cd21857c Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Fri, 9 Aug 2024 23:59:03 -0400 Subject: [PATCH] Implement GNU jobserver posix client support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The core principle of a jobserver is simple: Before starting a new job (edge in ninja-speak), a token must be acquired from an external entity as approval. Once a job is finished, the token must be returned to signal a free thread. In the case of GNU Make, this external entity is the parent process which has executed ninja and is managing the load capacity for all subprocesses which it has spawned. Introducing client support for this model allows ninja to give load capacity management to this parent process, allowing it to control the number of subprocesses that ninja spawns at any given time. This functionality is desirable when ninja is used as part of a bigger build, such as builds with Yocto/OpenEmbedded, Openwrt/Linux, Buildroot and Android. Here, multiple compile jobs are executed in parallel in order to maximize cpu utilization, but if each compile job uses all available cores, the system is overloaded. This implementation instantiates the client in real_main() and passes references to the class into other classes. All tokens are returned whenever the CommandRunner aborts, and the current number of tokens compared to the current number of running subprocesses controls the available load capacity, used to determine how many new tokens to attempt to acquire in order to start another job for each loop to find work. Calls to functions are excluded from Windows builds pending Windows-specific support for the jobserver. Co-authored-by: Martin Hundebøll Co-developed-by: Martin Hundebøll Signed-off-by: Martin Hundebøll Signed-off-by: Michael Pratt --- CMakeLists.txt | 5 +- configure.py | 1 + src/build.cc | 56 +++++++++++++-- src/build.h | 8 ++- src/build_test.cc | 68 ++++++++++-------- src/graph.h | 1 + src/jobserver-posix.cc | 160 +++++++++++++++++++++++++++++++++++++++++ src/jobserver.h | 107 +++++++++++++++++++++++++++ src/ninja.cc | 19 +++-- 9 files changed, 379 insertions(+), 46 deletions(-) create mode 100644 src/jobserver-posix.cc create mode 100644 src/jobserver.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b8fdee7d3a..f05d2cfcc5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,7 +169,10 @@ if(WIN32) # errors by telling windows.h to not define those two. add_compile_definitions(NOMINMAX) else() - target_sources(libninja PRIVATE src/subprocess-posix.cc) + target_sources(libninja PRIVATE + src/subprocess-posix.cc + src/jobserver-posix.cc + ) if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") target_sources(libninja PRIVATE src/getopt.c) # Build getopt.c, which can be compiled as either C or C++, as C++ diff --git a/configure.py b/configure.py index c88daad508..9d6175e35f 100755 --- a/configure.py +++ b/configure.py @@ -565,6 +565,7 @@ def has_re2c() -> bool: objs += cc('getopt') else: objs += cxx('subprocess-posix') + objs += cxx('jobserver-posix') if platform.is_aix(): objs += cc('getopt') if platform.is_msvc(): diff --git a/src/build.cc b/src/build.cc index deb8f04c8b..5a76e073fc 100644 --- a/src/build.cc +++ b/src/build.cc @@ -164,8 +164,24 @@ Edge* Plan::FindWork() { if (ready_.empty()) return NULL; +// TODO: jobserver client support for Windows +#ifndef _WIN32 + // Only initiate work if the jobserver can acquire a token. + if (builder_ && builder_->jobserver_.Enabled() && !builder_->jobserver_.Acquire()) { + return NULL; + } +#endif + Edge* work = ready_.top(); ready_.pop(); + +// TODO: jobserver client support for Windows +#ifndef _WIN32 + // Mark this edge as using a job token to be released when work is finished. + if (builder_) + work->has_job_token_ = builder_->jobserver_.Enabled(); +#endif + return work; } @@ -201,6 +217,15 @@ bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) { edge->pool()->EdgeFinished(*edge); edge->pool()->RetrieveReadyEdges(&ready_); +// TODO: jobserver client support for Windows +#ifndef _WIN32 + // If jobserver is used, return the token for this job. + if (builder_ && edge->has_job_token_) { + builder_->jobserver_.Release(); + edge->has_job_token_ = false; + } +#endif + // The rest of this function only applies to successful commands. if (result != kEdgeSucceeded) return true; @@ -594,7 +619,9 @@ void Plan::Dump() const { } struct RealCommandRunner : public CommandRunner { - explicit RealCommandRunner(const BuildConfig& config) : config_(config) {} + explicit RealCommandRunner(const BuildConfig& config, Jobserver& jobserver) : + config_(config), jobserver_(jobserver) {} + virtual ~RealCommandRunner() {} virtual size_t CanRunMore() const; virtual bool StartCommand(Edge* edge); @@ -603,6 +630,7 @@ struct RealCommandRunner : public CommandRunner { virtual void Abort(); const BuildConfig& config_; + Jobserver& jobserver_; SubprocessSet subprocs_; map subproc_to_edge_; }; @@ -617,6 +645,10 @@ vector RealCommandRunner::GetActiveEdges() { void RealCommandRunner::Abort() { subprocs_.Clear(); +// TODO: jobserver client support for Windows +#ifndef _WIN32 + jobserver_.Clear(); +#endif } size_t RealCommandRunner::CanRunMore() const { @@ -631,6 +663,18 @@ size_t RealCommandRunner::CanRunMore() const { capacity = load_capacity; } +// TODO: jobserver client support for Windows +#ifndef _WIN32 + int job_tokens = jobserver_.Tokens(); + + // When initialized, behave as if the implicit token is acquired already. + // Otherwise, this occurs after a token is released but before it is replaced, + // so the base capacity is represented by job_tokens + 1 when positive. + // Add an extra loop on capacity for each job in order to get an extra token. + if (job_tokens) + capacity = abs(job_tokens) - subproc_number + 2; +#endif + if (capacity < 0) capacity = 0; @@ -670,10 +714,10 @@ bool RealCommandRunner::WaitForCommand(Result* result) { return true; } -Builder::Builder(State* state, const BuildConfig& config, BuildLog* build_log, - DepsLog* deps_log, DiskInterface* disk_interface, - Status* status, int64_t start_time_millis) - : state_(state), config_(config), plan_(this), status_(status), +Builder::Builder(State* state, const BuildConfig& config, Jobserver& jobserver, + BuildLog* build_log, DepsLog* deps_log, DiskInterface* disk_interface, + Status* status, int64_t start_time_millis) : state_(state), + config_(config), jobserver_(jobserver), plan_(this), status_(status), start_time_millis_(start_time_millis), disk_interface_(disk_interface), explanations_(g_explaining ? new Explanations() : nullptr), scan_(state, build_log, deps_log, disk_interface, @@ -778,7 +822,7 @@ bool Builder::Build(string* err) { if (config_.dry_run) command_runner_.reset(new DryRunCommandRunner); else - command_runner_.reset(new RealCommandRunner(config_)); + command_runner_.reset(new RealCommandRunner(config_, jobserver_)); } // We are about to start the build process. diff --git a/src/build.h b/src/build.h index 9bb0c70b5c..117f0a5efe 100644 --- a/src/build.h +++ b/src/build.h @@ -24,6 +24,7 @@ #include "depfile_parser.h" #include "exit_status.h" #include "graph.h" +#include "jobserver.h" #include "util.h" // int64_t struct BuildLog; @@ -187,9 +188,9 @@ struct BuildConfig { /// Builder wraps the build process: starting commands, updating status. struct Builder { - Builder(State* state, const BuildConfig& config, BuildLog* build_log, - DepsLog* deps_log, DiskInterface* disk_interface, Status* status, - int64_t start_time_millis); + Builder(State* state, const BuildConfig& config, Jobserver& jobserver, + BuildLog* build_log, DepsLog* deps_log, DiskInterface* disk_interface, + Status* status, int64_t start_time_millis); ~Builder(); /// Clean up after interrupted commands by deleting output files. @@ -224,6 +225,7 @@ struct Builder { State* state_; const BuildConfig& config_; + Jobserver& jobserver_; Plan plan_; std::unique_ptr command_runner_; Status* status_; diff --git a/src/build_test.cc b/src/build_test.cc index c84190a040..2a8bb7de02 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -534,13 +534,15 @@ struct FakeCommandRunner : public CommandRunner { }; struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { - BuildTest() : config_(MakeConfig()), command_runner_(&fs_), status_(config_), - builder_(&state_, config_, NULL, NULL, &fs_, &status_, 0) { + BuildTest() : config_(MakeConfig()), jobserver_(MakeJobserver()), + command_runner_(&fs_), status_(config_), + builder_(&state_, config_, jobserver_, NULL, NULL, &fs_, &status_, 0) { } explicit BuildTest(DepsLog* log) - : config_(MakeConfig()), command_runner_(&fs_), status_(config_), - builder_(&state_, config_, NULL, log, &fs_, &status_, 0) {} + : config_(MakeConfig()), jobserver_(MakeJobserver()), + command_runner_(&fs_), status_(config_), + builder_(&state_, config_, jobserver_, NULL, log, &fs_, &status_, 0) {} virtual void SetUp() { StateTestWithBuiltinRules::SetUp(); @@ -577,7 +579,13 @@ struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { return config; } + Jobserver MakeJobserver() { + Jobserver jobserver; + return jobserver; + } + BuildConfig config_; + Jobserver jobserver_; FakeCommandRunner command_runner_; VirtualFileSystem fs_; StatusPrinter status_; @@ -610,7 +618,7 @@ void BuildTest::RebuildTarget(const string& target, const char* manifest, pdeps_log = &deps_log; } - Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_, &status_, 0); + Builder builder(pstate, config_, jobserver_, pbuild_log, pdeps_log, &fs_, &status_, 0); EXPECT_TRUE(builder.AddTarget(target, &err)); command_runner_.commands_ran_.clear(); @@ -2559,7 +2567,7 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2589,7 +2597,7 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2630,7 +2638,7 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2659,7 +2667,7 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2695,7 +2703,7 @@ TEST_F(BuildWithDepsLogTest, DepsIgnoredInDryRun) { // The deps log is NULL in dry runs. config_.dry_run = true; - Builder builder(&state, config_, NULL, NULL, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, NULL, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2730,7 +2738,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceCondition) { BuildLog::LogEntry* log_entry = NULL; { - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2750,7 +2758,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceCondition) { } { - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2772,7 +2780,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceCondition) { } { - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2811,7 +2819,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); { - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); // Run the build, out gets built, dep file is created @@ -2832,7 +2840,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { { // Trigger the build again - "out" will rebuild since its newest input mtime (header.h) // is newer than the recorded mtime of out in the build log - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2848,7 +2856,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { { // Trigger the build again - "out" won't rebuild since the file wasn't updated during // the previous build - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2867,7 +2875,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { { // Rebuild. This time, long-cc will cause header.h to be updated while the build is // in progress - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2883,7 +2891,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { { // Rebuild. Because header.h is now in the deplog for out, it should be detectable as // a change-while-in-progress and should cause a rebuild of out. - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2899,7 +2907,7 @@ TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { { // This time, the header.h file was not updated during the build, so the target should // not be considered dirty. - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2957,7 +2965,7 @@ TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2983,7 +2991,7 @@ TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) { ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -3016,7 +3024,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); @@ -3037,7 +3045,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); Edge* edge = state.edges_.back(); @@ -3087,7 +3095,7 @@ TEST_F(BuildWithDepsLogTest, DiscoveredDepDuringBuildChanged) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out2", &err)); EXPECT_FALSE(builder.AlreadyUpToDate()); @@ -3111,7 +3119,7 @@ TEST_F(BuildWithDepsLogTest, DiscoveredDepDuringBuildChanged) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out2", &err)); EXPECT_FALSE(builder.AlreadyUpToDate()); @@ -3134,7 +3142,7 @@ TEST_F(BuildWithDepsLogTest, DiscoveredDepDuringBuildChanged) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, &build_log, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out2", &err)); EXPECT_TRUE(builder.AlreadyUpToDate()); @@ -3162,7 +3170,7 @@ TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); ASSERT_EQ("", err); @@ -3185,7 +3193,7 @@ TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); state.GetNode("bar.h", 0)->MarkDirty(); // Mark bar.h as missing. @@ -4264,7 +4272,7 @@ TEST_F(BuildWithDepsLogTest, ValidationThroughDepfile) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out2", &err)); @@ -4300,7 +4308,7 @@ TEST_F(BuildWithDepsLogTest, ValidationThroughDepfile) { ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + Builder builder(&state, config_, jobserver_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out2", &err)); diff --git a/src/graph.h b/src/graph.h index 314c44296a..7f1496130c 100644 --- a/src/graph.h +++ b/src/graph.h @@ -227,6 +227,7 @@ struct Edge { bool deps_loaded_ = false; bool deps_missing_ = false; bool generated_by_dep_loader_ = false; + bool has_job_token_ = false; TimeStamp command_start_time_ = 0; const Rule& rule() const { return *rule_; } diff --git a/src/jobserver-posix.cc b/src/jobserver-posix.cc new file mode 100644 index 0000000000..ff3597baad --- /dev/null +++ b/src/jobserver-posix.cc @@ -0,0 +1,160 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jobserver.h" + +#include +#include + +#include +#include + +#include "util.h" + +Jobserver::Jobserver() { + assert(rfd_ < 0); + assert(wfd_ < 0); + + // Return early if no makeflags are passed in the environment + char* makeflags = std::getenv("MAKEFLAGS"); + if (makeflags == nullptr) { + return; + } + + // Tokenize string to characters in flag_, then words in flags_ + while (flag_char_ < strlen(makeflags)) { + while (flag_char_ < strlen(makeflags) && + !isblank(static_cast(makeflags[flag_char_]))) { + flag_.push_back(static_cast(makeflags[flag_char_])); + flag_char_++; + } + + if (flag_.size() > 0) + flags_.push_back(flag_); + + flag_.clear(); + flag_char_++; + } + + // Search for --jobserver-auth + for (size_t n = 0; n < flags_.size(); n++) + if (flags_[n].find(AUTH_KEY) == 0) + flag_ = flags_[n].substr(strlen(AUTH_KEY)); + + // Fallback to --jobserver-fds + if (flag_.empty()) + for (size_t n = 0; n < flags_.size(); n++) + if (flags_[n].find(FDS_KEY) == 0) + flag_ = flags_[n].substr(strlen(FDS_KEY)); + + jobserver_name_.assign(flag_); + + const char* jobserver = jobserver_name_.c_str(); + + // Return early if the flag's value is empty + if (jobserver_name_.empty()) { + Warning("invalid jobserver value: '%s'", jobserver); + return; + } + + jobserver_fifo_ = jobserver_name_.find(FIFO_KEY) == 0 ? true : false; + + // Return early if jobserver type is unknown (neither fifo nor pipe) + if (!jobserver_fifo_ && sscanf(jobserver, "%d,%d", &rfd_, &wfd_) != 2) { + Warning("invalid jobserver value: '%s'", jobserver); + return; + } + + if (jobserver_fifo_) { + rfd_ = open(jobserver + strlen(FIFO_KEY), O_RDONLY | O_NONBLOCK); + wfd_ = open(jobserver + strlen(FIFO_KEY), O_WRONLY); + } + + if (rfd_ == -1 || wfd_ == -1) + Fatal("failed to open jobserver: %s: %s", + jobserver, errno ? strerror(errno) : "Bad file descriptor"); + else if (rfd_ >= 0 && wfd_ >= 0) + Info("using jobserver: %s", jobserver); + else + Fatal("Make provided an invalid pipe: %s (mark the command as recursive)", + jobserver); + + // Signal that we have initialized but do not have a token yet + token_count_ = -1; +} + +Jobserver::~Jobserver() { + Clear(); + + if (rfd_ >= 0) + close(rfd_); + if (wfd_ >= 0) + close(wfd_); + + rfd_ = -1; + wfd_ = -1; +} + +bool Jobserver::Enabled() const { + return rfd_ >= 0 && wfd_ >= 0; +} + +int Jobserver::Tokens() const { + return token_count_; +} + +bool Jobserver::Acquire() { + // The first token is implicitly handed to a process + if (token_count_ <= 0) { + token_count_ = 1; + return true; + } + + char token; + int ret = read(rfd_, &token, 1); + if (ret < 0 && errno != EAGAIN && errno != EWOULDBLOCK) { + if (!jobserver_fifo_) + Warning("Make closed the pipe: %d (mark the command as recursive)", rfd_); + Fatal("failed to read token from jobserver: %d: %s", rfd_, strerror(errno)); + } + + if (ret > 0) + token_count_++; + + return ret > 0; +} + +void Jobserver::Release() { + if (token_count_ < 0) + token_count_ = 0; + if (token_count_ > 0) + token_count_--; + + // The first token is implicitly handed to a process + if (token_count_ == 0) + return; + + char token = '+'; + int ret = write(wfd_, &token, 1); + if (ret != 1) { + if (!jobserver_fifo_) + Warning("Make closed the pipe: %d (mark the command as recursive)", wfd_); + Fatal("failed to return token to jobserver: %d: %s", wfd_, strerror(errno)); + } +} + +void Jobserver::Clear() { + while (token_count_) + Release(); +} diff --git a/src/jobserver.h b/src/jobserver.h new file mode 100644 index 0000000000..f0b1a757ee --- /dev/null +++ b/src/jobserver.h @@ -0,0 +1,107 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#define AUTH_KEY "--jobserver-auth=" +#define FDS_KEY "--jobserver-fds=" +#define FIFO_KEY "fifo:" + +/// The GNU jobserver limits parallelism by assigning a token from an external +/// pool for each command. On posix systems, the pool is a fifo or simple pipe +/// with N characters. On windows systems, the pool is a semaphore initialized +/// to N. When a command is finished, the acquired token is released by writing +/// it back to the fifo or pipe or by increasing the semaphore count. +/// +/// The jobserver functionality is enabled by passing --jobserver-auth= +/// (previously --jobserver-fds= in older versions of Make) in the MAKEFLAGS +/// environment variable and creating the respective file descriptors or objects. +/// On posix systems, is 'fifo:' or ',' for pipes. +/// On windows systems, is the name of the semaphore. +/// +/// The class parses the MAKEFLAGS variable and opens the object handle if needed. +/// Once enabled, Acquire() must be called to acquire a token from the pool. +/// If a token is acquired, a new command can be started. +/// Once the command is completed, Release() must be called to return a token. +/// Make does not care which order a token is received or returned. +struct Jobserver { +// TODO: jobserver client support for Windows +#ifndef _WIN32 + /// Parse the MAKEFLAGS environment variable to receive the path / FDs / name + /// of the token pool, and open the handle to the pool if it is an object. + /// If a jobserver argument is found in the MAKEFLAGS environment variable, + /// and the handle is successfully opened, later calls to Enable() return true. + /// If a jobserver argument is found, but the handle fails to be opened, + /// the ninja process is aborted with an error. + Jobserver(); + + /// Before exiting Ninja, ensure that tokens are returned and handles closed. + ~Jobserver(); +#endif + + /// Return true if jobserver functionality is enabled and initialized. + bool Enabled() const; + + /// Return current token count or initialization signal if negative. + int Tokens() const; + + /// Implementation-specific method to acquire a token from the external pool + /// which is called for all tokens but returns early for the first token. + /// This method is called every time Ninja needs to start a command process. + /// Return true on success (token acquired), and false on failure (no tokens + /// available). First call always succeeds. Ninja is aborted on read errors. + bool Acquire(); + + /// Implementation-specific method to release a token to the external pool + /// which is called for all tokens but returns early for the last token. + /// Return a previously acquired token to the external token pool. + /// It must be called for each successful call to Acquire() after the command + /// even if subprocesses fail or in the case of errors causing Ninja to exit. + /// Ninja is aborted on write errors, and otherwise calls always succeed. + void Release(); + + /// Loop through Release() to return all tokens. Called before Ninja exits. + void Clear(); + +private: + /// The number of currently acquired tokens, or the jobserver status if negative. + /// Used to verify that all acquired tokens have been released before exiting, + /// and when the implicit (first) token has been acquired (initialization). + /// -1: initialized without a token + /// 0: uninitialized or disabled + /// +n: number of tokens in use + int token_count_ = 0; + + /// String of the parsed value of the jobserver flag passed to environment. + std::string jobserver_name_; + +// TODO: jobserver client support for Windows +#ifdef _WIN32 +#else + /// Whether the type of jobserver pipe supplied to ninja is named + bool jobserver_fifo_; + + /// File descriptors to communicate with upstream jobserver token pool. + int rfd_ = -1; + int wfd_ = -1; +#endif + + /// Helpers for initialization + std::vector flags_; + std::string flag_; + std::string::size_type flag_char_ = 0; +}; diff --git a/src/ninja.cc b/src/ninja.cc index 2902359f15..4351bca593 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -84,8 +84,8 @@ struct Options { /// The Ninja main() loads up a series of data structures; various tools need /// to poke into these, so store them as fields on an object. struct NinjaMain : public BuildLogUser { - NinjaMain(const char* ninja_command, const BuildConfig& config) : - ninja_command_(ninja_command), config_(config), + NinjaMain(const char* ninja_command, const BuildConfig& config, Jobserver& jobserver) : + ninja_command_(ninja_command), config_(config), jobserver_(jobserver), start_time_millis_(GetTimeMillis()) {} /// Command line used to run Ninja. @@ -94,6 +94,9 @@ struct NinjaMain : public BuildLogUser { /// Build configuration set from flags (e.g. parallelism). const BuildConfig& config_; + /// Reference to jobserver client. + Jobserver& jobserver_; + /// Loaded state (rules, nodes). State state_; @@ -267,7 +270,8 @@ bool NinjaMain::RebuildManifest(const char* input_file, string* err, if (!node) return false; - Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, + Builder builder(&state_, config_, jobserver_, + &build_log_, &deps_log_, &disk_interface_, status, start_time_millis_); if (!builder.AddTarget(node, err)) return false; @@ -1355,7 +1359,8 @@ int NinjaMain::RunBuild(int argc, char** argv, Status* status) { disk_interface_.AllowStatCache(g_experimental_statcache); - Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, + Builder builder(&state_, config_, jobserver_, + &build_log_, &deps_log_, &disk_interface_, status, start_time_millis_); for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { @@ -1542,6 +1547,8 @@ NORETURN void real_main(int argc, char** argv) { Status* status = Status::factory(config); + Jobserver jobserver; + if (options.working_dir) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for @@ -1558,14 +1565,14 @@ NORETURN void real_main(int argc, char** argv) { if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) { // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed // by other tools. - NinjaMain ninja(ninja_command, config); + NinjaMain ninja(ninja_command, config, jobserver); exit((ninja.*options.tool->func)(&options, argc, argv)); } // Limit number of rebuilds, to prevent infinite loops. const int kCycleLimit = 100; for (int cycle = 1; cycle <= kCycleLimit; ++cycle) { - NinjaMain ninja(ninja_command, config); + NinjaMain ninja(ninja_command, config, jobserver); ManifestParserOptions parser_opts; if (options.phony_cycle_should_err) {