Implement GNU jobserver posix client support

The core principle of a jobserver is simple:
Before starting a new job (edge in ninja-speak),
a token must be acquired from an external entity as approval.

Once a job is finished, the token must be returned to signal a free thread.
In the case of GNU Make, this external entity is the parent process
which has executed ninja and is managing the load capacity for
all subprocesses which it has spawned. Introducing client support
for this model allows ninja to give load capacity management
to this parent process, allowing it to control the number of
subprocesses that ninja spawns at any given time.

This functionality is desirable when ninja is used as part of a bigger build,
such as builds with Yocto/OpenEmbedded, Openwrt/Linux, Buildroot and Android.
Here, multiple compile jobs are executed in parallel
in order to maximize cpu utilization, but if each compile job
uses all available cores, the system is overloaded.

This implementation instantiates the client in real_main()
and passes references to the class into other classes.
All tokens are returned whenever the CommandRunner aborts,
and the current number of tokens compared to the current number
of running subprocesses controls the available load capacity,
used to determine how many new tokens to attempt to acquire
in order to start another job for each loop to find work.

Calls to functions are excluded from Windows builds
pending Windows-specific support for the jobserver.

Co-authored-by: Martin Hundebøll <[email protected]>
Co-developed-by: Martin Hundebøll <[email protected]>
Signed-off-by: Martin Hundebøll <[email protected]>
Signed-off-by: Michael Pratt <[email protected]>

CMakeLists.txt

@@ -169,7 +169,10 @@ if(WIN32)
 	# errors by telling windows.h to not define those two.
 	add_compile_definitions(NOMINMAX)
 else()
-	target_sources(libninja PRIVATE src/subprocess-posix.cc)
+	target_sources(libninja PRIVATE
+		src/subprocess-posix.cc
+		src/jobserver-posix.cc
+	)
 	if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
 		target_sources(libninja PRIVATE src/getopt.c)
 		# Build getopt.c, which can be compiled as either C or C++, as C++

configure.py

@@ -565,6 +565,7 @@ def has_re2c() -> bool:
     objs += cc('getopt')
 else:
     objs += cxx('subprocess-posix')
+    objs += cxx('jobserver-posix')
 if platform.is_aix():
     objs += cc('getopt')
 if platform.is_msvc():

src/build.cc

@@ -164,8 +164,24 @@ Edge* Plan::FindWork() {
   if (ready_.empty())
     return NULL;
 
+// TODO: jobserver client support for Windows
+#ifndef _WIN32
+  // Only initiate work if the jobserver can acquire a token.
+  if (builder_ && builder_->jobserver_.Enabled() && !builder_->jobserver_.Acquire()) {
+    return NULL;
+  }
+#endif
+
   Edge* work = ready_.top();
   ready_.pop();
+
+// TODO: jobserver client support for Windows
+#ifndef _WIN32
+  // Mark this edge as using a job token to be released when work is finished.
+  if (builder_)
+    work->has_job_token_ = builder_->jobserver_.Enabled();
+#endif
+
   return work;
 }
 
@@ -201,6 +217,15 @@ bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) {
     edge->pool()->EdgeFinished(*edge);
   edge->pool()->RetrieveReadyEdges(&ready_);
 
+// TODO: jobserver client support for Windows
+#ifndef _WIN32
+  // If jobserver is used, return the token for this job.
+  if (builder_ && edge->has_job_token_) {
+    builder_->jobserver_.Release();
+    edge->has_job_token_ = false;
+  }
+#endif
+
   // The rest of this function only applies to successful commands.
   if (result != kEdgeSucceeded)
     return true;
@@ -594,7 +619,9 @@ void Plan::Dump() const {
 }
 
 struct RealCommandRunner : public CommandRunner {
-  explicit RealCommandRunner(const BuildConfig& config) : config_(config) {}
+  explicit RealCommandRunner(const BuildConfig& config, Jobserver& jobserver) :
+      config_(config), jobserver_(jobserver) {}
+
   virtual ~RealCommandRunner() {}
   virtual size_t CanRunMore() const;
   virtual bool StartCommand(Edge* edge);
@@ -603,6 +630,7 @@ struct RealCommandRunner : public CommandRunner {
   virtual void Abort();
 
   const BuildConfig& config_;
+  Jobserver& jobserver_;
   SubprocessSet subprocs_;
   map<const Subprocess*, Edge*> subproc_to_edge_;
 };
@@ -617,6 +645,10 @@ vector<Edge*> RealCommandRunner::GetActiveEdges() {
 
 void RealCommandRunner::Abort() {
   subprocs_.Clear();
+// TODO: jobserver client support for Windows
+#ifndef _WIN32
+  jobserver_.Clear();
+#endif
 }
 
 size_t RealCommandRunner::CanRunMore() const {
@@ -631,6 +663,18 @@ size_t RealCommandRunner::CanRunMore() const {
       capacity = load_capacity;
   }
 
+// TODO: jobserver client support for Windows
+#ifndef _WIN32
+  int job_tokens = jobserver_.Tokens();
+
+  // When initialized, behave as if the implicit token is acquired already.
+  // Otherwise, this occurs after a token is released but before it is replaced,
+  // so the base capacity is represented by job_tokens + 1 when positive.
+  // Add an extra loop on capacity for each job in order to get an extra token.
+  if (job_tokens)
+    capacity = abs(job_tokens) - subproc_number + 2;
+#endif
+
   if (capacity < 0)
     capacity = 0;
 
@@ -670,10 +714,10 @@ bool RealCommandRunner::WaitForCommand(Result* result) {
   return true;
 }
 
-Builder::Builder(State* state, const BuildConfig& config, BuildLog* build_log,
-                 DepsLog* deps_log, DiskInterface* disk_interface,
-                 Status* status, int64_t start_time_millis)
-    : state_(state), config_(config), plan_(this), status_(status),
+Builder::Builder(State* state, const BuildConfig& config, Jobserver& jobserver,
+                 BuildLog* build_log, DepsLog* deps_log, DiskInterface* disk_interface,
+                 Status* status, int64_t start_time_millis) : state_(state),
+      config_(config), jobserver_(jobserver), plan_(this), status_(status),
       start_time_millis_(start_time_millis), disk_interface_(disk_interface),
       explanations_(g_explaining ? new Explanations() : nullptr),
       scan_(state, build_log, deps_log, disk_interface,
@@ -778,7 +822,7 @@ bool Builder::Build(string* err) {
     if (config_.dry_run)
       command_runner_.reset(new DryRunCommandRunner);
     else
-      command_runner_.reset(new RealCommandRunner(config_));
+      command_runner_.reset(new RealCommandRunner(config_, jobserver_));
   }
 
   // We are about to start the build process.

src/build.h

@@ -24,6 +24,7 @@
 #include "depfile_parser.h"
 #include "exit_status.h"
 #include "graph.h"
+#include "jobserver.h"
 #include "util.h"  // int64_t
 
 struct BuildLog;
@@ -187,9 +188,9 @@ struct BuildConfig {
 
 /// Builder wraps the build process: starting commands, updating status.
 struct Builder {
-  Builder(State* state, const BuildConfig& config, BuildLog* build_log,
-          DepsLog* deps_log, DiskInterface* disk_interface, Status* status,
-          int64_t start_time_millis);
+  Builder(State* state, const BuildConfig& config, Jobserver& jobserver,
+          BuildLog* build_log, DepsLog* deps_log, DiskInterface* disk_interface,
+          Status* status, int64_t start_time_millis);
   ~Builder();
 
   /// Clean up after interrupted commands by deleting output files.
@@ -224,6 +225,7 @@ struct Builder {
 
   State* state_;
   const BuildConfig& config_;
+  Jobserver& jobserver_;
   Plan plan_;
   std::unique_ptr<CommandRunner> command_runner_;
   Status* status_;