From cc239ed1d1ca387c443636a4a8185956c37b3100 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 3 Mar 2023 19:54:11 +0100 Subject: [PATCH] start implementing parallel job execution --- src/alias.rs | 5 +- src/analyzer.rs | 9 +-- src/dependency.rs | 3 +- src/justfile.rs | 120 +++++++++++++++++++++++++-------------- src/keyed.rs | 3 +- src/lib.rs | 1 - src/recipe_resolver.rs | 17 +++--- src/unresolved_recipe.rs | 3 +- 8 files changed, 101 insertions(+), 60 deletions(-) diff --git a/src/alias.rs b/src/alias.rs index 0c19fe5a5e..cc8f3a5f61 100644 --- a/src/alias.rs +++ b/src/alias.rs @@ -1,8 +1,9 @@ use super::*; +use std::sync::Arc; /// An alias, e.g. `name := target` #[derive(Debug, PartialEq, Clone, Serialize)] -pub(crate) struct Alias<'src, T = Rc>> { +pub(crate) struct Alias<'src, T = Arc>> { pub(crate) attributes: BTreeSet, pub(crate) name: Name<'src>, #[serde( @@ -17,7 +18,7 @@ impl<'src> Alias<'src, Name<'src>> { self.name.line } - pub(crate) fn resolve(self, target: Rc>) -> Alias<'src> { + pub(crate) fn resolve(self, target: Arc>) -> Alias<'src> { assert_eq!(self.target.lexeme(), target.name.lexeme()); Alias { diff --git a/src/analyzer.rs b/src/analyzer.rs index 6899796311..986f3a2531 100644 --- a/src/analyzer.rs +++ b/src/analyzer.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use {super::*, CompileErrorKind::*}; const VALID_ALIAS_ATTRIBUTES: [Attribute; 1] = [Attribute::Private]; @@ -71,11 +72,11 @@ impl<'src> Analyzer<'src> { first: recipes .values() .fold(None, |accumulator, next| match accumulator { - None => Some(Rc::clone(next)), + None => Some(Arc::clone(next)), Some(previous) => Some(if previous.line_number() < next.line_number() { previous } else { - Rc::clone(next) + Arc::clone(next) }), }), aliases, @@ -173,7 +174,7 @@ impl<'src> Analyzer<'src> { } fn resolve_alias( - recipes: &Table<'src, Rc>>, + recipes: &Table<'src, Arc>>, alias: Alias<'src, Name<'src>>, ) -> CompileResult<'src, Alias<'src>> { let token = alias.name.token(); @@ -187,7 +188,7 @@ impl<'src> Analyzer<'src> { // Make sure the target recipe exists match recipes.get(alias.target.lexeme()) { - Some(target) => Ok(alias.resolve(Rc::clone(target))), + Some(target) => Ok(alias.resolve(Arc::clone(target))), None => Err(token.error(UnknownAliasTarget { alias: alias.name.lexeme(), target: alias.target.lexeme(), diff --git a/src/dependency.rs b/src/dependency.rs index 2d1da1173d..f9de949972 100644 --- a/src/dependency.rs +++ b/src/dependency.rs @@ -1,10 +1,11 @@ use super::*; +use std::sync::Arc; #[derive(PartialEq, Debug, Serialize)] pub(crate) struct Dependency<'src> { pub(crate) arguments: Vec>, #[serde(serialize_with = "keyed::serialize")] - pub(crate) recipe: Rc>, + pub(crate) recipe: Arc>, } impl<'src> Display for Dependency<'src> { diff --git a/src/justfile.rs b/src/justfile.rs index 1919999548..3d13008526 100644 --- a/src/justfile.rs +++ b/src/justfile.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use {super::*, serde::Serialize}; #[derive(Debug, PartialEq, Serialize)] @@ -5,8 +6,8 @@ pub(crate) struct Justfile<'src> { pub(crate) aliases: Table<'src, Alias<'src>>, pub(crate) assignments: Table<'src, Assignment<'src>>, #[serde(serialize_with = "keyed::serialize_option")] - pub(crate) first: Option>>, - pub(crate) recipes: Table<'src, Rc>>, + pub(crate) first: Option>>, + pub(crate) recipes: Table<'src, Arc>>, pub(crate) settings: Settings<'src>, pub(crate) warnings: Vec, } @@ -253,10 +254,21 @@ impl<'src> Justfile<'src> { search, }; - let mut ran = BTreeSet::new(); - for (recipe, arguments) in grouped { - Self::run_recipe(&context, recipe, arguments, &dotenv, search, &mut ran)?; - } + // let mut ran = BTreeSet::new(); + std::thread::scope(|scope| -> RunResult<'src, ()> { + let mut threads = Vec::new(); + for (recipe, arguments) in grouped { + threads.push(scope.spawn(|| { + Self::run_recipe( + &context, recipe, arguments, &dotenv, search, /*&mut ran*/ + ) + })); + } + for thread in threads { + thread.join().unwrap()?; + } + Ok(()) + })?; Ok(()) } @@ -269,7 +281,7 @@ impl<'src> Justfile<'src> { self .recipes .get(name) - .map(Rc::as_ref) + .map(Arc::as_ref) .or_else(|| self.aliases.get(name).map(|alias| alias.target.as_ref())) } @@ -279,16 +291,16 @@ impl<'src> Justfile<'src> { arguments: &[&str], dotenv: &BTreeMap, search: &Search, - ran: &mut BTreeSet>, + // ran: &mut BTreeSet>, ) -> RunResult<'src, ()> { let mut invocation = vec![recipe.name().to_owned()]; for argument in arguments { invocation.push((*argument).to_string()); } - if ran.contains(&invocation) { - return Ok(()); - } + // if ran.contains(&invocation) { + // return Ok(()); + // } let (outer, positional) = Evaluator::evaluate_parameters( context.config, @@ -305,46 +317,70 @@ impl<'src> Justfile<'src> { let mut evaluator = Evaluator::recipe_evaluator(context.config, dotenv, &scope, context.settings, search); - for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) { - let arguments = arguments - .iter() - .map(|argument| evaluator.evaluate_expression(argument)) - .collect::>>()?; - - Self::run_recipe( - context, - recipe, - &arguments.iter().map(String::as_ref).collect::>(), - dotenv, - search, - ran, - )?; - } + std::thread::scope(|scope| -> RunResult<'src, ()> { + let mut threads = Vec::new(); + for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) { + let arguments = arguments + .iter() + .map(|argument| evaluator.evaluate_expression(argument)) + .collect::>>()?; + + threads.push(scope.spawn(move || { + Self::run_recipe( + context, + recipe, + &arguments.iter().map(String::as_ref).collect::>(), + dotenv, + search, + // ran, + ) + })); + } + + for thread in threads { + thread.join().unwrap()?; + } + Ok(()) + })?; recipe.run(context, dotenv, scope.child(), search, &positional)?; { - let mut ran = BTreeSet::new(); - - for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) { - let mut evaluated = Vec::new(); + // let mut ran = BTreeSet::new(); + + std::thread::scope(|scope| -> RunResult<'src, ()> { + let mut threads = Vec::new(); + for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) { + let mut evaluated = Vec::new(); + + for argument in arguments { + evaluated.push( + evaluator + .evaluate_expression(argument) + .expect("error evaluating expression"), + ); + } - for argument in arguments { - evaluated.push(evaluator.evaluate_expression(argument)?); + threads.push(scope.spawn(move || { + Self::run_recipe( + context, + recipe, + &evaluated.iter().map(String::as_ref).collect::>(), + dotenv, + search, + // &mut ran, + ) + })); } - Self::run_recipe( - context, - recipe, - &evaluated.iter().map(String::as_ref).collect::>(), - dotenv, - search, - &mut ran, - )?; - } + for thread in threads { + thread.join().unwrap()?; + } + Ok(()) + })?; } - ran.insert(invocation); + // ran.insert(invocation); Ok(()) } diff --git a/src/keyed.rs b/src/keyed.rs index e171451eb8..42bb34f8c4 100644 --- a/src/keyed.rs +++ b/src/keyed.rs @@ -1,10 +1,11 @@ use super::*; +use std::sync::Arc; pub(crate) trait Keyed<'key> { fn key(&self) -> &'key str; } -impl<'key, T: Keyed<'key>> Keyed<'key> for Rc { +impl<'key, T: Keyed<'key>> Keyed<'key> for Arc { fn key(&self) -> &'key str { self.as_ref().key() } diff --git a/src/lib.rs b/src/lib.rs index c69bfb6bc2..5d1cdb359a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,7 +50,6 @@ pub(crate) use { ops::{Index, Range, RangeInclusive}, path::{self, Path, PathBuf}, process::{self, Command, ExitStatus, Stdio}, - rc::Rc, str::{self, Chars}, sync::{Mutex, MutexGuard}, usize, vec, diff --git a/src/recipe_resolver.rs b/src/recipe_resolver.rs index 2715acc240..2873fa34b0 100644 --- a/src/recipe_resolver.rs +++ b/src/recipe_resolver.rs @@ -1,8 +1,9 @@ +use std::sync::Arc; use {super::*, CompileErrorKind::*}; pub(crate) struct RecipeResolver<'src: 'run, 'run> { unresolved_recipes: Table<'src, UnresolvedRecipe<'src>>, - resolved_recipes: Table<'src, Rc>>, + resolved_recipes: Table<'src, Arc>>, assignments: &'run Table<'src, Assignment<'src>>, } @@ -10,7 +11,7 @@ impl<'src: 'run, 'run> RecipeResolver<'src, 'run> { pub(crate) fn resolve_recipes( unresolved_recipes: Table<'src, UnresolvedRecipe<'src>>, assignments: &Table<'src, Assignment<'src>>, - ) -> CompileResult<'src, Table<'src, Rc>>> { + ) -> CompileResult<'src, Table<'src, Arc>>> { let mut resolver = RecipeResolver { resolved_recipes: Table::new(), unresolved_recipes, @@ -72,20 +73,20 @@ impl<'src: 'run, 'run> RecipeResolver<'src, 'run> { &mut self, stack: &mut Vec<&'src str>, recipe: UnresolvedRecipe<'src>, - ) -> CompileResult<'src, Rc>> { + ) -> CompileResult<'src, Arc>> { if let Some(resolved) = self.resolved_recipes.get(recipe.name()) { - return Ok(Rc::clone(resolved)); + return Ok(Arc::clone(resolved)); } stack.push(recipe.name()); - let mut dependencies: Vec> = Vec::new(); + let mut dependencies: Vec> = Vec::new(); for dependency in &recipe.dependencies { let name = dependency.recipe.lexeme(); if let Some(resolved) = self.resolved_recipes.get(name) { // dependency already resolved - dependencies.push(Rc::clone(resolved)); + dependencies.push(Arc::clone(resolved)); } else if stack.contains(&name) { let first = stack[0]; stack.push(first); @@ -113,8 +114,8 @@ impl<'src: 'run, 'run> RecipeResolver<'src, 'run> { stack.pop(); - let resolved = Rc::new(recipe.resolve(dependencies)?); - self.resolved_recipes.insert(Rc::clone(&resolved)); + let resolved = Arc::new(recipe.resolve(dependencies)?); + self.resolved_recipes.insert(Arc::clone(&resolved)); Ok(resolved) } } diff --git a/src/unresolved_recipe.rs b/src/unresolved_recipe.rs index 0a49443de8..0af5c666c8 100644 --- a/src/unresolved_recipe.rs +++ b/src/unresolved_recipe.rs @@ -1,11 +1,12 @@ use super::*; +use std::sync::Arc; pub(crate) type UnresolvedRecipe<'src> = Recipe<'src, UnresolvedDependency<'src>>; impl<'src> UnresolvedRecipe<'src> { pub(crate) fn resolve( self, - resolved: Vec>>, + resolved: Vec>>, ) -> CompileResult<'src, Recipe<'src>> { assert_eq!( self.dependencies.len(),