Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallel #1738

Closed
wants to merge 13 commits into from
Next Next commit
start implementing parallel job execution
syphar committed Mar 4, 2023

Verified

This commit was signed with the committer’s verified signature.
Lehmann-Fabian Fabian Lehmann
commit cc239ed1d1ca387c443636a4a8185956c37b3100
5 changes: 3 additions & 2 deletions src/alias.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use super::*;
use std::sync::Arc;

/// An alias, e.g. `name := target`
#[derive(Debug, PartialEq, Clone, Serialize)]
pub(crate) struct Alias<'src, T = Rc<Recipe<'src>>> {
pub(crate) struct Alias<'src, T = Arc<Recipe<'src>>> {
pub(crate) attributes: BTreeSet<Attribute>,
pub(crate) name: Name<'src>,
#[serde(
@@ -17,7 +18,7 @@ impl<'src> Alias<'src, Name<'src>> {
self.name.line
}

pub(crate) fn resolve(self, target: Rc<Recipe<'src>>) -> Alias<'src> {
pub(crate) fn resolve(self, target: Arc<Recipe<'src>>) -> Alias<'src> {
assert_eq!(self.target.lexeme(), target.name.lexeme());

Alias {
9 changes: 5 additions & 4 deletions src/analyzer.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use std::sync::Arc;
use {super::*, CompileErrorKind::*};

const VALID_ALIAS_ATTRIBUTES: [Attribute; 1] = [Attribute::Private];
@@ -71,11 +72,11 @@ impl<'src> Analyzer<'src> {
first: recipes
.values()
.fold(None, |accumulator, next| match accumulator {
None => Some(Rc::clone(next)),
None => Some(Arc::clone(next)),
Some(previous) => Some(if previous.line_number() < next.line_number() {
previous
} else {
Rc::clone(next)
Arc::clone(next)
}),
}),
aliases,
@@ -173,7 +174,7 @@ impl<'src> Analyzer<'src> {
}

fn resolve_alias(
recipes: &Table<'src, Rc<Recipe<'src>>>,
recipes: &Table<'src, Arc<Recipe<'src>>>,
alias: Alias<'src, Name<'src>>,
) -> CompileResult<'src, Alias<'src>> {
let token = alias.name.token();
@@ -187,7 +188,7 @@ impl<'src> Analyzer<'src> {

// Make sure the target recipe exists
match recipes.get(alias.target.lexeme()) {
Some(target) => Ok(alias.resolve(Rc::clone(target))),
Some(target) => Ok(alias.resolve(Arc::clone(target))),
None => Err(token.error(UnknownAliasTarget {
alias: alias.name.lexeme(),
target: alias.target.lexeme(),
3 changes: 2 additions & 1 deletion src/dependency.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
use super::*;
use std::sync::Arc;

#[derive(PartialEq, Debug, Serialize)]
pub(crate) struct Dependency<'src> {
pub(crate) arguments: Vec<Expression<'src>>,
#[serde(serialize_with = "keyed::serialize")]
pub(crate) recipe: Rc<Recipe<'src>>,
pub(crate) recipe: Arc<Recipe<'src>>,
}

impl<'src> Display for Dependency<'src> {
120 changes: 78 additions & 42 deletions src/justfile.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
use std::sync::Arc;
use {super::*, serde::Serialize};

#[derive(Debug, PartialEq, Serialize)]
pub(crate) struct Justfile<'src> {
pub(crate) aliases: Table<'src, Alias<'src>>,
pub(crate) assignments: Table<'src, Assignment<'src>>,
#[serde(serialize_with = "keyed::serialize_option")]
pub(crate) first: Option<Rc<Recipe<'src>>>,
pub(crate) recipes: Table<'src, Rc<Recipe<'src>>>,
pub(crate) first: Option<Arc<Recipe<'src>>>,
pub(crate) recipes: Table<'src, Arc<Recipe<'src>>>,
pub(crate) settings: Settings<'src>,
pub(crate) warnings: Vec<Warning>,
}
@@ -253,10 +254,21 @@ impl<'src> Justfile<'src> {
search,
};

let mut ran = BTreeSet::new();
for (recipe, arguments) in grouped {
Self::run_recipe(&context, recipe, arguments, &dotenv, search, &mut ran)?;
}
// let mut ran = BTreeSet::new();
std::thread::scope(|scope| -> RunResult<'src, ()> {
let mut threads = Vec::new();
for (recipe, arguments) in grouped {
threads.push(scope.spawn(|| {
Self::run_recipe(
&context, recipe, arguments, &dotenv, search, /*&mut ran*/
)
}));
}
for thread in threads {
thread.join().unwrap()?;
}
Ok(())
})?;

Ok(())
}
@@ -269,7 +281,7 @@ impl<'src> Justfile<'src> {
self
.recipes
.get(name)
.map(Rc::as_ref)
.map(Arc::as_ref)
.or_else(|| self.aliases.get(name).map(|alias| alias.target.as_ref()))
}

@@ -279,16 +291,16 @@ impl<'src> Justfile<'src> {
arguments: &[&str],
dotenv: &BTreeMap<String, String>,
search: &Search,
ran: &mut BTreeSet<Vec<String>>,
// ran: &mut BTreeSet<Vec<String>>,
) -> RunResult<'src, ()> {
let mut invocation = vec![recipe.name().to_owned()];
for argument in arguments {
invocation.push((*argument).to_string());
}

if ran.contains(&invocation) {
return Ok(());
}
// if ran.contains(&invocation) {
// return Ok(());
// }

let (outer, positional) = Evaluator::evaluate_parameters(
context.config,
@@ -305,46 +317,70 @@ impl<'src> Justfile<'src> {
let mut evaluator =
Evaluator::recipe_evaluator(context.config, dotenv, &scope, context.settings, search);

for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) {
let arguments = arguments
.iter()
.map(|argument| evaluator.evaluate_expression(argument))
.collect::<RunResult<Vec<String>>>()?;

Self::run_recipe(
context,
recipe,
&arguments.iter().map(String::as_ref).collect::<Vec<&str>>(),
dotenv,
search,
ran,
)?;
}
std::thread::scope(|scope| -> RunResult<'src, ()> {
let mut threads = Vec::new();
for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) {
let arguments = arguments
.iter()
.map(|argument| evaluator.evaluate_expression(argument))
.collect::<RunResult<Vec<String>>>()?;

threads.push(scope.spawn(move || {
Self::run_recipe(
context,
recipe,
&arguments.iter().map(String::as_ref).collect::<Vec<&str>>(),
dotenv,
search,
// ran,
)
}));
}

for thread in threads {
thread.join().unwrap()?;
}
Ok(())
})?;

recipe.run(context, dotenv, scope.child(), search, &positional)?;

{
let mut ran = BTreeSet::new();

for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) {
let mut evaluated = Vec::new();
// let mut ran = BTreeSet::new();

std::thread::scope(|scope| -> RunResult<'src, ()> {
let mut threads = Vec::new();
for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) {
let mut evaluated = Vec::new();

for argument in arguments {
evaluated.push(
evaluator
.evaluate_expression(argument)
.expect("error evaluating expression"),
);
}

for argument in arguments {
evaluated.push(evaluator.evaluate_expression(argument)?);
threads.push(scope.spawn(move || {
Self::run_recipe(
context,
recipe,
&evaluated.iter().map(String::as_ref).collect::<Vec<&str>>(),
dotenv,
search,
// &mut ran,
)
}));
}

Self::run_recipe(
context,
recipe,
&evaluated.iter().map(String::as_ref).collect::<Vec<&str>>(),
dotenv,
search,
&mut ran,
)?;
}
for thread in threads {
thread.join().unwrap()?;
}
Ok(())
})?;
}

ran.insert(invocation);
// ran.insert(invocation);
Ok(())
}

3 changes: 2 additions & 1 deletion src/keyed.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
use super::*;
use std::sync::Arc;

pub(crate) trait Keyed<'key> {
fn key(&self) -> &'key str;
}

impl<'key, T: Keyed<'key>> Keyed<'key> for Rc<T> {
impl<'key, T: Keyed<'key>> Keyed<'key> for Arc<T> {
fn key(&self) -> &'key str {
self.as_ref().key()
}
1 change: 0 additions & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
@@ -50,7 +50,6 @@ pub(crate) use {
ops::{Index, Range, RangeInclusive},
path::{self, Path, PathBuf},
process::{self, Command, ExitStatus, Stdio},
rc::Rc,
str::{self, Chars},
sync::{Mutex, MutexGuard},
usize, vec,
17 changes: 9 additions & 8 deletions src/recipe_resolver.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
use std::sync::Arc;
use {super::*, CompileErrorKind::*};

pub(crate) struct RecipeResolver<'src: 'run, 'run> {
unresolved_recipes: Table<'src, UnresolvedRecipe<'src>>,
resolved_recipes: Table<'src, Rc<Recipe<'src>>>,
resolved_recipes: Table<'src, Arc<Recipe<'src>>>,
assignments: &'run Table<'src, Assignment<'src>>,
}

impl<'src: 'run, 'run> RecipeResolver<'src, 'run> {
pub(crate) fn resolve_recipes(
unresolved_recipes: Table<'src, UnresolvedRecipe<'src>>,
assignments: &Table<'src, Assignment<'src>>,
) -> CompileResult<'src, Table<'src, Rc<Recipe<'src>>>> {
) -> CompileResult<'src, Table<'src, Arc<Recipe<'src>>>> {
let mut resolver = RecipeResolver {
resolved_recipes: Table::new(),
unresolved_recipes,
@@ -72,20 +73,20 @@ impl<'src: 'run, 'run> RecipeResolver<'src, 'run> {
&mut self,
stack: &mut Vec<&'src str>,
recipe: UnresolvedRecipe<'src>,
) -> CompileResult<'src, Rc<Recipe<'src>>> {
) -> CompileResult<'src, Arc<Recipe<'src>>> {
if let Some(resolved) = self.resolved_recipes.get(recipe.name()) {
return Ok(Rc::clone(resolved));
return Ok(Arc::clone(resolved));
}

stack.push(recipe.name());

let mut dependencies: Vec<Rc<Recipe>> = Vec::new();
let mut dependencies: Vec<Arc<Recipe>> = Vec::new();
for dependency in &recipe.dependencies {
let name = dependency.recipe.lexeme();

if let Some(resolved) = self.resolved_recipes.get(name) {
// dependency already resolved
dependencies.push(Rc::clone(resolved));
dependencies.push(Arc::clone(resolved));
} else if stack.contains(&name) {
let first = stack[0];
stack.push(first);
@@ -113,8 +114,8 @@ impl<'src: 'run, 'run> RecipeResolver<'src, 'run> {

stack.pop();

let resolved = Rc::new(recipe.resolve(dependencies)?);
self.resolved_recipes.insert(Rc::clone(&resolved));
let resolved = Arc::new(recipe.resolve(dependencies)?);
self.resolved_recipes.insert(Arc::clone(&resolved));
Ok(resolved)
}
}
3 changes: 2 additions & 1 deletion src/unresolved_recipe.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
use super::*;
use std::sync::Arc;

pub(crate) type UnresolvedRecipe<'src> = Recipe<'src, UnresolvedDependency<'src>>;

impl<'src> UnresolvedRecipe<'src> {
pub(crate) fn resolve(
self,
resolved: Vec<Rc<Recipe<'src>>>,
resolved: Vec<Arc<Recipe<'src>>>,
) -> CompileResult<'src, Recipe<'src>> {
assert_eq!(
self.dependencies.len(),