From c32a0604ce6a07d7a40b3a396f5ebaa62b9fa6a0 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sat, 27 Jan 2024 12:22:30 -0500 Subject: [PATCH 1/2] chore: Move recipe out to its own module (#18) --- src/commands/build.rs | 3 +- src/commands/local.rs | 3 +- src/commands/template.rs | 203 +------------------------------------ src/lib.rs | 1 + src/module_recipe.rs | 213 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 220 insertions(+), 203 deletions(-) diff --git a/src/commands/build.rs b/src/commands/build.rs index 3aeef382..cda7cc5b 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -30,10 +30,11 @@ use tokio::runtime::Runtime; use crate::{ commands::template::TemplateCommand, + module_recipe::Recipe, ops::{self, ARCHIVE_SUFFIX}, }; -use super::{template::Recipe, BlueBuildCommand}; +use super::BlueBuildCommand; #[derive(Debug, Clone, Args, TypedBuilder)] pub struct BuildCommand { diff --git a/src/commands/local.rs b/src/commands/local.rs index a9d3a6df..e4f8cebc 100644 --- a/src/commands/local.rs +++ b/src/commands/local.rs @@ -11,7 +11,8 @@ use typed_builder::TypedBuilder; use users::{Users, UsersCache}; use crate::{ - commands::{build::BuildCommand, template::Recipe}, + commands::build::BuildCommand, + module_recipe::Recipe, ops::{self, ARCHIVE_SUFFIX, LOCAL_BUILD}, }; diff --git a/src/commands/template.rs b/src/commands/template.rs index 345d9e52..8d7aa222 100644 --- a/src/commands/template.rs +++ b/src/commands/template.rs @@ -15,6 +15,8 @@ use serde::{Deserialize, Serialize}; use serde_yaml::Value; use typed_builder::TypedBuilder; +use crate::module_recipe::Recipe; + use super::BlueBuildCommand; #[derive(Debug, Clone, Template, TypedBuilder)] @@ -31,132 +33,6 @@ pub struct ContainerFileTemplate<'a> { #[template(path = "export.sh", escape = "none")] pub struct ExportsTemplate; -#[derive(Serialize, Clone, Deserialize, Debug, TypedBuilder)] -pub struct Recipe { - #[builder(setter(into))] - pub name: String, - - #[builder(setter(into))] - pub description: String, - - #[serde(alias = "base-image")] - #[builder(setter(into))] - pub base_image: String, - - #[serde(alias = "image-version")] - #[builder(setter(into))] - pub image_version: String, - - #[serde(alias = "blue-build-tag")] - #[builder(default, setter(into, strip_option))] - pub blue_build_tag: Option, - - #[serde(flatten)] - pub modules_ext: ModuleExt, - - #[serde(flatten)] - #[builder(setter(into))] - pub extra: IndexMap, -} - -impl Recipe { - #[must_use] - pub fn generate_tags(&self) -> Vec { - trace!("Recipe::generate_tags()"); - debug!("Generating image tags for {}", &self.name); - - let mut tags: Vec = Vec::new(); - let image_version = &self.image_version; - let timestamp = Local::now().format("%Y%m%d").to_string(); - - if let (Ok(commit_branch), Ok(default_branch), Ok(commit_sha), Ok(pipeline_source)) = ( - env::var("CI_COMMIT_REF_NAME"), - env::var("CI_DEFAULT_BRANCH"), - env::var("CI_COMMIT_SHORT_SHA"), - env::var("CI_PIPELINE_SOURCE"), - ) { - trace!("CI_COMMIT_REF_NAME={commit_branch}, CI_DEFAULT_BRANCH={default_branch},CI_COMMIT_SHORT_SHA={commit_sha}, CI_PIPELINE_SOURCE={pipeline_source}"); - warn!("Detected running in Gitlab, pulling information from CI variables"); - - if let Ok(mr_iid) = env::var("CI_MERGE_REQUEST_IID") { - trace!("CI_MERGE_REQUEST_IID={mr_iid}"); - if pipeline_source == "merge_request_event" { - debug!("Running in a MR"); - tags.push(format!("mr-{mr_iid}-{image_version}")); - } - } - - if default_branch == commit_branch { - debug!("Running on the default branch"); - tags.push(image_version.to_string()); - tags.push(format!("{image_version}-{timestamp}")); - tags.push(timestamp); - } else { - debug!("Running on branch {commit_branch}"); - tags.push(format!("{commit_branch}-{image_version}")); - } - - tags.push(format!("{commit_sha}-{image_version}")); - } else if let ( - Ok(github_event_name), - Ok(github_event_number), - Ok(github_sha), - Ok(github_ref_name), - ) = ( - env::var("GITHUB_EVENT_NAME"), - env::var("PR_EVENT_NUMBER"), - env::var("GITHUB_SHA"), - env::var("GITHUB_REF_NAME"), - ) { - trace!("GITHUB_EVENT_NAME={github_event_name},PR_EVENT_NUMBER={github_event_number},GITHUB_SHA={github_sha},GITHUB_REF_NAME={github_ref_name}"); - warn!("Detected running in Github, pulling information from GITHUB variables"); - - let mut short_sha = github_sha; - short_sha.truncate(7); - - if github_event_name == "pull_request" { - debug!("Running in a PR"); - tags.push(format!("pr-{github_event_number}-{image_version}")); - } else if github_ref_name == "live" { - tags.push(image_version.to_owned()); - tags.push(format!("{image_version}-{timestamp}")); - tags.push("latest".to_string()); - } else { - tags.push(format!("br-{github_ref_name}-{image_version}")); - } - tags.push(format!("{short_sha}-{image_version}")); - } else { - warn!("Running locally"); - tags.push(format!("{image_version}-local")); - } - info!("Finished generating tags!"); - debug!("Tags: {tags:#?}"); - tags - } -} - -#[derive(Serialize, Clone, Deserialize, Debug, Template, TypedBuilder)] -#[template(path = "Containerfile.module", escape = "none")] -pub struct ModuleExt { - #[builder(default, setter(into))] - pub modules: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] -pub struct Module { - #[serde(rename = "type")] - #[builder(default, setter(into, strip_option))] - pub module_type: Option, - - #[serde(rename = "from-file")] - #[builder(default, setter(into, strip_option))] - pub from_file: Option, - - #[serde(flatten)] - #[builder(default, setter(into))] - pub config: IndexMap, -} - #[derive(Debug, Clone, Args, TypedBuilder)] pub struct TemplateCommand { /// The recipe file to create a template from @@ -228,81 +104,6 @@ fn print_script(script_contents: &ExportsTemplate) -> String { ) } -fn get_containerfile_list(module: &Module) -> Option> { - if module.module_type.as_ref()? == "containerfile" { - Some( - module - .config - .get("containerfiles")? - .as_sequence()? - .iter() - .filter_map(|t| Some(t.as_str()?.to_owned())) - .collect(), - ) - } else { - None - } -} - -fn print_containerfile(containerfile: &str) -> String { - trace!("print_containerfile({containerfile})"); - debug!("Loading containerfile contents for {containerfile}"); - - let path = format!("config/containerfiles/{containerfile}/Containerfile"); - - let file = fs::read_to_string(&path).unwrap_or_else(|e| { - error!("Failed to read file {path}: {e}"); - process::exit(1); - }); - - trace!("Containerfile contents {path}:\n{file}"); - - file -} - -fn get_module_from_file(file_name: &str) -> String { - trace!("get_module_from_file({file_name})"); - - let io_err_fn = |e| { - error!("Failed to read module {file_name}: {e}"); - process::exit(1); - }; - - let file_path = PathBuf::from("config").join(file_name); - - let file = fs::read_to_string(file_path).unwrap_or_else(io_err_fn); - - let serde_err_fn = |e| { - error!("Failed to deserialize module {file_name}: {e}"); - process::exit(1); - }; - - let template_err_fn = |e| { - error!("Failed to render module {file_name}: {e}"); - process::exit(1); - }; - - serde_yaml::from_str::(file.as_str()).map_or_else( - |_| { - let module = serde_yaml::from_str::(file.as_str()).unwrap_or_else(serde_err_fn); - - ModuleExt::builder() - .modules(vec![module]) - .build() - .render() - .unwrap_or_else(template_err_fn) - }, - |module_ext| module_ext.render().unwrap_or_else(template_err_fn), - ) -} - -fn print_module_context(module: &Module) -> String { - serde_json::to_string(module).unwrap_or_else(|e| { - error!("Failed to parse module: {e}"); - process::exit(1); - }) -} - fn running_gitlab_actions() -> bool { trace!(" running_gitlab_actions()"); diff --git a/src/lib.rs b/src/lib.rs index 740a0ffc..82642f81 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,4 +9,5 @@ #![allow(clippy::module_name_repetitions)] pub mod commands; +pub mod module_recipe; mod ops; diff --git a/src/module_recipe.rs b/src/module_recipe.rs index 8b137891..818428b9 100644 --- a/src/module_recipe.rs +++ b/src/module_recipe.rs @@ -1 +1,214 @@ +use std::{env, fs, path::PathBuf, process}; +use askama::Template; +use chrono::Local; +use indexmap::IndexMap; +use log::{debug, error, info, trace, warn}; +use serde::{Deserialize, Serialize}; +use serde_yaml::Value; +use typed_builder::TypedBuilder; + +#[derive(Serialize, Clone, Deserialize, Debug, TypedBuilder)] +pub struct Recipe { + #[builder(setter(into))] + pub name: String, + + #[builder(setter(into))] + pub description: String, + + #[serde(alias = "base-image")] + #[builder(setter(into))] + pub base_image: String, + + #[serde(alias = "image-version")] + #[builder(setter(into))] + pub image_version: String, + + #[serde(alias = "blue-build-tag")] + #[builder(default, setter(into, strip_option))] + pub blue_build_tag: Option, + + #[serde(flatten)] + pub modules_ext: ModuleExt, + + #[serde(flatten)] + #[builder(setter(into))] + pub extra: IndexMap, +} + +impl Recipe { + #[must_use] + pub fn generate_tags(&self) -> Vec { + trace!("Recipe::generate_tags()"); + debug!("Generating image tags for {}", &self.name); + + let mut tags: Vec = Vec::new(); + let image_version = &self.image_version; + let timestamp = Local::now().format("%Y%m%d").to_string(); + + if let (Ok(commit_branch), Ok(default_branch), Ok(commit_sha), Ok(pipeline_source)) = ( + env::var("CI_COMMIT_REF_NAME"), + env::var("CI_DEFAULT_BRANCH"), + env::var("CI_COMMIT_SHORT_SHA"), + env::var("CI_PIPELINE_SOURCE"), + ) { + trace!("CI_COMMIT_REF_NAME={commit_branch}, CI_DEFAULT_BRANCH={default_branch},CI_COMMIT_SHORT_SHA={commit_sha}, CI_PIPELINE_SOURCE={pipeline_source}"); + warn!("Detected running in Gitlab, pulling information from CI variables"); + + if let Ok(mr_iid) = env::var("CI_MERGE_REQUEST_IID") { + trace!("CI_MERGE_REQUEST_IID={mr_iid}"); + if pipeline_source == "merge_request_event" { + debug!("Running in a MR"); + tags.push(format!("mr-{mr_iid}-{image_version}")); + } + } + + if default_branch == commit_branch { + debug!("Running on the default branch"); + tags.push(image_version.to_string()); + tags.push(format!("{image_version}-{timestamp}")); + tags.push(timestamp); + } else { + debug!("Running on branch {commit_branch}"); + tags.push(format!("{commit_branch}-{image_version}")); + } + + tags.push(format!("{commit_sha}-{image_version}")); + } else if let ( + Ok(github_event_name), + Ok(github_event_number), + Ok(github_sha), + Ok(github_ref_name), + ) = ( + env::var("GITHUB_EVENT_NAME"), + env::var("PR_EVENT_NUMBER"), + env::var("GITHUB_SHA"), + env::var("GITHUB_REF_NAME"), + ) { + trace!("GITHUB_EVENT_NAME={github_event_name},PR_EVENT_NUMBER={github_event_number},GITHUB_SHA={github_sha},GITHUB_REF_NAME={github_ref_name}"); + warn!("Detected running in Github, pulling information from GITHUB variables"); + + let mut short_sha = github_sha; + short_sha.truncate(7); + + if github_event_name == "pull_request" { + debug!("Running in a PR"); + tags.push(format!("pr-{github_event_number}-{image_version}")); + } else if github_ref_name == "live" { + tags.push(image_version.to_owned()); + tags.push(format!("{image_version}-{timestamp}")); + tags.push("latest".to_string()); + } else { + tags.push(format!("br-{github_ref_name}-{image_version}")); + } + tags.push(format!("{short_sha}-{image_version}")); + } else { + warn!("Running locally"); + tags.push(format!("{image_version}-local")); + } + info!("Finished generating tags!"); + debug!("Tags: {tags:#?}"); + tags + } +} + +#[derive(Serialize, Clone, Deserialize, Debug, Template, TypedBuilder)] +#[template(path = "Containerfile.module", escape = "none")] +pub struct ModuleExt { + #[builder(default, setter(into))] + pub modules: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] +pub struct Module { + #[serde(rename = "type")] + #[builder(default, setter(into, strip_option))] + pub module_type: Option, + + #[serde(rename = "from-file")] + #[builder(default, setter(into, strip_option))] + pub from_file: Option, + + #[serde(flatten)] + #[builder(default, setter(into))] + pub config: IndexMap, +} + +// ======================================================== // +// ========================= Helpers ====================== // +// ======================================================== // + +fn get_containerfile_list(module: &Module) -> Option> { + if module.module_type.as_ref()? == "containerfile" { + Some( + module + .config + .get("containerfiles")? + .as_sequence()? + .iter() + .filter_map(|t| Some(t.as_str()?.to_owned())) + .collect(), + ) + } else { + None + } +} + +fn print_containerfile(containerfile: &str) -> String { + trace!("print_containerfile({containerfile})"); + debug!("Loading containerfile contents for {containerfile}"); + + let path = format!("config/containerfiles/{containerfile}/Containerfile"); + + let file = fs::read_to_string(&path).unwrap_or_else(|e| { + error!("Failed to read file {path}: {e}"); + process::exit(1); + }); + + trace!("Containerfile contents {path}:\n{file}"); + + file +} + +fn get_module_from_file(file_name: &str) -> String { + trace!("get_module_from_file({file_name})"); + + let io_err_fn = |e| { + error!("Failed to read module {file_name}: {e}"); + process::exit(1); + }; + + let file_path = PathBuf::from("config").join(file_name); + + let file = fs::read_to_string(file_path).unwrap_or_else(io_err_fn); + + let serde_err_fn = |e| { + error!("Failed to deserialize module {file_name}: {e}"); + process::exit(1); + }; + + let template_err_fn = |e| { + error!("Failed to render module {file_name}: {e}"); + process::exit(1); + }; + + serde_yaml::from_str::(file.as_str()).map_or_else( + |_| { + let module = serde_yaml::from_str::(file.as_str()).unwrap_or_else(serde_err_fn); + + ModuleExt::builder() + .modules(vec![module]) + .build() + .render() + .unwrap_or_else(template_err_fn) + }, + |module_ext| module_ext.render().unwrap_or_else(template_err_fn), + ) +} + +fn print_module_context(module: &Module) -> String { + serde_json::to_string(module).unwrap_or_else(|e| { + error!("Failed to parse module: {e}"); + process::exit(1); + }) +} From 097dbb391a7a836f9800ded27839336f8b52644c Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sat, 27 Jan 2024 12:44:54 -0500 Subject: [PATCH 2/2] fix: Improve workflow for main branch and PRs (#17) This makes it so that the main branch will push images but PRs won't. This also puts integration tests back in the `+all` target. --- .github/workflows/build-pr.yml | 27 +++++++++++++++++++++++++++ .github/workflows/build.yml | 12 ++++++------ .github/workflows/tag.yml | 2 +- Earthfile | 4 ++++ 4 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-pr.yml diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml new file mode 100644 index 00000000..9e79c94e --- /dev/null +++ b/.github/workflows/build-pr.yml @@ -0,0 +1,27 @@ +name: Earthly PR +build + +on: + pull_request: + +env: + FORCE_COLOR: 1 + +jobs: + build: + timeout-minutes: 60 + runs-on: ubuntu-latest + + steps: + - uses: earthly/actions-setup@v1 + with: + use-cache: true + version: v0.8.2 + + # Setup repo and add caching + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.ref }} + + - name: Run build + run: earthly --ci -P +build + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15757646..ab41bccf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,9 +1,7 @@ -name: Earthly +build +name: Earthly main branch +build on: workflow_dispatch: - merge_group: - pull_request: push: branches: - main @@ -15,17 +13,19 @@ jobs: build: permissions: packages: write - timeout-minutes: 30 + timeout-minutes: 60 runs-on: ubuntu-latest steps: - uses: earthly/actions-setup@v1 with: use-cache: true - version: v0.8.0 + version: v0.8.2 # Setup repo and add caching - uses: actions/checkout@v4 + with: + ref: main - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -35,4 +35,4 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Run build - run: earthly --push --ci -P +all + run: earthly --push --ci -P +build diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index ee2bf0f2..034139bb 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -1,4 +1,4 @@ -name: Earthly +tag +name: Earthly tag +build on: push: diff --git a/Earthfile b/Earthfile index 31e301f2..c924376f 100644 --- a/Earthfile +++ b/Earthfile @@ -5,6 +5,10 @@ IMPORT github.com/blue-build/earthly-lib/cargo AS cargo ARG --global IMAGE=ghcr.io/blue-build/cli all: + BUILD +build + BUILD +integration-tests --NIGHTLY=true --NIGHTLY=false + +build: BUILD +default BUILD +nightly