From 993c613f954e6c6a6b17b6e8b75a4e5a49880ebe Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 7 Mar 2024 10:34:57 +1100 Subject: [PATCH 1/7] ci.yml: Loosened restrictions on branch names, now get version from spack.yaml, and restructured job dependencies --- .github/workflows/ci.yml | 267 ++++++++++++++++++++++----------------- 1 file changed, 150 insertions(+), 117 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b94101c..c96e393 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,52 +3,113 @@ on: pull_request: branches: - main + - backport/*.* paths: - config/** - spack.yaml +env: + SPACK_YAML_MODEL_YQ: .spack.specs[0] jobs: - branch-check: - name: Branch Name Compliance Check - # Branches that modify spack.yaml must be of the form: pre-*.*.* (ex: pre-2024.01.1) - # in order for the PR to access the '* Prerelease' GitHub Environments. - # We can't use regex in an `if` conditional, so we have to do more specific testing later. - if: startsWith(github.head_ref, 'pre-') + # There are a lot of interconnected jobs here. Here is a dependency diagram: + # validate-json ──> check-json ─────────┬─────────────────────────┐ + # ├> notifier ├> prerelease-deploy + # check-spack-yaml ──> deploy-versions ─┼> update-prerelease-tag ─┤ + # └─────────────────────────┘ + + # --------------------- + # | JSON-RELATED JOBS | + # --------------------- + validate-json: + name: Validate JSON + uses: access-nri/actions/.github/workflows/validate-json.yml@main + with: + src: "config" + + check-json: + name: Check JSON Fields + needs: + - validate-json runs-on: ubuntu-latest steps: - - name: Check + - uses: actions/checkout@v4 + + # The next two steps checkout the spack-{packages,config} repos to confirm that the versions in + # versions.json exist in the repositories. + - name: Setup + id: versions run: | - regex="pre-[0-9]+\.[0-9]+\.[0-9]+" - if [[ ! ${{ github.head_ref }} =~ $regex ]]; then - echo "::error::${{ github.head_ref }} doesn't match '$regex', so you will be unable to deploy prereleases. Please update the branch name to be in compliance." - exit 1 - fi + echo "packages=$(jq --compact-output --raw-output '."spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT + echo "config=$(jq --compact-output --raw-output '."spack-config"' ./config/versions.json)" >> $GITHUB_OUTPUT - changed: - name: Files Changed - runs-on: ubuntu-latest - needs: - - branch-check - outputs: - spack-yaml-changed: ${{ steps.filter.outputs.spack-yaml }} - json-changed: ${{ steps.filter.outputs.json }} - steps: - - uses: dorny/paths-filter@ad1ae68cd06927a8731fe67e877a2351c7a09691 #v2.9.3 - id: filter + - name: Spack Packages + id: spack-packages + continue-on-error: true + uses: actions/checkout@v4 + with: + repository: access-nri/spack-packages + ref: ${{ steps.versions.outputs.packages }} + path: packages + + - name: Spack Config + id: spack-config + continue-on-error: true + uses: actions/checkout@v4 with: - filters: | - spack-yaml: - - 'spack.yaml' - json: - - 'config/*.json' + repository: access-nri/spack-config + ref: ${{ steps.versions.outputs.config }} + path: config - spack-yaml-checks: + - name: Failure Notifier + if: contains(steps.*.outcome, 'failure') + run: | + if [[ "${{ steps.spack-packages.outcome }}" == "failure" ]]; then + echo "::error::spack-packages at the specified ref (${{ steps.versions.outputs.packages }}) doesn't exist." + fi + if [[ "${{ steps.spack-config.outcome }}" == "failure" ]]; then + echo "::error::spack-config at the specified ref (${{ steps.versions.outputs.config }}) doesn't exist." + fi + exit 1 + + # --------------------------- + # | SPACK.YAML-RELATED JOBS | + # --------------------------- + check-spack-yaml: name: Check spack.yaml runs-on: ubuntu-latest - needs: - - changed - if: ${{ needs.changed.outputs.spack-yaml-changed == 'true' }} + permissions: + pull-requests: write steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check Model Version Modified + id: version + run: | + git checkout ${{ github.base_ref }} + base_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) + + git checkout ${{ github.head_ref }} + current_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) + echo "current=${current_version}" >> $GITHUB_OUTPUT + + if [[ "${base_version}" == "${current_version}" ]]; then + echo "::warning::The version string hasn't been modified in this PR, but needs to be before merging." + exit 1 + fi + + - name: Same Model Version Failure Notifier + if: failure() && steps.version.outcome == 'failure' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BODY: | + The model version in the `spack.yaml` has not been updated. + Either update it manually, or comment the following to have it updated and committed automatically: + * `!bump major` for feature releases + * `!bump minor` for bugfixes + run: | + gh pr checkout ${{ github.event.pull_request.number }} + gh pr comment --body '${{ env.BODY }}' - name: Projection Version Matches # this step checks that the versions of the packages themselves match with the @@ -78,66 +139,11 @@ jobs: exit 1 fi - validate: - name: Validate JSON - needs: - - changed - if: ${{ needs.changed.outputs.json-changed == 'true' }} - uses: access-nri/actions/.github/workflows/validate-json.yml@main - with: - src: "config" - - check-versions-exist: - name: Check Versions Exist - runs-on: ubuntu-latest + get-versions: + name: Get Version and Build Number needs: - - changed - permissions: - pull-requests: write - steps: - - uses: actions/checkout@v4 - - - name: Setup - id: versions - run: | - echo "packages=$(jq --compact-output --raw-output '."spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT - echo "config=$(jq --compact-output --raw-output '."spack-config"' ./config/versions.json)" >> $GITHUB_OUTPUT - - # The next two steps checkout the spack-{packages,config} repos to confirm that the versions in - # versions.json exist in the repositories. - - name: Spack Packages - uses: actions/checkout@v4 - with: - repository: access-nri/spack-packages - ref: ${{ steps.versions.outputs.packages }} - path: packages - - - name: Spack Config - uses: actions/checkout@v4 - with: - repository: access-nri/spack-config - ref: ${{ steps.versions.outputs.config }} - path: config - - - name: Comment - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - BODY: | - This ${{ github.repository }} model will be deployed using: - * `access-nri/spack-packages` version [`${{ steps.versions.outputs.packages }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ steps.versions.outputs.packages }}) - * `access-nri/spack-config` version [`${{ steps.versions.outputs.config }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ steps.versions.outputs.config }}) - - If this is not what was expected, commit changes to `config/versions.json`. - run: | - gh pr checkout ${{ github.event.pull_request.number }} - gh pr comment --body '${{ env.BODY }}' - - prerelease-deploy-version: - name: Get Prerelease Number + - check-spack-yaml runs-on: ubuntu-latest - needs: - - branch-check outputs: version: ${{ steps.get-version.outputs.version-name }} version-build: ${{ steps.get-version-build.outputs.version-build-name }} @@ -149,24 +155,26 @@ jobs: - name: Generate Version Number id: get-version - # The step generates a general version number from the branch name, looking the + # The step generates a general version number from the spack.yaml, looking the # same as a regular release build. - # Ex. 'pre-2024.01.1' -> '2024.01.1' - run: version-name=$(cut --delimiter '-' --field 2 <<< "${{ github.head_ref }}") + # Ex. 'access-om2@git.2024.01.1' -> '2024.01.1' + run: | + access_om2_package=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) + echo "version-name=${access_om2_package/*@git./}" >> $GITHUB_OUTPUT - name: Generate Version-Build String id: get-version-build - # This step generates the version number for prereleases, which given a branch - # like `pre-`, looks like: `-`. - # Ex. `pre-2024.10.1` with 2 commits on branch -> `2024.10.1-2`. + # This step generates the version number for prereleases, + # which looks like: `-`. + # Ex. `2024.10.1` with 2 commits on branch -> `2024.10.1-2`. run: | - number-of-commits=$(git rev-list --count origin/main..HEAD) - echo "version-build-name=${{ steps.get-version.outputs.version-name }}-${number-of-commits}" >> $GITHUB_OUTPUT + number_of_commits=$(git rev-list --count ${{ github.event.pull_request.base.sha }}..HEAD) + echo "version-build-name=${{ steps.get-version.outputs.version-name }}-${number_of_commits}" >> $GITHUB_OUTPUT update-prerelease-tag: - name: Update Prerelease Tag ${{ needs.prerelease-deploy-version.outputs.version }} + name: Update Prerelease Tag ${{ needs.get-versions.outputs.version }} needs: - - prerelease-deploy-version + - get-versions runs-on: ubuntu-latest permissions: contents: write @@ -180,30 +188,55 @@ jobs: run: | git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} - git tag ${{ needs.prerelease-deploy-version.outputs.version }} --force + git tag ${{ needs.get-versions.outputs.version }} --force git push --tags --force - + # ----------------------------- + # | PRERELEASE DEPLOYMENT JOB | + # ----------------------------- prerelease-deploy: name: Deploy to Prerelease - # This will create a `spack` environment with the name `access-om2--` - # For example, `access-om2-2024.01.1-3` for the deployment based on the third commit on this - # `pre-2024.01.1` PR branch. + # This will create a `spack` environment with the name `access-om2--`. + # For example, `access-om2-2024_01_1-3` for the deployment based on the third commit on the PR branch. needs: - - prerelease-deploy-version - - spack-yaml-checks - - validate - - check-versions-exist - # The conditional below asserts that we should run this job as long as the result of the dependent jobs - # are not 'failures' or 'cancelled' (aka, either 'success' or 'skipped'). Skipped jobs may be created - # if certain files are not updated (for example, we don't need to check the validity of json if none - # has been updated in the PR), so we should treat 'skipped' jobs as a vacuous 'success'. - # The `always()` status function is there, because (if we do not give a different status function) - # by default there is an implicit 'if: success()' which would short-curcuit and skip the job. - if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + - get-versions # so we can give an appropriate version to the prerelease build + - update-prerelease-tag # implies all the spack.yaml-related checks have passed + - check-json # implies all the json-related checks have passed uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main with: type: prerelease ref: ${{ github.head_ref }} - version: ${{ needs.prerelease-deploy-version.outputs.version-build }} + version: ${{ needs.get-versions.outputs.version-build }} secrets: inherit + + notifier: + name: Notifier + runs-on: ubuntu-latest + needs: + - check-json + - get-versions + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + + - name: Comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + BODY: | + This `${{ github.repository }}` model will be deployed with the following versions: + * `${{ needs.get-versions.outputs.version }}` as a Release (when merged). + * `${{ needs.get-versions.outputs.version-build }}` as a Prerelease (during this PR). This can be accessed on `Gadi` via `spack` at `/g/data/vk83/prerelease/apps/spack/0.20/spack` once deployed. + + It will be deployed using: + * `access-nri/spack-packages` version [`${{ steps.versions.outputs.packages }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ steps.versions.outputs.packages }}) + * `access-nri/spack-config` version [`${{ steps.versions.outputs.config }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ steps.versions.outputs.config }}) + + If this is not what was expected, commit changes to `config/versions.json`. + run: | + gh pr checkout ${{ github.event.pull_request.number }} + gh pr comment --body '${{ env.BODY }}' From 9833e750d39c4124b263aac64e1170a20b79043a Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 7 Mar 2024 10:42:54 +1100 Subject: [PATCH 2/7] cd.yml: Loosened branch naming restriction and now get version from the spack.yaml --- .github/workflows/cd.yml | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 4e1442b..eb564bd 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,17 +3,17 @@ on: push: branches: - main + - backport/*.* paths: - config/** - spack.yaml - +env: + SPACK_YAML_MODEL_YQ: .spack.specs[0] jobs: generate-tag: name: Generate Tag Name - # Get the tag name from the branch that was merged into main, which - # is of the form `pre-`. - # We assume that this will always be the most recent merged PR, as - # this workflow kicks off immediately after the merge completes. + # Get the tag name from the `spack.yaml` that was merged into main, which + # is of the form `access-om2@git.`. runs-on: ubuntu-latest permissions: contents: write @@ -24,22 +24,12 @@ jobs: - name: Generate Tag id: tag - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - # We use the `gh` utility as it offers much easier traversal of GitHub- - # related things (prs, issues, etc...). - # In this merged_branch variable, we get the list of PRs that have been merged, - # and sort them by when they were merged, taking the most recent (last) and - # returning the branch name of that PR. - # The `cut` command splits the `pre-` branch into just ``, + # Get the tag name from the access-om2 spec in the `spack.yaml`. + # The `cut` command splits the `access-om2@git.` field into just ``, # which will be the new tag. run: | - merged_branch=$(gh pr list \ - --state merged \ - --json 'headRefName,mergedAt' \ - --jq 'sort_by(.mergedAt) | last | .headRefName') - echo "name=$(cut --delimiter '-' --fields 2 <<< '$merged_branch')" >> $GITHUB_OUTPUT + access_om2_package=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) + echo "name=${access_om2_package/*@git./}" >> $GITHUB_OUTPUT undeploy-prereleases: name: Undeploy Prereleases From 2cbeaf9f857200a8387683633f54e97c91a00db2 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 15 Mar 2024 13:47:33 +1100 Subject: [PATCH 3/7] cd.yml: Removed prerelease cleanup from cd --- .github/workflows/cd.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index eb564bd..6e8f129 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -15,8 +15,6 @@ jobs: # Get the tag name from the `spack.yaml` that was merged into main, which # is of the form `access-om2@git.`. runs-on: ubuntu-latest - permissions: - contents: write outputs: name: ${{ steps.tag.outputs.name }} steps: @@ -25,21 +23,10 @@ jobs: - name: Generate Tag id: tag # Get the tag name from the access-om2 spec in the `spack.yaml`. - # The `cut` command splits the `access-om2@git.` field into just ``, - # which will be the new tag. run: | access_om2_package=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) echo "name=${access_om2_package/*@git./}" >> $GITHUB_OUTPUT - undeploy-prereleases: - name: Undeploy Prereleases - needs: - - generate-tag - uses: access-nri/build-cd/.github/workflows/undeploy-1-setup.yml@main - with: - version-pattern: ${{ needs.generate-tag.outputs.name }}-* - secrets: inherit - push-tag: name: Tag Deployment needs: From c5b149d5fd4c5f52a18dfadee8f191eccd0280dd Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 7 Mar 2024 11:33:07 +1100 Subject: [PATCH 4/7] Added CONTRIBUTING.md and referenced this file in the README.md --- CONTRIBUTING.md | 41 +++++++++++++++++++++++++++++++++++++++++ README.md | 29 +++++++++++++++++------------ 2 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1b4be2a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Contributing to `ACCESS-OM2`s `spack` Environment + +## Via Pull Request + +### PRs for New Features + +For new features, the workflow is fairly straightforward. + +Simply open a pull request from your own branch to `main`, in which you make your modifications to the `spack.yaml` (and optionally the `config/versions.json`). The CI/CD will deploy these changes as a `Prerelease` version of `ACCESS-OM2` on `vk83/prerelease`, so you can verify that the changes will work before being merged formally into `main`. + +Some things to be mindful of: + +* Remember to update the version of the `access-om2` spec in the `spack.yaml` - this is the bit that looks like `- access-om2@git.VERSION`. It is this `VERSION` that will be eventually tagged to the commit. +* The `config/versions.json` file is used to bundle the deployment with a particular version of `spack-packages` and `spack-config`. Don't forget to change this if there are cool new features in either of the repositories that you want to incorporate into your deployment. + +### PRs for Backported Bugfixes + +Since we use the `main` branch as a place for the bleeding-edge `ACCESS-OM2` changes, how do we go about backporting bugfixes? + +We use dedicated `backport/*.*` branches for bugfixes and additions to past versions of `ACCESS-OM2`. For example, say we have `2024.01.1` version of `ACCESS-OM2` on `main`, which we need to backport a bugfix. + +We should: + +* Branch off that commit with a `backport/2024.01` branch (if it doesn't already exist) +* Open a PR off the `backport/2024.01` branch with the fixes, and when it is merged, will be tagged with `2024.01.2` on the `backport/2024.01` branch. + +We need to be mindful of the same things as merges of features into `main`: + +* Remember to update the version of the `access-om2` spec in the `spack.yaml` - this is the bit that looks like `- access-om2@git.VERSION`. It is this `VERSION` that will be eventually tagged to the commit. +* The `config/versions.json` file is used to bundle the deployment with a particular version of `spack-packages` and `spack-config`. Don't forget to change this if there are cool new features in either of the repositories that you want to incorporate into your deployment. + +> [!NOTE] +> For bugfixes, changing the versions for `spack-config` or `spack-packages` could have a large effect on the deployed model. Make sure this kind of fix is required. + +### More on the Prerelease Environment + +The `Prerelease` Environment is an ephemeral, completely separated place in `vk83` where you can test the changes you have made. Each commit in the PR is deployed to this environment, where you can test between versions in the PR, as well. + +These `spack env`s are of the form `access-om2-VERSION-BUILD`, such as `access-om2-2024_02_1-3` for the 3rd commit in the PR for `access-om2` version `2024.02.1`. + +These ephemeral `spack env`s are removed upon merging of the pull request, where the proper deployment is then done. diff --git a/README.md b/README.md index 326d831..fa9e172 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # ACCESS-OM2: ACCESS Ocean-Ice Model Release Configurations - + ## About the model - + ACCESS-OM2 is a global coupled ocean - sea ice model developed by [COSIMA](http://www.cosima.org.au). - + ACCESS-OM2 consists of the [MOM 5](https://github.com/ACCESS-NRI/MOM5) ocean model, [CICE 5](https://github.com/ACCESS-NRI/cice5) sea ice model, and a file-based atmosphere called [YATM](https://github.com/ACCESS-NRI/libaccessom2) coupled together using [OASIS3-MCT v2.0](https://github.com/ACCESS-NRI/oasis3-mct). ACCESS-OM2 builds on the ACCESS-OM ([Bi et al., 2013](http://www.bom.gov.au/jshess/docs/2013/bi2_hres.pdf)) and AusCOM ([Roberts et al., 2007](https://50years.acs.org.au/content/dam/acs/50-years/journals/jrpit/JRPIT39.2.137.pdf); [Bi and Marsland, 2010](https://www.cawcr.gov.au/technical-reports/CTR_027.pdf)) models originally developed at [CSIRO](http://www.csiro.au). - + The model code, configurations and performance were described in [Kiss et al. (2020)](https://doi.org/10.5194/gmd-13-401-2020), with further details in the draft [ACCESS-OM2 technical report](https://github.com/COSIMA/ACCESS-OM2-1-025-010deg-report). The current code and configurations differ from this version in a number of ways (biogeochemistry, updated forcing, improvements and bug fixes), as described by [Solodoch et al. (2022)](https://doi.org/10.1029/2021GL097211), [Hayashida et al. (2023)](https://dx.doi.org/10.1029/2023JC019697), [Menviel et al. (2023)](https://doi.org/10.5194/egusphere-2023-390) and [Wang et al. (2023)](https://doi.org/10.5194/gmd-2023-123). - + ## Support - + [ACCESS-NRI](https://www.access-nri.org.au) has assumed responsibility for supporting ACCESS-OM2 for the Australian Research Community. As part of this support ACCESS-NRI has developed a new build and deployment system for ACCESS-OM2 to align with plans for supporting a range of Earth System Models. Any questions about ACCESS-NRI releases of ACCESS-OM2 should be done through the [ACCESS-Hive Forum](https://forum.access-hive.org.au/). See the [ACCESS Help and Support topic](https://forum.access-hive.org.au/t/access-help-and-support/908) for details on how to do this. @@ -20,32 +20,37 @@ ACCESS-NRI is using [spack](https://spack.io), a build from source package manag Spack automatically builds all the components and their dependencies, producing model component executables. Spack already contains support for compiling thousands of common software packages. Spack packages for the components in ACCESS-OM2 are defined in the [spack packages repository](https://github.com/ACCESS-NRI/spack_packages/). -ACCESS-OM2 is built and deployed automatically to `gadi` on NCI (see below). However it is possible to use spack to compile the model using the `spack.yaml` environment file in this repository. To do so follow the [instructions on the ACCESS Forum for configuring spack on `gadi`](https://forum.access-hive.org.au/t/how-to-build-access-om2-on-gadi/1545). +ACCESS-OM2 is built and deployed automatically to `gadi` on NCI (see below). However it is possible to use spack to compile the model using the `spack.yaml` environment file in this repository. To do so follow the [instructions on the ACCESS Forum for configuring spack on `gadi`](https://forum.access-hive.org.au/t/how-to-build-access-om2-on-gadi/1545). Then clone this repository and run the following commands on `gadi`: -``` + +```bash spack env create access-om2 spack.yaml spack env activate access-om2 spack install ``` + to create a spack environment called `access-om2` and build all the ACCESS-OM2 components, the locations of which can be found using `spack find --paths`. -In contrast, the [COSIMA ACCESS-OM2 repository](https://github.com/COSIMA/access-om2) uses [submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to bring all the code dependencies into a single repository and build all the models together. +In contrast, the [COSIMA ACCESS-OM2 repository](https://github.com/COSIMA/access-om2) uses [submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to bring all the code dependencies into a single repository and build all the models together. ### Deployment - -ACCESS-OM2 is deployed automatically when a new version of the [`spack.yaml`](https://github.com/ACCESS-NRI/ACCESS-OM2/blob/main/spack.yaml) file is committed to this repository and tagged with a new version. All the ACCESS-OM2 components are built using `spack` on `gadi` and installed under the [`vk83`](https://my.nci.org.au/mancini/project/vk83) project in `/g/data/vk83`. It is necessary to be a member of [`vk83`](https://my.nci.org.au/mancini/project/vk83) project to use ACCESS-NRI deployments of ACCESS-OM2. + +ACCESS-OM2 is deployed automatically when a new version of the [`spack.yaml`](https://github.com/ACCESS-NRI/ACCESS-OM2/blob/main/spack.yaml) file is committed to `main` or a dedicated `backport/VERSION` branch. All the ACCESS-OM2 components are built using `spack` on `gadi` and installed under the [`vk83`](https://my.nci.org.au/mancini/project/vk83) project in `/g/data/vk83`. It is necessary to be a member of [`vk83`](https://my.nci.org.au/mancini/project/vk83) project to use ACCESS-NRI deployments of ACCESS-OM2. The deployment process also creates a GitHub release with the same tag. All releases are available under the [Releases page](https://github.com/ACCESS-NRI/ACCESS-OM2/releases). Each release has a changelog and meta-data with detailed information about the build and deployment, including: + - paths on `gadi` to all executables built in the deployment process (`spack.location`) - a `spack.lock` file, which is a complete build provenance document, listing all the components that were built and their dependencies, versions, compiler version, build flags and build architecture - the environment `spack.yaml` file used for deployment Additionally the deployment creates environment modulefiles, the [standard method for deploying software on `gadi`](https://opus.nci.org.au/display/Help/Environment+Modules). To view available ACCESS-OM2 versions: -``` + +```bash module use /g/data/vk83/apps/spack/0.20/release/modules/linux-rocky8-x86_64 module avail access-om2 ``` For users of ACCESS-OM2 model configurations released by ACCESS-NRI the exact location of the ACCESS-OM2 model executables is not required. Model configurations will be updated with new model components when necessary. +For information on contributing your own fixes to the ACCESS-OM2 `spack.yaml`, see the [CONTRIBUTING.md](./CONTRIBUTING.md) file. From 6f96e102583f8e55030fa5d76eb8c5144b572fb8 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 13 Mar 2024 16:40:09 +1100 Subject: [PATCH 5/7] Added comment.yml to handle pr comment commands, updated ci.ymls notifier job to get the spack-{packages,config} from the correct dependency --- .github/workflows/ci.yml | 7 ++- .github/workflows/comment.yml | 101 ++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/comment.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c96e393..2332eb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: needs: - validate-json runs-on: ubuntu-latest + outputs: + spack-packages-version: ${{ steps.versions.outputs.packages }} + spack-config-version: ${{ steps.versions.outputs.config }} steps: - uses: actions/checkout@v4 @@ -233,8 +236,8 @@ jobs: * `${{ needs.get-versions.outputs.version-build }}` as a Prerelease (during this PR). This can be accessed on `Gadi` via `spack` at `/g/data/vk83/prerelease/apps/spack/0.20/spack` once deployed. It will be deployed using: - * `access-nri/spack-packages` version [`${{ steps.versions.outputs.packages }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ steps.versions.outputs.packages }}) - * `access-nri/spack-config` version [`${{ steps.versions.outputs.config }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ steps.versions.outputs.config }}) + * `access-nri/spack-packages` version [`${{ needs.check-json.outputs.spack-packages-version }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.check-json.outputs.spack-packages-version }}) + * `access-nri/spack-config` version [`${{ needs.check-json.outputs.spack-config-version }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.check-json.outputs.spack-config-version }}) If this is not what was expected, commit changes to `config/versions.json`. run: | diff --git a/.github/workflows/comment.yml b/.github/workflows/comment.yml new file mode 100644 index 0000000..572ddc7 --- /dev/null +++ b/.github/workflows/comment.yml @@ -0,0 +1,101 @@ +name: Comment Command +on: + issue_comment: + types: + - created + - edited +env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SPACK_YAML_MODEL_YQ: .spack.specs[0] + SPACK_YAML_MODEL_PROJECTION_YQ: .spack.modules.default.tcl.projections.access-om2 +jobs: + bump-version: + name: Bump spack.yaml + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '!bump') + runs-on: ubuntu-latest + permissions: + pull-requests: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }} + + - name: Setup + id: setup + # outputs: + # original-version: The version contained within the spack.yaml + # version: The version that will be bumped (could be latest tag instead of original-version) + # bump: The bump type (major, minor or current as specified in the bump-version action) + run: | + # Get the version of access-om2 from the spack.yaml in the PR the comment was written in + gh pr checkout ${{ github.event.issue.number }} + access_om2_package=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) + original_version=${access_om2_package/*@git./} + echo "original-version=${original_version}" >> $GITHUB_OUTPUT + + # Validate the comment + if [[ "${{ contains(github.event.comment.body, 'major') }}" == "true" ]]; then + # Compare the current date (year-month) with the latest git tag (year-month) + # to determine the next valid tag. We do this because especially feature-rich + # months might increment the date part beyond the current date. + + d="$(date +%Y-%m)-01" + d_s=$(date --date "$d" +%s) + + latest_tag=$(git describe --tags --abbrev=0 | tr '.' '-') + tag_date=${latest_tag%-*}-01 + tag_date_s=$(date --date "$tag_date" +%s) + + echo "Comparing current date ${d} with ${tag_date} (tag looks like ${latest_tag})" + + if (( d_s <= tag_date_s )); then + echo "version=${tag_date}" >> $GITHUB_OUTPUT + echo "bump=major" >> $GITHUB_OUTPUT + else + echo "version=${original_version}" >> $GITHUB_OUTPUT + echo "bump=current" >> $GITHUB_OUTPUT + fi + elif [[ "${{ contains(github.event.comment.body, 'minor')}}" == "true" ]]; then + echo "version=${original_version}" >> $GITHUB_OUTPUT + echo "bump=minor" >> $GITHUB_OUTPUT + else + echo "::warning::Usage: `!bump [major|minor]`, got `${{ github.event.comment.body }}`" + exit 1 + fi + + - name: Bump Version + id: bump + uses: access-nri/actions/.github/actions/bump-version@main + with: + version: ${{ steps.setup.outputs.version }} + versioning-scheme: calver-minor + bump-type: ${{ steps.setup.outputs.bump }} + + - name: Update, Commit and Push the Bump + run: | + git config user.name ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} + git config user.email ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} + + yq -i '${{ env.SPACK_YAML_MODEL_YQ }} = "access-om2@git.${{ steps.bump.outputs.after }}"' spack.yaml + yq -i '${{ env.SPACK_YAML_MODEL_PROJECTION_YQ }} = "{name}/${{ steps.bump.outputs.after }}"' spack.yaml + git add spack.yaml + git commit -m "spack.yaml: Updated access-om2 package version from ${{ steps.setup.outputs.original-version }} to ${{ steps.bump.outputs.after }}" + git push + + - name: Success Notifier + env: + BODY: | + :white_check_mark: Version bumped from `${{ steps.setup.outputs.original-version }}` to `${{ steps.bump.outputs.after }}` :white_check_mark: + run: | + gh pr comment --body '${{ env.BODY }}' + + - name: Failure Notifier + if: failure() + env: + BODY: | + :x: Failed to bump version or commit changes, see ${{ env.RUN_URL }} :x: + run: gh pr comment --body '${{ env.BODY }}' From 6318f680c5556e9f5d9cebe109e1cf7719088ba8 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 15 Mar 2024 13:48:03 +1100 Subject: [PATCH 6/7] Added pr-closed.yml: Responsible for Prerelease cleanup after PR closed --- .github/workflows/pr-closed.yml | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/pr-closed.yml diff --git a/.github/workflows/pr-closed.yml b/.github/workflows/pr-closed.yml new file mode 100644 index 0000000..5d904de --- /dev/null +++ b/.github/workflows/pr-closed.yml @@ -0,0 +1,42 @@ +name: PR Closed Cleanup +# Remove prereleases that were part of a closed PR, so we save space +# on our deployment targets. If needed, one can still get the +# spack.yaml as part of the closed PR and revive it themselves. +on: + pull_request: + types: + - closed + branches: + - main + - backport/*.* + paths: + - config/** + - spack.yaml +env: + SPACK_YAML_MODEL_YQ: .spack.specs[0] +jobs: + get-prerelease-tag-pattern: + name: Get Prerelease Tag Pattern + # Get the tag name from the `spack.yaml` that was in the PR that was closed + # which is of the form `access-om2@git.`. + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.name }} + steps: + - uses: actions/checkout@v4 + + - name: Get Tag + id: tag + # Get the tag name from the access-om2 spec in the `spack.yaml`. + run: | + access_om2_package=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) + echo "name=${access_om2_package/*@git./}" >> $GITHUB_OUTPUT + + undeploy-prereleases: + name: Undeploy Prereleases + needs: + - get-prerelease-tag-pattern + uses: access-nri/build-cd/.github/workflows/undeploy-1-setup.yml@main + with: + version-pattern: ${{ needs.get-prerelease-tag-pattern.outputs.tag }}-* + secrets: inherit From 25028c4236bad83b93eff9f0da769177da402e0c Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 15 Mar 2024 14:10:59 +1100 Subject: [PATCH 7/7] CONTRIBUTING.md: Removed explicit requirement for version updating --- CONTRIBUTING.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b4be2a..c2d3d35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,6 @@ Simply open a pull request from your own branch to `main`, in which you make you Some things to be mindful of: -* Remember to update the version of the `access-om2` spec in the `spack.yaml` - this is the bit that looks like `- access-om2@git.VERSION`. It is this `VERSION` that will be eventually tagged to the commit. * The `config/versions.json` file is used to bundle the deployment with a particular version of `spack-packages` and `spack-config`. Don't forget to change this if there are cool new features in either of the repositories that you want to incorporate into your deployment. ### PRs for Backported Bugfixes @@ -26,7 +25,6 @@ We should: We need to be mindful of the same things as merges of features into `main`: -* Remember to update the version of the `access-om2` spec in the `spack.yaml` - this is the bit that looks like `- access-om2@git.VERSION`. It is this `VERSION` that will be eventually tagged to the commit. * The `config/versions.json` file is used to bundle the deployment with a particular version of `spack-packages` and `spack-config`. Don't forget to change this if there are cool new features in either of the repositories that you want to incorporate into your deployment. > [!NOTE] @@ -38,4 +36,4 @@ The `Prerelease` Environment is an ephemeral, completely separated place in `vk8 These `spack env`s are of the form `access-om2-VERSION-BUILD`, such as `access-om2-2024_02_1-3` for the 3rd commit in the PR for `access-om2` version `2024.02.1`. -These ephemeral `spack env`s are removed upon merging of the pull request, where the proper deployment is then done. +These ephemeral `spack env`s are removed upon closing of the pull request, where the proper deployment is then done if the PR is merged.