From 750446fad9609cba9d6af9d3f8b4cdb2c2444d01 Mon Sep 17 00:00:00 2001 From: Chris Tomkins-Tinch Date: Fri, 3 Mar 2023 11:08:26 -0500 Subject: [PATCH] migration from Travis CI to GitHub Actions (#30) * commit changes beginning migration from Travis CI to GitHub Actions * update GitHub cache action update GitHub cache action (v2 -> v3) https://github.com/marketplace/actions/cache in connection with deprecation of the save-state command ( https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ ) * update docker login action v1 -> v2 * bump docker/setup-buildx-action v1 -> v2 * update GitHub checkout action v2 -> v3 * update GitHub setup-python action v2 -> v4 * update docker/setup-buildx-action@v1 -> v2 in docs build action * bump ReadTheDocs python version 3.6 -> 3.8 * update viral-baseimage to 2.2.0 containing upgrade from Ubuntu 18.04 LTS (bionic) to Ubuntu 22.04 LTS (jammy) * remove Travis CI-related files * update README build status badge to reflect GitHub Actions state * change gap2seq version to gap2seq 3.1.1a to reflect new build on bioconda; bump viral-core to 2.2.1 --- .github/workflows/build.yml | 245 ++++++++++++++++++ .readthedocs.yml | 3 +- .travis.yml | 83 ------ Dockerfile | 2 +- README.md | 2 +- docs/conf.py | 2 +- docs/requirements.txt | 15 +- github_actions_ci/deploy-docker.sh | 10 + .../install-pip-docs.sh | 0 .../list-docker-tags.sh | 18 +- requirements-conda.txt | 2 +- travis/deploy-docker.sh | 25 -- 12 files changed, 278 insertions(+), 129 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml create mode 100755 github_actions_ci/deploy-docker.sh rename {travis => github_actions_ci}/install-pip-docs.sh (100%) rename {travis => github_actions_ci}/list-docker-tags.sh (76%) delete mode 100755 travis/deploy-docker.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..a4487c5d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,245 @@ +name: "viral-assemble CI" + +on: + push: + pull_request: + branches: + - master + release: + types: + - created + schedule: + - cron: '0 18 * * 1' # weekly at 18:00 on Mondays + +env: + HOME: "${{ github.workspace }}" + CACHE_DIR: "${{ github.workspace }}/misc_cache" + MINICONDA_DIR: "${{ github.workspace }}/miniconda" + GATK_PATH: "${{ github.workspace }}/misc_cache" + PYTHONIOENCODING: UTF8 + + DOCKER_REGISTRY: "quay.io" + DOCKER_REPO_PROD: "quay.io/broadinstitute/viral-assemble" + DOCKER_REPO_DEV: "quay.io/broadinstitute/viral-assemble" + + #BOTO_CONFIG: /dev/null # bogus value to override config on travis + _JAVA_OPTIONS: "-Xmx3g" # overrides jvm opts set elsewhere; see: https://stackoverflow.com/questions/28327620/difference-between-java-options-java-tool-options-and-java-opts + + # TravisCI variables described here: + # https://docs.travis-ci.com/user/environment-variables/ + # GitHub Actions environment variables and context described here: + # https://docs.github.com/en/actions/reference/environment-variables + # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#env-context + GITHUB_ACTIONS_COMMIT: ${{ github.sha }} + GITHUB_ACTIONS_BUILD_DIR: ${{ github.workspace }} + #GITHUB_ACTIONS_BRANCH: ${{ github.base_ref }} + GITHUB_ACTIONS_BRANCH: ${{ github.ref }} + GITHUB_ACTIONS_PULL_REQUEST: ${{ github.event.number }} + GITHUB_ACTIONS_PULL_REQUEST_BRANCH: ${{ github.head_ref }} + GITHUB_ACTIONS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }} + GITHUB_ACTIONS_BASE_REF: ${{ github.base_ref }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + build_docker: + runs-on: ubuntu-20.04 + steps: + - name: checkout repository + uses: actions/checkout@v3 + # fetch git tags (tagged releases) because + # actions/checkout@v3 does either a full checkout or a shallow checkout without tags + - name: fetch tags + run: git fetch --prune --unshallow --tags + - name: Programmatic environment setup + run: | + set -e -x + # $GITHUB_ENV is available for subsequent steps + GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') + echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV + # + # Set GITHUB_ACTIONS_BRANCH + # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) + # for push builds, or builds not triggered by a pull request, this is the name of the branch. + # for builds triggered by a pull request this is the name of the branch targeted by the pull request. + # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). + # if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build + if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then + GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} + # if event_name=="release", this is a tagged build + elif [[ "${{ github.event_name }}" == "release" ]]; then + GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG + else + GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) + fi + echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" + echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV + + - name: Login to docker registry + uses: docker/login-action@v2 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + # cache; see: https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows + - name: setup cache + id: docker_cache + uses: actions/cache@v3 + env: + cache-name: old-docker-tag + with: + path: "$CACHE_DIR" + key: ${{ runner.os }}-${{ env.cache-name }} + # - name: Pull old docker image from cache + # if: steps.docker_cache.outputs.cache-hit != 'true' + # run: | + - name: Attempt to pull older image from cache + if: steps.docker_cache.outputs.cache-hit == 'true' + shell: bash + run: | + set -e + # restore old tag from cache if present + if [ -f "$CACHE_DIR/old-docker-tag.txt" ]; then + OLD_DOCKER_TAG=$(cat $CACHE_DIR/old-docker-tag.txt) + else + OLD_DOCKER_TAG=$DOCKER_REPO_PROD + fi + echo "old docker tag = $OLD_DOCKER_TAG" + + # attempt to pull tag from cache + if docker pull $OLD_DOCKER_TAG; then + echo _CACHE_FROM="--cache-from $OLD_DOCKER_TAG" >> $GITHUB_ENV + else + echo "_CACHE_FROM=" >> $GITHUB_ENV + fi + - name: Build docker image + shell: bash + run: | + set -e -x + git describe --tags --always | tee VERSION + + if [ -n "$GITHUB_ACTIONS_TAG" ]; then + echo "Release $GITHUB_ACTIONS_TAG" + elif [ -n "$GITHUB_ACTIONS_PULL_REQUEST_BRANCH" ]; then + echo "LABEL quay.expires-after=10w" >> Dockerfile + elif [[ "$GITHUB_ACTIONS_BRANCH" != "master" ]]; then + echo "LABEL quay.expires-after=10w" >> Dockerfile + fi + + echo "Building with cache from: $_CACHE_FROM" + docker build -t local/build-container:build $_CACHE_FROM . + - name: Deploy docker image + run: | + github_actions_ci/deploy-docker.sh + - name: store tag ID in cache + run: | + mkdir -p $CACHE_DIR + github_actions_ci/list-docker-tags.sh | tail -1 | tee $CACHE_DIR/old-docker-tag.txt + + test_py36_in_docker: + needs: build_docker + runs-on: ubuntu-20.04 + env: + GITHUB_ACTIONS_PYTHON_VERSION: 3.8 + PYTEST_ADDOPTS: "-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov read_utils --cov reports --cov tools --cov util --cov file_utils" + steps: + - name: checkout repository + uses: actions/checkout@v3 + # fetch git tags (tagged releases) because + # actions/checkout@v3 does either a full checkout or a shallow checkout without tags + - name: fetch tags + run: git fetch --prune --unshallow --tags + - name: Programmatic environment setup + run: | + set -e -x + # $GITHUB_ENV is available for subsequent steps + GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') + echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV + # + # Set GITHUB_ACTIONS_BRANCH + # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) + # for push builds, or builds not triggered by a pull request, this is the name of the branch. + # for builds triggered by a pull request this is the name of the branch targeted by the pull request. + # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). + # if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build + if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then + GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} + # if event_name=="release", this is a tagged build + elif [[ "${{ github.event_name }}" == "release" ]]; then + GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG + else + GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) + fi + echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" + echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV + - name: install python + uses: actions/setup-python@v4 + with: + python-version: "${{ env.GITHUB_ACTIONS_PYTHON_VERSION }}" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: pull docker image + run: | + set -e -x + DOCKER_TAG=$(github_actions_ci/list-docker-tags.sh | tail -1) + echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV + echo "pulling $DOCKER_TAG" + docker pull $DOCKER_TAG + mkdir coverage + - name: test with docker + run: | + docker run -e _JAVA_OPTIONS -e PYTEST_ADDOPTS -v `pwd`/coverage:/coverage -v `pwd`/test:/opt/viral-ngs/source/test --entrypoint /bin/bash $DOCKER_TAG -c 'set -e; cd /opt/viral-ngs/source; pytest test/unit; cp .coverage /coverage' + - name: run coveralls + run: | + mv coverage/.coverage . + pip install coveralls>=1.3.0 + coveralls --service=github + + ## note: this test_docs job does not actually produce the output on readthedocs + ## readthedocs does its own build trigger. this job exists simply to alert us + ## of build failures of the docs because otherwise we would never know. + test_docs: + needs: build_docker + runs-on: ubuntu-20.04 + steps: + - name: checkout repository + uses: actions/checkout@v3 + # fetch git tags (tagged releases) because + # actions/checkout@v3 does either a full checkout or a shallow checkout without tags + - name: fetch tags + run: git fetch --prune --unshallow --tags + - name: Programmatic environment setup + run: | + set -e -x + # $GITHUB_ENV is available for subsequent steps + GITHUB_ACTIONS_TAG=$(git describe --tags --exact-match && sed 's/^v//g' || echo '') + echo "GITHUB_ACTIONS_TAG=$GITHUB_ACTIONS_TAG" >> $GITHUB_ENV + # + # Set GITHUB_ACTIONS_BRANCH + # TRAVIS_BRANCH: (via https://docs.travis-ci.com/user/environment-variables/ ) + # for push builds, or builds not triggered by a pull request, this is the name of the branch. + # for builds triggered by a pull request this is the name of the branch targeted by the pull request. + # for builds triggered by a tag, this is the same as the name of the tag (TRAVIS_TAG). + # if GITHUB_ACTIONS_PULL_REQUEST_BRANCH is set, this is a pull request build + if [[ $GITHUB_ACTIONS_PULL_REQUEST_BRANCH ]]; then + GITHUB_ACTIONS_BRANCH=${GITHUB_ACTIONS_BASE_REF##*/} + # if event_name=="release", this is a tagged build + elif [[ "${{ github.event_name }}" == "release" ]]; then + GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_TAG + else + GITHUB_ACTIONS_BRANCH=$(git rev-parse --abbrev-ref HEAD) + fi + echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" + echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: pull docker image + run: | + set -e -x + DOCKER_TAG=$(github_actions_ci/list-docker-tags.sh | tail -1) + echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV + echo "pulling $DOCKER_TAG" + docker pull $DOCKER_TAG + - name: test building docs + run: | + set -e -x + docker run --entrypoint /bin/bash -v `pwd`:/opt/viral-ngs/source $DOCKER_TAG -c 'set -e; cd /opt/viral-ngs/source; github_actions_ci/install-pip-docs.sh; cd docs; make html' diff --git a/.readthedocs.yml b/.readthedocs.yml index 4110ffff..be45417d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,6 +7,7 @@ version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: + builder: html configuration: docs/conf.py # Build documentation with MkDocs @@ -18,6 +19,6 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: 3.6 + version: 3.8 install: - requirements: docs/requirements.txt \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8acccb0e..00000000 --- a/.travis.yml +++ /dev/null @@ -1,83 +0,0 @@ -dist: bionic -os: linux - -git: - depth: 150 - -env: - global: - - CACHE_DIR="$HOME/misc_cache" - - MINICONDA_DIR="$HOME/miniconda" - - GATK_PATH="$CACHE_DIR" - - PYTHONIOENCODING=UTF8 - - - DOCKER_REGISTRY="quay.io" - - DOCKER_REPO_PROD="quay.io/broadinstitute/viral-assemble" - - DOCKER_REPO_DEV="quay.io/broadinstitute/viral-assemble" - - - BOTO_CONFIG=/dev/null # bogus value to override config on travis - - _JAVA_OPTIONS="-Xmx3g" # overrides jvm opts set elsewhere; see: https://stackoverflow.com/questions/28327620/difference-between-java-options-java-tool-options-and-java-opts - -cache: - directories: - - $HOME/misc_cache - - $HOME/miniconda - timeout: 1000 - -stages: - - build - - test - -jobs: - fast_finish: true - include: - - - language: generic - stage: build - env: - - TRAVIS_JOB=build_docker - # DOCKER_USER - - secure: "b7p66b6qjL1eHCbr3v+duObYXJEPBqZg4oHHstYy72lBTE9A5u4/qkLvCU7wwcH8UGnMpcqLVM1x2vGpqCSJ1SEx3+alaLJnR9WlZ98dMDfX+kOgsR9kUhhpLnjs97ahLsFQVhcJ81IolDtHxlv2b6igKJHI4ntZtRW2JaNVuApiPnxycunEA4emBcWLlSm8E6mkUx/e1dmxi+JXWnjUI2CeayI+OLHAFi3WJkowpEZCd/sQ6rHdLwFN/+EABENX0H/nyHdm7G1Sl7IpoUyACZWV51IkrHqH7E/OY5pK0y/SLXKYq21w0VOGKcOMrKsk3TuwT9yg4OZF5ZYZeXUAifxJGx6BZvoqwEpySvMEN+gpPfu+93lPKvYg5q7Hqupv3ER6KSsx1If5DHBRaCsRvFP4s3XBgRrv3tjRKNNWNrf4tem3ybBu03AqWkjMAo2+ixAiGmYHIj/AXibuI14NJbbs/6cto1NgD7hUChT9GyYVZoyGLRlAh9BXh94JU0krV+A4MVSWHpOurWRjcSwkTfvgkFbUoBXL+s0aC24wyZ1N/JDTshsAQ53pArwB99cAy3+7NnkGW44IAc+O20uvND4ANa9lZDECmJOn4K5uIG5EWMs537lpG7ThQgtCsr0O44yHn6XovxA3+jxpa2VpocTg5yjbZ0o1SBVUK2k0K6w=" - # DOCKER_PASS - - secure: "1LPm6+IV9ji066QlThCKRojalFlN6z1O+oZncgE/V9FQ82x/T+LCdfFyPDPOhVswTYtzh0GCPhKuhpbiQiGTzicY2wIOiu/dAXmTqesCMI/VttNZIl//Q8/3mfcGKewYLx8zhdWd4ihkuumyqK28fcLs4AmZLEqbAlO8hE87lSCjHzj3wMEB0ggMzLM29Cv3GWAcLbQ4+eQcRgfrUSRmUXlf2jwf+C1YQ/PrtB46uRHpX/Gr9Pwi38U7nzLBor/TMvSbCoJ78g/0uuZgbnYg5J5BrWl5n2DLx0EpL2Vf//7wIQGi1+cvX3Uk8uBBb93LdfCADL99pIhSEfMNQ7+0Ux4uWvK/Q+mFP6gclZWbvsi5o1EP24Ra8YQFfV2bIf2KUIspiS/wmmESF2GbOzBLS2H2hQC4l+at2NlaR+CCfXnYvZaQk2UJrWBWb0kMgDzY+avqC6bblM4L7IPyIDgKiMQpwCS4YdROTFmeUFOrWK04ymkUOiDiBrHSG8nHAWHMjNyGFkq6DyBE9FKFkjeS59q/4kDwHLS32esimn46s/a7g31++wiLMKQgqN9fbvXJ11cGecfyT66stukU5GfT1gaALj61LGB46RyeZe/AX9S5o5/2TlHLtgWcjG3huIvccxGJ3HkCnq+thKFErJmyBWwDjomjWcNg1k3+hp3A7PE=" - script: - - set -e - - if [ -f "$CACHE_DIR/old-docker-tag.txt" ]; then OLD_DOCKER_TAG=$(cat $CACHE_DIR/old-docker-tag.txt); else OLD_DOCKER_TAG=$DOCKER_REPO_PROD; fi; echo "old docker tag = $OLD_DOCKER_TAG" - - _CACHE_FROM="--cache-from $OLD_DOCKER_TAG"; docker pull $OLD_DOCKER_TAG || _CACHE_FROM="" - - git describe --tags --always | tee VERSION - - if [ -n "$TRAVIS_TAG" ]; then echo "Release $TRAVIS_TAG"; elif [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ]; then echo "LABEL quay.expires-after=10w" >> Dockerfile; elif [[ "$TRAVIS_BRANCH" != "master" ]]; then echo "LABEL quay.expires-after=10w" >> Dockerfile; fi - - docker build -t local/build-container:build $_CACHE_FROM . - - travis/deploy-docker.sh - before_cache: - - travis/list-docker-tags.sh | tail -1 | tee $CACHE_DIR/old-docker-tag.txt - - - language: python - stage: test - env: - - TRAVIS_JOB=test_py36_in_docker - - PYTEST_ADDOPTS="-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov assembly" - install: - - DOCKER_TAG=`travis/list-docker-tags.sh | tail -1` - - echo "pulling $DOCKER_TAG" - - docker pull $DOCKER_TAG - - mkdir coverage - script: - - docker run -e _JAVA_OPTIONS -e PYTEST_ADDOPTS -v `pwd`/test:/opt/viral-ngs/source/test -v `pwd`/coverage:/coverage --entrypoint /bin/bash $DOCKER_TAG -c 'set -e; cd /opt/viral-ngs/source; pytest test/unit; cp .coverage /coverage' - after_success: - - mv coverage/.coverage . - - pip install coveralls==1.1 - - coveralls - - - language: generic - stage: test - env: - - TRAVIS_JOB=test_docs - ## note: this test_docs job does not actually produce the output on readthedocs - ## readthedocs does its own build trigger. this travis job exists simply to alert us - ## of build failures of the docs because otherwise we would never know. - install: - - DOCKER_TAG=`travis/list-docker-tags.sh | tail -1` - - echo "pulling $DOCKER_TAG" - - docker pull $DOCKER_TAG - script: - - docker run --entrypoint /bin/bash -v `pwd`:/opt/viral-ngs/viral-assemble $DOCKER_TAG -c 'set -e; cd /opt/viral-ngs/viral-assemble; travis/install-pip-docs.sh; cd docs; make html' diff --git a/Dockerfile b/Dockerfile index e2833887..0134e617 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/broadinstitute/viral-core:2.1.33 +FROM quay.io/broadinstitute/viral-core:2.2.1 LABEL maintainer "viral-ngs@broadinstitute.org" diff --git a/README.md b/README.md index e9774911..6d867ad3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Docker Repository on Quay](https://quay.io/repository/broadinstitute/viral-assemble/status "Docker Repository on Quay")](https://quay.io/repository/broadinstitute/viral-assemble) -[![Build Status](https://travis-ci.com/broadinstitute/viral-assemble.svg?branch=master)](https://travis-ci.com/broadinstitute/viral-assemble) +[![Build Status](https://github.com/broadinstitute/viral-assemble/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/broadinstitute/viral-assemble/actions) [![Documentation Status](https://readthedocs.org/projects/viral-assemble/badge/?version=latest)](https://viral-assemble.readthedocs.io/en/latest/?badge=latest)