test-and-deploy.yml

name: Test, Build and Deploy Images

env:
  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}

on:
    workflow_dispatch:
    pull_request:
    push:
    schedule:
    # Run every 6 days to keep our caches alive
    - cron: '0 0 */6 * *'

jobs:
  python-tests:
    runs-on: ubuntu-latest

    env:
      python-version: 3.9 # Our base image has Python 3.9

    steps:
    - name: Checkout
      uses: actions/checkout@v3

    - name: Set up Python ${{ env.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.python-version }}

    - name: Install dependencies
      run: |
        sudo ./core/tools/mavlink_router/bootstrap.sh
        ./core/tools/ardupilot_tools/bootstrap.sh
        sudo apt install shellcheck
        npm install \
          eslint@7 eslint-plugin-vue eslint-plugin-html @vue/eslint-config-typescript @typescript-eslint/parser \
          @typescript-eslint/eslint-plugin typescript @vue/eslint-config-airbnb@^5.3.0 @vue/cli-service \
          eslint-plugin-simple-import-sort \
          --save-dev
        python -m pip install --upgrade pip
        pip install wheel # wheel need to be installed before mavproxy
        pip install mavproxy
        pip install pyfakefs pytest-cov pytest-timeout pylint pylint-pydantic mypy isort black==22.1.0 asyncmock types-requests \
          pytest-mock nmeasim geographiclib ruff
        pip install git+https://github.com/patrickelectric/pykson.git@master_fixes
        ## Install our own libraries
        python ./core/libs/bridges/setup.py install
        python ./core/libs/commonwealth/setup.py install
        ## We need to install loguru and appdirs since they may be used inside setup.py
        python -m pip install --user appdirs==1.4.4 loguru==0.5.3
        find . -type f -name "setup.py" | xargs --max-lines=1 --replace=% python % install --user
        ## Fix Click version since its being updated to an old version by our packages
        pip install click==8.0.0

    - name: Run tests
      run: |
        ./.hooks/pre-push

    - name: Upload coverage
      uses: actions/upload-artifact@v3
      with:
        name: coverage
        path: htmlcov


  frontend-tests:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive

      - name: Yarn install
        uses: borales/actions-yarn@v4
        with:
          cmd: --cwd ./core/frontend install --frozen-lockfile --ignore-engines

      - name: Yarn lint
        uses: borales/actions-yarn@v4
        with:
          cmd: --cwd ./core/frontend lint --no-fix --ignore-engines --max-warnings=0

      - name: Yarn build
        uses: borales/actions-yarn@v4
        with:
          cmd: --cwd ./core/frontend build --ignore-engines


  deploy-docker-images:
    runs-on: ubuntu-latest
    needs: [python-tests, frontend-tests]
    strategy:
      matrix:
        docker: [bootstrap, core]
        project: [blueos]
        platforms: ["linux/arm/v7,linux/arm64/v8,linux/amd64"]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0 #Number of commits to fetch. 0 indicates all history for all branches and tags.
          submodules: recursive

      - name: Prepare
        id: prepare
        run: |
          # Deploy image with the name of the branch, if the build is a git tag, replace tag with the tag name.
          # If git tag matches semver, append latest tag to the push.
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          VERSION=${GITHUB_REF##*/}

          if [[ $GITHUB_REF == refs/tags/* ]]; then
            VERSION=${GITHUB_REF#refs/tags/}
          fi

          TAGS="--tag ${DOCKER_IMAGE}:${VERSION}"
          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
            TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest"
          fi

          echo "docker_image=${DOCKER_IMAGE}" >> $GITHUB_OUTPUT
          echo "version=${VERSION}" >> $GITHUB_OUTPUT
          echo "buildx_args=\
            --build-arg GIT_DESCRIBE_TAGS=$(git describe --tags --long --always) \
            --build-arg VUE_APP_GIT_DESCRIBE=$(git describe --long --always --dirty --all) \
            --cache-from 'type=local,src=/tmp/.buildx-cache' \
            --cache-to 'type=local,dest=/tmp/.buildx-cache' \
            ${TAGS} \
            --file ${{ matrix.docker }}/Dockerfile ./${{ matrix.docker }}" >> $GITHUB_OUTPUT

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
        with:
          platforms: all

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: latest

      - name: Cache Docker layers
        uses: actions/cache@v3
        id: cache
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }}
          restore-keys: |
            ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }}
            ${{ runner.os }}-buildx-${{ matrix.docker }}

      - name: Docker Buildx (build)
        run: |
          # Pull latest version of image to help with build speed
          for platform in $(echo ${{ matrix.platforms }} | tr ',' '\n'); do
            docker pull --platform ${platform} ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}:master || true
          done
          docker buildx build \
            --output "type=image,push=false" \
            --platform ${{ matrix.platforms }} \
            ${{ steps.prepare.outputs.buildx_args }}

      - name: Check core size
        if: ${{ matrix.docker == 'core' }}
        run: |
          # Check if the image size is lower than our limit
          docker image list
          IMAGE_ID=$(docker images -q ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }} | head -n 1)
          LIMIT_SIZE_MB=700
          IMAGE_SIZE_MB=$(( $(docker inspect $IMAGE_ID --format {{.Size}})/(2**20) ))
          echo "Core size is: $IMAGE_SIZE_MB MB"
          ((IMAGE_SIZE_MB < LIMIT_SIZE_MB))

      - name: Login to DockerHub
        if: success() && github.event_name != 'pull_request'
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Docker Buildx (push)
        if: success() && github.event_name != 'pull_request'
        run: |
          docker buildx build \
            --output "type=image,push=true" \
            --platform ${{ matrix.platforms }} \
            ${{ steps.prepare.outputs.buildx_args }}

      - name: Inspect image
        if: always() && github.event_name != 'pull_request'
        run: |
          docker buildx imagetools \
            inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }}

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/arm64/v8"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm64-v8.tar" \

      - name: Upload artifact arm64-v8
        uses: actions/upload-artifact@v3
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-arm64-v8
          path: '*arm64-v8.tar'

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/arm/v7"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm-v7.tar" \

      - name: Upload artifact arm-v7
        uses: actions/upload-artifact@v3
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-arm-v7
          path: '*arm-v7.tar'

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/amd64"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-amd64.tar" \

      - name: Upload artifact amd64
        uses: actions/upload-artifact@v3
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-amd64
          path: '*amd64.tar'

      - name: Upload docker image for release
        uses: svenstaro/upload-release-action@v2
        if: startsWith(github.ref, 'refs/tags/') && success() && matrix.docker == 'core'
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: '*.tar'
          tag: ${{ github.ref }}
          overwrite: true
          prerelease: true
          file_glob: true

  deploy-raspberry-image:
    needs: deploy-docker-images
    if: github.event_name != 'pull_request'
    runs-on: self-hosted
    # The runner for this job is a Raspberry Pi OS Bullseye.
    # Just install docker (curl -sSL https://get.docker.com/ | sh) and follow the instructions for setting up a new runner in
    # https://github.com/bluerobotics/BlueOS-docker/settings/actions/runners/new

    strategy:
      matrix:
        platforms: ["linux/arm/v7"]
    steps:
      - name: Install git
        run: sudo apt install -y git

      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive


      # We use our own pimod as upstream doesn't provide armv7 images
      - name: Pimod Build
        run: |
          VERSION=$GITHUB_REPOSITORY
          VERSION=${VERSION:-master}
          wget https://raw.githubusercontent.com/williangalvani/pimod/master/pimod.sh && chmod +x pimod.sh
          docker run --rm --privileged \
            -v $PWD:/files \
            -e PATH=/pimod:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
            -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY \
            -e VERSION=$GITHUB_REF_NAME \
            --workdir=/files \
            --platform ${{ matrix.platforms }} williangalvani/pimod:latest pimod.sh deploy/pimod/blueos.Pifile

      # TODO: add GITHUB_REF_NAME after https://github.com/actions/upload-artifact/issues/231 is fixed
      # name: blueos-${{ env.GITHUB_REF_NAME }}.zip
      - name: Zip image
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          sudo apt install zip
          zip BlueOS-raspberry.zip deploy/pimod/blueos.img

      - name: Upload artifact
        uses: actions/upload-artifact@v3
        with:
          # We use the .img to avoid a zip of a zip
          name: BlueOS-raspberry.zip
          path: deploy/pimod/blueos.img
          if-no-files-found: error
          retention-days: 7

      - name: Upload raspberry image for release
        uses: svenstaro/upload-release-action@v2
        if: startsWith(github.ref, 'refs/tags/')
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: BlueOS-raspberry.zip
          tag: ${{ github.ref }}
          overwrite: true
          prerelease: true

      # This is required because docker has root permissions, which means the runner is unable to clear this cache normally
      - name: Cleanup
        if: ${{ always() }}
        run: sudo rm -rf .cache