operator.yml

---
# yamllint disable rule:line-length

name: operator
on:  # yamllint disable-line rule:truthy
  push:
    branches: ["main", "ocm-*", "release-*"]
    tags: ["*"]
  pull_request:
    branches: ["main", "ocm-*", "release-*"]
  schedule:
    - cron: "15 4 * * 1"  # 4:15 every Monday

env:
  GO_VERSION: "1.17"
  KIND_VERSION: "0.11.1"
  GO111MODULE: "on"
  OPERATOR_IMAGE: "quay.io/backube/volsync"
  RCLONE_IMAGE: "quay.io/backube/volsync-mover-rclone"
  RESTIC_IMAGE: "quay.io/backube/volsync-mover-restic"
  RSYNC_IMAGE: "quay.io/backube/volsync-mover-rsync"
  SYNCTHING_IMAGE: "quay.io/backube/volsync-mover-syncthing"

jobs:
  lint:
    name: Lint
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout source
        uses: actions/checkout@v2

      - name: Install prereqs
        run: |
          echo 'APT::Acquire::Retries "5";' | sudo tee /etc/apt/apt.conf.d/80-retries
          sudo apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python3-pip ruby
          sudo gem install asciidoctor mdl
          sudo pip3 install yamllint
      - name: Run linters
        run: ./.ci-scripts/pre-commit.sh --require-all

  build-operator:
    name: Build-operator
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout source
        uses: actions/checkout@v2

      - name: Install Go
        uses: actions/setup-go@v1
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Build operator container
        run: make docker-build IMG=${OPERATOR_IMAGE}

      - name: Upload test coverage
        uses: codecov/codecov-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./cover.out
          fail_ci_if_error: true

      - name: Export container image
        run: docker save -o /tmp/image.tar ${OPERATOR_IMAGE}

      - name: Save container as artifact
        uses: actions/upload-artifact@v1
        with:
          name: volsync-operator
          path: /tmp/image.tar

      - name: Build cli
        run: make cli

      - name: Save cli as artifact
        uses: actions/upload-artifact@v1
        with:
          name: kubectl-volsync
          path: bin/kubectl-volsync

  build-rclone:
    name: Build-mover-rclone
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout source
        uses: actions/checkout@v2

      - name: Build operator container
        run: make -C mover-rclone image

      - name: Export container image
        run: docker save -o /tmp/image.tar ${RCLONE_IMAGE}

      - name: Save container as artifact
        uses: actions/upload-artifact@v1
        with:
          name: volsync-mover-rclone-container
          path: /tmp/image.tar

  build-restic:
    name: Build-mover-restic
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout source
        uses: actions/checkout@v2

      - name: Build operator container
        run: make -C mover-restic image

      - name: Export container image
        run: docker save -o /tmp/image.tar ${RESTIC_IMAGE}

      - name: Save container as artifact
        uses: actions/upload-artifact@v1
        with:
          name: volsync-mover-restic-container
          path: /tmp/image.tar

  build-rsync:
    name: Build-mover-rsync
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout source
        uses: actions/checkout@v2

      - name: Build operator container
        run: make -C mover-rsync image

      - name: Export container image
        run: docker save -o /tmp/image.tar ${RSYNC_IMAGE}

      - name: Save container as artifact
        uses: actions/upload-artifact@v1
        with:
          name: volsync-mover-rsync-container
          path: /tmp/image.tar

  build-syncthing:
    name: build-mover-syncthing
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout source
        uses: actions/checkout@v2

      - name: Build operator container
        run: make -C mover-syncthing image

      - name: Export container image
        run: docker save -o /tmp/image.tar ${SYNCTHING_IMAGE}

      - name: Save container as artifact
        uses: actions/upload-artifact@v1
        with:
          name: volsync-mover-syncthing-container
          path: /tmp/image.tar

  e2e:
    name: End-to-end
    needs: [build-operator, build-rclone, build-restic, build-rsync]
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        # There must be kindest/node images for these versions
        # See: https://hub.docker.com/r/kindest/node/tags?page=1&ordering=name
        # Or: skopeo list-tags docker://kindest/node
        KUBERNETES_VERSIONS:
          - "1.19.11"  # OCP 4.6
          - "1.20.7"   # OCP 4.7
          - "1.21.2"   # OCP 4.8
          - "1.22.2"   # OCP 4.9
    env:
      KUBECONFIG: /tmp/kubeconfig
      KUBERNETES_VERSION: ${{ matrix.KUBERNETES_VERSIONS }}
    steps:
      - name: Checkout source
        uses: actions/checkout@v2

      # We set bash as the default shell (instead of dash) because the kuttl
      # test steps require bash, but the "script" directive executes them as "sh
      # -c ..."
      - name: Set bash as default shell
        run: |
          sudo ln -s bash /bin/sh.bash && sudo mv /bin/sh.bash /bin/sh
          sudo ln -s bash /usr/bin/sh.bash && sudo mv /usr/bin/sh.bash /usr/bin/sh

      - name: Install kubectl
        run: |
          curl -LO "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
          sudo install ./kubectl /usr/local/bin/
          kubectl version --short --client
          kubectl version --short --client | grep -q ${KUBERNETES_VERSION}

      - name: Install helm
        run: make helm

      - name: Install kind
        run: |
          curl -L -o kind https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-linux-amd64
          sudo install ./kind /usr/local/bin && rm kind
          kind version
          kind version | grep -q ${KIND_VERSION}

      - name: Create Kubernetes cluster
        run: |
          ./hack/setup-kind-cluster.sh "${KUBERNETES_VERSION}"

      - name: Start MinIO
        run: |
          ./hack/run-minio.sh

      - name: Load operator container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-operator
          path: /tmp

      - name: Import container image into cluster
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${OPERATOR_IMAGE}
          docker tag ${OPERATOR_IMAGE} ${OPERATOR_IMAGE}:ci-build
          kind load docker-image "${OPERATOR_IMAGE}:ci-build"

      - name: Load rclone container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-mover-rclone-container
          path: /tmp

      - name: Import container image into cluster
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${RCLONE_IMAGE}
          docker tag ${RCLONE_IMAGE} ${RCLONE_IMAGE}:ci-build
          kind load docker-image "${RCLONE_IMAGE}:ci-build"

      - name: Load restic container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-mover-restic-container
          path: /tmp

      - name: Import container image into cluster
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${RESTIC_IMAGE}
          docker tag ${RESTIC_IMAGE} ${RESTIC_IMAGE}:ci-build
          kind load docker-image "${RESTIC_IMAGE}:ci-build"

      - name: Load rsync container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-mover-rsync-container
          path: /tmp

      - name: Import container image into cluster
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${RSYNC_IMAGE}
          docker tag ${RSYNC_IMAGE} ${RSYNC_IMAGE}:ci-build
          kind load docker-image "${RSYNC_IMAGE}:ci-build"

      - name: Start operator
        run: |
          helm install --create-namespace -n volsync-system \
              --set image.tag=ci-build \
              --set rclone.tag=ci-build \
              --set rsync.tag=ci-build \
              --set restic.tag=ci-build \
              --wait --timeout=300s \
              volsync-ghaction ./helm/volsync

      - name: Load cli artifact
        uses: actions/download-artifact@v1
        with:
          name: kubectl-volsync
          path: bin

      - name: Make cli executable
        run: chmod a+x bin/kubectl-volsync

      - name: Run e2e tests
        run: |
          kubectl -n minio wait --for=condition=Available --timeout=300s deploy/minio
          make test-e2e

  # This is a dummy job that can be used to determine success of CI:
  # - by Mergify instead of having to list a bunch of other jobs
  # - by the push jobs to ensure all pre-reqs pass before ANY containers are
  #   pushed.
  e2e-success:
    name: Successful e2e tests
    needs: [build-operator, e2e, lint]
    runs-on: ubuntu-20.04
    steps:
      - name: Success
        run: echo "Previous steps were successful"

  push-operator:
    name: Push operator container to registry
    needs: e2e-success
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' ||
       startsWith(github.ref, 'refs/heads/ocm-') ||
       startsWith(github.ref, 'refs/heads/release-') ||
       startsWith(github.ref, 'refs/tags/v'))
    runs-on: ubuntu-20.04

    steps:
      - name: Load container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-operator
          path: /tmp

      - name: Import container image
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${OPERATOR_IMAGE}
      - name: Login to registry
        # If the registry server is specified in the image name, we use that.
        # If the server isn't in the image name, default to docker.io
        run: |
          [[ "${OPERATOR_IMAGE}" =~ ^([^/]+)/[^/]+/[^/]+ ]] && REGISTRY="${BASH_REMATCH[1]}" || REGISTRY="docker.io"
          echo "Attempting docker login to: ${REGISTRY}"
          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin ${REGISTRY}
      - name: Push to registry (latest)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          github.ref == 'refs/heads/main'
        run: |
          docker push "${OPERATOR_IMAGE}"
      - name: Push to registry (version tag)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          startsWith(github.ref, 'refs/tags/v')
        run: |
          [[ "${{ github.ref }}" =~ ^refs/tags/v([0-9]+\..*) ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${OPERATOR_IMAGE}" "${OPERATOR_IMAGE}:${TAG}"
          docker push "${OPERATOR_IMAGE}:${TAG}"
      - name: Push to registry (release branch)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          (startsWith(github.ref, 'refs/heads/ocm-') ||
           startsWith(github.ref, 'refs/heads/release-'))
        run: |
          [[ "${{ github.ref }}" =~ ^refs/heads/(.+)$ ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${OPERATOR_IMAGE}" "${OPERATOR_IMAGE}:${TAG}"
          docker push "${OPERATOR_IMAGE}:${TAG}"

  push-rclone:
    name: Push rclone container to registry
    needs: e2e-success
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' ||
       startsWith(github.ref, 'refs/heads/ocm-') ||
       startsWith(github.ref, 'refs/heads/release-') ||
       startsWith(github.ref, 'refs/tags/v'))
    runs-on: ubuntu-20.04

    steps:
      - name: Load container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-mover-rclone-container
          path: /tmp

      - name: Import container image
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${RCLONE_IMAGE}
      - name: Login to registry
        # If the registry server is specified in the image name, we use that.
        # If the server isn't in the image name, default to docker.io
        run: |
          [[ "${RCLONE_IMAGE}" =~ ^([^/]+)/[^/]+/[^/]+ ]] && REGISTRY="${BASH_REMATCH[1]}" || REGISTRY="docker.io"
          echo "Attempting docker login to: ${REGISTRY}"
          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin ${REGISTRY}
      - name: Push to registry (latest)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          github.ref == 'refs/heads/main'
        run: |
          docker push "${RCLONE_IMAGE}"
      - name: Push to registry (version tag)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          startsWith(github.ref, 'refs/tags/v')
        run: |
          [[ "${{ github.ref }}" =~ ^refs/tags/v([0-9]+\..*) ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${RCLONE_IMAGE}" "${RCLONE_IMAGE}:${TAG}"
          docker push "${RCLONE_IMAGE}:${TAG}"
      - name: Push to registry (release branch)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          (startsWith(github.ref, 'refs/heads/ocm-') ||
           startsWith(github.ref, 'refs/heads/release-'))
        run: |
          [[ "${{ github.ref }}" =~ ^refs/heads/(.+)$ ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${RCLONE_IMAGE}" "${RCLONE_IMAGE}:${TAG}"
          docker push "${RCLONE_IMAGE}:${TAG}"

  push-restic:
    name: Push restic container to registry
    needs: e2e-success
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' ||
       startsWith(github.ref, 'refs/heads/ocm-') ||
       startsWith(github.ref, 'refs/heads/release-') ||
       startsWith(github.ref, 'refs/tags/v'))
    runs-on: ubuntu-20.04

    steps:
      - name: Load container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-mover-restic-container
          path: /tmp

      - name: Import container image
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${RESTIC_IMAGE}
      - name: Login to registry
        # If the registry server is specified in the image name, we use that.
        # If the server isn't in the image name, default to docker.io
        run: |
          [[ "${RESTIC_IMAGE}" =~ ^([^/]+)/[^/]+/[^/]+ ]] && REGISTRY="${BASH_REMATCH[1]}" || REGISTRY="docker.io"
          echo "Attempting docker login to: ${REGISTRY}"
          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin ${REGISTRY}
      - name: Push to registry (latest)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          github.ref == 'refs/heads/main'
        run: |
          docker push "${RESTIC_IMAGE}"
      - name: Push to registry (version tag)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          startsWith(github.ref, 'refs/tags/v')
        run: |
          [[ "${{ github.ref }}" =~ ^refs/tags/v([0-9]+\..*) ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${RESTIC_IMAGE}" "${RESTIC_IMAGE}:${TAG}"
          docker push "${RESTIC_IMAGE}:${TAG}"
      - name: Push to registry (release branch)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          (startsWith(github.ref, 'refs/heads/ocm-') ||
           startsWith(github.ref, 'refs/heads/release-'))
        run: |
          [[ "${{ github.ref }}" =~ ^refs/heads/(.+)$ ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${RESTIC_IMAGE}" "${RESTIC_IMAGE}:${TAG}"
          docker push "${RESTIC_IMAGE}:${TAG}"

  push-rsync:
    name: Push rsync container to registry
    needs: e2e-success
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' ||
       startsWith(github.ref, 'refs/heads/ocm-') ||
       startsWith(github.ref, 'refs/heads/release-') ||
       startsWith(github.ref, 'refs/tags/v'))
    runs-on: ubuntu-20.04

    steps:
      - name: Load container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-mover-rsync-container
          path: /tmp

      - name: Import container image
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${RSYNC_IMAGE}
      - name: Login to registry
        # If the registry server is specified in the image name, we use that.
        # If the server isn't in the image name, default to docker.io
        run: |
          [[ "${RSYNC_IMAGE}" =~ ^([^/]+)/[^/]+/[^/]+ ]] && REGISTRY="${BASH_REMATCH[1]}" || REGISTRY="docker.io"
          echo "Attempting docker login to: ${REGISTRY}"
          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin ${REGISTRY}
      - name: Push to registry (latest)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          github.ref == 'refs/heads/main'
        run: |
          docker push "${RSYNC_IMAGE}"
      - name: Push to registry (version tag)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          startsWith(github.ref, 'refs/tags/v')
        run: |
          [[ "${{ github.ref }}" =~ ^refs/tags/v([0-9]+\..*) ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${RSYNC_IMAGE}" "${RSYNC_IMAGE}:${TAG}"
          docker push "${RSYNC_IMAGE}:${TAG}"
      - name: Push to registry (release branch)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          (startsWith(github.ref, 'refs/heads/ocm-') ||
           startsWith(github.ref, 'refs/heads/release-'))
        run: |
          [[ "${{ github.ref }}" =~ ^refs/heads/(.+)$ ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${RSYNC_IMAGE}" "${RSYNC_IMAGE}:${TAG}"
          docker push "${RSYNC_IMAGE}:${TAG}"

  push-syncthing:
    name: Push syncthing container to registry
    needs: e2e-success
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' ||
       startsWith(github.ref, 'refs/heads/ocm-') ||
       startsWith(github.ref, 'refs/heads/release-') ||
       startsWith(github.ref, 'refs/tags/v'))
    runs-on: ubuntu-20.04

    steps:
      - name: Load container artifact
        uses: actions/download-artifact@v1
        with:
          name: volsync-mover-syncthing-container
          path: /tmp

      - name: Import container image
        run: |
          docker load -i /tmp/image.tar
          docker inspect ${SYNCTHING_IMAGE}
      - name: Login to registry
        # If the registry server is specified in the image name, we use that.
        # If the server isn't in the image name, default to docker.io
        run: |
          [[ "${SYNCTHING_IMAGE}" =~ ^([^/]+)/[^/]+/[^/]+ ]] && REGISTRY="${BASH_REMATCH[1]}" || REGISTRY="docker.io"
          echo "Attempting docker login to: ${REGISTRY}"
          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin ${REGISTRY}
      - name: Push to registry (latest)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          github.ref == 'refs/heads/main'
        run: |
          docker push "${SYNCTHING_IMAGE}"
      - name: Push to registry (version tag)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          startsWith(github.ref, 'refs/tags/v')
        run: |
          [[ "${{ github.ref }}" =~ ^refs/tags/v([0-9]+\..*) ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${SYNCTHING_IMAGE}" "${SYNCTHING_IMAGE}:${TAG}"
          docker push "${SYNCTHING_IMAGE}:${TAG}"
      - name: Push to registry (release branch)
        if: >
          (github.event_name == 'push' || github.event_name == 'schedule') &&
          (startsWith(github.ref, 'refs/heads/ocm-') ||
           startsWith(github.ref, 'refs/heads/release-'))
        run: |
          [[ "${{ github.ref }}" =~ ^refs/heads/(.+)$ ]] || exit 0
          TAG="${BASH_REMATCH[1]}"
          echo "Pushing to $TAG"
          docker tag "${SYNCTHING_IMAGE}" "${SYNCTHING_IMAGE}:${TAG}"
          docker push "${SYNCTHING_IMAGE}:${TAG}"