feat(backend): backend arm builds, create dockerfile

.github/workflows/backend.yml → .github/workflows/backend-image.yml

@@ -1,15 +1,36 @@
-name: backend
+name: backend-image
 
 on:
+  pull_request:
   push:
+    branches:
+      - main
   workflow_dispatch:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: false
+  workflow_call:
+    inputs:
+      build_arm:
+        type: string
+        description: "Build for ARM as well"
+        default: "false"
+        required: true
+
+env:
+  DOCKER_IMAGE_NAME: ghcr.io/loculus-project/backend
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}
 
 concurrency:
   group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-backend
   cancel-in-progress: true
 
 jobs:
-  dockerImage:
+  backend-image:
     name: Build Backend Docker Image # Don't change: Referenced by .github/workflows/update-argocd-metadata.yml
     runs-on: ubuntu-latest
     timeout-minutes: 15
@@ -25,8 +46,8 @@ jobs:
       - name: Generate files hash
         id: files-hash
         run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('backend/**', '.github/workflows/backend.yml') }})
-          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+          DIR_HASH=$(echo -n ${{ hashFiles('backend/**', '.github/workflows/backend-image.yml') }})
+          echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
 
       - name: Setup Docker metadata
         id: dockerMetadata
@@ -38,7 +59,7 @@ jobs:
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
             type=ref,event=branch
             type=sha,prefix=commit-
-
+            type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -52,6 +73,9 @@ jobs:
           EXISTS=$(docker manifest inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
           echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Set up JDK
         if: env.CACHE_HIT == 'false'
         uses: actions/setup-java@v4
@@ -63,22 +87,24 @@ jobs:
         if: env.CACHE_HIT == 'false'
         uses: gradle/actions/setup-gradle@v3
 
-      - name: Build Docker Image For Branch
+      - name: Build Backend
         if: env.CACHE_HIT == 'false'
-        run: ./gradlew bootBuildImage --imageName=${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }}
         working-directory: ./backend
-        env:
-          USER: ${{ github.actor }}
-          TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: ./gradlew bootJar
 
-      - name: Push Docker Image
+      - name: Build and push image if input files changed
         if: env.CACHE_HIT == 'false'
-        run: docker push ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/build-push-action@v5
+        with:
+          context: ./backend
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          cache-from: type=gha,scope=backend-${{ github.ref }}
+          cache-to: type=gha,mode=max,scope=backend-${{ github.ref }}
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
 
-      - name: Tag and push existing images
+      - name: Retag and push existing image if cache hit
+        if: env.CACHE_HIT == 'true'
         run: |
           TAGS=(${{ steps.dockerMetadata.outputs.tags }})
           for TAG in "${TAGS[@]}"; do

.github/workflows/build-arm-images.yaml

@@ -0,0 +1,45 @@
+# Trigger a build of all docker images including ARM images
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+  checks: read
+
+jobs:
+  trigger-backend:
+    uses: ./.github/workflows/backend.yml
+    with:
+      build_arm: true
+
+  trigger-config-preprocessor:
+    uses: ./.github/workflows/config-preprocessor-build.yml
+    with:
+      build_arm: true
+
+  trigger-dummy-preprocessing:
+    uses: ./.github/workflows/dummyPreprocessing.yml
+    with:
+      build_arm: true
+
+  trigger-ingest:
+    uses: ./.github/workflows/ingest.yml
+    with:
+      build_arm: true
+
+  trigger-keycloakify:
+    uses: ./.github/workflows/keycloakify-build.yml
+    with:
+      build_arm: true
+
+  trigger-preprocessing-nextclade:
+    uses: ./.github/workflows/preprocessing-nextclade.yml
+    with:
+      build_arm: true
+
+  trigger-website:
+    uses: ./.github/workflows/website.yml
+    with:
+      build_arm: true

.github/workflows/config-preprocessor-build.yml → .github/workflows/config-preprocessor-image.yml

@@ -1,29 +1,40 @@
-name: config-processor-build
+name: config-processor-image
 
 on:
+  pull_request:
   push:
+    branches:
+      - main
   workflow_dispatch:
     inputs:
       build_arm:
         type: boolean
         description: "Build for ARM as well"
         default: false
-        required: false
+        required: true
+  workflow_call:
+    inputs:
+      build_arm:
+        type: string
+        description: "Build for ARM as well"
+        default: "false"
+        required: true
 
 env:
   DOCKER_IMAGE_NAME: ghcr.io/loculus-project/config-processor
-  BUILD_ARM: ${{ github.ref == 'refs/heads/main' || github.event.inputs.build_arm }} # When to build for arm as well
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}
 
 defaults:
   run:
     working-directory: ./kubernetes/config-processor
 
 concurrency:
-  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-config-processor
+  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-config-processor-${{github.event.inputs.build_arm}}
   cancel-in-progress: true
 
 jobs:
-  dockerImage:
+  config-processor-image:
     name: Build config-processor Docker Image # Don't change: Referenced by .github/workflows/update-argocd-metadata.yml
     runs-on: ubuntu-latest
     timeout-minutes: 15
@@ -50,6 +61,7 @@ jobs:
             type=ref,event=branch
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
             type=sha,prefix=commit-
+            type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
       - name: Build and push image
         uses: docker/build-push-action@v5
         with:
@@ -58,4 +70,4 @@ jobs:
           tags: ${{ steps.dockerMetadata.outputs.tags }}
           cache-from: type=gha,scope=config-preprocessor-${{ github.ref }}
           cache-to: type=gha,mode=max,scope=config-preprocessor-${{ github.ref }}
-          platforms: ${{ env.BUILD_ARM && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}

.github/workflows/docs-build.yml → .github/workflows/docs-test.yml

@@ -1,10 +1,14 @@
 name: Docs Build Check
 
 on:
-  push:
+  pull_request:
     paths:
       - "docs/**"
       - .github/workflows/docs-build-check.yml
+  push:
+    branches:
+      - main
+
   workflow_dispatch:
 
 concurrency:

.github/workflows/e2e-k3d.yml

@@ -2,7 +2,7 @@ name: E2E test (on kubernetes)
 
 on:
   workflow_dispatch:
-  push:
+  pull_request:
     paths:
       - "backend/**"
       - "keycloak/**"
@@ -11,6 +11,9 @@ on:
       - "deploy.py"
       - ".github/scripts/**"
       - ".github/workflows/**"
+  push:
+    branches:
+      - main
 
 concurrency:
   group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-e2e-k3d
@@ -93,40 +96,41 @@ jobs:
         run: cd website && npx playwright install-deps
         if: steps.playwright-cache.outputs.cache-hit == 'true'
 
-      # Waits are identical to the update-argocd-metadata.yml file
-      # Mirror changes to that file
-      - name: Wait for Config Processor Docker Image
-        uses: lewagon/[email protected]
-        with:
-          ref: ${{ github.sha }}
-          check-name: Build config-processor Docker Image
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          wait-interval: 2
-
-      - name: Wait for Backend Docker Image
-        uses: lewagon/[email protected]
-        with:
-          ref: ${{ github.sha }}
-          check-name: Build Backend Docker Image
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          wait-interval: 2
-
-      - name: Wait for Website Docker Image
-        uses: lewagon/[email protected]
-        with:
-          ref: ${{ github.sha }}
-          check-name: Build Website Docker Image
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          wait-interval: 2
-
-      - name: Wait for Keycloakify Docker Image
-        uses: lewagon/[email protected]
-        with:
-          ref: ${{ github.sha }}
-          check-name: Build keycloakify Docker Image
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          wait-interval: 2
-      # End of wait block
+      # Action misbehaved
+      # # Waits are identical to the update-argocd-metadata.yml file
+      # # Mirror changes to that file
+      # - name: Wait for Config Processor Docker Image
+      #   uses: lewagon/[email protected]
+      #   with:
+      #     ref: ${{ github.sha }}
+      #     check-name: Build config-processor Docker Image
+      #     repo-token: ${{ secrets.GITHUB_TOKEN }}
+      #     wait-interval: 2
+
+      # - name: Wait for Backend Docker Image
+      #   uses: lewagon/[email protected]
+      #   with:
+      #     ref: ${{ github.sha }}
+      #     check-name: Build Backend Docker Image
+      #     repo-token: ${{ secrets.GITHUB_TOKEN }}
+      #     wait-interval: 2
+
+      # - name: Wait for Website Docker Image
+      #   uses: lewagon/[email protected]
+      #   with:
+      #     ref: ${{ github.sha }}
+      #     check-name: Build Website Docker Image
+      #     repo-token: ${{ secrets.GITHUB_TOKEN }}
+      #     wait-interval: 2
+
+      # - name: Wait for Keycloakify Docker Image
+      #   uses: lewagon/[email protected]
+      #   with:
+      #     ref: ${{ github.sha }}
+      #     check-name: Build keycloakify Docker Image
+      #     repo-token: ${{ secrets.GITHUB_TOKEN }}
+      #     wait-interval: 2
+      # # End of wait block
 
       - name: Wait for the pods to be ready (timeout 480s)
         run: ./.github/scripts/wait_for_pods_to_be_ready.py

.github/workflows/ingest.yml → .github/workflows/ingest-image.yml

@@ -1,24 +1,35 @@
-name: ingest
+name: ingest-image
 on:
+  pull_request:
   push:
+    branches:
+      - main
   workflow_dispatch:
     inputs:
       build_arm:
         type: boolean
         description: "Build for ARM as well"
         default: false
         required: false
+  workflow_call:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: false
 
 env:
   DOCKER_IMAGE_NAME: ghcr.io/loculus-project/ingest
-  BUILD_ARM: ${{ github.ref == 'refs/heads/main' || github.event.inputs.build_arm }}
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}
 
 concurrency:
-  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-ingest
+  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-ingest-${{github.event.inputs.build_arm}}
   cancel-in-progress: true
 
 jobs:
-  dockerImage:
+  ingest-image:
     name: Build ingest Docker Image # Don't change: Referenced by .github/workflows/update-argocd-metadata.yml
     runs-on: ubuntu-latest
     timeout-minutes: 15
@@ -32,8 +43,8 @@ jobs:
       - name: Generate files hash
         id: files-hash
         run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('ingest/**', '.github/workflows/ingest.yml') }})
-          echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM && '-arm'|| '' }}" >> $GITHUB_ENV
+          DIR_HASH=$(echo -n ${{ hashFiles('ingest/**', '.github/workflows/ingest-image.yml') }})
+          echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
 
       - name: Setup Docker metadata
         id: dockerMetadata
@@ -45,6 +56,7 @@ jobs:
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
             type=ref,event=branch
             type=sha,prefix=commit-
+            type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
@@ -71,7 +83,7 @@ jobs:
           tags: ${{ steps.dockerMetadata.outputs.tags }}
           cache-from: type=gha,scope=ingest-${{ github.ref }}
           cache-to: type=gha,mode=max,scope=ingest-${{ github.ref }}
-          platforms: ${{ env.BUILD_ARM && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
 
       - name: Retag and push existing image if cache hit
         if: env.CACHE_HIT == 'true'