Merge branch 'develop' into reduce-any

.github/workflows/build-images-from-branch.yml

@@ -9,16 +9,18 @@
 name: Publish images to Dockerhub from any branch
 
 on:
-  workflow_dispatch:
+  workflow_call:
     inputs:
       branch_name:
         description: Branch to build from
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      branch_name:
         default: develop
         required: true
-  pull_request:
-    branches-ignore:
-      - 'dependabot/**'
-      - 'renovate/**'
+        description: Branch to build from
   push:
     branches:
       - develop
@@ -48,6 +50,11 @@ jobs:
           export VERSION=`git log -1 --pretty=format:%h`
           echo "Pushing version $VERSION"
           echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+          # the event_name is the event that triggered the caller workflow
+          # and not "workflow_call" like how it was supposed to be when
+          # another workflow is calling this one
+
           if [ "${{ github.event_name }}" == 'push' ]; then
           BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
           elif [ "${{ github.event_name }}" == 'pull_request' ]; then

.github/workflows/deploy-to-feature-environment.yml

@@ -102,6 +102,14 @@ jobs:
             }
             core.setOutput('stack', slugify('${{ env.BRANCH_NAME }}'));
 
+  trigger-build:
+    if: ${{ (github.event_name == 'workflow_dispatch') || (!contains(github.actor, 'bot') && github.event.pull_request.head.repo.fork == false) }}
+    needs: generate_stack_name_and_branch
+    uses: ./.github/workflows/build-images-from-branch.yml
+    with:
+      branch_name: ${{  needs.generate_stack_name_and_branch.outputs.branch_name  }}
+    secrets: inherit
+
   trigger-e2e:
     if: ${{ (github.event_name == 'workflow_dispatch') || (!contains(github.actor, 'bot') && github.event.pull_request.head.repo.fork == false) }}
     runs-on: ubuntu-22.04

.github/workflows/lint-and-test.yml

@@ -85,17 +85,36 @@ jobs:
         with:
           node-version-file: .nvmrc
 
+      - name: Extract dependencies for ${{ matrix.package }}
+        if: steps.check-scripts.outputs.skip != 'true'
+        id: extract-dependencies
+        run: |
+          DEPENDENCIES=$(node -e "
+            const { execSync } = require('child_process');
+            const output = execSync('yarn --silent workspaces info', { encoding: 'utf-8' });
+            const json = JSON.parse(output.replaceAll('@opencrvs', 'packages'));
+
+            const getDependencies = (pkg) =>
+              json[pkg].workspaceDependencies.concat(
+                json[pkg].workspaceDependencies.flatMap(getDependencies)
+              );
+
+            console.log(
+              getDependencies('${{ matrix.package }}').join(' ')
+            );
+          ")
+          echo "DEPENDENCIES=${DEPENDENCIES}" >> $GITHUB_ENV
+          echo "Found dependencies: $DEPENDENCIES"
+
       - name: Remove other package directories
         if: steps.check-scripts.outputs.skip != 'true'
         run: |
           for dir in packages/*; do
-            if [ "$dir" != "${{ matrix.package }}" ] && [ "$dir" != "packages/commons" ] && [ "$dir" != "packages/components" ]; then
-              if [ "${{ matrix.package }}" == "packages/client" ] && [ "$dir" == "packages/gateway" ] ; then
-                echo "Skipping $dir"
-              else
-                echo "Removing $dir"
-                rm -rf "$dir"
-              fi
+            if echo "${{ matrix.package }} $DEPENDENCIES" | grep -q -w "$dir"; then
+              echo "Skipping $dir"
+            else
+              echo "Removing $dir"
+              rm -rf "$dir"
             fi
           done
 
@@ -119,15 +138,13 @@ jobs:
 
       # TODO: Move out of the matrix to be built once and shared
       - name: Build common package
-        if: steps.check-scripts.outputs.skip != 'true'
+        if: steps.check-scripts.outputs.skip != 'true' && contains(env.DEPENDENCIES, 'packages/commons')
         run: cd packages/commons && yarn build
 
       - name: Build components client and login
-        if: steps.check-scripts.outputs.skip != 'true'
+        if: steps.check-scripts.outputs.skip != 'true' && contains(env.DEPENDENCIES, 'packages/components')
         run: |
-          if [[ "${{ matrix.package }}" == "packages/client" || "${{ matrix.package }}" == "packages/login" ]]; then
-            cd packages/components && yarn build
-          fi
+          cd packages/components && yarn build
 
       # TODO: should run parallel to unit tests as can take as much as unit tests
       - name: Run linting
@@ -166,7 +183,7 @@ jobs:
       - name: Run knip on base branch
         id: knip_base
         run: |
-          npx knip --no-exit-code --exports --reporter=markdown > knip_report.md
+          npx knip --tags=-knipignore --no-exit-code --exports --reporter=markdown > knip_report.md
           TOTAL=$(grep -oP '## [A-Za-z\s]+ \(\K[0-9]+' knip_report.md | awk '{sum+=$1} END {print sum}')
           echo "Total $TOTAL issue(s) on base branch."
           echo "total=${TOTAL}" >> $GITHUB_OUTPUT
@@ -175,7 +192,7 @@ jobs:
       - name: Run knip on PR branch
         id: knip_pr
         run: |
-          npx knip --no-exit-code --exports --reporter=markdown > knip_report.md
+          npx knip --tags=-knipignore --no-exit-code --exports --reporter=markdown > knip_report.md
           TOTAL=$(grep -oP '## [A-Za-z\s]+ \(\K[0-9]+' knip_report.md | awk '{sum+=$1} END {print sum}')
           echo "Total $TOTAL issue(s) on PR branch."
           echo "total=${TOTAL}" >> $GITHUB_OUTPUT

.github/workflows/publish-release.yml

@@ -48,19 +48,37 @@ jobs:
       - name: Get list of services
         id: get-services
         run: |
-          services=$(grep "^  [^ ]" docker-compose.yml | grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//')
+          services=$(grep "^  [^ ]" docker-compose.yml | grep -v base| grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//')
           services_json=$(echo $services | tr '\n' ',' | sed 's/,$//' | jq -R 'split(" ")' | tr -d '\n')
           # Set the list of service names as an output variable
           echo "services=$services_json" >> $GITHUB_OUTPUT
           echo "services=$services_json"
 
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push base image
+        uses: docker/build-push-action@v6
+        with:
+          file: packages/Dockerfile.base
+          context: .
+          push: true
+          tags: |
+            opencrvs/ocrvs-base:${{ steps.set-version.outputs.version }}
+          cache-from: type=registry,ref=opencrvs/ocrvs-base:${{ steps.set-version.outputs.version }}
+          cache-to: type=inline
+
     outputs:
       services: ${{ steps.get-services.outputs.services }}
       version: ${{ steps.set-version.outputs.version }}
 
   build:
     needs: base
     strategy:
+      fail-fast: false
       matrix:
         service: ${{ fromJSON(needs.base.outputs.services) }}
     runs-on: ubuntu-22.04
@@ -74,17 +92,23 @@ jobs:
         if: github.event_name == 'push'
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Build ${{ matrix.service }}
-        run: |
-          echo ${{ matrix.service }}
-          export VERSION=${{ github.event.inputs.release_version }}
-          docker compose build ${{ matrix.service }}
-          docker compose push ${{ matrix.service }}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          file: packages/${{ matrix.service }}/Dockerfile
+          build-args: |
+            VERSION=${{ needs.base.outputs.version }}
+          push: true
+          context: .
+          tags: |
+            opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}
+          cache-from: type=registry,ref=opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version}}
+          cache-to: type=inline
 
   security-scans:
     needs: [base, build]