cache macOS binaries

phusion · Oct 6, 2024 · caeb42c · caeb42c
1 parent 6a4df12
commit caeb42c
Show file tree

Hide file tree

Showing 3 changed files with 113 additions and 68 deletions.
diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml
@@ -1,4 +1,4 @@
-name: Passenger generic binaries tests CI tests
+name: Passenger generic binaries CI
 
 env:
   ENTERPRISE: 0
@@ -48,34 +48,15 @@ jobs:
           - runner: macos-14
             arch: arm64
     runs-on: ${{ matrix.runner }}
-    environment: dev
     permissions:
       id-token: write
     env:
-      WORKSPACE: ${{ github.workspace }}
-      OUTPUT_DIR: ${{ github.workspace }}/output
-      ARCHITECTURE: ${{ matrix.arch }}
-      CACHE_DIR: ${{ github.workspace }}/cache/macos-${{ matrix.arch }}/executor-${{ github.run_id }}
-      RUNTIME_DIR: ${{ github.workspace }}/cache/macos-${{ matrix.arch }}/executor-${{ github.run_id }}/runtime
+      CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
 
-      # - uses: azure/login@v2
-      #   with:
-      #     tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      #     client-id: ${{ vars.AZURE_CLIENT_ID }}
-      #     subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }}
-
-
-      - name: Install RVM
-        run: set -o pipefail && curl -fsSL https://get.rvm.io | bash
-
-      - name: Install Rubies
-        run: xargs -I{} "$HOME/.rvm/bin/rvm" install ruby-{} < packaging/binaries/shared/definitions/ruby_versions
-
-
       - run: mkdir cache runtime
         working-directory: packaging/binaries
 
@@ -86,65 +67,50 @@ jobs:
           echo "version=$RUNTIME_VERSION" >> "$GITHUB_OUTPUT"
         working-directory: packaging/binaries
 
-      - name: Check whether precompiled runtime exists
-        id: check_precompiled_runtime_exists
-        run: |
-          OUTPUT=$(
-            az storage blob exists \
-            -c "$AZURE_CI_STORAGE_CONTAINER_NAME" \
-            -n "macos-runtime-$SUFFIX.tar.zstd" \
-            -o tsv
-          )
-          echo "exists=$OUTPUT" >> "$GITHUB_OUTPUT"
+
+      - name: Fetch RVM cache
+        run: CACHE_PATH="$HOME/.rvm" ./dev/ci/fetch-cache-az-blob-storage
+        id: fetch_rvm_cache
         env:
           AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }}
-          AZURE_CI_STORAGE_CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }}
-          SUFFIX: ${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}
-
-      - name: Download precompiled runtime
-        run: >
-          az storage blob download
-          -c "$AZURE_CI_STORAGE_CONTAINER_NAME"
-          -n "macos-runtime-$SUFFIX.tar.zstd"
-          -f macos-runtime.tar.zstd
-          --no-progress
+          BLOB_NAME: "rvm-${{ hashFiles('packaging/binaries/shared/definitions/ruby_versions') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd"
+
+      - name: Install RVM
+        run: set -o pipefail && curl -fsSL https://get.rvm.io | bash
+
+      - name: Install Rubies
+        run: xargs -I{} "$HOME/.rvm/bin/rvm" install ruby-{} < packaging/binaries/shared/definitions/ruby_versions
+        if: steps.fetch_rvm_cache.outputs.cache-hit != 'true'
+
+      - name: Update runtime cache
+        run: CACHE_PATH="$HOME/.rvm" ./dev/ci/update-cache-az-blob-storage
+        if: steps.fetch_rvm_cache.outputs.cache-hit != 'true'
         env:
           AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }}
-          AZURE_CI_STORAGE_CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }}
-          SUFFIX: ${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}
-        if: steps.check_precompiled_runtime_exists.outputs.exists == 'True'
-        working-directory: packaging/binaries
+          BLOB_NAME: "rvm-${{ hashFiles('packaging/binaries/shared/definitions/ruby_versions') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd"
 
-      - name: Extract precompiled runtime
-        run: tar -C runtime -x --zstd -f macos-runtime.tar.zstd && rm macos-runtime.tar.zstd
-        if: steps.check_precompiled_runtime_exists.outputs.exists == 'True'
-        working-directory: packaging/binaries
+
+      - name: Fetch runtime cache
+        run: ./dev/ci/fetch-cache-az-blob-storage
+        id: fetch_runtime_cache
+        env:
+          AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }}
+          BLOB_NAME: macos-runtime-${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd
+          CACHE_PATH: packaging/binaries/runtime
 
       - name: Compile runtime
         run: ./macos/setup-runtime -c cache -o runtime -j 4
-        if: steps.check_precompiled_runtime_exists.outputs.exists != 'True'
-        working-directory: packaging/binaries
-
-      - name: Package compiled runtime
-        run: tar -C runtime -c --zstd -f macos-runtime.tar.zstd .
-        if: steps.check_precompiled_runtime_exists.outputs.exists != 'True'
+        if: steps.fetch_runtime_cache.outputs.cache-hit != 'true'
         working-directory: packaging/binaries
 
-      - name: Store compiled runtime
-        run: >
-          az storage blob upload
-          -c "$AZURE_CI_STORAGE_CONTAINER_NAME"
-          -n "macos-runtime-$SUFFIX.tar.zstd"
-          -f macos-runtime.tar.zstd
-          --overwrite
-          &&
-          rm macos-runtime.tar.zstd
+      - name: Update runtime cache
+        run: ./dev/ci/update-cache-az-blob-storage
+        if: steps.fetch_runtime_cache.outputs.cache-hit != 'true'
         env:
           AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }}
-          AZURE_CI_STORAGE_CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }}
-          SUFFIX: ${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}
-        if: steps.check_precompiled_runtime_exists.outputs.exists != 'True'
-        working-directory: packaging/binaries
+          BLOB_NAME: macos-runtime-${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd
+          CACHE_PATH: packaging/binaries/runtime
+
 
       - run: ./dev/ci/tests/binaries/build-macos
       - uses: actions/upload-artifact@v4

diff --git a/dev/ci/fetch-cache-az-blob-storage b/dev/ci/fetch-cache-az-blob-storage
@@ -0,0 +1,48 @@
+#!/usr/bin/bash
+set -eo pipefail
+
+if [[ -z "$AZURE_STORAGE_CONNECTION_STRING" ]]; then
+	echo "AZURE_STORAGE_CONNECTION_STRING is required"
+	exit 1
+fi
+
+if [[ -z "$CONTAINER_NAME" ]]; then
+	echo "CONTAINER_NAME is required"
+	exit 1
+fi
+
+if [[ -z "$BLOB_NAME" ]]; then
+		echo "BLOB_NAME is required"
+		exit 1
+fi
+
+CACHE_PATH=${CACHE_PATH:-$BLOB_NAME}
+
+
+echo "--> Checking whether blob exists"
+EXISTS=$(
+	az storage blob exists \
+	--container-name "$CONTAINER_NAME" \
+	--name "$BLOB_NAME" \
+	--connection-string "$AZURE_STORAGE_CONNECTION_STRING" \
+	--output tsv
+)
+echo "$EXISTS"
+
+if [[ "$EXISTS" = True ]]; then
+	echo "--> Downloading blob"
+	az storage blob download \
+		--container-name "$CONTAINER_NAME" \
+		--name "$BLOB_NAME" \
+		--file "$CACHE_PATH.tar.zstd" \
+		--no-progress
+
+	echo "--> Extracting blob"
+	mkdir -p "$CACHE_PATH"
+	tar -C "$CACHE_PATH" -x --zstd -f "$CACHE_PATH.tar.zstd"
+	rm "$CACHE_PATH.tar.zstd"
+
+	echo "cache-hit=true" >> "$GITHUB_OUTPUT"
+else
+	echo "cache-hit=false" >> "$GITHUB_OUTPUT"
+fi
diff --git a/dev/ci/update-cache-az-blob-storage b/dev/ci/update-cache-az-blob-storage
@@ -0,0 +1,31 @@
+#!/usr/bin/bash
+set -eo pipefail
+
+if [[ -z "$AZURE_STORAGE_CONNECTION_STRING" ]]; then
+	echo "AZURE_STORAGE_CONNECTION_STRING is required"
+	exit 1
+fi
+
+if [[ -z "$CONTAINER_NAME" ]]; then
+	echo "CONTAINER_NAME is required"
+	exit 1
+fi
+
+if [[ -z "$BLOB_NAME" ]]; then
+	echo "BLOB_NAME is required"
+	exit 1
+fi
+
+CACHE_PATH=${CACHE_PATH:-$BLOB_NAME}
+
+
+echo "--> Packaging cache directory"
+tar -C "$CACHE_PATH" -c --zstd -f "$CACHE_PATH.tar.zstd" .
+
+echo "--> Uploading blob"
+az storage blob upload \
+	-c "$CONTAINER_NAME" \
+	-n "$BLOB_NAME" \
+	-f "$CACHE_PATH.tar.zstd" \
+	--overwrite \
+	--no-progress