diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 245b784402..d17279577b 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -1,41 +1,43 @@ -name: Passenger generic binaries tests CI tests +name: Passenger generic binaries CI env: ENTERPRISE: 0 + FORCE_COLOR: 1 + RUST_LOG_STYLE: always on: push: {} pull_request: {} jobs: - # build_linux: - # name: "Binary automation Linux-${{ matrix.arch }}" - # strategy: - # fail-fast: false - # matrix: - # include: - # - arch: x86_64 - # runner: ubuntu-24.04 - # - arch: aarch64 - # runner: passenger-ubuntu-24.04-arm64-4cpu - # runs-on: ${{ matrix.runner }} - # env: - # WORKSPACE: ${{ github.workspace }} - # OUTPUT_DIR: ${{ github.workspace }}/output-linux-${{ matrix.arch }} - # ARCHITECTURE: ${{ matrix.arch }} - # CACHE_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }} - # RUNTIME_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }}/runtime - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - uses: docker/setup-buildx-action@v3 - # - run: ./dev/ci/tests/binaries/build-linux - # - uses: actions/upload-artifact@v4 - # with: - # name: binaries-linux-${{ matrix.arch }} - # path: 'output-linux-${{ matrix.arch }}/**/*' - # - run: ./dev/ci/tests/binaries/test-linux + build_linux: + name: Binaries Linux ${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + arch: x86_64 + - runner: passenger-ubuntu-24.04-arm64-4cpu + arch: arm64 + runs-on: ${{ matrix.runner }} + env: + WORKSPACE: ${{ github.workspace }} + OUTPUT_DIR: ${{ github.workspace }}/output-linux-${{ matrix.arch }} + ARCHITECTURE: ${{ matrix.arch }} + CACHE_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }} + RUNTIME_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }}/runtime + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: docker/setup-buildx-action@v3 + - run: ./dev/ci/tests/binaries/build-linux + - uses: actions/upload-artifact@v4 + with: + name: binaries-linux-${{ matrix.arch }} + path: 'output-linux-${{ matrix.arch }}/**/*' + - run: ./dev/ci/tests/binaries/test-linux build_macos: name: Binaries macOS ${{ matrix.arch }} @@ -45,110 +47,129 @@ jobs: include: - runner: macos-13 arch: x86_64 + sccache_arch_and_os: x86_64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-13-x86_64 - runner: macos-14 arch: arm64 + sccache_arch_and_os: aarch64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-14-arm runs-on: ${{ matrix.runner }} - environment: dev - permissions: - id-token: write env: - WORKSPACE: ${{ github.workspace }} - OUTPUT_DIR: ${{ github.workspace }}/output - ARCHITECTURE: ${{ matrix.arch }} - CACHE_DIR: ${{ github.workspace }}/cache/macos-${{ matrix.arch }}/executor-${{ github.run_id }} - RUNTIME_DIR: ${{ github.workspace }}/cache/macos-${{ matrix.arch }}/executor-${{ github.run_id }}/runtime + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.sccache_azure_key_prefix }} steps: - uses: actions/checkout@v4 with: submodules: true - # - uses: azure/login@v2 - # with: - # tenant-id: ${{ vars.AZURE_TENANT_ID }} - # client-id: ${{ vars.AZURE_CLIENT_ID }} - # subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - name: Install dependencies + run: brew install zlib automake libtool coreutils + + - run: mkdir work runtime-output output + working-directory: packaging/binaries/macos + + - name: Determine runtime version + id: determine_runtime_version + run: | + RUNTIME_VERSION=$(cat ../shared/definitions/macos_runtime_version) + echo "version=$RUNTIME_VERSION" >> "$GITHUB_OUTPUT" + working-directory: packaging/binaries/macos + + - name: Fetch RVM cache + run: CACHE_PATH="$HOME/.rvm" ./dev/ci/fetch-cache-az-blob-storage + id: fetch_rvm_cache + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + BLOB_NAME: "binaries/rvm-${{ hashFiles('packaging/binaries/shared/definitions/ruby_versions') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd" - name: Install RVM - run: set -o pipefail && curl -fsSL https://get.rvm.io | bash + run: set -o pipefail && curl -fsSL https://get.rvm.io/stable | bash - name: Install Rubies run: xargs -I{} "$HOME/.rvm/bin/rvm" install ruby-{} < packaging/binaries/shared/definitions/ruby_versions + if: steps.fetch_rvm_cache.outputs.cache-hit != 'true' + - name: Cleanup RVM + run: ~/.rvm/bin/rvm cleanup all + if: steps.fetch_rvm_cache.outputs.cache-hit != 'true' - - run: mkdir cache runtime - working-directory: packaging/binaries + - name: Update RVM cache + run: CACHE_PATH="$HOME/.rvm" ./dev/ci/update-cache-az-blob-storage + if: steps.fetch_rvm_cache.outputs.cache-hit != 'true' + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + BLOB_NAME: "binaries/rvm-${{ hashFiles('packaging/binaries/shared/definitions/ruby_versions') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd" - - name: Determine runtime version - id: determine_runtime_version - run: | - RUNTIME_VERSION=$(cat shared/definitions/macos_runtime_version) - echo "version=$RUNTIME_VERSION" >> "$GITHUB_OUTPUT" - working-directory: packaging/binaries - - name: Check whether precompiled runtime exists - id: check_precompiled_runtime_exists - run: | - OUTPUT=$( - az storage blob exists \ - -c "$AZURE_CI_STORAGE_CONTAINER_NAME" \ - -n "macos-runtime-$SUFFIX.tar.zstd" \ - -o tsv - ) - echo "exists=$OUTPUT" >> "$GITHUB_OUTPUT" + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 env: - AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} - AZURE_CI_STORAGE_CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} - SUFFIX: ${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }} - - - name: Download precompiled runtime - run: > - az storage blob download - -c "$AZURE_CI_STORAGE_CONTAINER_NAME" - -n "macos-runtime-$SUFFIX.tar.zstd" - -f macos-runtime.tar.zstd - --no-progress + ARCH_AND_OS: ${{ matrix.sccache_arch_and_os }} + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + + - name: Fetch runtime cache + run: ./dev/ci/fetch-cache-az-blob-storage + id: fetch_runtime_cache env: AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} - AZURE_CI_STORAGE_CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} - SUFFIX: ${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }} - if: steps.check_precompiled_runtime_exists.outputs.exists == 'True' - working-directory: packaging/binaries - - - name: Extract precompiled runtime - run: tar -C runtime -x --zstd -f macos-runtime.tar.zstd && rm macos-runtime.tar.zstd - if: steps.check_precompiled_runtime_exists.outputs.exists == 'True' - working-directory: packaging/binaries + BLOB_NAME: binaries/macos-runtime-${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd + CACHE_PATH: packaging/binaries/macos/runtime-output - name: Compile runtime - run: ./macos/setup-runtime -c cache -o runtime -j 4 - if: steps.check_precompiled_runtime_exists.outputs.exists != 'True' - working-directory: packaging/binaries - - - name: Package compiled runtime - run: tar -C runtime -c --zstd -f macos-runtime.tar.zstd . - if: steps.check_precompiled_runtime_exists.outputs.exists != 'True' - working-directory: packaging/binaries - - - name: Store compiled runtime - run: > - az storage blob upload - -c "$AZURE_CI_STORAGE_CONTAINER_NAME" - -n "macos-runtime-$SUFFIX.tar.zstd" - -f macos-runtime.tar.zstd - --overwrite - && - rm macos-runtime.tar.zstd + run: ./setup-runtime -w work -o runtime-output -j 4 + if: steps.fetch_runtime_cache.outputs.cache-hit != 'true' + working-directory: packaging/binaries/macos + + - name: Update runtime cache + run: ./dev/ci/update-cache-az-blob-storage + if: steps.fetch_runtime_cache.outputs.cache-hit != 'true' env: AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} - AZURE_CI_STORAGE_CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} - SUFFIX: ${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }} - if: steps.check_precompiled_runtime_exists.outputs.exists != 'True' - working-directory: packaging/binaries + BLOB_NAME: binaries/macos-runtime-${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd + CACHE_PATH: packaging/binaries/macos/runtime-output + + + - name: Build binaries + run: ./build -p "$WORKSPACE" -r runtime-output -w work -o output -j 4 passenger nginx + env: + WORKSPACE: ${{ github.workspace }} + working-directory: packaging/binaries/macos - - run: ./dev/ci/tests/binaries/build-macos - uses: actions/upload-artifact@v4 with: name: binaries-macos-${{ matrix.arch }} - path: 'output-macos-${{ matrix.arch }}/**/*' - - run: ./dev/ci/tests/binaries/test-macos + path: 'output/**/*' + + + - name: Package binaries + run: ./package -i output -o output + working-directory: packaging/binaries/macos + + - name: Cache test suite gem bundle + uses: actions/cache@v4 + with: + path: packaging/binaries/macos/runtime-output/gems + key: test-gems-${{ hashFiles('packaging/binaries/shared/Gemfile.lock', 'packaging/binaries/shared/definitions/ruby_versions') }}-${{ matrix.runner }}-${{ matrix.arch }} + + - name: Test + run: ./test -p "$WORKSPACE" -r runtime-output -i output -I output + env: + WORKSPACE: ${{ github.workspace }} + working-directory: packaging/binaries/macos + + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: always() + + - name: Archive configure logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: generic-binaries-logs-macos + path: packaging/binaries/macos/work/*/config.log + if-no-files-found: ignore diff --git a/dev/ci/fetch-cache-az-blob-storage b/dev/ci/fetch-cache-az-blob-storage new file mode 100755 index 0000000000..e41fa5b4b4 --- /dev/null +++ b/dev/ci/fetch-cache-az-blob-storage @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -eo pipefail + +if [[ -z "$AZURE_STORAGE_CONNECTION_STRING" ]]; then + echo "AZURE_STORAGE_CONNECTION_STRING is required" + exit 1 +fi + +if [[ -z "$CONTAINER_NAME" ]]; then + echo "CONTAINER_NAME is required" + exit 1 +fi + +if [[ -z "$BLOB_NAME" ]]; then + echo "BLOB_NAME is required" + exit 1 +fi + +CACHE_PATH=${CACHE_PATH:-$BLOB_NAME} + + +echo "--> Checking whether blob exists" +EXISTS=$( + az storage blob exists \ + --container-name "$CONTAINER_NAME" \ + --name "$BLOB_NAME" \ + --connection-string "$AZURE_STORAGE_CONNECTION_STRING" \ + --output tsv +) +echo "$EXISTS" + +if [[ "$EXISTS" = True ]]; then + echo "--> Downloading and extracting blob" + mkdir -p "$CACHE_PATH" + az storage blob download \ + --container-name "$CONTAINER_NAME" \ + --name "$BLOB_NAME" \ + --no-progress \ + | tar -C "$CACHE_PATH" -x --zstd -f - + echo "Extracted" + + echo "cache-hit=true" >> "$GITHUB_OUTPUT" +else + echo "cache-hit=false" >> "$GITHUB_OUTPUT" +fi diff --git a/dev/ci/update-cache-az-blob-storage b/dev/ci/update-cache-az-blob-storage new file mode 100755 index 0000000000..f97aa59653 --- /dev/null +++ b/dev/ci/update-cache-az-blob-storage @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -eo pipefail + +if [[ -z "$AZURE_STORAGE_CONNECTION_STRING" ]]; then + echo "AZURE_STORAGE_CONNECTION_STRING is required" + exit 1 +fi + +if [[ -z "$CONTAINER_NAME" ]]; then + echo "CONTAINER_NAME is required" + exit 1 +fi + +if [[ -z "$BLOB_NAME" ]]; then + echo "BLOB_NAME is required" + exit 1 +fi + +CACHE_PATH=${CACHE_PATH:-$BLOB_NAME} + + +echo "--> Packaging cache directory" +tar -C "$CACHE_PATH" -c --zstd -f "$CACHE_PATH.tar.zstd" . + +echo "--> Uploading blob" +az storage blob upload \ + -c "$CONTAINER_NAME" \ + -n "$BLOB_NAME" \ + -f "$CACHE_PATH.tar.zstd" \ + --overwrite \ + --no-progress diff --git a/packaging/binaries b/packaging/binaries index 258988c3eb..18054db278 160000 --- a/packaging/binaries +++ b/packaging/binaries @@ -1 +1 @@ -Subproject commit 258988c3eb02a80861a5ed3c606c7959857163db +Subproject commit 18054db2780b939be2ffd1ea7004b4ef14fb0a07