Build arm images on demand #2885
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LAPIS-SILO | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies | |
| DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo | |
| jobs: | |
| dockerImage: | |
| name: Build and cache docker image with dependencies installed | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| arch: | |
| ['amd64', 'arm64'] | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - name: Generate dependency files hash | |
| run: | | |
| DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }}) | |
| echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV | |
| - name: Docker metadata | |
| id: dockerMetadataDependencies | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch | |
| type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} | |
| type=raw,value=filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} | |
| type=raw,value=commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| - name: Determine platform build necessity | |
| run: | | |
| # Default PLATFORM_BUILD to false | |
| echo "PLATFORM_BUILD=false" >> $GITHUB_ENV | |
| # Check if architecture is amd64 | |
| if [[ "${{ matrix.arch }}" == "amd64" ]]; then | |
| echo "PLATFORM_BUILD=true" >> $GITHUB_ENV | |
| fi | |
| # Check if branch is main | |
| if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then | |
| echo "PLATFORM_BUILD=true" >> $GITHUB_ENV | |
| echo "ARM_BUILD=true" >> $GITHUB_ENV | |
| fi | |
| # Write the current labels to a file | |
| cat <<EOF > event.json | |
| ${{ toJSON(github.event.pull_request.labels) }} | |
| EOF | |
| # Check if the PR has the label 'arm-image' | |
| if jq -e '.[] | select(.name == "arm-image")' event.json > /dev/null; then | |
| echo "PLATFORM_BUILD=true" >> $GITHUB_ENV | |
| echo "ARM_BUILD=true" >> $GITHUB_ENV | |
| fi | |
| - name: Check if image exists | |
| if: env.PLATFORM_BUILD == 'true' | |
| run: | | |
| EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} > /dev/null 2>&1 && echo "true" || echo "false") | |
| echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV | |
| - name: Build and push image if input files changed | |
| if: env.CACHE_HIT == 'false' && env.PLATFORM_BUILD == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: Dockerfile_dependencies | |
| push: true | |
| tags: ${{ steps.dockerMetadataDependencies.outputs.tags }} | |
| cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| platforms: "linux/${{ matrix.arch }}" | |
| - name: Retag and push existing image if cache hit | |
| if: env.CACHE_HIT == 'true' && env.PLATFORM_BUILD == 'true' | |
| run: | | |
| TAGS=(${{ steps.dockerMetadataDependencies.outputs.tags }}) | |
| for TAG in "${TAGS[@]}"; do | |
| docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} | |
| done | |
| - name: Docker metadata | |
| if: env.PLATFORM_BUILD == 'true' | |
| id: dockerMetadataImage | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch | |
| type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | |
| type=raw,value=commit-${{ env.HEAD_SHA }} | |
| - name: Add architecture suffix to tags | |
| if: env.PLATFORM_BUILD == 'true' | |
| run: | | |
| echo "$DOCKER_METADATA_OUTPUT_JSON" | |
| echo "$DOCKER_METADATA_OUTPUT_JSON" | \ | |
| echo "TAGS_WITH_ARCH=$(jq '.tags | map(. + "-${{ matrix.arch }}") | join(",")')" >> $GITHUB_ENV | |
| echo "$TAGS_WITH_ARCH" | |
| - name: Build and push production image | |
| if: env.PLATFORM_BUILD == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| platforms: "linux/${{ matrix.arch }}" | |
| cache-from: type=gha,ref=${{ github.ref_name }}-image-cache | |
| cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache | |
| tags: ${{ env.TAGS_WITH_ARCH }} | |
| build-args: | | |
| DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| unitTests: | |
| name: Unit tests | |
| needs: dockerImage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build unit test image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| target: builder | |
| tags: builder | |
| load: true | |
| cache-from: type=gha,ref=${{ github.ref_name }}-image-cache | |
| cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache | |
| build-args: | | |
| DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| - name: Run unit tests | |
| run: | | |
| docker run \ | |
| --entrypoint "./silo_test" \ | |
| builder | |
| multiPlatformImages: | |
| name: Create the multi-platform image from the platform-specific ones | |
| needs: dockerImage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Tag and push existing image if cache hit | |
| run: | | |
| TAGS=$(jq -cr '.tags | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") | |
| for TAG in "${TAGS[@]}"; do | |
| if [ "${{ env.ARM_BUILD }}" == "1" ]; then | |
| docker buildx imagetools create --tag $TAG $TAG-arm64 $TAG-amd64 | |
| else | |
| docker buildx imagetools create --tag $TAG $TAG-amd64 | |
| fi | |
| done | |
| endToEndTests: | |
| name: Run End To End Tests | |
| needs: dockerImage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18.x | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" | |
| - name: npm install | |
| run: cd endToEndTests && npm ci | |
| - name: Check Format | |
| run: cd endToEndTests && npm run check-format | |
| - name: Start Docker Container and preprocess data | |
| run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up | |
| env: | |
| SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
| - name: Start Docker Container and run api | |
| run: docker compose -f docker-compose-for-tests-api.yml up -d --wait | |
| env: | |
| SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
| - name: Run Tests | |
| run: cd endToEndTests && SILO_URL=localhost:8080 npm run test | |
| linterChanges: | |
| name: Build/Run linter on changed files | |
| needs: dockerImage | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| env: | |
| PR_NUMBER: ${{ github.event.number }} | |
| container: | |
| image: ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ github.event.pull_request.head.sha }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - shell: bash | |
| name: Configure and run clang-tidy on changed files | |
| run: | | |
| mv /src/build . | |
| cmake -DBUILD_WITH_CLANG_TIDY=on -D CMAKE_BUILD_TYPE=Debug -B build/Debug | |
| echo "Successfully configured cmake" | |
| files="" | |
| PAGE=1 | |
| while true; do | |
| page_files=$(curl -s \ | |
| "https://api.github.com/repos/${{ github.repository }}/pulls/${PR_NUMBER}/files?per_page=100&page=$PAGE" \ | |
| | jq -r '.[] | select(.status != "removed") | .filename') | |
| # If there are no more files, break the loop | |
| if [[ -z "$page_files" ]]; then | |
| break | |
| fi | |
| files+="$page_files"$'\n' | |
| PAGE=$((PAGE + 1)) | |
| done | |
| echo "Changed files of this PR:" | |
| echo "$files" | |
| IFS=$'\n' | |
| for file in $files; do | |
| echo "Check ending for file: $file" | |
| if [[ $file == *.cpp ]]; then | |
| echo "Now linting the file: $file" | |
| echo "cmake --build build/Debug --target ${file%.cpp}.o" | |
| cmake --build build/Debug --target ${file%.cpp}.o | |
| fi | |
| done | |
| linterAll: | |
| name: Build/Run linter on all files | |
| needs: dockerImage | |
| if: github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ github.sha }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - shell: bash | |
| name: Configure and run clang-tidy on changed files | |
| run: | | |
| mv /src/build . | |
| cmake -DBUILD_WITH_CLANG_TIDY=on -D CMAKE_BUILD_TYPE=Debug -B build/Debug | |
| cmake --build build/Debug |