Ondemand arm image alternative approach #2912
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LAPIS-SILO | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies | |
| DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo | |
| jobs: | |
| dockerImage: | |
| name: Build docker images | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - name: Generate dependency files hash | |
| run: | | |
| DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }}) | |
| echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV | |
| - name: Check if image exists | |
| run: | | |
| EXISTS=$(docker manifest inspect \ | |
| ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") | |
| echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV | |
| - name: Determine platforms for build | |
| run: | | |
| echo "PLATFORMS=amd64 " >> $GITHUB_ENV | |
| # Write the current labels to a file | |
| cat <<EOF > event.json | |
| ${{ toJSON(github.event.pull_request.labels) }} | |
| EOF | |
| # Check if we are on main or the PR has the label 'arm-image' | |
| if [[ "$GITHUB_REF" == "refs/heads/main" ]] || jq -e '.[] | select(.name == "arm-image")' event.json > /dev/null; then | |
| echo "PLATFORMS+=arm64" >> $GITHUB_ENV | |
| EXISTS=$(docker manifest inspect \ | |
| ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }} \ | |
| | jq '.manifests | map(.platform.architecture) | contains(["arm64"])') > /dev/null 2>&1 && echo "true" || echo "false" | |
| echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV | |
| fi | |
| - name: Build and push image if input files changed | |
| if: env.CACHE_HIT == 'false' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: Dockerfile_dependencies | |
| push: true | |
| tags: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }} | |
| cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| platforms: ${{ env.PLATFORMS }} | |
| - name: Tag dependency image with commit hash | |
| run: | | |
| docker buildx imagetools create \ | |
| --tag ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} \ | |
| ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }} | |
| - name: Docker metadata | |
| id: dockerMetadataImage | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch | |
| type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} | |
| type=raw,value=commit-${{ env.HEAD_SHA }} | |
| - name: Build and push production image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| platforms: ${{ env.PLATFORMS }} | |
| cache-from: type=gha,ref=${{ github.ref_name }}-image-cache | |
| cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache | |
| tags: ${{ steps.dockerMetadataImage.outputs.tags }} | |
| build-args: | | |
| DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
| - name: Build unit test image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| target: builder | |
| tags: builder | |
| load: true | |
| cache-from: type=gha,ref=${{ github.ref_name }}-image-cache | |
| cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache | |
| build-args: | | |
| DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
| - name: Run unit tests | |
| run: | | |
| docker run \ | |
| --entrypoint "./silo_test" \ | |
| builder | |
| endToEndTests: | |
| name: Run End To End Tests | |
| needs: dockerImage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18.x | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" | |
| - name: npm install | |
| run: cd endToEndTests && npm ci | |
| - name: Check Format | |
| run: cd endToEndTests && npm run check-format | |
| - name: Start Docker Container and preprocess data | |
| run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up | |
| env: | |
| SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
| - name: Start Docker Container and run api | |
| run: docker compose -f docker-compose-for-tests-api.yml up -d --wait | |
| env: | |
| SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }} | |
| - name: Run Tests | |
| run: cd endToEndTests && SILO_URL=localhost:8080 npm run test | |
| linterChanges: | |
| name: Build/Run linter on changed files | |
| needs: dockerImage | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| env: | |
| PR_NUMBER: ${{ github.event.number }} | |
| container: | |
| image: ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ github.event.pull_request.head.sha }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - shell: bash | |
| name: Configure and run clang-tidy on changed files | |
| run: | | |
| mv /src/build . | |
| cmake -DBUILD_WITH_CLANG_TIDY=on -D CMAKE_BUILD_TYPE=Debug -B build/Debug | |
| echo "Successfully configured cmake" | |
| files="" | |
| PAGE=1 | |
| while true; do | |
| page_files=$(curl -s \ | |
| "https://api.github.com/repos/${{ github.repository }}/pulls/${PR_NUMBER}/files?per_page=100&page=$PAGE" \ | |
| | jq -r '.[] | select(.status != "removed") | .filename') | |
| # If there are no more files, break the loop | |
| if [[ -z "$page_files" ]]; then | |
| break | |
| fi | |
| files+="$page_files"$'\n' | |
| PAGE=$((PAGE + 1)) | |
| done | |
| echo "Changed files of this PR:" | |
| echo "$files" | |
| IFS=$'\n' | |
| for file in $files; do | |
| echo "Check ending for file: $file" | |
| if [[ $file == *.cpp ]]; then | |
| echo "Now linting the file: $file" | |
| echo "cmake --build build/Debug --target ${file%.cpp}.o" | |
| cmake --build build/Debug --target ${file%.cpp}.o | |
| fi | |
| done |