Skip to content

Commit

Permalink
Implement initial E2E test suite setup (#775)
Browse files Browse the repository at this point in the history
Signed-off-by: wen.rui <[email protected]>
  • Loading branch information
Rei1010 authored Jan 9, 2025
1 parent d04fc8b commit 594ef64
Show file tree
Hide file tree
Showing 22 changed files with 1,144 additions and 26 deletions.
1 change: 1 addition & 0 deletions .github/workflows/auto-release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ jobs:
uses: ./.github/workflows/call-e2e.yaml
with:
ref: ${{ needs.ensure-tag.outputs.tag }}
type: "release"

# excute a compatibility test when hami release
release-e2e-upgrade:
Expand Down
70 changes: 65 additions & 5 deletions .github/workflows/call-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,75 @@ on:
workflow_call:
inputs:
ref:
description: 'Reference id to run tests'
required: true
type: string
permissions: write-all
type:
description: 'E2E type'
required: true
type: string
default: pullrequest

jobs:
e2e:
runs-on: ubuntu-latest
e2e-test:
strategy:
matrix:
include:
- device: nvidia
type: tesla-p4
# - device: nvidia
# type: rtx-4090
# - device: huawei
# type: ascend-910b
runs-on: [ "${{ matrix.device }}", "${{ matrix.type }}" ]
environment: ${{ matrix.device }}
env:
E2E_TYPE: ${{ inputs.type }}
steps:
- name: checkout code
uses: actions/checkout@v4

- name: install Go
uses: actions/setup-go@v5
with:
go-version: "1.21"

- name: setup e2e env
run: |
make e2e-env-setup
- name: download hami helm
if: inputs.type == 'pullrequest'
uses: actions/download-artifact@v4
with:
name: chart_package_artifact
path: charts/

- name: download hami image
if: inputs.type == 'pullrequest'
uses: actions/download-artifact@v4
with:
name: hami-image
path: ./image

- name: load e2e image
if: inputs.type == 'pullrequest'
run: |
echo "Loading Docker image from image.tar..."
if [ -z "${VSPHERE_GPU_VM_IP}" ]; then
echo "Error: VSPHERE_GPU_VM_IP is not defined!"
exit 1
fi
scp ./image/image.tar root@$VSPHERE_GPU_VM_IP:/home/
ssh root@$VSPHERE_GPU_VM_IP "nerdctl load -i /home/image.tar"
ssh root@$VSPHERE_GPU_VM_IP "nerdctl image ls | grep hami"
- name: deploy hami helm
env:
HAMI_VERSION: ${{ inputs.ref }}
run: |
make helm-deploy
- name: e2e test
# https://github.com/actions/virtual-environments/issues/709
run: |
echo "Need to add e2e test"
make e2e-test
122 changes: 105 additions & 17 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ on:
branches-ignore:
- 'dependabot/**'
permissions:
contents: read # for actions/checkout to fetch code
contents: read # for actions/checkout to fetch code

env:
REGISTRY: docker.io
Expand Down Expand Up @@ -38,10 +38,11 @@ jobs:
run: make lint
- name: import alias
run: hack/verify-import-aliases.sh

test:
name: Unit test
needs: lint # rely on lint successful completion
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v4
Expand Down Expand Up @@ -71,11 +72,29 @@ jobs:
fail_ci_if_error: false
verbose: true

get_version:
name: get_version
runs-on: ubuntu-22.04
outputs:
version: ${{ steps.parse_version.outputs.version }}
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Parse the version
id: parse_version
run: |
value=$(make -f Makefile.defs print-version)
suffix="$(git rev-parse --short HEAD)"
tag="${value}-${suffix}"
echo "Generated tag: ${tag}"
echo "version=${tag}" >> $GITHUB_OUTPUT
build:
name: compile
runs-on: ubuntu-latest
needs: test # rely on test successful completion
name: Compile
runs-on: ubuntu-22.04
environment: nvidia
needs: [ test, get_version ]
steps:
- uses: actions/checkout@master
- name: Free disk space
Expand All @@ -88,22 +107,16 @@ jobs:
echo "=========after clean up, the left CI disk space"
df -h
- name: Get the version
id: get_version
run: |
tag="$(git rev-parse --short HEAD)"
echo ::set-output name=VERSION::${tag}
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Checkout submodule
uses: Mushus/[email protected]
with:
basePath: # optional, default is .
submodulePath: libvgpu
submodulePath: libvgpu

- name: Set up QEMU
uses: docker/setup-qemu-action@v3
Expand All @@ -124,13 +137,88 @@ jobs:
with:
context: .
file: ${{ env.IMAGE_ROOT_PATH }}/Dockerfile
labels: ${{ steps.meta.outputs.labels }}
platforms: ${{ env.BUILD_PLATFORM }}
labels: ${{ needs.get_version.outputs.version }}
build-args: |
VERSION=${{ steps.get_version.outputs.VERSION }}
VERSION=${{ needs.get_version.outputs.version }}
GOLANG_IMAGE=golang:1.22.5-bullseye
NVIDIA_IMAGE=nvidia/cuda:12.2.0-devel-ubuntu20.04
DEST_DIR=/usr/local
tags: ${{ steps.meta.outputs.tags }}
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_REPO }}:${{ needs.get_version.outputs.version }}
push: false
load: true
github-token: ${{ env.REGISTER_PASSWORD }}

- name: List images
run: |
docker images
- name: Save Docker image to image.tar
run: |
docker save ${{ env.REGISTRY }}/${{ env.IMAGE_REPO }}:${{ needs.get_version.outputs.version }} -o image.tar
- name: Upload image.tar as artifact
uses: actions/upload-artifact@v4
with:
name: hami-image
path: image.tar
retention-days: 5
if-no-files-found: error

# packages tgz from /charts of original branch, deploy to /charts of target branch
package_chart:
runs-on: ubuntu-22.04
needs: [ test, get_version ]
env:
HELM_VERSION: v3.8.1
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ needs.get_ref.outputs.ref }}

- name: Configure Git
run: |
git config user.name "$GITHUB_ACTOR"
git config user.email "[email protected]"
- name: Install Helm
uses: azure/setup-helm@v4
with:
version: ${{ env.HELM_VERSION }}

- name: Lint helm chart
run: |
make lint_chart
- name: Package Chart
continue-on-error: false
env:
VERSION: ${{ needs.get_version.outputs.version }}
run: |
cd charts
make clean
make
if ! ls *.tgz &>/dev/null ; then
echo "failed to generate chart"
exit 1
fi
cd ..
mkdir -p tmp
mv charts/*.tgz tmp
- name: Upload Artifact
uses: actions/[email protected]
with:
name: chart_package_artifact
path: tmp/*
retention-days: 5
if-no-files-found: error

# execute a full e2e test when hami code merge
e2e_test:
uses: ./.github/workflows/call-e2e.yaml
needs: [ package_chart, get_version, build ]
with:
ref: ${{ needs.get_version.outputs.version }}
type: "pullrequest"
12 changes: 12 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,15 @@ lint_chart:
aquasec/trivy:$(TRIVY_VERSION) config --exit-code 1 --severity $(LINT_TRIVY_SEVERITY_LEVEL) /tmp/src/charts ; \
(($$?==0)) || { echo "error, failed to check chart trivy" && exit 1 ; } ; \
echo "chart trivy check: pass"

.PHONY: e2e-env-setup
e2e-env-setup:
./hack/e2e-test-setup.sh

.PHONY: helm-deploy
helm-deploy:
./hack/deploy-helm.sh "${E2E_TYPE}" "${KUBE_CONF}" "${HAMI_VERSION}"

.PHONY: e2e-test
e2e-test:
./hack/e2e-test.sh "${E2E_TYPE}" "${KUBE_CONF}"
9 changes: 8 additions & 1 deletion Makefile.defs
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,16 @@ TARGETARCH ?= amd64
DESTDIR_BIN ?= $(ROOT_DIR)/output/$(TARGETARCH)/bin
DESTDIR_BASH_COMPLETION ?= $(ROOT_DIR)/output/$(TARGETARCH)/bash-completion

VERSION = $(shell cat $(dir $(lastword $(MAKEFILE_LIST)))/VERSION)
VERSION?=""
ifeq ($(VERSION), "")
VERSION=$(shell cat $(dir $(lastword $(MAKEFILE_LIST)))/VERSION)
endif

ECHO_GEN=echo " GEN $(RELATIVE_DIR)/"

LINT_TRIVY_SEVERITY_LEVEL ?= CRITICAL
TRIVY_VERSION=0.36.0

.PHONY: print-version
print-version:
@echo $(VERSION)
7 changes: 5 additions & 2 deletions charts/Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# get VERSION
.DEFAULT_GOAL := all
include ../Makefile.defs

VERSION_REGEX := '[vV]*[0-9]\+\.[0-9]\+\.[0-9]\+.*'
Expand All @@ -11,6 +12,7 @@ all: update-versions lint package
#update version in chart
update-versions:
$(ECHO_GEN) " Updating Chart version to $(VERSION)"
echo "VERSION=$(VERSION)"
echo "VERSION_MAJOR=$(VERSION_MAJOR)"
echo "GIT_VERSION=$(GIT_VERSION)"
echo "FULL_BUILD_VERSION=$(FULL_BUILD_VERSION)"
Expand All @@ -22,10 +24,11 @@ update-versions:
sed -i 's/version: "*'$(VERSION_REGEX)'"*/version: "'$$hami_version'"/g' $(VALUES_FILE)

lint: update-versions
helm lint --with-subcharts --values ./hami/values.yaml ./hami
helm lint --with-subcharts --values ./hami/values.yaml ./hami --debug

package: lint
helm package ./hami
helm package ./hami --debug

clean:
rm -f *.tgz

7 changes: 6 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ require (
github.com/fsnotify/fsnotify v1.7.0
github.com/google/uuid v1.6.0
github.com/julienschmidt/httprouter v1.3.0
github.com/onsi/ginkgo/v2 v2.17.1
github.com/onsi/gomega v1.32.0
github.com/opencontainers/runtime-spec v1.2.0
github.com/prometheus/client_golang v1.18.0
github.com/sirupsen/logrus v1.9.3
Expand All @@ -23,6 +25,7 @@ require (
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
google.golang.org/grpc v1.63.2
google.golang.org/protobuf v1.33.0
gopkg.in/yaml.v2 v2.4.0
gotest.tools/v3 v3.5.1
k8s.io/api v0.29.3
k8s.io/apimachinery v0.29.3
Expand All @@ -45,12 +48,14 @@ require (
github.com/go-openapi/jsonpointer v0.20.2 // indirect
github.com/go-openapi/jsonreference v0.20.4 // indirect
github.com/go-openapi/swag v0.22.9 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand Down Expand Up @@ -79,7 +84,6 @@ require (
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20240227032403-f107216b40e2 // indirect
k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
Expand All @@ -92,6 +96,7 @@ require (
replace (
github.com/Project-HAMi/HAMi/pkg/api => ./pkg/api
github.com/Project-HAMi/HAMi/pkg/device-plugin => ./pkg/device-plugin
github.com/Project-HAMi/HAMi/test/utils => ./test/utils
k8s.io/api => k8s.io/api v0.28.3
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.3
k8s.io/apimachinery => k8s.io/apimachinery v0.28.3
Expand Down
Loading

0 comments on commit 594ef64

Please sign in to comment.