🌱 Use vm operator v1.8.6 for supervisor e2e testing (#2894)

* Use vm-operator v1.8.6 for supervisor mode testing

* Use VSPHERE_NETWORK instead of NAMED_NETWORK

* Add content library items

* Address comments

* make generate-modules

* make lint-fix

---------

Co-authored-by: Stefan Bueringer <[email protected]>

Makefile

@@ -58,6 +58,7 @@ BIN_DIR := bin
 BUILD_DIR := .build
 TEST_DIR := test
 VCSIM_DIR := test/infrastructure/vcsim
+NETOP_DIR := test/infrastructure/net-operator
 TOOLS_DIR := hack/tools
 TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR))
 FLAVOR_DIR := $(ROOT_DIR)/templates
@@ -223,9 +224,15 @@ VM_OPERATOR_IMAGE_NAME ?= extra/vm-operator
 VM_OPERATOR_CONTROLLER_IMG ?= $(STAGING_REGISTRY)/$(VM_OPERATOR_IMAGE_NAME)
 VM_OPERATOR_DIR := test/infrastructure/vm-operator
 VM_OPERATOR_TMP_DIR ?= vm-operator.tmp
-VM_OPERATOR_VERSION ?= v1.8.1
+# note: this is the commit from 1.8.6 tag
+VM_OPERATOR_COMMIT ?= de75746a9505ef3161172d99b735d6593c54f0c5
+VM_OPERATOR_VERSION ?= v1.8.6-0-gde75746a
 VM_OPERATOR_ALL_ARCH = amd64 arm64
 
+# net operator
+NET_OPERATOR_IMAGE_NAME ?= cluster-api-net-operator
+NET_OPERATOR_IMG ?= $(STAGING_REGISTRY)/$(NET_OPERATOR_IMAGE_NAME)
+
 # It is set by Prow GIT_TAG, a git-based tag of the form vYYYYMMDD-hash, e.g., v20210120-v0.3.10-308-gc61521971
 
 TAG ?= dev
@@ -255,6 +262,7 @@ VCSIM_CRD_ROOT ?= $(VCSIM_DIR)/config/crd/bases
 WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
 RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 VCSIM_RBAC_ROOT ?= $(VCSIM_DIR)/config/rbac
+NETOP_RBAC_ROOT ?= $(NETOP_DIR)/config/rbac
 VERSION ?= $(shell cat clusterctl-settings.json | jq .config.nextVersion -r)
 OVERRIDES_DIR := $(HOME)/.cluster-api/overrides/infrastructure-vsphere/$(VERSION)
 
@@ -294,16 +302,21 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
 		paths=./apis/vmware/v1beta1 \
 		crd:crdVersions=v1 \
 		output:crd:dir=$(SUPERVISOR_CRD_ROOT)
-	# vm-operator crds are loaded to be used for integration tests.
+	# vm-operator crds are used for test.
 	$(CONTROLLER_GEN) \
 		paths=github.com/vmware-tanzu/vm-operator/api/v1alpha1/... \
 		crd:crdVersions=v1 \
 		output:crd:dir=$(VMOP_CRD_ROOT)
+	# net-operator is used for tests
+	$(CONTROLLER_GEN) \
+		paths=./$(NETOP_DIR)/controllers/... \
+        output:rbac:dir=$(NETOP_RBAC_ROOT) \
+        rbac:roleName=manager-role
 	# vcsim crds are used for tests.
 	$(CONTROLLER_GEN) \
-    		paths=./$(VCSIM_DIR)/api/v1alpha1 \
-    		crd:crdVersions=v1 \
-    		output:crd:dir=$(VCSIM_CRD_ROOT)
+		paths=./$(VCSIM_DIR)/api/v1alpha1 \
+		crd:crdVersions=v1 \
+		output:crd:dir=$(VCSIM_CRD_ROOT)
 	$(CONTROLLER_GEN) \
 		paths=./$(VCSIM_DIR)/ \
 		paths=./$(VCSIM_DIR)/controllers/... \
@@ -317,8 +330,8 @@ generate-go-deepcopy: $(CONTROLLER_GEN) ## Generate deepcopy go code for core
 		object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \
 		paths=./apis/...
 	$(CONTROLLER_GEN) \
-    		object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \
-    		paths=./$(VCSIM_DIR)/api/...
+    	object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \
+    	paths=./$(VCSIM_DIR)/api/...
 
 .PHONY: generate-go-conversions
 generate-go-conversions: $(CONTROLLER_GEN) $(CONVERSION_GEN) ## Runs Go related generate targets
@@ -539,6 +552,15 @@ docker-build-vcsim: docker-pull-prerequisites ## Build the docker image for vcsi
 		$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(VCSIM_DIR)/config/default/manager_pull_policy.yaml"; \
     fi
 
+.PHONY: docker-build-net-operator
+docker-build-net-operator: docker-pull-prerequisites ## Build the docker image for net-operator controller manager
+## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
+	cat $(NETOP_DIR)/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(NET_OPERATOR_IMG)-$(ARCH):$(TAG) --file -
+	@if [ "${DOCKER_BUILD_MODIFY_MANIFESTS}" = "true" ]; then \
+		$(MAKE) set-manifest-image MANIFEST_IMG=$(NET_OPERATOR_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./$(NETOP_DIR)/config/default/manager_image_patch.yaml"; \
+		$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(NETOP_DIR)/config/default/manager_pull_policy.yaml"; \
+	fi
+
 ## --------------------------------------
 ## Testing
 ## --------------------------------------
@@ -586,6 +608,7 @@ e2e-images: ## Build the e2e manager image
     # also the same settings must exist in e2e.sh
 	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build
 	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-vcsim
+	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-net-operator
 
 .PHONY: e2e
 e2e: e2e-images generate-e2e-templates
@@ -785,13 +808,15 @@ vm-operator-checkout:
 	@if [ -z "${VM_OPERATOR_VERSION}" ]; then echo "VM_OPERATOR_VERSION is not set"; exit 1; fi
 	@if [ -d "$(VM_OPERATOR_TMP_DIR)" ]; then \
 		echo "$(VM_OPERATOR_TMP_DIR) exists, skipping clone"; \
-		cd "$(VM_OPERATOR_TMP_DIR)"; \
-		if [ "$$(git describe --match "v[0-9]*")" != "$(VM_OPERATOR_VERSION)" ]; then \
-			echo "ERROR: checked out version $$(git describe --match "v[0-9]*") does not match expected version $(VM_OPERATOR_VERSION)"; \
-			exit 1; \
-		fi \
 	else \
-		git clone --depth 1 --branch "$(VM_OPERATOR_VERSION)" "https://github.com/vmware-tanzu/vm-operator.git" "$(VM_OPERATOR_TMP_DIR)"; \
+		git clone "https://github.com/vmware-tanzu/vm-operator.git" "$(VM_OPERATOR_TMP_DIR)"; \
+		cd "$(VM_OPERATOR_TMP_DIR)"; \
+		git checkout "$(VM_OPERATOR_COMMIT)"; \
+	fi
+	@cd "$(ROOT_DIR)/$(VM_OPERATOR_TMP_DIR)"; \
+	if [ "$$(git describe --dirty 2> /dev/null)" != "$(VM_OPERATOR_VERSION)" ]; then \
+		echo "ERROR: checked out version $$(git describe --dirty 2> /dev/null) does not match expected version $(VM_OPERATOR_VERSION)"; \
+		exit 1; \
 	fi
 
 .PHONY: vm-operator-manifest-build

hack/e2e.sh

@@ -83,7 +83,7 @@ export VSPHERE_SSH_AUTHORIZED_KEY="${VM_SSH_PUB_KEY:-}"
 export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key"
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere.yaml"
 export E2E_CONF_OVERRIDE_FILE=""
-export E2E_VM_OPERATOR_VERSION="${VM_OPERATOR_VERSION:-v1.8.1}"
+export E2E_VM_OPERATOR_VERSION="${VM_OPERATOR_VERSION:-v1.8.6-0-gde75746a}"
 export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
 export DOCKER_IMAGE_TAR="/tmp/images/image.tar"
 export GC_KIND="false"

internal/test/helpers/vcsim/model.go

@@ -89,3 +89,8 @@ func NetworkFolderName(datacenter int) string {
 func NetworkPath(datacenter int, network string) string {
 	return fmt.Sprintf("/%s/%s", NetworkFolderName(datacenter), network)
 }
+
+// DistributedPortGroupName provide a function to compute vcsim distribute port group names in a datacenter.
+func DistributedPortGroupName(datacenter int, distributedPortGroup int) string {
+	return fmt.Sprintf("%s_DVPG%d", DatacenterName(datacenter), distributedPortGroup)
+}

internal/test/helpers/vcsim/model_test.go

@@ -28,6 +28,7 @@ func Test_vcsim_NamesAndPath(t *testing.T) {
 	datacenter := 5
 	cluster := 3
 	datastore := 7
+	distributedPortGroup := 4
 
 	g.Expect(DatacenterName(datacenter)).To(Equal("DC5"))
 	g.Expect(ClusterName(datacenter, cluster)).To(Equal("DC5_C3"))
@@ -39,4 +40,5 @@ func Test_vcsim_NamesAndPath(t *testing.T) {
 	g.Expect(VMPath(datacenter, "my-mv")).To(Equal("/DC5/vm/my-mv"))
 	g.Expect(NetworkFolderName(datacenter)).To(Equal("DC5/network"))
 	g.Expect(NetworkPath(datacenter, "my-network")).To(Equal("/DC5/network/my-network"))
+	g.Expect(DistributedPortGroupName(datacenter, distributedPortGroup)).To(Equal("DC5_DVPG4"))
 }

test/e2e/config/vsphere.yaml

@@ -20,7 +20,9 @@ images:
     loadBehavior: mustLoad
   - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vcsim-controller-{ARCH}:dev
     loadBehavior: mustLoad
-  - name: gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:v1.8.1
+  - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-{ARCH}:dev
+    loadBehavior: mustLoad
+  - name: gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:v1.8.6-0-gde75746a
     loadBehavior: tryLoad
   - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2
     loadBehavior: tryLoad
@@ -213,9 +215,9 @@ providers:
   - name: vm-operator
     type: RuntimeExtensionProvider # vm-operator isn't a provider, but we fake it is so it can be handled by the clusterctl machinery.
     versions:
-      - name: v1.8.1
+      - name: v1.8.6-0-gde75746a
         # Use manifest from source files
-        value: "https://storage.googleapis.com/artifacts.k8s-staging-capi-vsphere.appspot.com/vm-operator/v1.8.1.yaml"
+        value: "https://storage.googleapis.com/artifacts.k8s-staging-capi-vsphere.appspot.com/vm-operator/v1.8.6-0-gde75746a.yaml"
         type: "url"
         contract: v1beta1
         files:
@@ -224,6 +226,19 @@ providers:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
 
+  - name: net-operator
+    type: RuntimeExtensionProvider # net-operator isn't a provider, but we fake it is so it can be handled by the clusterctl machinery.
+    versions:
+      - name: v1.10.99
+        # Use manifest from source files
+        value: ../../../../cluster-api-provider-vsphere/test/infrastructure/net-operator/config/default
+        contract: v1beta1
+        files:
+          - sourcePath: "../data/shared/capv/main/metadata.yaml"
+        replacements:
+          - old: "imagePullPolicy: Always"
+            new: "imagePullPolicy: IfNotPresent"
+
 variables:
   # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
   KUBERNETES_VERSION: "v1.29.0"
@@ -252,8 +267,10 @@ variables:
   VSPHERE_MACHINE_CLASS_CPU_CONFORMANCE: "8"
   VSPHERE_MACHINE_CLASS_MEMORY_CONFORMANCE: "8Gi"
   VSPHERE_CONTENT_LIBRARY: "capv"
+  VSPHERE_CONTENT_LIBRARY_ITEMS: "ubuntu-2204-kube-v1.28.0,ubuntu-2204-kube-v1.29.0"
   VSPHERE_IMAGE_NAME: "ubuntu-2204-kube-v1.29.0"
   VSPHERE_NETWORK: "sddc-cgw-network-6"
+  VSPHERE_DISTRIBUTED_PORT_GROUP: "/SDDC-Datacenter/network/sddc-cgw-network-6"
   VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.29.0"
   FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3602.2.3-kube-v1.29.0"
   VSPHERE_INSECURE_CSI: "true"

test/e2e/e2e_setup_test.go

@@ -273,7 +273,7 @@ func setupNamespaceWithVMOperatorDependenciesVCenter(managementClusterProxy fram
 					// NOTE: when running on vCenter the vm-operator automatically creates VirtualMachine objects for the content library.
 					Items: []vcsimv1.ContentLibraryItemConfig{},
 				},
-				NetworkName: e2eConfig.GetVariable("VSPHERE_NETWORK"),
+				DistributedPortGroupName: e2eConfig.GetVariable("VSPHERE_DISTRIBUTED_PORT_GROUP"),
 			},
 			StorageClasses: []vcsimv1.StorageClass{
 				{
@@ -296,6 +296,16 @@ func setupNamespaceWithVMOperatorDependenciesVCenter(managementClusterProxy fram
 		},
 	}
 
+	items := e2eConfig.GetVariable("VSPHERE_CONTENT_LIBRARY_ITEMS")
+	if items != "" {
+		for _, i := range strings.Split(e2eConfig.GetVariable("VSPHERE_CONTENT_LIBRARY_ITEMS"), ",") {
+			dependenciesConfig.Spec.VCenter.ContentLibrary.Items = append(dependenciesConfig.Spec.VCenter.ContentLibrary.Items, vcsimv1.ContentLibraryItemConfig{
+				Name:     i,
+				ItemType: "ovf",
+			})
+		}
+	}
+
 	err := vmoperator.ReconcileDependencies(ctx, c, dependenciesConfig)
 	Expect(err).ToNot(HaveOccurred(), "Failed to reconcile VMOperatorDependencies")
 }

test/framework/framework.go

@@ -122,6 +122,21 @@ func LoadE2EConfig(ctx context.Context, configPath string, configOverridesPath,
 				break
 			}
 		}
+
+		Byf("Dropping net-operator from the e2e config")
+		for i := range config.Providers {
+			if config.Providers[i].Name == "net-operator" {
+				config.Providers = append(config.Providers[:i], config.Providers[i+1:]...)
+				break
+			}
+		}
+
+		for i := range config.Images {
+			if strings.Contains(config.Images[i].Name, "net-operator") {
+				config.Images = append(config.Images[:i], config.Images[i+1:]...)
+				break
+			}
+		}
 	} else {
 		// In case we are testing supervisor, change the folder we build manifest from
 		Byf("Overriding source folder for vsphere provider to /config/supervisor in the e2e config")