feat: Support for Azure CNI without overlay & BYO CNI

.github/actions/e2e/create-cluster/action.yaml

@@ -1,10 +1,10 @@
 name: CreateCluster
 description: 'Installs Go Downloads and installs Karpenter Dependencies'
 inputs:
-#   k8s_version:
-#     description: 'Version of Kubernetes to use for the launched cluster'
-#     required: false
-#     default: "1.27"
+  # k8s_version:
+  #   description: 'Version of Kubernetes to use for the launched cluster'
+  #   required: false
+  #   default: "1.27"
   client-id:
     description: 
     required: true
@@ -32,35 +32,51 @@ inputs:
 runs:
   using: "composite"
   steps:
-  - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-    with:
-      ref: ${{ inputs.git_ref }}
-  - name: az login
-    uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
-    with:
-      client-id: ${{ inputs.client-id }}
-      tenant-id: ${{ inputs.tenant-id }}
-      subscription-id: ${{ inputs.subscription-id }}
-  - name: az set sub
-    shell: bash
-    run: az account set --subscription ${{ inputs.subscription-id }}
-  - name: create workload msi
-    shell: bash
-    run: AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_LOCATION=${{ inputs.location }} make az-create-workload-msi
-  - name: create cluster
-    shell: bash
-    run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} AZURE_LOCATION=${{ inputs.location }} make az-mkaks-cilium
-  - name: az login 2
-    uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
-    with:
-      client-id: ${{ inputs.client-id }}
-      tenant-id: ${{ inputs.tenant-id }}
-      subscription-id: ${{ inputs.subscription-id }}
-  - name: create federated cred
-    shell: bash
-    run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-create-federated-cred
-  - name: update azure perms
-    shell: bash
-    run: |
-      AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_LOCATION=${{ inputs.location }} make az-perm
-      AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} make az-perm-acr
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        ref: ${{ inputs.git_ref }}
+    - name: az login
+      uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 
+      with:
+        client-id: ${{ inputs.client-id }}
+        tenant-id: ${{ inputs.tenant-id }}
+        subscription-id: ${{ inputs.subscription-id }}
+    - name: az set sub
+      shell: bash
+      run: az account set --subscription ${{ inputs.subscription-id }}
+    - name: create workload msi
+      shell: bash
+      env:
+        AZURE_RESOURCE_GROUP: ${{ inputs.resource_group }}
+        AZURE_LOCATION: ${{ inputs.location }}
+      run: make az-create-workload-msi
+    - name: create cluster
+      shell: bash
+      env:
+        AZURE_CLUSTER_NAME: ${{ inputs.cluster_name }}
+        AZURE_RESOURCE_GROUP: ${{ inputs.resource_group }}
+        AZURE_ACR_NAME: ${{ inputs.acr_name }} 
+        AZURE_LOCATION: ${{ inputs.location }}
+      run: make az-mkaks-cilium
+    - name: az login 2
+      uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 
+      with:
+        client-id: ${{ inputs.client-id }}
+        tenant-id: ${{ inputs.tenant-id }}
+        subscription-id: ${{ inputs.subscription-id }}
+    - name: create federated cred
+      shell: bash
+      env:
+        AZURE_CLUSTER_NAME: ${{ inputs.cluster_name }}
+        AZURE_RESOURCE_GROUP: ${{ inputs.resource_group }}
+      run: make az-create-federated-cred
+    - name: update azure perms
+      shell: bash
+      env:
+        AZURE_CLUSTER_NAME: ${{ inputs.cluster_name }}
+        AZURE_RESOURCE_GROUP: ${{ inputs.resource_group }}
+        AZURE_LOCATION: ${{ inputs.location }}
+        AZURE_ACR_NAME: ${{ inputs.acr_name }}
+      run: |
+        make az-perm
+        make az-perm-acr

Makefile-az.mk

@@ -18,11 +18,16 @@ KARPENTER_FEDERATED_IDENTITY_CREDENTIAL_NAME ?= KARPENTER_FID
 CUSTOM_VNET_NAME ?= $(AZURE_CLUSTER_NAME)-vnet
 CUSTOM_SUBNET_NAME ?= nodesubnet
 
-az-all:             az-login az-create-workload-msi az-mkaks-cilium      az-create-federated-cred az-perm               az-perm-acr az-configure-values             az-build az-run          az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload
-az-all-custom-vnet: az-login az-create-workload-msi az-mkaks-custom-vnet az-create-federated-cred az-perm-subnet-custom az-perm-acr az-configure-values-custom-vnet az-build az-run          az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload
-az-all-user:	    az-login                        az-mkaks-user                                                                   az-configure-values             az-helm-install-snapshot az-run-sample ## Provision the cluster and deploy Karpenter snapshot release
+az-all:              az-login az-create-workload-msi az-mkaks-cilium      az-create-federated-cred az-perm               az-perm-acr az-configure-values             az-build az-run          az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload 
+
+az-all-cniv1:        az-login az-create-workload-msi az-mkaks-cniv1       az-create-federated-cred az-perm               az-perm-acr az-configure-values             az-build az-run          az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload
+
+az-all-cni-overlay:  az-login az-create-workload-msi az-mkaks-overlay     az-create-federated-cred az-perm               az-perm-acr az-configure-values             az-build az-run          az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload
+
+az-all-custom-vnet:  az-login az-create-workload-msi az-mkaks-custom-vnet az-create-federated-cred az-perm-subnet-custom az-perm-acr az-configure-values-custom-vnet az-build az-run          az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload
+az-all-user:	     az-login                        az-mkaks-user                                                                   az-configure-values             az-helm-install-snapshot az-run-sample ## Provision the cluster and deploy Karpenter snapshot release
 # TODO: az-all-savm case is not currently built to support workload identity, need to re-evaluate
-az-all-savm:        az-login                        az-mkaks-savm                                 az-perm-savm                      az-configure-values             az-build az-run          az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload - StandaloneVirtualMachines
+az-all-savm:         az-login                        az-mkaks-savm                                 az-perm-savm                      az-configure-values             az-build az-run          az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload - StandaloneVirtualMachines
 
 az-login: ## Login into Azure
 	az account show -o none || az login
@@ -53,13 +58,29 @@ az-mkaks: az-mkacr ## Create test AKS cluster (with --vm-set-type AvailabilitySe
 	az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing
 	skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter
 
+az-mkaks-cniv1: az-mkacr ## Create test AKS cluster (with --network-plugin azure)
+	az aks create          --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) \
+		--enable-managed-identity --node-count 3 --generate-ssh-keys -o none --network-plugin azure \
+		--enable-oidc-issuer --enable-workload-identity
+	az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing
+	skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter
+
+
 az-mkaks-cilium: az-mkacr ## Create test AKS cluster (with --network-dataplane cilium, --network-plugin cilium, and --network-plugin-mode overlay)
 	az aks create          --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) \
 		--enable-managed-identity --node-count 3 --generate-ssh-keys -o none --network-dataplane cilium --network-plugin azure --network-plugin-mode overlay \
 		--enable-oidc-issuer --enable-workload-identity
 	az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing
 	skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter
 
+az-mkaks-overlay: az-mkacr ## Create test AKS cluster (with --network-plugin-mode overlay)
+	az aks create          --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) \
+		--enable-managed-identity --node-count 3 --generate-ssh-keys -o none --network-plugin azure --network-plugin-mode overlay \
+		--enable-oidc-issuer --enable-workload-identity
+	az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing
+	skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter
+
+
 az-mkvnet: ## Create a VNet with address range of 10.1.0.0/16
 	az network vnet create --name $(CUSTOM_VNET_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --location $(AZURE_LOCATION) --address-prefixes "10.1.0.0/16"
 

hack/deploy/configure-values.sh

@@ -35,13 +35,18 @@ if [[ ! -v VNET_SUBNET_ID ]]; then
     VNET_SUBNET_ID=$(jq -r ".subnets[0].id" <<< "$VNET_JSON")
 fi
 
+# The // empty ensures that if the files is 'null' or not prsent jq will output nothing
+# If the value returned is none, its from jq and not the aks api in this case we return ""
+NETWORK_PLUGIN=$(jq -r ".networkProfile.networkPlugin // empty | if . == \"none\" then \"\" else . end" <<< "$AKS_JSON")
+NETWORK_PLUGIN_MODE=$(jq -r ".networkProfile.networkPluginMode // empty | if . == \"none\" then \"\" else . end" <<< "$AKS_JSON")
+NETWORK_POLICY=$(jq -r ".networkProfile.networkPolicy // empty | if . == \"none\" then \"\" else . end" <<< "$AKS_JSON")
+
 NODE_IDENTITIES=$(jq -r ".identityProfile.kubeletidentity.resourceId" <<< "$AKS_JSON")
 
 KARPENTER_USER_ASSIGNED_CLIENT_ID=$(az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'clientId' -otsv)
 
 export CLUSTER_NAME AZURE_LOCATION AZURE_RESOURCE_GROUP_MC KARPENTER_SERVICE_ACCOUNT_NAME \
-    CLUSTER_ENDPOINT BOOTSTRAP_TOKEN SSH_PUBLIC_KEY VNET_SUBNET_ID KARPENTER_USER_ASSIGNED_CLIENT_ID NODE_IDENTITIES
-
+    CLUSTER_ENDPOINT BOOTSTRAP_TOKEN SSH_PUBLIC_KEY VNET_SUBNET_ID KARPENTER_USER_ASSIGNED_CLIENT_ID NODE_IDENTITIES NETWORK_PLUGIN NETWORK_PLUGIN_MODE NETWORK_POLICY
 # get karpenter-values-template.yaml, if not already present (e.g. outside of repo context)
 if [ ! -f karpenter-values-template.yaml ]; then
     curl -sO https://raw.githubusercontent.com/Azure/karpenter/main/karpenter-values-template.yaml

karpenter-values-template.yaml

@@ -10,7 +10,6 @@ controller:
     # with this in place write (and read) QPS can be increased too
     #- name: GODEBUG
     #  value: http2client=0
-
     # options
     - name: CLUSTER_NAME
       value: ${CLUSTER_NAME}
@@ -21,9 +20,11 @@ controller:
     - name: SSH_PUBLIC_KEY
       value: "${SSH_PUBLIC_KEY}"
     - name: NETWORK_PLUGIN
-      value: "azure"
+      value: ${NETWORK_PLUGIN} 
+    - name: NETWORK_PLUGIN_MODE
+      value: ${NETWORK_PLUGIN_MODE}
     - name: NETWORK_POLICY
-      value: ""
+      value: ${NETWORK_POLICY}
     - name: VNET_SUBNET_ID
       value: ${VNET_SUBNET_ID}
     - name: NODE_IDENTITIES

pkg/consts/consts.go

@@ -0,0 +1,31 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consts
+
+const (
+	NetworkPluginAzure = "azure"
+	NetworkPluginNone  = "none"
+
+	NetworkPluginModeOverlay = "overlay"
+	NetworkPluginModeNone    = ""
+
+	NetworkDataplaneNone   = ""
+	NetworkDataplaneCilium = "cilium"
+	NetworkDataplaneAzure  = "azure"
+
+	DefaultKubernetesMaxPods = 250
+)

pkg/operator/options/options.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/Azure/karpenter-provider-azure/pkg/consts"
 	"k8s.io/apimachinery/pkg/util/sets"
 	coreoptions "sigs.k8s.io/karpenter/pkg/operator/options"
 	"sigs.k8s.io/karpenter/pkg/utils/env"
@@ -66,6 +67,7 @@ type Options struct {
 	SSHPublicKey                   string // ssh.publicKeys.keyData => VM SSH public key // TODO: move to v1alpha2.AKSNodeClass?
 	NetworkPlugin                  string // => NetworkPlugin in bootstrap
 	NetworkPolicy                  string // => NetworkPolicy in bootstrap
+	NetworkPluginMode              string // => Network Plugin Mode is used to control the mode the network plugin should operate in. For example, "overlay" used with --network-plugin=azure will use an overlay network (non-VNET IPs) for pods in the cluster. Learn more about overlay networking here: https://learn.microsoft.com/en-us/azure/aks/azure-cni-overlay?tabs=kubectl#overview-of-overlay-networking
 	NetworkDataplane               string
 	NodeIdentities                 []string // => Applied onto each VM
 
@@ -79,7 +81,8 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
 	fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.")
 	fs.StringVar(&o.KubeletClientTLSBootstrapToken, "kubelet-bootstrap-token", env.WithDefaultString("KUBELET_BOOTSTRAP_TOKEN", ""), "[REQUIRED] The bootstrap token for new nodes to join the cluster.")
 	fs.StringVar(&o.SSHPublicKey, "ssh-public-key", env.WithDefaultString("SSH_PUBLIC_KEY", ""), "[REQUIRED] VM SSH public key.")
-	fs.StringVar(&o.NetworkPlugin, "network-plugin", env.WithDefaultString("NETWORK_PLUGIN", "azure"), "The network plugin used by the cluster.")
+	fs.StringVar(&o.NetworkPlugin, "network-plugin", env.WithDefaultString("NETWORK_PLUGIN", consts.NetworkPluginAzure), "The network plugin used by the cluster.")
+	fs.StringVar(&o.NetworkPluginMode, "network-plugin-mode", env.WithDefaultString("NETWORK_PLUGIN_MODE", consts.NetworkPluginModeOverlay), "network plugin mode of the cluster")
 	fs.StringVar(&o.NetworkPolicy, "network-policy", env.WithDefaultString("NETWORK_POLICY", ""), "The network policy used by the cluster.")
 	fs.StringVar(&o.NetworkDataplane, "network-dataplane", env.WithDefaultString("NETWORK_DATAPLANE", "cilium"), "The network dataplane used by the cluster.")
 	fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR")

pkg/operator/options/options_validation.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/Azure/karpenter-provider-azure/pkg/consts"
 	"github.com/Azure/karpenter-provider-azure/pkg/utils"
 	"github.com/go-playground/validator/v10"
 	"go.uber.org/multierr"
@@ -30,13 +31,30 @@ func (o Options) Validate() error {
 	return multierr.Combine(
 		o.validateRequiredFields(),
 		o.validateEndpoint(),
+		o.validateNetworkingOptions(),
 		o.validateVMMemoryOverheadPercent(),
-		o.validateNetworkDataplane(),
 		o.validateVnetSubnetID(),
 		validate.Struct(o),
 	)
 }
 
+func (o Options) validateNetworkingOptions() error {
+	if o.NetworkPlugin != consts.NetworkPluginAzure && o.NetworkPlugin != consts.NetworkPluginNone {
+		return fmt.Errorf("network-plugin %v is invalid. network-plugin must equal 'azure' or 'none'", o.NetworkPlugin)
+	}
+	if o.NetworkPluginMode != consts.NetworkPluginModeOverlay && o.NetworkPluginMode != consts.NetworkPluginModeNone {
+		return fmt.Errorf("network-plugin-mode %s is invalid. network-plugin-mode must equal 'overlay' or ''", o.NetworkPluginMode)
+	}
+	if o.NetworkDataplane != consts.NetworkDataplaneAzure && o.NetworkDataplane != consts.NetworkDataplaneCilium && o.NetworkDataplane != consts.NetworkDataplaneNone {
+		return fmt.Errorf("network dataplane %s is not a valid network dataplane, valid dataplanes are ('azure', 'cilium')", o.NetworkDataplane)
+	}
+
+	if o.NetworkPlugin == consts.NetworkPluginNone && o.NetworkPluginMode != consts.NetworkPluginModeNone {
+		return fmt.Errorf("network-plugin-mode '%s' is invalid when network-plugin is 'none'. network-plugin-mode must be empty", o.NetworkPluginMode)
+	}
+	return nil
+}
+
 func (o Options) validateVnetSubnetID() error {
 	_, err := utils.GetVnetSubnetIDComponents(o.SubnetID)
 	if err != nil {
@@ -45,12 +63,6 @@ func (o Options) validateVnetSubnetID() error {
 	return nil
 }
 
-func (o Options) validateNetworkDataplane() error {
-	if o.NetworkDataplane != "azure" && o.NetworkDataplane != "cilium" {
-		return fmt.Errorf("network dataplane %s is not a valid network dataplane, valid dataplanes are ('azure', 'cilium')", o.NetworkDataplane)
-	}
-	return nil
-}
 func (o Options) validateEndpoint() error {
 	if o.ClusterEndpoint == "" {
 		return nil