Merge branch 'main' into charliedmcb/useWorkloadIdForKarpenter

README.md

@@ -6,14 +6,39 @@
 [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/Azure/karpenter/issues)
 
 ---
+## Features Overview 
+The AKS Karpenter Provider enables node autoprovisioning using [Karpenter](https://karpenter.sh/) on your AKS cluster.
+Karpenter improves the efficiency and cost of running workloads on Kubernetes clusters by:
 
-# AKS Karpenter Provider
+* **Watching** for pods that the Kubernetes scheduler has marked as unschedulable
+* **Evaluating** scheduling constraints (resource requests, node selectors, affinities, tolerations, and topology-spread constraints) requested by the pods
+* **Provisioning** nodes that meet the requirements of the pods
+* **Removing** the nodes when they are no longer needed
+* **Consolidating** existing nodes onto cheaper nodes with higher utilization per node
 
-The AKS Karpenter Provider enables node autoprovisioning using [Karpenter](https://karpenter.sh/) on your AKS cluster.
+## Production Readiness Status
+- API-Version: AKS Karpenter Provider is currently in alpha (`v1alpha2`).
+- AKS-Feature State: [Node Auto Provisioning is currently in preview](https://learn.microsoft.com/en-gb/azure/aks/node-autoprovision?tabs=azure-cli)
 
-## Status of Project:
+## Installation: Managed Karpenter (AKA Node Auto Provisioning) 
+The Node Auto Provisioning Preview, runs Karpenter as a managed addon similar to Managed Cluster Autoscaler. 
+
+To get started, just go through the prerequisites of [installing the Preview CLI](https://learn.microsoft.com/en-gb/azure/aks/node-autoprovision?tabs=azure-cli#install-the-aks-preview-cli-extension), and [register the NodeAutoProvisioningPreview feature flag](https://learn.microsoft.com/en-gb/azure/aks/node-autoprovision?tabs=azure-cli#register-the-nodeautoprovisioningpreview-feature-flag).
+
+### Enable node autoprovisioning 
+To enable node autoprovisioning, create a new cluster using the `az aks create` command and set --node-provisioning-mode to "Auto". You'll also need to use overlay networking and the cilium network policy for now.
+
+```bash 
+az aks create --name myFirstNap --resource-group napTest --node-provisioning-mode Auto --network-plugin azure --network-plugin-mode overlay --network-dataplane cilium
+```
 
-The API for AKS Karpenter Provider is currently alpha (`v1alpha2`).
+[View Limitations of the node autoprovisioning preview here](https://learn.microsoft.com/en-gb/azure/aks/node-autoprovision?tabs=azure-cli#limitations)
+
+### NAP Usage
+- [Learn More about Configuring the Nodepool CRDs to be more or less flexible](https://learn.microsoft.com/en-gb/azure/aks/node-autoprovision?tabs=azure-cli#node-pools)
+
+## Helm Chart
+a self-hosted experience similar to [aws/karpenter](https://artifacthub.io/packages/helm/karpenter/karpenter) is coming soon...
 
 ## Development
 
@@ -59,14 +84,6 @@ A: Oftentimes, especially for pre-existing tests, running `make toolchain` will
 
 ---
 
-Karpenter is an open-source node provisioning project built for Kubernetes.
-Karpenter improves the efficiency and cost of running workloads on Kubernetes clusters by:
-
-* **Watching** for pods that the Kubernetes scheduler has marked as unschedulable
-* **Evaluating** scheduling constraints (resource requests, nodeselectors, affinities, tolerations, and topology spread constraints) requested by the pods
-* **Provisioning** nodes that meet the requirements of the pods
-* **Removing** the nodes when the nodes are no longer needed
-
 ### Source Attribution
 
 Notice: Files in this source code originated from a fork of https://github.com/aws/karpenter
@@ -77,7 +94,7 @@ Many thanks to @ellistarn, @jonathan-innis, @tzneal, @bwagner5, @njtran, and man
 Many thanks to @Bryce-Soghigian, @rakechill, @charliedmcb, @jackfrancis, @comtalyst, @aagusuab, @matthchr, @gandhipr, @dtzar for contributing to AKS Karpenter Provider!
 
 ---
-
+### Community, discussion, contribution, and support 
 This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
 For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
 or contact [[email protected]](mailto:[email protected]) with any additional questions or comments.

hack/codegen.sh

@@ -32,7 +32,7 @@ skugen() {
   NO_UPDATE=" pkg/fake/zz_generated.sku.$location.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)"
   SUBJECT="SKUGEN"
 
-  go run hack/code/instancetype_testdata_gen.go -- "${GENERATED_FILE}" "$location" "Standard_B1s,Standard_A0,Standard_D2_v2,Standard_D2_v3,Standard_DS2_v2,Standard_D2s_v3,Standard_D2_v5,Standard_F16s_v2,Standard_NC24ads_A100_v4,Standard_M8-2ms,Standard_D4s_v3,Standard_D64s_v3,Standard_DC8s_v3"
+  go run hack/code/instancetype_testdata_gen.go -- "${GENERATED_FILE}" "$location" "Standard_B1s,Standard_A0,Standard_D2_v2,Standard_D2_v3,Standard_DS2_v2,Standard_D2s_v3,Standard_D2_v5,Standard_F16s_v2,Standard_NC6s,Standard_NC6s_v3,Standard_NC16as_T4_v3,Standard_NC24ads_A100_v4,Standard_M8-2ms,Standard_D4s_v3,Standard_D64s_v3,Standard_DC8s_v3"
   go fmt "${GENERATED_FILE}"
 
   GIT_DIFF=$(git diff --stat "${GENERATED_FILE}")

pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml

@@ -46,6 +46,7 @@ spec:
                 description: ImageFamily is the image family that instances use.
                 enum:
                 - Ubuntu2204
+                - AzureLinux
                 type: string
               imageVersion:
                 description: ImageVersion is the image version that instances use.

pkg/apis/v1alpha2/aksnodeclass.go

@@ -36,7 +36,7 @@ type AKSNodeClassSpec struct {
 	ImageID *string `json:"-"`
 	// ImageFamily is the image family that instances use.
 	// +kubebuilder:default=Ubuntu2204
-	// +kubebuilder:validation:Enum:={Ubuntu2204}
+	// +kubebuilder:validation:Enum:={Ubuntu2204,AzureLinux}
 	ImageFamily *string `json:"imageFamily,omitempty"`
 	// ImageVersion is the image version that instances use.
 	// +optional

pkg/apis/v1alpha2/labels.go

@@ -122,3 +122,8 @@ var (
 
 	NodeClaimLinkedAnnotationKey = v1alpha5.MachineLinkedAnnotationKey // still using the one from v1alpha5
 )
+
+const (
+	Ubuntu2204ImageFamily = "Ubuntu2204"
+	AzureLinuxImageFamily = "AzureLinux"
+)

pkg/fake/azureresourcegraphapi.go

@@ -39,7 +39,7 @@ type AzureResourceGraphBehavior struct {
 }
 
 // assert that the fake implements the interface
-var _ instance.AzureResourceGraphAPI = (*AzureResourceGraphAPI)(nil)
+var _ instance.AzureResourceGraphAPI = &AzureResourceGraphAPI{}
 
 type AzureResourceGraphAPI struct {
 	AzureResourceGraphBehavior

pkg/fake/communityimageversionsapi.go

@@ -30,7 +30,7 @@ type CommunityGalleryImageVersionsAPI struct {
 }
 
 // assert that the fake implements the interface
-var _ imagefamily.CommunityGalleryImageVersionsAPI = (*CommunityGalleryImageVersionsAPI)(nil)
+var _ imagefamily.CommunityGalleryImageVersionsAPI = &CommunityGalleryImageVersionsAPI{}
 
 // NewListPager returns a new pager to return the next page of CommunityGalleryImageVersionsClientListResponse
 func (c *CommunityGalleryImageVersionsAPI) NewListPager(_ string, _ string, _ string, _ *armcompute.CommunityGalleryImageVersionsClientListOptions) *runtime.Pager[armcompute.CommunityGalleryImageVersionsClientListResponse] {

pkg/fake/networkinterfaceapi.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/Azure/azure-sdk-for-go-extensions/pkg/errors"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 	"github.com/Azure/go-autorest/autorest/to"
@@ -34,13 +36,18 @@ type NetworkInterfaceCreateOrUpdateInput struct {
 	Options           *armnetwork.InterfacesClientBeginCreateOrUpdateOptions
 }
 
+type NetworkInterfaceDeleteInput struct {
+	ResourceGroupName, InterfaceName string
+}
+
 type NetworkInterfacesBehavior struct {
 	NetworkInterfacesCreateOrUpdateBehavior MockedLRO[NetworkInterfaceCreateOrUpdateInput, armnetwork.InterfacesClientCreateOrUpdateResponse]
+	NetworkInterfacesDeleteBehavior         MockedLRO[NetworkInterfaceDeleteInput, armnetwork.InterfacesClientDeleteResponse]
 	NetworkInterfaces                       sync.Map
 }
 
 // assert that the fake implements the interface
-var _ instance.NetworkInterfacesAPI = (*NetworkInterfacesAPI)(nil)
+var _ instance.NetworkInterfacesAPI = &NetworkInterfacesAPI{}
 
 type NetworkInterfacesAPI struct {
 	// instance.NetworkInterfacesAPI
@@ -79,17 +86,23 @@ func (c *NetworkInterfacesAPI) Get(_ context.Context, resourceGroupName string,
 	id := mkNetworkInterfaceID(resourceGroupName, interfaceName)
 	iface, ok := c.NetworkInterfaces.Load(id)
 	if !ok {
-		return armnetwork.InterfacesClientGetResponse{}, fmt.Errorf("not found")
+		return armnetwork.InterfacesClientGetResponse{}, &azcore.ResponseError{ErrorCode: errors.ResourceNotFound}
 	}
 	return armnetwork.InterfacesClientGetResponse{
 		Interface: iface.(armnetwork.Interface),
 	}, nil
 }
 
 func (c *NetworkInterfacesAPI) BeginDelete(_ context.Context, resourceGroupName string, interfaceName string, _ *armnetwork.InterfacesClientBeginDeleteOptions) (*runtime.Poller[armnetwork.InterfacesClientDeleteResponse], error) {
-	id := mkNetworkInterfaceID(resourceGroupName, interfaceName)
-	c.NetworkInterfaces.Delete(id)
-	return nil, nil
+	input := &NetworkInterfaceDeleteInput{
+		ResourceGroupName: resourceGroupName,
+		InterfaceName:     interfaceName,
+	}
+	return c.NetworkInterfacesDeleteBehavior.Invoke(input, func(input *NetworkInterfaceDeleteInput) (*armnetwork.InterfacesClientDeleteResponse, error) {
+		id := mkNetworkInterfaceID(resourceGroupName, interfaceName)
+		c.NetworkInterfaces.Delete(id)
+		return &armnetwork.InterfacesClientDeleteResponse{}, nil
+	})
 }
 
 func mkNetworkInterfaceID(resourceGroupName, interfaceName string) string {

pkg/fake/pricingapi.go

@@ -36,6 +36,9 @@ type PricingBehavior struct {
 	ProductsPricePage AtomicPtr[client.ProductsPricePage]
 }
 
+// assert that the fake implements the interface
+var _ client.PricingAPI = &PricingAPI{}
+
 func (p *PricingAPI) Reset() {
 	p.NextError.Reset()
 	p.ProductsPricePage.Reset()

pkg/fake/skus_test.go

@@ -0,0 +1,71 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fake
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+func TestSKUExistence(t *testing.T) {
+	testSKUExistenceForRegion(t, "eastus", sets.New(
+		"Standard_A0",
+		"Standard_B1s",
+		"Standard_D2s_v3",
+		"Standard_D2_v2",
+		"Standard_D2_v3",
+		"Standard_D2_v5",
+		"Standard_D4s_v3",
+		"Standard_D64s_v3",
+		"Standard_DC8s_v3",
+		"Standard_DS2_v2",
+		"Standard_F16s_v2",
+		"Standard_M8-2ms",
+		"Standard_NC24ads_A100_v4",
+		"Standard_NC6s_v3",
+		"Standard_NC16as_T4_v3",
+	))
+	testSKUExistenceForRegion(t, "westcentralus", sets.New(
+		"Standard_A0",
+		"Standard_B1s",
+		"Standard_D2s_v3",
+		"Standard_D2_v2",
+		"Standard_D2_v3",
+		"Standard_D2_v5",
+		"Standard_D4s_v3",
+		"Standard_D64s_v3",
+		"Standard_DS2_v2",
+		"Standard_F16s_v2",
+	))
+}
+
+func testSKUExistenceForRegion(t *testing.T, region string, expectedSKUs sets.Set[string]) {
+	generatedSKUs := ResourceSkus[region]
+
+	skuSets := make(sets.Set[string])
+	for _, sku := range generatedSKUs {
+		skuName := *sku.Name
+		skuSets.Insert(skuName)
+	}
+
+	for _, expectedSKU := range expectedSKUs.UnsortedList() {
+		if exists := skuSets.Has(expectedSKU); !exists {
+			t.Errorf("SKU not found in %v: %v", region, expectedSKU)
+		}
+	}
+}

pkg/fake/virtualmachineextensionsapi.go

@@ -40,7 +40,7 @@ type VirtualMachineExtensionsBehavior struct {
 }
 
 // assert that ComputeAPI implements ARMComputeAPI
-var _ instance.VirtualMachineExtensionsAPI = (*VirtualMachineExtensionsAPI)(nil)
+var _ instance.VirtualMachineExtensionsAPI = &VirtualMachineExtensionsAPI{}
 
 type VirtualMachineExtensionsAPI struct {
 	// instance.VirtualMachineExtensionsAPI

pkg/fake/virtualmachinesapi.go

@@ -66,7 +66,7 @@ type VirtualMachinesBehavior struct {
 }
 
 // assert that the fake implements the interface
-var _ instance.VirtualMachinesAPI = (*VirtualMachinesAPI)(nil)
+var _ instance.VirtualMachinesAPI = &VirtualMachinesAPI{}
 
 type VirtualMachinesAPI struct {
 	// TODO: document the implications of embedding vs. not embedding the interface here