From df9cd4024d0b4b3eb609b24d7a4c35300d94a746 Mon Sep 17 00:00:00 2001 From: Bryce Soghigian Date: Tue, 28 May 2024 13:26:14 -0700 Subject: [PATCH] feat: adding support for using out of tree credentials in karpenter --- pkg/providers/imagefamily/bootstrap/aksbootstrap.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go index e51038d85..6cc793359 100644 --- a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go +++ b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go @@ -104,6 +104,7 @@ type NodeBootstrapVariables struct { KubernetesVersion string // ? cluster/node pool specific, derived from user input HyperkubeURL string // - should be unnecessary KubeBinaryURL string // - necessary only for non-cached versions / static-ish + CredentialProviderDownloadURL string // - necessary only for non-cached versions / static-ish CustomKubeBinaryURL string // - unnecessary KubeproxyURL string // - should be unnecessary or bug APIServerPublicKey string // - unique per cluster, actually not sure best way to extract? [should not be needed on agent nodes] @@ -239,12 +240,12 @@ var ( // source note: unique per nodepool. partially user-specified, static, and RP-generated // removed --image-pull-progress-deadline=30m (not in 1.24?) // removed --network-plugin=cni (not in 1.24?) - // removed "--azure-container-registry-config" (not in 1.30) kubeletFlagsBase = map[string]string{ "--address": "0.0.0.0", "--anonymous-auth": "false", "--authentication-token-webhook": "true", "--authorization-mode": "Webhook", + "--azure-container-registry-config": "/etc/kubernetes/azure.json", "--cgroups-per-qos": "true", "--client-ca-file": "/etc/kubernetes/certs/ca.crt", "--cloud-config": "/etc/kubernetes/azure.json", @@ -385,7 +386,6 @@ var ( KubenetTemplate: base64.StdEncoding.EncodeToString(kubenetTemplate), // s ContainerdConfigContent: "", // kd IsKata: false, // n - NeedsCgroupV2: true, // s only static for karpenter } ) @@ -439,6 +439,7 @@ func (a AKS) applyOptions(nbv *NodeBootstrapVariables) { nbv.VNETCNILinuxPluginsURL = fmt.Sprintf("%s/azure-cni/v1.4.32/binaries/azure-vnet-cni-linux-%s-v1.4.32.tgz", globalAKSMirror, a.Arch) nbv.CNIPluginsURL = fmt.Sprintf("%s/cni-plugins/v1.1.1/binaries/cni-plugins-linux-%s-v1.1.1.tgz", globalAKSMirror, a.Arch) + nbv.CredentialProviderDownloadURL = fmt.Sprintf("https://acs-mirror.azureedge.net/cloud-provider-azure/%s/binaries/azure-acr-credential-provider-linux-amd64-v%s.tar.gz", nbv.KubernetesVersion, nbv.KubernetesVersion) // calculated values nbv.EnsureNoDupePromiscuousBridge = nbv.NeedsContainerd && nbv.NetworkPlugin == "kubenet" && nbv.NetworkPolicy != "calico" nbv.NetworkSecurityGroup = fmt.Sprintf("aks-agentpool-%s-nsg", a.ClusterID) @@ -451,10 +452,14 @@ func (a AKS) applyOptions(nbv *NodeBootstrapVariables) { nbv.GPUImageSHA = a.GPUImageSHA } - if semver.MustParse(a.KubernetesVersion).Minor < 30 { + minorVersion := semver.MustParse(a.KubernetesVersion).Minor + if minorVersion < 30 { kubeletFlagsBase["--azure-container-registry-config"] = "/etc/kubernetes/azure.json" } - + if minorVersion >= 30 { + kubeletFlagsBase["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml" + kubeletFlagsBase["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider" + } // merge and stringify labels kubeletLabels := lo.Assign(kubeletNodeLabelsBase, a.Labels) getAgentbakerGeneratedLabels(a.ResourceGroup, kubeletLabels)