diff --git a/controllers/devbox/api/v1alpha1/devbox_types.go b/controllers/devbox/api/v1alpha1/devbox_types.go index aea3a051da0..165cbea6660 100644 --- a/controllers/devbox/api/v1alpha1/devbox_types.go +++ b/controllers/devbox/api/v1alpha1/devbox_types.go @@ -18,17 +18,10 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type ResourceName string - const ( - // ResourceCPU CPU, in cores. (500m = .5 cores) - ResourceCPU ResourceName = "cpu" - // ResourceMemory Memory, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) - ResourceMemory ResourceName = "memory" // FinalizerName is the finalizer for Devbox FinalizerName = "devbox.sealos.io/finalizer" DevBoxPartOf = "devbox" @@ -52,7 +45,12 @@ const ( NetworkTypeTailnet NetworkType = "Tailnet" ) -type ResourceList map[ResourceName]resource.Quantity +type RuntimeRef struct { + // +kubebuilder:validation:Required + Name string `json:"name"` + // +kubebuilder:validation:Optional + Namespace string `json:"namespace,omitempty"` +} type NetworkSpec struct { // +kubebuilder:validation:Required @@ -109,7 +107,7 @@ type DevboxSpec struct { // +kubebuilder:validation:Enum=Running;Stopped State DevboxState `json:"state"` // +kubebuilder:validation:Required - Resource ResourceList `json:"resource"` + Resource corev1.ResourceList `json:"resource"` // +kubebuilder:validation:Optional // +kubebuilder:default=false @@ -127,6 +125,10 @@ type DevboxSpec struct { // +kubebuilder:validation:Required NetworkSpec NetworkSpec `json:"network,omitempty"` + // +kubebuilder:validation:Optional + RuntimeClassName string `json:"runtimeClassName,omitempty"` + // +kubebuilder:validation:Optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` // +kubebuilder:validation:Optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` // +kubebuilder:validation:Optional diff --git a/controllers/devbox/api/v1alpha1/zz_generated.deepcopy.go b/controllers/devbox/api/v1alpha1/zz_generated.deepcopy.go index 2bcdb420198..d1590df7e11 100644 --- a/controllers/devbox/api/v1alpha1/zz_generated.deepcopy.go +++ b/controllers/devbox/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -276,13 +276,20 @@ func (in *DevboxSpec) DeepCopyInto(out *DevboxSpec) { *out = *in if in.Resource != nil { in, out := &in.Resource, &out.Resource - *out = make(ResourceList, len(*in)) + *out = make(v1.ResourceList, len(*in)) for key, val := range *in { (*out)[key] = val.DeepCopy() } } in.Config.DeepCopyInto(&out.Config) in.NetworkSpec.DeepCopyInto(&out.NetworkSpec) + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) @@ -461,22 +468,16 @@ func (in *OperationRequestStatus) DeepCopy() *OperationRequestStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in ResourceList) DeepCopyInto(out *ResourceList) { - { - in := &in - *out = make(ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } +func (in *RuntimeRef) DeepCopyInto(out *RuntimeRef) { + *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceList. -func (in ResourceList) DeepCopy() ResourceList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeRef. +func (in *RuntimeRef) DeepCopy() *RuntimeRef { if in == nil { return nil } - out := new(ResourceList) + out := new(RuntimeRef) in.DeepCopyInto(out) - return *out + return out } diff --git a/controllers/devbox/cmd/main.go b/controllers/devbox/cmd/main.go index 62416e9441d..6a9b84efef1 100644 --- a/controllers/devbox/cmd/main.go +++ b/controllers/devbox/cmd/main.go @@ -27,6 +27,7 @@ import ( "k8s.io/client-go/rest" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -42,7 +43,9 @@ import ( devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" "github.com/labring/sealos/controllers/devbox/internal/controller" + "github.com/labring/sealos/controllers/devbox/internal/controller/utils/matcher" "github.com/labring/sealos/controllers/devbox/internal/controller/utils/registry" + utilresource "github.com/labring/sealos/controllers/devbox/internal/controller/utils/resource" // +kubebuilder:scaffold:imports ) @@ -65,19 +68,24 @@ func main() { var secureMetrics bool var enableHTTP2 bool var tlsOpts []func(*tls.Config) + // debug flag + var debugMode bool + // registry flag var registryAddr string var registryUser string var registryPassword string - var authAddr string + // resource flag var requestCPURate float64 var requestMemoryRate float64 var requestEphemeralStorage string var limitEphemeralStorage string - var debugMode bool - flag.StringVar(®istryAddr, "registry-addr", "sealos.hub:5000", "The address of the registry") - flag.StringVar(®istryUser, "registry-user", "admin", "The user of the registry") - flag.StringVar(®istryPassword, "registry-password", "passw0rd", "The password of the registry") - flag.StringVar(&authAddr, "auth-addr", "sealos.hub:5000", "The address of the auth") + var maximumLimitEphemeralStorage string + // pod matcher flag + var enablePodResourceMatcher bool + var enablePodEnvMatcher bool + var enablePodPortMatcher bool + var enablePodEphemeralStorageMatcher bool + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -88,11 +96,24 @@ func main() { "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") + // debug flag flag.BoolVar(&debugMode, "debug", false, "If set, debug mode will be enabled") + // registry flag + flag.StringVar(®istryAddr, "registry-addr", "sealos.hub:5000", "The address of the registry") + flag.StringVar(®istryUser, "registry-user", "admin", "The user of the registry") + flag.StringVar(®istryPassword, "registry-password", "passw0rd", "The password of the registry") + // resource flag flag.Float64Var(&requestCPURate, "request-cpu-rate", 10, "The request rate of cpu limit in devbox.") flag.Float64Var(&requestMemoryRate, "request-memory-rate", 10, "The request rate of memory limit in devbox.") - flag.StringVar(&requestEphemeralStorage, "request-ephemeral-storage", "500Mi", "The request value of ephemeral storage in devbox.") - flag.StringVar(&limitEphemeralStorage, "limit-ephemeral-storage", "10Gi", "The limit value of ephemeral storage in devbox.") + flag.StringVar(&requestEphemeralStorage, "request-ephemeral-storage", "500Mi", "The default request value of ephemeral storage in devbox.") + flag.StringVar(&limitEphemeralStorage, "limit-ephemeral-storage", "10Gi", "The default limit value of ephemeral storage in devbox.") + flag.StringVar(&maximumLimitEphemeralStorage, "maximum-limit-ephemeral-storage", "50Gi", "The maximum limit value of ephemeral storage in devbox.") + // pod matcher flag, pod resource matcher, env matcher, port matcher will be enabled by default, ephemeral storage matcher will be disabled by default + flag.BoolVar(&enablePodResourceMatcher, "enable-pod-resource-matcher", true, "If set, pod resource matcher will be enabled") + flag.BoolVar(&enablePodEnvMatcher, "enable-pod-env-matcher", true, "If set, pod env matcher will be enabled") + flag.BoolVar(&enablePodPortMatcher, "enable-pod-port-matcher", true, "If set, pod port matcher will be enabled") + flag.BoolVar(&enablePodEphemeralStorageMatcher, "enable-pod-ephemeral-storage-matcher", false, "If set, pod ephemeral storage matcher will be enabled") + opts := zap.Options{ Development: true, } @@ -182,16 +203,36 @@ func main() { os.Exit(1) } + podMatchers := []matcher.PodMatcher{} + if enablePodResourceMatcher { + podMatchers = append(podMatchers, matcher.ResourceMatcher{}) + } + if enablePodEnvMatcher { + podMatchers = append(podMatchers, matcher.EnvVarMatcher{}) + } + if enablePodPortMatcher { + podMatchers = append(podMatchers, matcher.PortMatcher{}) + } + if enablePodEphemeralStorageMatcher { + podMatchers = append(podMatchers, matcher.EphemeralStorageMatcher{}) + } + if err = (&controller.DevboxReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - CommitImageRegistry: registryAddr, - Recorder: mgr.GetEventRecorderFor("devbox-controller"), - RequestCPURate: requestCPURate, - RequestMemoryRate: requestMemoryRate, - RequestEphemeralStorage: requestEphemeralStorage, - LimitEphemeralStorage: limitEphemeralStorage, - DebugMode: debugMode, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + CommitImageRegistry: registryAddr, + Recorder: mgr.GetEventRecorderFor("devbox-controller"), + RequestRate: utilresource.RequestRate{ + CPU: requestCPURate, + Memory: requestMemoryRate, + }, + EphemeralStorage: utilresource.EphemeralStorage{ + DefaultRequest: resource.MustParse(requestEphemeralStorage), + DefaultLimit: resource.MustParse(limitEphemeralStorage), + MaximumLimit: resource.MustParse(maximumLimitEphemeralStorage), + }, + PodMatchers: podMatchers, + DebugMode: debugMode, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Devbox") os.Exit(1) diff --git a/controllers/devbox/config/crd/bases/devbox.sealos.io_devboxes.yaml b/controllers/devbox/config/crd/bases/devbox.sealos.io_devboxes.yaml index ff22615fd47..0a0d64cb467 100644 --- a/controllers/devbox/config/crd/bases/devbox.sealos.io_devboxes.yaml +++ b/controllers/devbox/config/crd/bases/devbox.sealos.io_devboxes.yaml @@ -2803,6 +2803,10 @@ spec: required: - type type: object + nodeSelector: + additionalProperties: + type: string + type: object resource: additionalProperties: anyOf: @@ -2810,7 +2814,10 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) pairs. type: object + runtimeClassName: + type: string squash: default: false type: boolean diff --git a/controllers/devbox/config/manager/manager.yaml b/controllers/devbox/config/manager/manager.yaml index d8098831880..a571e871f39 100644 --- a/controllers/devbox/config/manager/manager.yaml +++ b/controllers/devbox/config/manager/manager.yaml @@ -80,7 +80,6 @@ spec: - --registry-addr={{ .registryAddr }} - --registry-user={{ .registryUser }} - --registry-password={{ .registryPassword }} - - --auth-addr={{ .authAddr }} image: controller:latest name: manager securityContext: diff --git a/controllers/devbox/config/samples/devbox_v1alpha1_devbox.yaml b/controllers/devbox/config/samples/devbox_v1alpha1_devbox.yaml index b0b194ef79c..71e5bf070b0 100644 --- a/controllers/devbox/config/samples/devbox_v1alpha1_devbox.yaml +++ b/controllers/devbox/config/samples/devbox_v1alpha1_devbox.yaml @@ -18,19 +18,23 @@ metadata: labels: app.kubernetes.io/name: devbox app.kubernetes.io/managed-by: kustomize - name: devbox-sample + name: devbox-gpu-sample spec: state: Running + runtimeClassName: nvidia resource: cpu: 2 memory: 4000Mi + nvidia.com/gpu: 1 runtimeRef: - name: go-1-22-5 + name: go-1-22-5-2024-11-12-0651 namespace: devbox-system + nodeSelector: + nvidia.com/gpu.product: Tesla-P40 network: type: NodePort extraPorts: - containerPort: 443 name: 'https' - containerPort: 80 - name: 'http' \ No newline at end of file + name: 'http' diff --git a/controllers/devbox/config/samples/devbox_v1alpha1_runtime.yaml b/controllers/devbox/config/samples/devbox_v1alpha1_runtime.yaml deleted file mode 100644 index 774a7a5947e..00000000000 --- a/controllers/devbox/config/samples/devbox_v1alpha1_runtime.yaml +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright © 2024 sealos. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: devbox.sealos.io/v1alpha1 -kind: Runtime -metadata: - name: go-1-22-5 - namespace: devbox-system -spec: - classRef: go - title: go1.22.5 - description: go1.22.5 - config: - image: ghcr.io/cbluebird/devbox/go1.22.5:2f4067 - workingDir: /home/sealos/project - releaseCommand: - - /bin/bash - - -c - releaseArgs: - - /home/sealos/project/entrypoint.sh - category: - - ubuntu - - go ---- -apiVersion: devbox.sealos.io/v1alpha1 -kind: Runtime -metadata: - name: go-1-23-0 - namespace: devbox-system -spec: - classRef: go - config: - image: ghcr.io/labring-actions/devbox/go-1.23.0:409348 - ports: - - containerPort: 22 - name: devbox-ssh-port - protocol: TCP - appPorts: - - name: devbox-app-port - port: 8080 - protocol: TCP - user: sealos - workingDir: /home/sealos/project - releaseCommand: - - /bin/bash - - -c - releaseArgs: - - /home/sealos/project/entrypoint.sh - description: go 1.23.0 - version: "1.23.0" ---- -apiVersion: devbox.sealos.io/v1alpha1 -kind: Runtime -metadata: - name: gin - namespace: devbox-system -spec: - classRef: gin - title: gin - description: gin - config: - image: ghcr.io/cbluebird/devbox/gin:2f4067 - category: - - ubuntu - - go - - gin ---- -apiVersion: devbox.sealos.io/v1alpha1 -kind: Runtime -metadata: - name: spring-boot - namespace: devbox-system -spec: - classRef: spring-boot - title: Spring Boot - description: Spring Boot - config: - image: ghcr.io/cbluebird/devbox/spring-boot:2f4067 - category: - - ubuntu - - java - - spring-boot diff --git a/controllers/devbox/config/samples/devbox_v1alpha1_runtimeclass.yaml b/controllers/devbox/config/samples/devbox_v1alpha1_runtimeclass.yaml deleted file mode 100644 index 3886893f263..00000000000 --- a/controllers/devbox/config/samples/devbox_v1alpha1_runtimeclass.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright © 2024 sealos. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: devbox.sealos.io/v1alpha1 -kind: RuntimeClass -metadata: - name: go - namespace: devbox-system -spec: - kind: Language - title: go - description: go ---- -apiVersion: devbox.sealos.io/v1alpha1 -kind: RuntimeClass -metadata: - name: gin - namespace: devbox-system -spec: - kind: Framework - title: gin - description: gin ---- -apiVersion: devbox.sealos.io/v1alpha1 -kind: RuntimeClass -metadata: - name: java - namespace: devbox-system -spec: - kind: Language - title: java - description: java ---- -apiVersion: devbox.sealos.io/v1alpha1 -kind: RuntimeClass -metadata: - name: spring-boot - namespace: devbox-system -spec: - kind: Framework - title: Spring Boot - description: Spring Boot ---- -apiVersion: devbox.sealos.io/v1alpha1 -kind: RuntimeClass -metadata: - name: python - namespace: devbox-system -spec: - kind: Language - title: python - description: python ---- -apiVersion: devbox.sealos.io/v1alpha1 -kind: RuntimeClass -metadata: - name: nodejs - namespace: devbox-system -spec: - kind: Language - title: node.js - description: node.js \ No newline at end of file diff --git a/controllers/devbox/deploy/manifests/deploy.yaml.tmpl b/controllers/devbox/deploy/manifests/deploy.yaml.tmpl index 4ed4905ba93..910d8310bf1 100644 --- a/controllers/devbox/deploy/manifests/deploy.yaml.tmpl +++ b/controllers/devbox/deploy/manifests/deploy.yaml.tmpl @@ -3587,7 +3587,6 @@ spec: - --registry-addr={{ .registryAddr }} - --registry-user={{ .registryUser }} - --registry-password={{ .registryPassword }} - - --auth-addr={{ .authAddr }} command: - /manager image: ghcr.io/labring/sealos-devbox-controller:latest diff --git a/controllers/devbox/internal/controller/devbox_controller.go b/controllers/devbox/internal/controller/devbox_controller.go index df05630234a..65fe7e292ef 100644 --- a/controllers/devbox/internal/controller/devbox_controller.go +++ b/controllers/devbox/internal/controller/devbox_controller.go @@ -23,6 +23,8 @@ import ( devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" "github.com/labring/sealos/controllers/devbox/internal/controller/helper" + "github.com/labring/sealos/controllers/devbox/internal/controller/utils/matcher" + "github.com/labring/sealos/controllers/devbox/internal/controller/utils/resource" "github.com/labring/sealos/controllers/devbox/label" corev1 "k8s.io/api/core/v1" @@ -44,11 +46,12 @@ import ( // DevboxReconciler reconciles a Devbox object type DevboxReconciler struct { - CommitImageRegistry string - RequestCPURate float64 - RequestMemoryRate float64 - RequestEphemeralStorage string - LimitEphemeralStorage string + CommitImageRegistry string + + RequestRate resource.RequestRate + EphemeralStorage resource.EphemeralStorage + + PodMatchers []matcher.PodMatcher DebugMode bool @@ -294,7 +297,7 @@ func (r *DevboxReconciler) syncPod(ctx context.Context, devbox *devboxv1alpha1.D logger.Info("pod has been deleted") return r.handlePodDeleted(ctx, devbox, pod) } - switch helper.PodMatchExpectations(expectPod, pod) { + switch matcher.PodMatchExpectations(expectPod, pod, r.PodMatchers...) { case true: // pod match expectations logger.Info("pod match expectations") @@ -420,11 +423,25 @@ func (r *DevboxReconciler) syncService(ctx context.Context, devbox *devboxv1alph // create a new pod, add predicated status to nextCommitHistory func (r *DevboxReconciler) createPod(ctx context.Context, devbox *devboxv1alpha1.Devbox, expectPod *corev1.Pod, nextCommitHistory *devboxv1alpha1.CommitHistory) error { + logger := log.FromContext(ctx) + + logger.Info("creating pod", + "podName", expectPod.Name, + "namespace", expectPod.Namespace, + "nextCommitHistory", nextCommitHistory) + nextCommitHistory.Status = devboxv1alpha1.CommitStatusPending nextCommitHistory.PredicatedStatus = devboxv1alpha1.CommitStatusPending + + if expectPod.Name == "" { + return fmt.Errorf("pod name cannot be empty") + } + if err := r.Create(ctx, expectPod); err != nil { + logger.Error(err, "failed to create pod") return err } + devbox.Status.CommitHistory = append(devbox.Status.CommitHistory, nextCommitHistory) return nil } @@ -532,13 +549,20 @@ func (r *DevboxReconciler) generateDevboxPod(devbox *devboxv1alpha1.Devbox, next WorkingDir: helper.GetWorkingDir(devbox), Command: helper.GetCommand(devbox), Args: helper.GetArgs(devbox), - Resources: helper.GenerateResourceRequirements(devbox, r.RequestCPURate, r.RequestMemoryRate, r.RequestEphemeralStorage, r.LimitEphemeralStorage), - }, + Resources: helper.GenerateResourceRequirements(devbox, r.RequestRate, r.EphemeralStorage)}, } terminationGracePeriodSeconds := 300 automountServiceAccountToken := false + runtimeClassName := devbox.Spec.RuntimeClassName + var runtimeClassNamePtr *string + if runtimeClassName == "" { + runtimeClassNamePtr = nil + } else { + runtimeClassNamePtr = ptr.To(runtimeClassName) + } + expectPod := &corev1.Pod{ ObjectMeta: objectMeta, Spec: corev1.PodSpec{ @@ -550,8 +574,11 @@ func (r *DevboxReconciler) generateDevboxPod(devbox *devboxv1alpha1.Devbox, next Containers: containers, Volumes: volumes, - Tolerations: devbox.Spec.Tolerations, - Affinity: devbox.Spec.Affinity, + RuntimeClassName: runtimeClassNamePtr, + + NodeSelector: devbox.Spec.NodeSelector, + Tolerations: devbox.Spec.Tolerations, + Affinity: devbox.Spec.Affinity, }, } // set controller reference and finalizer diff --git a/controllers/devbox/internal/controller/helper/devbox.go b/controllers/devbox/internal/controller/helper/devbox.go index c1d989bf872..1a7afcb02ba 100644 --- a/controllers/devbox/internal/controller/helper/devbox.go +++ b/controllers/devbox/internal/controller/helper/devbox.go @@ -16,7 +16,6 @@ package helper import ( "fmt" - "log/slog" "sort" "strings" @@ -31,6 +30,7 @@ import ( "k8s.io/utils/ptr" devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" + utilsresource "github.com/labring/sealos/controllers/devbox/internal/controller/utils/resource" "github.com/labring/sealos/controllers/devbox/label" ) @@ -190,10 +190,6 @@ func podContainerID(pod *corev1.Pod) string { } return "" } - -// PredicateCommitStatus returns the commit status of the pod -// if the pod container id is empty, it means the pod is pending or has't started, we can assume the image has not been committed -// otherwise, it means the pod has been started, we can assume the image has been committed func PredicateCommitStatus(pod *corev1.Pod) devboxv1alpha1.CommitStatus { if podContainerID(pod) == "" { return devboxv1alpha1.CommitStatusPending @@ -201,85 +197,6 @@ func PredicateCommitStatus(pod *corev1.Pod) devboxv1alpha1.CommitStatus { return devboxv1alpha1.CommitStatusSuccess } -func PodMatchExpectations(expectPod *corev1.Pod, pod *corev1.Pod) bool { - if len(pod.Spec.Containers) == 0 { - slog.Info("Pod has no containers") - return false - } - container := pod.Spec.Containers[0] - expectContainer := expectPod.Spec.Containers[0] - - // Check CPU and memory limits - if container.Resources.Requests.Cpu().Cmp(*expectContainer.Resources.Requests.Cpu()) != 0 { - slog.Info("CPU requests are not equal") - return false - } - if container.Resources.Limits.Cpu().Cmp(*expectContainer.Resources.Limits.Cpu()) != 0 { - slog.Info("CPU limits are not equal") - return false - } - if container.Resources.Requests.Memory().Cmp(*expectContainer.Resources.Requests.Memory()) != 0 { - slog.Info("Memory requests are not equal") - return false - } - if container.Resources.Limits.Memory().Cmp(*expectContainer.Resources.Limits.Memory()) != 0 { - slog.Info("Memory limits are not equal") - return false - } - - // Check Ephemeral Storage changes - if container.Resources.Requests.StorageEphemeral().Cmp(*expectContainer.Resources.Requests.StorageEphemeral()) != 0 { - slog.Info("Ephemeral-Storage requests are not equal") - return false - } - if container.Resources.Limits.StorageEphemeral().Cmp(*expectContainer.Resources.Limits.StorageEphemeral()) != 0 { - slog.Info("Ephemeral-Storage limits are not equal") - return false - } - - // Check environment variables - if len(container.Env) != len(expectContainer.Env) { - return false - } - for _, env := range container.Env { - found := false - for _, expectEnv := range expectContainer.Env { - if env.Name == "SEALOS_COMMIT_IMAGE_NAME" { - found = true - break - } - if env.Name == expectEnv.Name && env.Value == expectEnv.Value { - found = true - break - } - } - if !found { - slog.Info("Environment variables are not equal", "env not found", env.Name, "env value", env.Value) - return false - } - } - - // Check ports - if len(container.Ports) != len(expectContainer.Ports) { - return false - } - for _, expectPort := range expectContainer.Ports { - found := false - for _, podPort := range container.Ports { - if expectPort.ContainerPort == podPort.ContainerPort && expectPort.Protocol == podPort.Protocol { - found = true - break - } - } - if !found { - slog.Info("Ports are not equal") - return false - } - } - - return true -} - func GenerateDevboxEnvVars(devbox *devboxv1alpha1.Devbox, nextCommitHistory *devboxv1alpha1.CommitHistory) []corev1.EnvVar { // if devbox.Spec.Squash is true, and devbox.Status.CommitHistory has success commit history, we need to set SEALOS_COMMIT_IMAGE_SQUASH to true doSquash := false @@ -388,53 +305,45 @@ func GenerateSSHVolume(devbox *devboxv1alpha1.Devbox) corev1.Volume { } } -func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, - requestCPURate, requestMemoryRate float64, - requestEphemeralStorage, limitEphemeralStorage string, -) corev1.ResourceRequirements { +// GenerateResourceRequirements generates the resource requirements for the Devbox pod +func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, requestRate utilsresource.RequestRate, ephemeralStorage utilsresource.EphemeralStorage) corev1.ResourceRequirements { return corev1.ResourceRequirements{ - Requests: calculateResourceRequest( - corev1.ResourceList{ - corev1.ResourceCPU: devbox.Spec.Resource["cpu"], - corev1.ResourceMemory: devbox.Spec.Resource["memory"], - corev1.ResourceEphemeralStorage: resource.MustParse(requestEphemeralStorage), - }, - requestCPURate, requestMemoryRate, - ), - Limits: corev1.ResourceList{ - corev1.ResourceCPU: devbox.Spec.Resource["cpu"], - corev1.ResourceMemory: devbox.Spec.Resource["memory"], - corev1.ResourceEphemeralStorage: resource.MustParse(limitEphemeralStorage), - }, + Limits: calculateResourceLimit(devbox.Spec.Resource, ephemeralStorage), + Requests: calculateResourceRequest(devbox.Spec.Resource, requestRate, ephemeralStorage), } } -func IsExceededQuotaError(err error) bool { - return strings.Contains(err.Error(), "exceeded quota") +func calculateResourceLimit(original corev1.ResourceList, ephemeralStorage utilsresource.EphemeralStorage) corev1.ResourceList { + limit := original.DeepCopy() + // If ephemeral storage limit is not set, set it to default limit + if l, ok := limit[corev1.ResourceEphemeralStorage]; !ok { + limit[corev1.ResourceEphemeralStorage] = ephemeralStorage.DefaultLimit + } else { + // Check if the resource limit for ephemeral storage is set and compare it, if it is exceeded the maximum limit, set it to maximum limit + if l.AsApproximateFloat64() > ephemeralStorage.MaximumLimit.AsApproximateFloat64() { + limit[corev1.ResourceEphemeralStorage] = ephemeralStorage.MaximumLimit + } + } + return limit } -func calculateResourceRequest(limit corev1.ResourceList, requestCPURate, requestMemoryRate float64) corev1.ResourceList { - if limit == nil { - return nil - } - request := make(corev1.ResourceList) +func calculateResourceRequest(original corev1.ResourceList, requestRate utilsresource.RequestRate, ephemeralStorage utilsresource.EphemeralStorage) corev1.ResourceList { + // deep copy limit to request, only cpu and memory are calculated + request := original.DeepCopy() // Calculate CPU request - if cpu, ok := limit[corev1.ResourceCPU]; ok { + if cpu, ok := original[corev1.ResourceCPU]; ok { cpuValue := cpu.AsApproximateFloat64() - cpuRequest := cpuValue / requestCPURate + cpuRequest := cpuValue / requestRate.CPU request[corev1.ResourceCPU] = *resource.NewMilliQuantity(int64(cpuRequest*1000), resource.DecimalSI) } // Calculate memory request - if memory, ok := limit[corev1.ResourceMemory]; ok { + if memory, ok := original[corev1.ResourceMemory]; ok { memoryValue := memory.AsApproximateFloat64() - memoryRequest := memoryValue / requestMemoryRate + memoryRequest := memoryValue / requestRate.Memory request[corev1.ResourceMemory] = *resource.NewQuantity(int64(memoryRequest), resource.BinarySI) } - - if ephemeralStorage, ok := limit[corev1.ResourceEphemeralStorage]; ok { - request[corev1.ResourceEphemeralStorage] = ephemeralStorage - } - + // Set ephemeral storage request to default request + request[corev1.ResourceEphemeralStorage] = ephemeralStorage.DefaultRequest return request } @@ -452,3 +361,7 @@ func GetCommand(devbox *devboxv1alpha1.Devbox) []string { func GetArgs(devbox *devboxv1alpha1.Devbox) []string { return devbox.Spec.Config.Args } + +func IsExceededQuotaError(err error) bool { + return strings.Contains(err.Error(), "exceeded quota") +} diff --git a/controllers/devbox/internal/controller/utils/matcher/matcher.go b/controllers/devbox/internal/controller/utils/matcher/matcher.go new file mode 100644 index 00000000000..dfad8e59f9f --- /dev/null +++ b/controllers/devbox/internal/controller/utils/matcher/matcher.go @@ -0,0 +1,154 @@ +// Copyright © 2024 sealos. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package matcher + +import ( + "log/slog" + + corev1 "k8s.io/api/core/v1" +) + +type PodMatcher interface { + Match(expectPod *corev1.Pod, pod *corev1.Pod) bool +} + +type ResourceMatcher struct{} + +func (r ResourceMatcher) Match(expectPod *corev1.Pod, pod *corev1.Pod) bool { + if len(pod.Spec.Containers) == 0 { + slog.Info("Pod has no containers") + return false + } + container := pod.Spec.Containers[0] + expectContainer := expectPod.Spec.Containers[0] + + if container.Resources.Requests.Cpu().Cmp(*expectContainer.Resources.Requests.Cpu()) != 0 { + slog.Info("CPU requests are not equal") + return false + } + if container.Resources.Limits.Cpu().Cmp(*expectContainer.Resources.Limits.Cpu()) != 0 { + slog.Info("CPU limits are not equal") + return false + } + if container.Resources.Requests.Memory().Cmp(*expectContainer.Resources.Requests.Memory()) != 0 { + slog.Info("Memory requests are not equal") + return false + } + if container.Resources.Limits.Memory().Cmp(*expectContainer.Resources.Limits.Memory()) != 0 { + slog.Info("Memory limits are not equal") + return false + } + return true +} + +type EphemeralStorageMatcher struct{} + +func (e EphemeralStorageMatcher) Match(expectPod *corev1.Pod, pod *corev1.Pod) bool { + if len(pod.Spec.Containers) == 0 { + slog.Info("Pod has no containers") + return false + } + container := pod.Spec.Containers[0] + expectContainer := expectPod.Spec.Containers[0] + + if container.Resources.Limits.StorageEphemeral().Cmp(*expectContainer.Resources.Limits.StorageEphemeral()) != 0 { + slog.Info("Ephemeral-Storage limits are not equal") + return false + } + if container.Resources.Requests.StorageEphemeral().Cmp(*expectContainer.Resources.Requests.StorageEphemeral()) != 0 { + slog.Info("Ephemeral-Storage requests are not equal") + return false + } + return true +} + +type EnvVarMatcher struct{} + +func (e EnvVarMatcher) Match(expectPod *corev1.Pod, pod *corev1.Pod) bool { + if len(pod.Spec.Containers) == 0 { + slog.Info("Pod has no containers") + return false + } + container := pod.Spec.Containers[0] + expectContainer := expectPod.Spec.Containers[0] + + if len(container.Env) != len(expectContainer.Env) { + slog.Info("Environment variable count mismatch") + return false + } + + for _, env := range container.Env { + found := false + for _, expectEnv := range expectContainer.Env { + if env.Name == "SEALOS_COMMIT_IMAGE_NAME" { + found = true + break + } + if env.Name == expectEnv.Name && env.Value == expectEnv.Value { + found = true + break + } + } + if !found { + slog.Info("Environment variables are not equal", "env not found", env.Name, "env value", env.Value) + return false + } + } + return true +} + +type PortMatcher struct{} + +func (p PortMatcher) Match(expectPod *corev1.Pod, pod *corev1.Pod) bool { + if len(pod.Spec.Containers) == 0 { + slog.Info("Pod has no containers") + return false + } + container := pod.Spec.Containers[0] + expectContainer := expectPod.Spec.Containers[0] + + if len(container.Ports) != len(expectContainer.Ports) { + slog.Info("Port count mismatch") + return false + } + + for _, expectPort := range expectContainer.Ports { + found := false + for _, podPort := range container.Ports { + if expectPort.ContainerPort == podPort.ContainerPort && expectPort.Protocol == podPort.Protocol { + found = true + break + } + } + if !found { + slog.Info("Ports are not equal") + return false + } + } + return true +} + +// PredicateCommitStatus returns the commit status of the pod +// if the pod container id is empty, it means the pod is pending or has't started, we can assume the image has not been committed +// otherwise, it means the pod has been started, we can assume the image has been committed + +func PodMatchExpectations(expectPod *corev1.Pod, pod *corev1.Pod, matchers ...PodMatcher) bool { + for _, matcher := range matchers { + if !matcher.Match(expectPod, pod) { + return false + } + } + return true +} diff --git a/controllers/devbox/internal/controller/helper/devbox_test.go b/controllers/devbox/internal/controller/utils/matcher/matcher_test.go similarity index 95% rename from controllers/devbox/internal/controller/helper/devbox_test.go rename to controllers/devbox/internal/controller/utils/matcher/matcher_test.go index f3563b743fb..1317615196a 100644 --- a/controllers/devbox/internal/controller/helper/devbox_test.go +++ b/controllers/devbox/internal/controller/utils/matcher/matcher_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helper +package matcher import ( "testing" @@ -156,9 +156,16 @@ func TestPodMatchExpectations(t *testing.T) { }, } + matchers := []PodMatcher{ + ResourceMatcher{}, + EnvVarMatcher{}, + PortMatcher{}, + EphemeralStorageMatcher{}, + } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := PodMatchExpectations(expectPod, tt.pod) + result := PodMatchExpectations(expectPod, tt.pod, matchers...) if result != tt.expected { t.Errorf("CheckPodConsistency() = %v, expected %v", result, tt.expected) } diff --git a/controllers/devbox/internal/controller/utils/resource/resource.go b/controllers/devbox/internal/controller/utils/resource/resource.go new file mode 100644 index 00000000000..3e0e4781360 --- /dev/null +++ b/controllers/devbox/internal/controller/utils/resource/resource.go @@ -0,0 +1,30 @@ +// Copyright © 2024 sealos. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "k8s.io/apimachinery/pkg/api/resource" +) + +type RequestRate struct { + CPU float64 + Memory float64 +} + +type EphemeralStorage struct { + DefaultRequest resource.Quantity + DefaultLimit resource.Quantity + MaximumLimit resource.Quantity +}