diff --git a/api/bases/cinder.openstack.org_cinders.yaml b/api/bases/cinder.openstack.org_cinders.yaml index 440f13b17..c26991de1 100644 --- a/api/bases/cinder.openstack.org_cinders.yaml +++ b/api/bases/cinder.openstack.org_cinders.yaml @@ -365,8 +365,21 @@ spec: databaseUser: default: cinder type: string + dbPurge: + properties: + age: + default: 30 + minimum: 1 + type: integer + schedule: + default: 1 0 * * * + type: string + type: object debug: properties: + dbPurge: + default: false + type: boolean dbSync: default: false type: boolean diff --git a/api/v1beta1/cinder_types.go b/api/v1beta1/cinder_types.go index c6bebd759..f360aaebe 100644 --- a/api/v1beta1/cinder_types.go +++ b/api/v1beta1/cinder_types.go @@ -18,12 +18,16 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" - "github.com/openstack-k8s-operators/lib-common/modules/common/util" "github.com/openstack-k8s-operators/lib-common/modules/storage" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( + // CinderUserID - Kolla's cinder UID + CinderUserID = 42407 + // CinderGroupID - Kolla's cinder GID + CinderGroupID = 42407 + // DbSyncHash hash DbSyncHash = "dbsync" @@ -40,6 +44,11 @@ const ( CinderSchedulerContainerImage = "quay.io/podified-antelope-centos9/openstack-cinder-scheduler:current-podified" // CinderVolumeContainerImage is the fall-back container image for CinderVolume CinderVolumeContainerImage = "quay.io/podified-antelope-centos9/openstack-cinder-volume:current-podified" + + // DBPurgeDefaultAge - Default age, in days, for purging deleted DB records + DBPurgeDefaultAge = 30 + // DBPurgeDefaultSchedule - Default cron schedule for purging the DB + DBPurgeDefaultSchedule = "1 0 * * *" ) // CinderSpec defines the desired state of Cinder @@ -123,6 +132,10 @@ type CinderSpec struct { // NodeSelector here acts as a default value and can be overridden by service // specific NodeSelector Settings. NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // DBPurge parameters - + DBPurge DBPurge `json:"dbPurge,omitempty"` } // CinderStatus defines the observed state of Cinder @@ -172,6 +185,32 @@ type Cinder struct { Status CinderStatus `json:"status,omitempty"` } +// DBPurge struct is used to model the parameters exposed to the Manila API CronJob +type DBPurge struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=30 + // +kubebuilder:validation:Minimum=1 + // Age is the DBPurgeAge parameter and indicates the number of days of purging DB records + Age int `json:"age"` + // +kubebuilder:validation:Optional + // +kubebuilder:default="1 0 * * *" + // Schedule defines the crontab format string to schedule the DBPurge cronJob + Schedule string `json:"schedule"` +} + +// CinderDebug indicates whether certain stages of Cinder deployment should +// pause in debug mode, or if debug is enabled when purging the DB. +type CinderDebug struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // dbSync enable debug + DBSync bool `json:"dbSync"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // DBPurge enable debug + DBPurge bool `json:"dbPurge"` +} + //+kubebuilder:object:root=true // CinderList contains a list of Cinder @@ -231,16 +270,3 @@ func (instance Cinder) RbacNamespace() string { func (instance Cinder) RbacResourceName() string { return "cinder-" + instance.Name } - -// SetupDefaults - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) -func SetupDefaults() { - // Acquire environmental defaults and initialize Cinder defaults with them - cinderDefaults := CinderDefaults{ - APIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CINDER_API_IMAGE_URL_DEFAULT", CinderAPIContainerImage), - BackupContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CINDER_BACKUP_IMAGE_URL_DEFAULT", CinderBackupContainerImage), - SchedulerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CINDER_SCHEDULER_IMAGE_URL_DEFAULT", CinderSchedulerContainerImage), - VolumeContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CINDER_VOLUME_IMAGE_URL_DEFAULT", CinderVolumeContainerImage), - } - - SetupCinderDefaults(cinderDefaults) -} diff --git a/api/v1beta1/cinder_webhook.go b/api/v1beta1/cinder_webhook.go index 2e8cc002a..4be5fc134 100644 --- a/api/v1beta1/cinder_webhook.go +++ b/api/v1beta1/cinder_webhook.go @@ -26,6 +26,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" ) // CinderDefaults - @@ -34,6 +35,8 @@ type CinderDefaults struct { BackupContainerImageURL string SchedulerContainerImageURL string VolumeContainerImageURL string + DBPurgeAge int + DBPurgeSchedule string } var cinderDefaults CinderDefaults @@ -41,10 +44,18 @@ var cinderDefaults CinderDefaults // log is for logging in this package. var cinderlog = logf.Log.WithName("cinder-resource") -// SetupCinderDefaults - initialize Cinder spec defaults for use with either internal or external webhooks -func SetupCinderDefaults(defaults CinderDefaults) { - cinderDefaults = defaults - cinderlog.Info("Cinder defaults initialized", "defaults", defaults) +// SetupDefaults - initialize Cinder spec defaults for use with either internal or external webhooks +func SetupDefaults() { + cinderDefaults = CinderDefaults{ + APIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CINDER_API_IMAGE_URL_DEFAULT", CinderAPIContainerImage), + BackupContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CINDER_BACKUP_IMAGE_URL_DEFAULT", CinderBackupContainerImage), + SchedulerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CINDER_SCHEDULER_IMAGE_URL_DEFAULT", CinderSchedulerContainerImage), + VolumeContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CINDER_VOLUME_IMAGE_URL_DEFAULT", CinderVolumeContainerImage), + DBPurgeAge: DBPurgeDefaultAge, + DBPurgeSchedule: DBPurgeDefaultSchedule, + } + + cinderlog.Info("Cinder defaults initialized", "defaults", cinderDefaults) } // SetupWebhookWithManager sets up the webhook with the Manager @@ -86,6 +97,11 @@ func (spec *CinderSpec) Default() { // This is required, as the loop variable is a by-value copy spec.CinderVolumes[index] = cinderVolume } + + if (spec.DBPurge.Age == 0) && (spec.DBPurge.Schedule == "") { + spec.DBPurge.Age = cinderDefaults.DBPurgeAge + spec.DBPurge.Schedule = cinderDefaults.DBPurgeSchedule + } } // TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 14d76fe00..e61f8f922 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -101,15 +101,6 @@ type PasswordSelector struct { Service string `json:"service"` } -// CinderDebug indicates whether certain stages of Cinder deployment should -// pause in debug mode -type CinderDebug struct { - // +kubebuilder:validation:Optional - // +kubebuilder:default=false - // dbSync enable debug - DBSync bool `json:"dbSync"` -} - // CinderServiceDebug indicates whether certain stages of Cinder service // deployment should pause in debug mode type CinderServiceDebug struct { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index aeeb981ba..13e35fa29 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -722,6 +722,7 @@ func (in *CinderSpec) DeepCopyInto(out *CinderSpec) { (*out)[key] = val } } + out.DBPurge = in.DBPurge } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CinderSpec. @@ -958,6 +959,21 @@ func (in *CinderVolumeTemplate) DeepCopy() *CinderVolumeTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DBPurge) DeepCopyInto(out *DBPurge) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DBPurge. +func (in *DBPurge) DeepCopy() *DBPurge { + if in == nil { + return nil + } + out := new(DBPurge) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PasswordSelector) DeepCopyInto(out *PasswordSelector) { *out = *in diff --git a/config/crd/bases/cinder.openstack.org_cinders.yaml b/config/crd/bases/cinder.openstack.org_cinders.yaml index 440f13b17..c26991de1 100644 --- a/config/crd/bases/cinder.openstack.org_cinders.yaml +++ b/config/crd/bases/cinder.openstack.org_cinders.yaml @@ -365,8 +365,21 @@ spec: databaseUser: default: cinder type: string + dbPurge: + properties: + age: + default: 30 + minimum: 1 + type: integer + schedule: + default: 1 0 * * * + type: string + type: object debug: properties: + dbPurge: + default: false + type: boolean dbSync: default: false type: boolean diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 945dcb619..b5707cfee 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -51,6 +51,18 @@ rules: - patch - update - watch +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - batch resources: diff --git a/controllers/cinder_controller.go b/controllers/cinder_controller.go index 8557dc2a3..26e7ffbb1 100644 --- a/controllers/cinder_controller.go +++ b/controllers/cinder_controller.go @@ -42,6 +42,7 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + cronjob "github.com/openstack-k8s-operators/lib-common/modules/common/cronjob" "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -109,6 +110,7 @@ type CinderReconciler struct { // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete; // service account, role, rolebinding // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update @@ -183,6 +185,7 @@ func (r *CinderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res condition.UnknownCondition(cinderv1beta1.CinderSchedulerReadyCondition, condition.InitReason, cinderv1beta1.CinderSchedulerReadyInitMessage), condition.UnknownCondition(cinderv1beta1.CinderBackupReadyCondition, condition.InitReason, cinderv1beta1.CinderBackupReadyInitMessage), condition.UnknownCondition(cinderv1beta1.CinderVolumeReadyCondition, condition.InitReason, cinderv1beta1.CinderVolumeReadyInitMessage), + condition.UnknownCondition(condition.CronJobReadyCondition, condition.InitReason, condition.CronJobReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), // service account, role, rolebinding conditions @@ -300,6 +303,7 @@ func (r *CinderReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&cinderv1beta1.CinderVolume{}). Owns(&rabbitmqv1.TransportURL{}). Owns(&batchv1.Job{}). + Owns(&batchv1.CronJob{}). Owns(&corev1.Secret{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). @@ -819,6 +823,27 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder return ctrl.Result{}, err } + // create CronJob + cronjobDef := cinder.CronJob(instance, serviceLabels, serviceAnnotations) + cronjob := cronjob.NewCronJob( + cronjobDef, + 5*time.Second, + ) + + ctrlResult, err = cronjob.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CronJobReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CronJobReadyErrorMessage, + err.Error())) + return ctrlResult, err + } + + instance.Status.Conditions.MarkTrue(condition.CronJobReadyCondition, condition.CronJobReadyMessage) + // create CronJob - end + r.Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) return ctrl.Result{}, nil } diff --git a/pkg/cinder/cronjob.go b/pkg/cinder/cronjob.go new file mode 100644 index 000000000..af88b1687 --- /dev/null +++ b/pkg/cinder/cronjob.go @@ -0,0 +1,131 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cinder + +import ( + cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" + + "fmt" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +// CronJob func +func CronJob( + instance *cinderv1.Cinder, + labels map[string]string, + annotations map[string]string, +) *batchv1.CronJob { + cinderUser := int64(cinderv1.CinderUserID) + cinderGroup := int64(cinderv1.CinderGroupID) + config0644AccessMode := int32(0644) + + debugArg := "" + if instance.Spec.Debug.DBPurge { + debugArg = " --debug" + } + + dbPurgeCommand := fmt.Sprintf( + "/usr/bin/cinder-manage%s --config-dir /etc/cinder/cinder.conf.d db purge %d", + debugArg, + instance.Spec.DBPurge.Age) + + args := []string{"-c", dbPurgeCommand} + + cronJobVolumes := []corev1.Volume{ + { + Name: "db-purge-config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: instance.Name + "-config-data", + Items: []corev1.KeyToPath{ + { + Key: DefaultsConfigFileName, + Path: DefaultsConfigFileName, + }, + { + Key: CustomConfigFileName, + Path: CustomConfigFileName, + }, + }, + }, + }, + }, + } + cronJobVolumeMounts := []corev1.VolumeMount{ + { + Name: "db-purge-config-data", + MountPath: "/etc/cinder/cinder.conf.d", + ReadOnly: true, + }, + } + + cronjob := &batchv1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-db-purge", ServiceName), + Namespace: instance.Namespace, + Annotations: annotations, + Labels: labels, + }, + Spec: batchv1.CronJobSpec{ + Schedule: instance.Spec.DBPurge.Schedule, + ConcurrencyPolicy: batchv1.ForbidConcurrent, + JobTemplate: batchv1.JobTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Parallelism: ptr.To(int32(1)), + Completions: ptr.To(int32(1)), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: ServiceName + "-db-purge", + Image: instance.Spec.CinderAPI.ContainerImage, + Command: []string{ + "/bin/bash", + }, + Args: args, + VolumeMounts: cronJobVolumeMounts, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &cinderUser, + RunAsGroup: &cinderGroup, + }, + }, + }, + Volumes: cronJobVolumes, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: instance.RbacResourceName(), + }, + }, + }, + }, + }, + } + if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { + cronjob.Spec.JobTemplate.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector + } + return cronjob +} diff --git a/pkg/cinderbackup/statefulset.go b/pkg/cinderbackup/statefulset.go index 519d26f65..e29a0ccff 100644 --- a/pkg/cinderbackup/statefulset.go +++ b/pkg/cinderbackup/statefulset.go @@ -16,7 +16,7 @@ limitations under the License. package cinderbackup import ( - cinderv1beta1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" + cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" cinder "github.com/openstack-k8s-operators/cinder-operator/pkg/cinder" common "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" @@ -35,7 +35,7 @@ const ( // StatefulSet func func StatefulSet( - instance *cinderv1beta1.CinderBackup, + instance *cinderv1.CinderBackup, configHash string, labels map[string]string, annotations map[string]string, @@ -44,8 +44,8 @@ func StatefulSet( rootUser := int64(0) // Cinder's uid and gid magic numbers come from the 'cinder-user' in // https://github.com/openstack/kolla/blob/master/kolla/common/users.py - cinderUser := int64(42407) - cinderGroup := int64(42407) + cinderUser := int64(cinderv1.CinderUserID) + cinderGroup := int64(cinderv1.CinderGroupID) // TODO until we determine how to properly query for these livenessProbe := &corev1.Probe{ diff --git a/pkg/cinderscheduler/statefulset.go b/pkg/cinderscheduler/statefulset.go index 5f6b5a706..33b680a1e 100644 --- a/pkg/cinderscheduler/statefulset.go +++ b/pkg/cinderscheduler/statefulset.go @@ -16,7 +16,7 @@ limitations under the License. package cinderscheduler import ( - cinderv1beta1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" + cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" cinder "github.com/openstack-k8s-operators/cinder-operator/pkg/cinder" common "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" @@ -35,7 +35,7 @@ const ( // StatefulSet func func StatefulSet( - instance *cinderv1beta1.CinderScheduler, + instance *cinderv1.CinderScheduler, configHash string, labels map[string]string, annotations map[string]string, @@ -43,8 +43,8 @@ func StatefulSet( rootUser := int64(0) // Cinder's uid and gid magic numbers come from the 'cinder-user' in // https://github.com/openstack/kolla/blob/master/kolla/common/users.py - cinderUser := int64(42407) - cinderGroup := int64(42407) + cinderUser := int64(cinderv1.CinderUserID) + cinderGroup := int64(cinderv1.CinderGroupID) // TODO until we determine how to properly query for these livenessProbe := &corev1.Probe{ diff --git a/pkg/cindervolume/statefulset.go b/pkg/cindervolume/statefulset.go index 90b086e9e..e77823ff0 100644 --- a/pkg/cindervolume/statefulset.go +++ b/pkg/cindervolume/statefulset.go @@ -16,7 +16,7 @@ limitations under the License. package cindervolume import ( - cinderv1beta1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" + cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" cinder "github.com/openstack-k8s-operators/cinder-operator/pkg/cinder" common "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" @@ -35,7 +35,7 @@ const ( // StatefulSet func func StatefulSet( - instance *cinderv1beta1.CinderVolume, + instance *cinderv1.CinderVolume, configHash string, labels map[string]string, annotations map[string]string, @@ -44,8 +44,8 @@ func StatefulSet( rootUser := int64(0) // Cinder's uid and gid magic numbers come from the 'cinder-user' in // https://github.com/openstack/kolla/blob/master/kolla/common/users.py - cinderUser := int64(42407) - cinderGroup := int64(42407) + cinderUser := int64(cinderv1.CinderUserID) + cinderGroup := int64(cinderv1.CinderGroupID) // TODO until we determine how to properly query for these livenessProbe := &corev1.Probe{ diff --git a/test/functional/cinder_controller_test.go b/test/functional/cinder_controller_test.go index c9c5b95f4..de2b48ce9 100644 --- a/test/functional/cinder_controller_test.go +++ b/test/functional/cinder_controller_test.go @@ -44,7 +44,7 @@ var _ = Describe("Cinder controller", func() { It("initializes the status fields", func() { Eventually(func(g Gomega) { cinder := GetCinder(cinderName) - g.Expect(cinder.Status.Conditions).To(HaveLen(16)) + g.Expect(cinder.Status.Conditions).To(HaveLen(17)) g.Expect(cinder.Status.DatabaseHostname).To(Equal("")) }, timeout*2, interval).Should(Succeed()) @@ -78,6 +78,7 @@ var _ = Describe("Cinder controller", func() { }) It("should have Unknown Conditions initialized", func() { for _, cond := range []condition.Type{ + condition.CronJobReadyCondition, condition.DBReadyCondition, condition.DBSyncReadyCondition, condition.InputReadyCondition,