Merge pull request #227 from omersch381/add_predict_ips_mdns

Implement predictable IP generation for miniDNS

api/bases/designate.openstack.org_designates.yaml

@@ -940,6 +940,11 @@ spec:
                 required:
                 - containerImage
                 type: object
+              designateNetworkAttachment:
+                default: designate
+                description: DesignateNetworkAttachment is a NetworkAttachment resource
+                  name for the Designate Control Network
+                type: string
               designateProducer:
                 description: DesignateProducer - Spec definition for the Producer
                   service of this Designate deployment
@@ -1473,6 +1478,7 @@ spec:
             - designateBackendbind9
             - designateCentral
             - designateMdns
+            - designateNetworkAttachment
             - designateProducer
             - designateWorker
             - rabbitMqClusterName

api/v1beta1/designate_types.go

@@ -176,6 +176,11 @@ type DesignateSpecBase struct {
 	// Resources - Compute Resources required by this service (Limits/Requests).
 	// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default=designate
+	// DesignateNetworkAttachment is a NetworkAttachment resource name for the Designate Control Network
+	DesignateNetworkAttachment string `json:"designateNetworkAttachment"`
 }
 
 // DesignateStatus defines the observed state of Designate

config/crd/bases/designate.openstack.org_designates.yaml

@@ -940,6 +940,11 @@ spec:
                 required:
                 - containerImage
                 type: object
+              designateNetworkAttachment:
+                default: designate
+                description: DesignateNetworkAttachment is a NetworkAttachment resource
+                  name for the Designate Control Network
+                type: string
               designateProducer:
                 description: DesignateProducer - Spec definition for the Producer
                   service of this Designate deployment
@@ -1473,6 +1478,7 @@ spec:
             - designateBackendbind9
             - designateCentral
             - designateMdns
+            - designateNetworkAttachment
             - designateProducer
             - designateWorker
             - rabbitMqClusterName

config/rbac/role.yaml

@@ -5,6 +5,13 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

controllers/designate_controller.go

@@ -115,6 +115,7 @@ type DesignateReconciler struct {
 // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch
 // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list
 
 // service account, role, rolebinding
 // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch
@@ -713,6 +714,112 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des
 	}
 	Log.Info("Deployment API task reconciled")
 
+	nad, err := nad.GetNADWithName(ctx, helper, instance.Spec.DesignateNetworkAttachment, instance.Namespace)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	networkParameters, err := designate.GetNetworkParametersFromNAD(nad)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	nodeConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      designate.MdnsPredIPConfigMap,
+			Namespace: instance.GetNamespace(),
+			Labels:    labels.GetLabels(instance, labels.GetGroupLabel(instance.ObjectMeta.Name), map[string]string{}),
+		},
+		Data: make(map[string]string),
+	}
+
+	// Look for existing config map and if exists, read existing data and match
+	// against nodes.
+	foundMap := &corev1.ConfigMap{}
+	err = helper.GetClient().Get(ctx, types.NamespacedName{Name: designate.MdnsPredIPConfigMap, Namespace: instance.GetNamespace()},
+		foundMap)
+	if err != nil {
+		if k8s_errors.IsNotFound(err) {
+			Log.Info(fmt.Sprintf("Ip map %s doesn't exist, creating.", designate.MdnsPredIPConfigMap))
+		} else {
+			return ctrl.Result{}, err
+		}
+	} else {
+		Log.Info("Retrieved existing map, updating..")
+		nodeConfigMap.Data = foundMap.Data
+	}
+
+	//
+	// Predictable IPs.
+	//
+	// NOTE(oschwart): refactoring this might be nice. This could also  be
+	// optimized but the data sets are small (nodes an IP ranges are less than
+	// 100) so optimization might be a waste.
+	//
+	predictableIPParams, err := designate.GetPredictableIPAM(networkParameters)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	// Get a list of the nodes in the cluster
+
+	// TODO(oschwart):
+	// * confirm whether or not this lists only the nodes we want (i.e. ones
+	// that will host the daemonset)
+	// * do we want to provide a mechanism to temporarily disabling this list
+	// for maintenance windows where nodes might be "coming and going"
+
+	nodes, err := helper.GetKClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	updatedMap := make(map[string]string)
+	allocatedIPs := make(map[string]bool)
+	var predictableIPsRequired []string
+
+	// First scan existing allocations so we can keep existing allocations.
+	// Keeping track of what's required and what already exists. If a node is
+	// removed from the cluster, it's IPs will not be added to the allocated
+	// list and are effectively recycled.
+	for _, node := range nodes.Items {
+		nodeName := fmt.Sprintf("mdns_%s", node.Name)
+		if ipValue, ok := nodeConfigMap.Data[nodeName]; ok {
+			updatedMap[nodeName] = ipValue
+			allocatedIPs[ipValue] = true
+			Log.Info(fmt.Sprintf("%s has IP mapping %s: %s", node.Name, nodeName, ipValue))
+		} else {
+			predictableIPsRequired = append(predictableIPsRequired, nodeName)
+		}
+	}
+	// Get new IPs using the range from predictableIPParmas minus the
+	// allocatedIPs captured above.
+	Log.Info(fmt.Sprintf("Allocating %d predictable IPs", len(predictableIPsRequired)))
+	for _, nodeName := range predictableIPsRequired {
+		nodeIP, err := designate.GetNextIP(predictableIPParams, allocatedIPs)
+		if err != nil {
+			// An error here is really unexpected- it means either we have
+			// messed up the allocatedIPs list or the range we are assuming is
+			// too small for the number of mdns pod.
+			return ctrl.Result{}, err
+		}
+		updatedMap[nodeName] = nodeIP
+	}
+
+	mapLabels := labels.GetLabels(instance, labels.GetGroupLabel(instance.ObjectMeta.Name), map[string]string{})
+	_, err = controllerutil.CreateOrPatch(ctx, helper.GetClient(), nodeConfigMap, func() error {
+		nodeConfigMap.Labels = util.MergeStringMaps(nodeConfigMap.Labels, mapLabels)
+		nodeConfigMap.Data = updatedMap
+		err := controllerutil.SetControllerReference(instance, nodeConfigMap, helper.GetScheme())
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	if err != nil {
+		Log.Info("Unable to create config map for mdns ips...")
+		return ctrl.Result{}, err
+	}
+
 	// deploy designate-central
 	designateCentral, op, err := r.centralDeploymentCreateOrUpdate(ctx, instance)
 	if err != nil {

controllers/designatemdns_controller.go

@@ -43,6 +43,7 @@ import (
 	designatemdns "github.com/openstack-k8s-operators/designate-operator/pkg/designatemdns"
 	"github.com/openstack-k8s-operators/lib-common/modules/common"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/daemonset"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
@@ -489,7 +490,7 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance
 	// create hash over all the different input resources to identify if any those changed
 	// and a restart/recreate is required.
 	//
-	inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars)
+	inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, helper, instance, configMapVars)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ServiceConfigReadyCondition,
@@ -803,13 +804,30 @@ func (r *DesignateMdnsReconciler) generateServiceConfigMaps(
 // returns the hash, whether the hash changed (as a bool) and any error
 func (r *DesignateMdnsReconciler) createHashOfInputHashes(
 	ctx context.Context,
+	h *helper.Helper,
 	instance *designatev1beta1.DesignateMdns,
 	envVars map[string]env.Setter,
 ) (string, bool, error) {
 	Log := r.GetLogger(ctx)
 
 	var hashMap map[string]string
 	changed := false
+
+	// If MdnsPredIPConfigMap exists, add its hash to status hash
+	mdnsPredIPCM := &corev1.ConfigMap{}
+	err := h.GetClient().Get(ctx, types.NamespacedName{
+		Name:      designate.MdnsPredIPConfigMap,
+		Namespace: instance.Namespace,
+	}, mdnsPredIPCM)
+	if err != nil {
+		Log.Error(err, "Unable to retrieve Mdns predictable IPs ConfigMap")
+		return "", false, err
+	}
+	mdnsPredIPCMHash, err := configmap.Hash(mdnsPredIPCM)
+	if err != nil {
+		return mdnsPredIPCMHash, changed, err
+	}
+
 	mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars)
 	hash, err := util.ObjectHash(mergedMapVars)
 	if err != nil {

controllers/designateworker_controller.go

@@ -487,7 +487,7 @@ func (r *DesignateWorkerReconciler) reconcileNormal(ctx context.Context, instanc
 	// create hash over all the different input resources to identify if any those changed
 	// and a restart/recreate is required.
 	//
-	inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars)
+	inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, helper, instance, configMapVars)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ServiceConfigReadyCondition,
@@ -798,12 +798,30 @@ func (r *DesignateWorkerReconciler) generateServiceConfigMaps(
 // returns the hash, whether the hash changed (as a bool) and any error
 func (r *DesignateWorkerReconciler) createHashOfInputHashes(
 	ctx context.Context,
+	h *helper.Helper,
 	instance *designatev1beta1.DesignateWorker,
 	envVars map[string]env.Setter,
 ) (string, bool, error) {
 	Log := r.GetLogger(ctx)
 	var hashMap map[string]string
 	changed := false
+
+	// If DesignateBindKeySecret exists, add its hash to status hash
+	rndcSecret := &corev1.Secret{}
+	err := h.GetClient().Get(ctx, types.NamespacedName{
+		Name:      designate.DesignateBindKeySecret,
+		Namespace: instance.Namespace,
+	}, rndcSecret)
+	if err != nil {
+		Log.Error(err, "Unable to retrieve rndc key secret")
+		return "", false, err
+	}
+	secretHash, err := secret.Hash(rndcSecret)
+	if err != nil {
+		return secretHash, changed, err
+	}
+
+	envVars[designate.DesignateBindKeySecret] = env.SetValue(secretHash)
 	mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars)
 	hash, err := util.ObjectHash(mergedMapVars)
 	if err != nil {

pkg/designate/bind_ctrl_network.go

@@ -0,0 +1,56 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+@you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package designate
+
+import (
+	"fmt"
+)
+
+// GetPredictableIPAM returns a struct describing the available IP range. If the
+// IP pool size does not fit in given networkParameters CIDR it will return an
+// error instead.
+func GetPredictableIPAM(networkParameters *NetworkParameters) (*NADIpam, error) {
+	predParams := &NADIpam{}
+	predParams.CIDR = networkParameters.CIDR
+	predParams.RangeStart = networkParameters.ProviderAllocationEnd.Next()
+	endRange := predParams.RangeStart
+	for i := 0; i < BindProvPredictablePoolSize; i++ {
+		if !predParams.CIDR.Contains(endRange) {
+			return nil, fmt.Errorf("predictable IPs: cannot allocate %d IP addresses in %s", BindProvPredictablePoolSize, predParams.CIDR)
+		}
+		endRange = endRange.Next()
+	}
+	predParams.RangeEnd = endRange
+	return predParams, nil
+}
+
+// GetNextIP picks the next available IP from the range defined by a NADIpam,
+// skipping ones that are already used appear as keys in the currentValues map.
+func GetNextIP(predParams *NADIpam, currentValues map[string]bool) (string, error) {
+	candidateAddress := predParams.RangeStart
+	for alloced := true; alloced; {
+
+		if _, ok := currentValues[candidateAddress.String()]; ok {
+			if candidateAddress == predParams.RangeEnd {
+				return "", fmt.Errorf("predictable IPs: out of available addresses")
+			}
+			candidateAddress = candidateAddress.Next()
+		} else {
+			alloced = false
+		}
+	}
+	currentValues[candidateAddress.String()] = true
+	return candidateAddress.String(), nil
+}

pkg/designate/const.go

@@ -38,4 +38,6 @@ const (
 	DesignateBindKeySecret = "designate-bind-secret"
 
 	DesignateRndcKey = "rndc-key"
+
+	MdnsPredIPConfigMap = "designate-mdns-ip-map"
 )

pkg/designate/network_consts.go

@@ -0,0 +1,26 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package designate
+
+// NOTE: Strictly speaking, these don't have to be package scope constants, but having them externally
+// accessible might aide constructing functional tests later on.
+
+const (
+	// Common consts for Control network
+
+	// BindProvPredictablePoolSize  -
+	BindProvPredictablePoolSize = 25
+)