Add EnsureNetworksAnnotation as an alternative for CreateNetworksAnno…

…tation For BGP setup there is the need to set the default gateway to the additional interface defined via the multus annotations. To allow this a user can configure `ipam.gateway` in the NAD. CreateNetworksAnnotation() will override the pod network default route by reading the NAD. If `ipam.gateway` is defined and not "", it gets set on the networks annotation as the `default-route`. Jira: https://issues.redhat.com/browse/OSPRH-8680 Signed-off-by: Martin Schuppert <[email protected]>
openstack-k8s-operators · Nov 11, 2024 · 9e433ca · 9e433ca
1 parent 71a0e9d
commit 9e433ca
Show file tree

Hide file tree

Showing 3 changed files with 195 additions and 0 deletions.
diff --git a/modules/common/networkattachment/networkattachment.go b/modules/common/networkattachment/networkattachment.go
@@ -17,14 +17,17 @@ limitations under the License.
 package networkattachment
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"net"
 
 	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/jsonpath"
 )
 
 // GetNADWithName - Get network-attachment-definition with name in namespace
@@ -49,6 +52,7 @@ func GetNADWithName(
 
 // CreateNetworksAnnotation returns pod annotation for network-attachment-definition list
 // e.g. k8s.v1.cni.cncf.io/networks: '[{"name": "internalapi", "namespace": "openstack"},{"name": "storage", "namespace": "openstack"}]'
+// NOTE: Deprecated, use EnsureNetworksAnnotation
 func CreateNetworksAnnotation(namespace string, nads []string) (map[string]string, error) {
 
 	netAnnotations := []networkv1.NetworkSelectionElement{}
@@ -136,3 +140,67 @@ func VerifyNetworkStatusFromAnnotation(
 
 	return networkReady, networkAttachmentStatus, nil
 }
+
+// EnsureNetworksAnnotation returns pod annotation for network-attachment-definition list
+// e.g. k8s.v1.cni.cncf.io/networks: '[{"name": "internalapi", "namespace": "openstack"},{"name": "storage", "namespace": "openstack"}]'
+// If `ipam.gateway` is defined in the NAD, the annotation will contain the `default-route` for that network:
+// e.g. k8s.v1.cni.cncf.io/networks: '[{"name":"internalapi","namespace":"openstack","interface":"internalapi","default-route":["10.1.2.200"]}]'
+func EnsureNetworksAnnotation(
+	nadList []networkv1.NetworkAttachmentDefinition,
+) (map[string]string, error) {
+
+	annotationString := map[string]string{}
+	netAnnotations := []networkv1.NetworkSelectionElement{}
+	for _, nad := range nadList {
+		gateway := ""
+
+		var data interface{}
+		if err := json.Unmarshal([]byte(nad.Spec.Config), &data); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal JSON data: %w", err)
+		}
+
+		// use jsonpath to parse the cni config
+		jp := jsonpath.New(nad.Name)
+		jp.AllowMissingKeys(true) // Allow missing keys, for when no gateway configured
+
+		// Parse the JSONPath template, for now just `ipam.gateway`
+		err := jp.Parse(`{.ipam.gateway}`)
+		if err != nil {
+			return annotationString, fmt.Errorf("parse template error: %w", err)
+		}
+
+		buf := new(bytes.Buffer)
+		// get the gateway from the config
+		err = jp.Execute(buf, data)
+		if err != nil {
+			return annotationString, fmt.Errorf("parse execute template against nad %+v error: %w", nad.Spec.Config, err)
+		}
+
+		gateway = buf.String()
+
+		gatewayReq := []net.IP{}
+		if gateway != "" {
+			gatewayReq = append(gatewayReq, net.ParseIP(gateway))
+
+		}
+
+		netAnnotations = append(
+			netAnnotations,
+			networkv1.NetworkSelectionElement{
+				Name:             nad.Name,
+				Namespace:        nad.Namespace,
+				InterfaceRequest: GetNetworkIFName(nad.Name),
+				GatewayRequest:   gatewayReq,
+			},
+		)
+	}
+
+	networks, err := json.Marshal(netAnnotations)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode networks %v into json: %w", nadList, err)
+	}
+
+	annotationString[networkv1.NetworkAttachmentAnnot] = string(networks)
+
+	return annotationString, nil
+}
diff --git a/modules/common/networkattachment/networkattachment_test.go b/modules/common/networkattachment/networkattachment_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	. "github.com/onsi/gomega"
 )
@@ -169,3 +170,124 @@ func TestGetNetworkIFName(t *testing.T) {
 		})
 	}
 }
+
+func TestEnsureNetworksAnnotation(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		nadList []networkv1.NetworkAttachmentDefinition
+		want    map[string]string
+	}{
+		{
+			name:    "No network",
+			nadList: []networkv1.NetworkAttachmentDefinition{},
+			want:    map[string]string{networkv1.NetworkAttachmentAnnot: "[]"},
+		},
+		{
+			name: "Single network",
+			nadList: []networkv1.NetworkAttachmentDefinition{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "one", Namespace: "foo"},
+					Spec: networkv1.NetworkAttachmentDefinitionSpec{
+						Config: `
+{
+  "cniVersion": "0.3.1",
+  "name": "internalapi",
+  "type": "macvlan",
+  "master": "internalapi",
+  "ipam": {
+    "type": "whereabouts",
+    "range": "172.17.0.0/24",
+    "range_start": "172.17.0.30",
+    "range_end": "172.17.0.70"
+  }
+}
+`,
+					},
+				},
+			},
+			want: map[string]string{networkv1.NetworkAttachmentAnnot: "[{\"name\":\"one\",\"namespace\":\"foo\",\"interface\":\"one\"}]"},
+		},
+		{
+			name: "Multiple networks",
+			nadList: []networkv1.NetworkAttachmentDefinition{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "one", Namespace: "foo"},
+					Spec: networkv1.NetworkAttachmentDefinitionSpec{
+						Config: `
+{
+  "cniVersion": "0.3.1",
+  "name": "internalapi",
+  "type": "macvlan",
+  "master": "internalapi",
+  "ipam": {
+    "type": "whereabouts",
+    "range": "172.17.0.0/24",
+    "range_start": "172.17.0.30",
+    "range_end": "172.17.0.70"
+  }
+}
+`,
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "two", Namespace: "foo"},
+					Spec: networkv1.NetworkAttachmentDefinitionSpec{
+						Config: `
+{
+  "cniVersion": "0.3.1",
+  "name": "tenant",
+  "type": "macvlan",
+  "master": "tenant",
+  "ipam": {
+    "type": "whereabouts",
+    "range": "172.19.0.0/24",
+    "range_start": "172.19.0.30",
+    "range_end": "172.19.0.70"
+  }
+}
+`,
+					},
+				},
+			},
+			want: map[string]string{networkv1.NetworkAttachmentAnnot: "[{\"name\":\"one\",\"namespace\":\"foo\",\"interface\":\"one\"},{\"name\":\"two\",\"namespace\":\"foo\",\"interface\":\"two\"}]"},
+		},
+		{
+			name: "With gateway defined",
+			nadList: []networkv1.NetworkAttachmentDefinition{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "one", Namespace: "foo"},
+					Spec: networkv1.NetworkAttachmentDefinitionSpec{
+						Config: `
+{
+  "cniVersion": "0.3.1",
+  "name": "internalapi",
+  "type": "macvlan",
+  "master": "internalapi",
+  "ipam": {
+    "type": "whereabouts",
+    "range": "172.17.0.0/24",
+    "range_start": "172.17.0.30",
+    "range_end": "172.17.0.70",
+    "gateway": "172.17.0.1"
+  }
+}
+`,
+					},
+				},
+			},
+			want: map[string]string{networkv1.NetworkAttachmentAnnot: "[{\"name\":\"one\",\"namespace\":\"foo\",\"interface\":\"one\",\"default-route\":[\"172.17.0.1\"]}]"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			networkAnnotation, err := EnsureNetworksAnnotation(tt.nadList)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(networkAnnotation).To(HaveLen(len(tt.want)))
+			g.Expect(networkAnnotation).To(BeEquivalentTo(tt.want))
+		})
+	}
+}
diff --git a/modules/storage/zz_generated.deepcopy.go b/modules/storage/zz_generated.deepcopy.go