Add SDN node subnet gateway IP to host-network address_set

During live SDN migration, host-to-pod traffic originating from
SDN nodes will use the first IP address of the hybrid overlay node
subnet. These IPs are being added to ensure proper functionality of
host network policies.

Signed-off-by: Peng Liu <[email protected]>

go-controller/pkg/ovn/base_network_controller.go

@@ -3,6 +3,7 @@ package ovn
 import (
 	"fmt"
 	"net"
+	"os"
 	"sync"
 	"time"
 
@@ -38,6 +39,8 @@ import (
 	utilnet "k8s.io/utils/net"
 )
 
+const migrationEnvVar = "NODE_CNI"
+
 // CommonNetworkControllerInfo structure is place holder for all fields shared among controllers.
 type CommonNetworkControllerInfo struct {
 	client       clientset.Interface
@@ -66,6 +69,9 @@ type CommonNetworkControllerInfo struct {
 
 	// Northbound database zone name to which this Controller is connected to - aka local zone
 	zone string
+
+	// is running in SDN live migration mode
+	inMigrationMode bool
 }
 
 // BaseNetworkController structure holds per-network fields and network specific configuration
@@ -182,6 +188,7 @@ func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeO
 	if err != nil {
 		return nil, fmt.Errorf("error getting NB zone name : err - %w", err)
 	}
+	_, inMigration := os.LookupEnv(migrationEnvVar)
 	return &CommonNetworkControllerInfo{
 		client:             client,
 		kube:               kube,
@@ -194,6 +201,7 @@ func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeO
 		multicastSupport:   multicastSupport,
 		svcTemplateSupport: svcTemplateSupport,
 		zone:               zone,
+		inMigrationMode:    inMigration,
 	}, nil
 }
 

go-controller/pkg/ovn/default_network_controller.go

@@ -896,9 +896,12 @@ func (h *defaultNetworkControllerEventHandler) UpdateResource(oldObj, newObj int
 		if config.HybridOverlay.Enabled {
 			if util.NoHostSubnet(newNode) && !util.NoHostSubnet(oldNode) {
 				klog.Infof("Node %s has been updated to be a remote/unmanaged hybrid overlay node", newNode.Name)
+				// need to reset the host network address set, as the address is different in ovn and sdn.
+				h.oc.syncHostNetAddrSetFailed.Store(newNode.Name, true)
 				return h.oc.addUpdateHoNodeEvent(newNode)
 			} else if !util.NoHostSubnet(newNode) && util.NoHostSubnet(oldNode) {
 				klog.Infof("Node %s has been updated to be an ovn-kubernetes managed node", newNode.Name)
+				h.oc.syncHostNetAddrSetFailed.Store(newNode.Name, true)
 				if err := h.oc.deleteHoNodeEvent(newNode); err != nil {
 					return err
 				}

go-controller/pkg/ovn/master.go

@@ -915,6 +915,38 @@ func (oc *DefaultNetworkController) deleteHoNodeEvent(node *kapi.Node) error {
 			return fmt.Errorf("failed to remove hybrid overlay static routes and route policy: %w", err)
 		}
 	}
+	if oc.inMigrationMode {
+		// Remove SDN node subnet GW IP from address_set specific to HostNetworkNamespace
+		hoHostNetworkPolicyIPs, err := oc.getHostNamespaceAddressesForHoNode(node)
+		if err != nil {
+			parsedErr := err
+			if !oc.isLocalZoneNode(node) {
+				parsedErr = types.NewSuppressedError(err)
+			}
+			return fmt.Errorf("error parsing annotation for node %s: %w", node.Name, parsedErr)
+		}
+		if len(hoHostNetworkPolicyIPs) > 0 {
+			// delete the host network IPs for this ho node from host network namespace's address set
+			if err = func() error {
+				hostNetworkNamespace := config.Kubernetes.HostNetworkNamespace
+				if hostNetworkNamespace != "" {
+					nsInfo, nsUnlock, err := oc.ensureNamespaceLocked(hostNetworkNamespace, true, nil)
+					if err != nil {
+						return fmt.Errorf("failed to ensure namespace locked: %v", err)
+					}
+					defer nsUnlock()
+					if err = nsInfo.addressSet.DeleteAddresses(util.StringSlice(hoHostNetworkPolicyIPs)); err != nil &&
+						!errors.Is(err, libovsdbclient.ErrNotFound) {
+						return err
+					}
+				}
+				return nil
+			}(); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }
 

go-controller/pkg/ovn/namespace.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/ovn-org/libovsdb/ovsdb"
+	hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
 	libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
@@ -320,20 +321,45 @@ func (oc *DefaultNetworkController) getAllHostNamespaceAddresses() []net.IP {
 	} else {
 		ips = make([]net.IP, 0, len(existingNodes))
 		for _, node := range existingNodes {
+			var hostNetworkIPs []net.IP
 			if config.HybridOverlay.Enabled && util.NoHostSubnet(node) {
-				// skip hybrid overlay nodes
-				continue
-			}
-			hostNetworkIPs, err := oc.getHostNamespaceAddressesForNode(node)
-			if err != nil {
-				klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err)
+				if oc.inMigrationMode {
+					hostNetworkIPs, err = oc.getHostNamespaceAddressesForHoNode(node)
+					if err != nil {
+						klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err)
+					}
+				} else {
+					continue
+				}
+			} else {
+				hostNetworkIPs, err = oc.getHostNamespaceAddressesForNode(node)
+				if err != nil {
+					klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err)
+				}
 			}
 			ips = append(ips, hostNetworkIPs...)
 		}
 	}
 	return ips
 }
 
+func (oc *DefaultNetworkController) getHostNamespaceAddressesForHoNode(node *kapi.Node) ([]net.IP, error) {
+	var ips []net.IP
+	// during SDN live migration, add the SDN node GW IP to the host network address set.
+	hoSubnet, ok := node.Annotations[hotypes.HybridOverlayNodeSubnet]
+	if !ok {
+		// skip hybrid overlay nodes without per-node subnet
+		return nil, nil
+	}
+	_, subnet, err := net.ParseCIDR(hoSubnet)
+	if err != nil {
+		klog.Errorf("Error parsing hybrid overlay subnet %s for node %s: %v", hoSubnet, node.Name, err)
+	}
+	gwIP := util.GetNodeGatewayIfAddr(subnet)
+	ips = append(ips, gwIP.IP)
+	return ips, nil
+}
+
 // getHostNamespaceAddressesForNode retrives management port and gateway router LRP
 // IP of a specific node
 func (oc *DefaultNetworkController) getHostNamespaceAddressesForNode(node *kapi.Node) ([]net.IP, error) {

go-controller/pkg/ovn/namespace_test.go

@@ -341,6 +341,121 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() {
 			fakeOvn.asf.EventuallyExpectAddressSetWithAddresses(hostNetworkNamespace, allowIPs)
 		})
 
+		ginkgo.It("creates an address set for hybrid overlay nodes when the host network traffic namespace is created", func() {
+			config.HybridOverlay.Enabled = true
+			config.Kubernetes.NoHostSubnetNodes, _ = metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+				MatchLabels: map[string]string{"hybrid-overlay-node": "true"},
+			})
+			config.Gateway.Mode = config.GatewayModeShared
+			config.Gateway.NodeportEnable = true
+			var err error
+			config.Default.ClusterSubnets, err = config.ParseClusterSubnetEntries(clusterCIDR)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			node1 := tNode{
+				Name:       "node1",
+				NodeIP:     "1.2.3.4",
+				NodeSubnet: "10.1.1.0/24",
+				NodeGWIP:   "10.1.1.1/24",
+			}
+			// create a test node and annotate it with host subnet
+			testNode := v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:   node1.Name,
+					Labels: map[string]string{"hybrid-overlay-node": "true"},
+					Annotations: map[string]string{
+						"k8s.ovn.org/hybrid-overlay-node-subnet": node1.NodeSubnet,
+						"k8s.ovn.org/node-subnets":               fmt.Sprintf("{\"default\":\"%s\"}", node1.NodeSubnet),
+					},
+				},
+			}
+
+			hostNetworkNamespace := "test-host-network-ns"
+			config.Kubernetes.HostNetworkNamespace = hostNetworkNamespace
+
+			expectedClusterLBGroup := newLoadBalancerGroup(ovntypes.ClusterLBGroupName)
+			expectedSwitchLBGroup := newLoadBalancerGroup(ovntypes.ClusterSwitchLBGroupName)
+			expectedRouterLBGroup := newLoadBalancerGroup(ovntypes.ClusterRouterLBGroupName)
+			expectedOVNClusterRouter := newOVNClusterRouter()
+			expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID})
+			expectedClusterRouterPortGroup := newRouterPortGroup()
+			expectedClusterPortGroup := newClusterPortGroup()
+
+			fakeOvn.startWithDBSetup(
+				libovsdbtest.TestSetup{
+					NBData: []libovsdbtest.TestData{
+						newClusterJoinSwitch(),
+						expectedOVNClusterRouter,
+						expectedNodeSwitch,
+						expectedClusterRouterPortGroup,
+						expectedClusterPortGroup,
+						expectedClusterLBGroup,
+						expectedSwitchLBGroup,
+						expectedRouterLBGroup,
+					},
+				},
+				&v1.NamespaceList{
+					Items: []v1.Namespace{
+						*newNamespace(hostNetworkNamespace),
+					},
+				},
+				&v1.NodeList{
+					Items: []v1.Node{
+						testNode,
+					},
+				},
+			)
+			fakeOvn.controller.multicastSupport = false
+			fakeOvn.controller.SCTPSupport = true
+			fakeOvn.controller.inMigrationMode = true
+
+			err = fakeOvn.controller.WatchNamespaces()
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			err = fakeOvn.controller.WatchNodes()
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			err = fakeOvn.controller.StartServiceController(wg, false)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			// check the namespace again and ensure the address set
+			// being created with the right set of IPs in it.
+			ip, _, _ := net.ParseCIDR(node1.NodeGWIP)
+			allowIPs := []string{ip.String()}
+			fakeOvn.asf.EventuallyExpectAddressSetWithAddresses(hostNetworkNamespace, allowIPs)
+
+			// switch the node from a ho node to a ovn node
+			ovn_node1 := tNode{
+				Name:                 "node1",
+				NodeIP:               "1.2.3.4",
+				NodeLRPMAC:           "0a:58:0a:01:01:01",
+				LrpIP:                "100.64.0.2",
+				LrpIPv6:              "fd98::2",
+				DrLrpIP:              "100.64.0.1",
+				PhysicalBridgeMAC:    "11:22:33:44:55:66",
+				SystemID:             "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6",
+				NodeSubnet:           "10.1.1.0/24",
+				GWRouter:             ovntypes.GWRouterPrefix + "node1",
+				GatewayRouterIPMask:  "172.16.16.2/24",
+				GatewayRouterIP:      "172.16.16.2",
+				GatewayRouterNextHop: "172.16.16.1",
+				PhysicalBridgeName:   "br-eth0",
+				NodeGWIP:             "10.1.1.1/24",
+				NodeMgmtPortIP:       "10.1.1.2",
+				NodeMgmtPortMAC:      "0a:58:0a:01:01:02",
+				DnatSnatIP:           "169.254.0.1",
+			}
+			ovnTestNode := ovn_node1.k8sNode("2")
+			ovnTestNode.Annotations["k8s.ovn.org/hybrid-overlay-node-subnet"] = testNode.Annotations["k8s.ovn.org/hybrid-overlay-node-subnet"]
+			ovnTestNode.Annotations["k8s.ovn.org/node-subnets"] = testNode.Annotations["k8s.ovn.org/node-subnets"]
+
+			fakeOvn.fakeClient.GetNodeClientset().KubeClient.CoreV1().Nodes().Update(context.TODO(), &ovnTestNode, metav1.UpdateOptions{})
+			// check the namespace again and ensure the ho node gateway IP is removed from the address_set
+			// and the ovn ips are added instead.
+			allowIPs = []string{"10.1.1.2", "100.64.0.2"}
+			fakeOvn.asf.EventuallyExpectAddressSetWithAddresses(hostNetworkNamespace, allowIPs)
+		})
+
 		ginkgo.It("reconciles an existing namespace port group, without updating it", func() {
 			// this flag will create namespaced port group
 			config.OVNKubernetesFeature.EnableEgressFirewall = true