Merge pull request #12 from 21kyu/fix/setup-vxlan-device

fix(agent): setup vxlan device

agent/src/main.rs

@@ -11,9 +11,10 @@ use std::sync::Arc;
 use clap::Parser;
 use ipnet::IpNet;
 use log::{debug, info};
+use sinabro_config::generate_mac_addr;
 use sinabro_netlink::netlink::Netlink;
 use sinabro_netlink::route::addr::AddressBuilder;
-use sinabro_netlink::route::link::{Kind, Link, LinkAttrs};
+use sinabro_netlink::route::link::{Kind, Link, LinkAttrs, VxlanAttrs};
 use sinabro_netlink::route::routing::{RoutingBuilder, Via};
 use tokio::sync::Notify;
 use tracing::Level;
@@ -73,19 +74,8 @@ async fn main() -> anyhow::Result<()> {
 
     let mut netlink = Netlink::new();
 
-    // create and configure the bridge with the cni0 name
     let bridge = Kind::new_bridge(bridge_name);
-
-    if let Err(e) = netlink.link_add(&bridge) {
-        if e.to_string().contains("File exists") {
-            info!("cni0 interface already exists");
-        } else {
-            return Err(e);
-        }
-    }
-
-    let bridge = netlink.link_get(bridge.attrs())?;
-    netlink.link_up(&bridge)?;
+    let bridge = netlink.ensure_link(&bridge)?;
 
     let address = AddressBuilder::default()
         .ip(bridge_ip.as_str().parse::<IpNet>()?)
@@ -103,7 +93,6 @@ async fn main() -> anyhow::Result<()> {
     let eth0 = netlink.link_get(&eth0_attrs)?;
     netlink.link_up(&eth0)?;
 
-    // setup additional route rule
     node_routes
         .iter()
         .filter(|node_route| node_route.ip != host_ip)
@@ -126,6 +115,22 @@ async fn main() -> anyhow::Result<()> {
             }
         })?;
 
+    let vxlan_mac = generate_mac_addr()?;
+    let vxlan_dev = Kind::Vxlan {
+        attrs: LinkAttrs {
+            name: "sinabro_vxlan".to_string(),
+            mtu: 1500,
+            hw_addr: vxlan_mac,
+            ..Default::default()
+        },
+        vxlan_attrs: VxlanAttrs {
+            flow_based: true,
+            port: Some(8472),
+            ..Default::default()
+        },
+    };
+    let _vxlan_dev = netlink.ensure_link(&vxlan_dev)?;
+
     sinabro_config::Config::new(&cluster_cidr, &host_route.pod_cidr)
         .write("/etc/cni/net.d/10-sinabro.conf")?;
 

cni/src/command/add.rs

@@ -6,7 +6,7 @@ use ipnet::IpNet;
 use nix::sched::{setns, CloneFlags};
 use rand::Rng;
 use serde::Serialize;
-use sinabro_config::Config;
+use sinabro_config::{generate_mac_addr, Config};
 use sinabro_netlink::{
     netlink::Netlink,
     route::{
@@ -18,8 +18,6 @@ use sinabro_netlink::{
 use tokio::task::spawn_blocking;
 use tracing::info;
 
-use crate::command::generate_mac_addr;
-
 use super::CniCommand;
 
 pub struct AddCommand;

cni/src/command/mod.rs

@@ -1,6 +1,4 @@
-use anyhow::Result;
 use async_trait::async_trait;
-use rand::Rng;
 use sinabro_config::Config;
 
 use self::{add::AddCommand, delete::DeleteCommand};
@@ -20,13 +18,3 @@ pub fn cni_command_from(command: &str) -> anyhow::Result<Box<dyn CniCommand>> {
         _ => anyhow::bail!("unknown command: {}", command),
     }
 }
-
-pub fn generate_mac_addr() -> Result<Vec<u8>> {
-    let mut rng = rand::thread_rng();
-    let mut buf = [0u8; 6];
-    rng.fill(&mut buf[..]);
-
-    buf[0] = (buf[0] | 0x02) & 0xfe;
-
-    Ok(buf.to_vec())
-}

config/Cargo.toml

@@ -14,3 +14,4 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 tracing = "0.1"
 tracing-appender = "0.2"
 tracing-subscriber = "0.3"
+rand = "0.8.5"

config/src/lib.rs

@@ -1,5 +1,7 @@
 use std::path::Path;
 
+use anyhow::Result;
+use rand::Rng;
 use serde::{Deserialize, Serialize};
 use tracing::level_filters::LevelFilter;
 use tracing_appender::{non_blocking, rolling};
@@ -67,6 +69,16 @@ pub fn setup_tracing_to_file(
     Ok(guard)
 }
 
+pub fn generate_mac_addr() -> Result<Vec<u8>> {
+    let mut rng = rand::thread_rng();
+    let mut buf = [0u8; 6];
+    rng.fill(&mut buf[..]);
+
+    buf[0] = (buf[0] | 0x02) & 0xfe;
+
+    Ok(buf.to_vec())
+}
+
 #[cfg(test)]
 mod tests {
     use tracing::Level;
@@ -116,4 +128,12 @@ mod tests {
 
         std::fs::remove_file(&file_name).unwrap();
     }
+
+    #[test]
+    fn test_generate_mac_addr() {
+        let mac_addr = generate_mac_addr().unwrap();
+        assert_eq!(mac_addr.len(), 6);
+        assert_eq!(mac_addr[0] & 0x01, 0);
+        assert_eq!(mac_addr[0] & 0x02, 2);
+    }
 }

justfile

@@ -24,3 +24,6 @@ cargo-check:
 
 e2e-test: build-image
     kubectl kuttl test --config ./tests/kuttl-test.yaml
+
+launch-rust-env:
+    docker run --rm --privileged -it -v $(pwd):/source rust sh

netlink/Cargo.toml

@@ -14,3 +14,4 @@ thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 nix = { version = "0.27.1", features = ["sched", "user"] }
 derive_builder = "0.13.0"
+sysctl = "0.5"

netlink/src/netlink.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 
 use anyhow::Result;
+use sysctl::Sysctl;
 
 use crate::{
     handle::sock_handle::SocketHandle,
@@ -21,6 +22,46 @@ impl Netlink {
         Self::default()
     }
 
+    pub fn ensure_link<T: Link + ?Sized>(&mut self, link: &T) -> Result<Box<dyn Link>> {
+        let link = self.link_get(link.attrs()).or_else(|_| {
+            self.link_add(link)?;
+            self.link_get(link.attrs())
+        })?;
+
+        self.enable_forwarding(&link, true, true)?;
+        Ok(link)
+    }
+
+    pub fn enable_forwarding<T: Link + ?Sized>(
+        &mut self,
+        link: &T,
+        enable_ipv6: bool,
+        enable_ipv4: bool,
+    ) -> Result<()> {
+        self.link_up(link)?;
+
+        let if_name = &link.attrs().name;
+        let mut sys_settings = Vec::new();
+
+        if enable_ipv6 {
+            sys_settings.push((format!("net.ipv6.conf.{}.forwarding", if_name), "1"));
+        }
+
+        if enable_ipv4 {
+            sys_settings.push((format!("net.ipv4.conf.{}.forwarding", if_name), "1"));
+            sys_settings.push((format!("net.ipv4.conf.{}.rp_filter", if_name), "0"));
+            sys_settings.push((format!("net.ipv4.conf.{}.accept_local", if_name), "1"));
+            sys_settings.push((format!("net.ipv4.conf.{}.send_redirects", if_name), "0"));
+        }
+
+        for setting in sys_settings {
+            let ctl = sysctl::Ctl::new(&setting.0)?;
+            ctl.set_value_string(setting.1)?;
+        }
+
+        Ok(())
+    }
+
     pub fn link_get(&mut self, attr: &LinkAttrs) -> Result<Box<dyn Link>> {
         self.sockets
             .entry(libc::NETLINK_ROUTE)
@@ -171,4 +212,15 @@ mod tests {
         let link = netlink.link_get(&LinkAttrs::new("foo")).unwrap();
         assert_ne!(link.attrs().oper_state, 2);
     }
+
+    #[test]
+    fn test_ensure_link() {
+        test_setup!();
+        let mut netlink = Netlink::new();
+
+        let link = Kind::new_bridge("foo");
+        let link = netlink.ensure_link(&link);
+
+        assert!(link.is_ok());
+    }
 }

netlink/src/route/link.rs

@@ -17,6 +17,7 @@ pub enum Namespace {
     Fd(i32),
 }
 
+#[derive(Default)]
 pub struct VxlanAttrs {
     pub id: u32,
     pub group: Option<Vec<u8>>,