From c6b72c80ae9c9f8f63708f539aa36f5164ee91ea Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Mon, 20 Jan 2025 13:46:28 +0100 Subject: [PATCH 1/7] Check for fwmark and ip rule support and unify env vars --- client/iface/configurer/usp.go | 2 +- client/internal/connect.go | 3 + .../routemanager/systemops/systemops_linux.go | 29 +---- util/grpc/dialer.go | 6 +- util/net/env.go | 21 ++-- util/net/env_generic.go | 11 ++ util/net/env_linux.go | 108 ++++++++++++++++++ util/net/net_linux.go | 20 +--- 8 files changed, 147 insertions(+), 53 deletions(-) create mode 100644 util/net/env_generic.go create mode 100644 util/net/env_linux.go diff --git a/client/iface/configurer/usp.go b/client/iface/configurer/usp.go index 21d65ab2a5d..391269dd09e 100644 --- a/client/iface/configurer/usp.go +++ b/client/iface/configurer/usp.go @@ -362,7 +362,7 @@ func toWgUserspaceString(wgCfg wgtypes.Config) string { } func getFwmark() int { - if runtime.GOOS == "linux" && !nbnet.CustomRoutingDisabled() { + if nbnet.AdvancedRouting() { return nbnet.NetbirdFwmark } return 0 diff --git a/client/internal/connect.go b/client/internal/connect.go index a1e8f0f8c04..32cb6bc872e 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -31,6 +31,7 @@ import ( relayClient "github.com/netbirdio/netbird/relay/client" signal "github.com/netbirdio/netbird/signal/client" "github.com/netbirdio/netbird/util" + nbnet "github.com/netbirdio/netbird/util/net" "github.com/netbirdio/netbird/version" ) @@ -114,6 +115,8 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, probes *ProbeHold log.Infof("starting NetBird client version %s on %s/%s", version.NetbirdVersion(), runtime.GOOS, runtime.GOARCH) + nbnet.Init() + backOff := &backoff.ExponentialBackOff{ InitialInterval: time.Second, RandomizationFactor: 1, diff --git a/client/internal/routemanager/systemops/systemops_linux.go b/client/internal/routemanager/systemops/systemops_linux.go index 1da92cc8059..d724cb1a7ab 100644 --- a/client/internal/routemanager/systemops/systemops_linux.go +++ b/client/internal/routemanager/systemops/systemops_linux.go @@ -53,20 +53,6 @@ type ruleParams struct { description string } -// isLegacy determines whether to use the legacy routing setup -func isLegacy() bool { - return os.Getenv("NB_USE_LEGACY_ROUTING") == "true" || nbnet.CustomRoutingDisabled() || nbnet.SkipSocketMark() -} - -// setIsLegacy sets the legacy routing setup -func setIsLegacy(b bool) { - if b { - os.Setenv("NB_USE_LEGACY_ROUTING", "true") - } else { - os.Unsetenv("NB_USE_LEGACY_ROUTING") - } -} - func getSetupRules() []ruleParams { return []ruleParams{ {100, -1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V4, false, 0, "rule with suppress prefixlen v4"}, @@ -87,7 +73,7 @@ func getSetupRules() []ruleParams { // This table is where a default route or other specific routes received from the management server are configured, // enabling VPN connectivity. func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager.Manager) (_ nbnet.AddHookFunc, _ nbnet.RemoveHookFunc, err error) { - if isLegacy() { + if !nbnet.AdvancedRouting() { log.Infof("Using legacy routing setup") return r.setupRefCounter(initAddresses, stateManager) } @@ -103,11 +89,6 @@ func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager rules := getSetupRules() for _, rule := range rules { if err := addRule(rule); err != nil { - if errors.Is(err, syscall.EOPNOTSUPP) { - log.Warnf("Rule operations are not supported, falling back to the legacy routing setup") - setIsLegacy(true) - return r.setupRefCounter(initAddresses, stateManager) - } return nil, nil, fmt.Errorf("%s: %w", rule.description, err) } } @@ -130,7 +111,7 @@ func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager // It systematically removes the three rules and any associated routing table entries to ensure a clean state. // The function uses error aggregation to report any errors encountered during the cleanup process. func (r *SysOps) CleanupRouting(stateManager *statemanager.Manager) error { - if isLegacy() { + if !nbnet.AdvancedRouting() { return r.cleanupRefCounter(stateManager) } @@ -168,7 +149,7 @@ func (r *SysOps) removeFromRouteTable(prefix netip.Prefix, nexthop Nexthop) erro } func (r *SysOps) AddVPNRoute(prefix netip.Prefix, intf *net.Interface) error { - if isLegacy() { + if !nbnet.AdvancedRouting() { return r.genericAddVPNRoute(prefix, intf) } @@ -191,7 +172,7 @@ func (r *SysOps) AddVPNRoute(prefix netip.Prefix, intf *net.Interface) error { } func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error { - if isLegacy() { + if !nbnet.AdvancedRouting() { return r.genericRemoveVPNRoute(prefix, intf) } @@ -504,7 +485,7 @@ func getAddressFamily(prefix netip.Prefix) int { } func hasSeparateRouting() ([]netip.Prefix, error) { - if isLegacy() { + if !nbnet.AdvancedRouting() { return GetRoutesFromTable() } return nil, ErrRoutingIsSeparate diff --git a/util/grpc/dialer.go b/util/grpc/dialer.go index 4fbffe34245..ec2acc58b5c 100644 --- a/util/grpc/dialer.go +++ b/util/grpc/dialer.go @@ -4,13 +4,14 @@ import ( "context" "crypto/tls" "fmt" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "net" "os/user" "runtime" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/cenkalti/backoff/v4" log "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -37,7 +38,6 @@ func WithCustomDialer() grpc.DialOption { } } - log.Debug("Using nbnet.NewDialer()") conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) if err != nil { log.Errorf("Failed to dial: %s", err) diff --git a/util/net/env.go b/util/net/env.go index 099da39b760..32425665dea 100644 --- a/util/net/env.go +++ b/util/net/env.go @@ -2,6 +2,7 @@ package net import ( "os" + "strconv" log "github.com/sirupsen/logrus" @@ -10,20 +11,24 @@ import ( const ( envDisableCustomRouting = "NB_DISABLE_CUSTOM_ROUTING" - envSkipSocketMark = "NB_SKIP_SOCKET_MARK" ) +// CustomRoutingDisabled returns true if custom routing is disabled. +// This will fall back to the operation mode before the exit node functionality was implemented. +// In particular exclusion routes won't be set up and all dialers and listeners will use net.Dial and net.Listen, respectively. func CustomRoutingDisabled() bool { if netstack.IsEnabled() { return true } - return os.Getenv(envDisableCustomRouting) == "true" -} -func SkipSocketMark() bool { - if skipSocketMark := os.Getenv(envSkipSocketMark); skipSocketMark == "true" { - log.Infof("%s is set to true, skipping SO_MARK", envSkipSocketMark) - return true + var customRoutingDisabled bool + if val := os.Getenv(envDisableCustomRouting); val != "" { + var err error + customRoutingDisabled, err = strconv.ParseBool(val) + if err != nil { + log.Warnf("failed to parse %s: %v", envDisableCustomRouting, err) + } } - return false + + return customRoutingDisabled } diff --git a/util/net/env_generic.go b/util/net/env_generic.go new file mode 100644 index 00000000000..ecf9180306d --- /dev/null +++ b/util/net/env_generic.go @@ -0,0 +1,11 @@ +//go:build !linux || android + +package net + +func Init() { +} + +func AdvancedRouting() bool { + // non-linux currently doesn't support advanced routing + return false +} diff --git a/util/net/env_linux.go b/util/net/env_linux.go new file mode 100644 index 00000000000..4499d365f84 --- /dev/null +++ b/util/net/env_linux.go @@ -0,0 +1,108 @@ +//go:build linux && !android + +package net + +import ( + "errors" + "os" + "strconv" + "syscall" + + log "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + + "github.com/netbirdio/netbird/client/iface/netstack" +) + +const ( + // these have the same effect, skip socket env supported for backward compatibility + envSkipSocketMark = "NB_SKIP_SOCKET_MARK" + envUseLegacyRouting = "NB_USE_LEGACY_ROUTING" +) + +var advancedRoutingSupported bool + +func Init() { + advancedRoutingSupported = checkAdvancedRoutingSupport() +} + +func AdvancedRouting() bool { + return advancedRoutingSupported +} + +func checkAdvancedRoutingSupport() bool { + var err error + + var legacyRouting bool + if val := os.Getenv("NB_USE_LEGACY_ROUTING"); val != "" { + legacyRouting, err = strconv.ParseBool(val) + if err != nil { + log.Warnf("failed to parse %s: %v", envUseLegacyRouting, err) + } + } + + var skipSocketMark bool + if val := os.Getenv(envSkipSocketMark); val != "" { + skipSocketMark, err = strconv.ParseBool(val) + if err != nil { + log.Warnf("failed to parse %s: %v", envSkipSocketMark, err) + } + } + + // requested to disable advanced routing + if legacyRouting || skipSocketMark || + // envCustomRoutingDisabled disables the custom dialers. + // There is no point in using advanced routing without those, as they set up fwmarks on the sockets. + CustomRoutingDisabled() || + // netstack mode doesn't need routing at all + netstack.IsEnabled() { + + log.Info("advanced routing has been requested to be disabled") + return false + } + + if !CheckFwmarkSupport() || !CheckRuleOperationsSupport() { + log.Warn("system doesn't support required routing features, falling back to legacy routing") + return false + } + + return true +} + +func CheckFwmarkSupport() bool { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0) + if err != nil { + log.Warnf("failed to create test socket: %v", err) + return false + } + defer syscall.Close(fd) + + err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, 1) + if err != nil { + log.Warnf("fwmark is not supported: %v", err) + return false + } + return true +} + +func CheckRuleOperationsSupport() bool { + rule := netlink.NewRule() + // low precedence, semi-random + rule.Priority = 32321 + rule.Table = syscall.RT_TABLE_MAIN + rule.Family = netlink.FAMILY_V4 + + if err := netlink.RuleAdd(rule); err != nil { + if errors.Is(err, syscall.EOPNOTSUPP) { + log.Warn("IP rule operations are not supported") + return false + } + log.Warnf("failed to test rule support: %v", err) + return false + } + + if err := netlink.RuleDel(rule); err != nil { + log.Warnf("failed to delete test rule: %v", err) + } + return true +} diff --git a/util/net/net_linux.go b/util/net/net_linux.go index fc486ebd496..eae483a26a9 100644 --- a/util/net/net_linux.go +++ b/util/net/net_linux.go @@ -5,13 +5,11 @@ package net import ( "fmt" "syscall" - - log "github.com/sirupsen/logrus" ) // SetSocketMark sets the SO_MARK option on the given socket connection func SetSocketMark(conn syscall.Conn) error { - if isSocketMarkDisabled() { + if !AdvancedRouting() { return nil } @@ -25,7 +23,7 @@ func SetSocketMark(conn syscall.Conn) error { // SetSocketOpt sets the SO_MARK option on the given file descriptor func SetSocketOpt(fd int) error { - if isSocketMarkDisabled() { + if !AdvancedRouting() { return nil } @@ -36,7 +34,7 @@ func setRawSocketMark(conn syscall.RawConn) error { var setErr error err := conn.Control(func(fd uintptr) { - if isSocketMarkDisabled() { + if !AdvancedRouting() { return } setErr = setSocketOptInt(int(fd)) @@ -55,15 +53,3 @@ func setRawSocketMark(conn syscall.RawConn) error { func setSocketOptInt(fd int) error { return syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, NetbirdFwmark) } - -func isSocketMarkDisabled() bool { - if CustomRoutingDisabled() { - log.Infof("Custom routing is disabled, skipping SO_MARK") - return true - } - - if SkipSocketMark() { - return true - } - return false -} From b46423be6d7002ea2f6e269a4c7610e75de752dc Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Mon, 20 Jan 2025 17:17:57 +0100 Subject: [PATCH 2/7] Fix lint issue --- util/net/env_generic.go | 1 + 1 file changed, 1 insertion(+) diff --git a/util/net/env_generic.go b/util/net/env_generic.go index ecf9180306d..6d142a8387c 100644 --- a/util/net/env_generic.go +++ b/util/net/env_generic.go @@ -3,6 +3,7 @@ package net func Init() { + // nothing to do on non-linux } func AdvancedRouting() bool { From 024dad23b2ef874d65eb4ab3ba1c99690c7b9514 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Mon, 20 Jan 2025 17:28:26 +0100 Subject: [PATCH 3/7] Fix test --- client/internal/routemanager/systemops/systemops_unix_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/internal/routemanager/systemops/systemops_unix_test.go b/client/internal/routemanager/systemops/systemops_unix_test.go index a6000d963a4..d88c1ab6bca 100644 --- a/client/internal/routemanager/systemops/systemops_unix_test.go +++ b/client/internal/routemanager/systemops/systemops_unix_test.go @@ -85,6 +85,7 @@ var testCases = []testCase{ } func TestRouting(t *testing.T) { + nbnet.Init() for _, tc := range testCases { // todo resolve test execution on freebsd if runtime.GOOS == "freebsd" { From 415cb0295bcc52901b490a7d7cb61f2704b2d414 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Mon, 20 Jan 2025 23:29:10 +0100 Subject: [PATCH 4/7] Use netbird fwmark for test --- util/net/env_linux.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/util/net/env_linux.go b/util/net/env_linux.go index 4499d365f84..c64849ae7d0 100644 --- a/util/net/env_linux.go +++ b/util/net/env_linux.go @@ -66,6 +66,8 @@ func checkAdvancedRoutingSupport() bool { return false } + log.Info("system supports advanced routing") + return true } @@ -77,7 +79,7 @@ func CheckFwmarkSupport() bool { } defer syscall.Close(fd) - err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, 1) + err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, NetbirdFwmark) if err != nil { log.Warnf("fwmark is not supported: %v", err) return false From 7c62240ae4cb64cb0f340a11c8e05b8c2cd8d6e6 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Wed, 22 Jan 2025 11:52:38 +0100 Subject: [PATCH 5/7] Use env var --- util/net/env_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/net/env_linux.go b/util/net/env_linux.go index c64849ae7d0..b4b0da2efed 100644 --- a/util/net/env_linux.go +++ b/util/net/env_linux.go @@ -34,7 +34,7 @@ func checkAdvancedRoutingSupport() bool { var err error var legacyRouting bool - if val := os.Getenv("NB_USE_LEGACY_ROUTING"); val != "" { + if val := os.Getenv(envUseLegacyRouting); val != "" { legacyRouting, err = strconv.ParseBool(val) if err != nil { log.Warnf("failed to parse %s: %v", envUseLegacyRouting, err) From 5a11ca0cf8ad0a45e40fcbac1650eecc7d0f922b Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Wed, 22 Jan 2025 23:59:23 +0100 Subject: [PATCH 6/7] Enable default routes for netstack mode --- client/internal/routemanager/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 6f73fb166c6..fff19aa302e 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -454,7 +454,7 @@ func (m *DefaultManager) initialClientRoutes(initialRoutes []*route.Route) []*ro } func isRouteSupported(route *route.Route) bool { - if !nbnet.CustomRoutingDisabled() || route.IsDynamic() { + if netstack.IsEnabled() || !nbnet.CustomRoutingDisabled() || route.IsDynamic() { return true } From 7e8bd9d050f15792157f52fa341ad38ea49f4c02 Mon Sep 17 00:00:00 2001 From: Viktor Liu Date: Thu, 23 Jan 2025 00:05:33 +0100 Subject: [PATCH 7/7] Avoid panic when selecting routes with client routes disabled --- client/internal/routemanager/manager.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index fff19aa302e..0f07a2f03ea 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -113,13 +113,14 @@ func NewManager(config ManagerConfig) *DefaultManager { disableServerRoutes: config.DisableServerRoutes, } + useNoop := netstack.IsEnabled() || config.DisableClientRoutes + dm.setupRefCounters(useNoop) + // don't proceed with client routes if it is disabled if config.DisableClientRoutes { return dm } - dm.setupRefCounters() - if runtime.GOOS == "android" { cr := dm.initialClientRoutes(config.InitialRoutes) dm.notifier.SetInitialClientRoutes(cr) @@ -127,7 +128,7 @@ func NewManager(config ManagerConfig) *DefaultManager { return dm } -func (m *DefaultManager) setupRefCounters() { +func (m *DefaultManager) setupRefCounters(useNoop bool) { m.routeRefCounter = refcounter.New( func(prefix netip.Prefix, _ struct{}) (struct{}, error) { return struct{}{}, m.sysOps.AddVPNRoute(prefix, m.wgInterface.ToInterface()) @@ -137,7 +138,7 @@ func (m *DefaultManager) setupRefCounters() { }, ) - if netstack.IsEnabled() { + if useNoop { m.routeRefCounter = refcounter.New( func(netip.Prefix, struct{}) (struct{}, error) { return struct{}{}, refcounter.ErrIgnore