feat: Use socket to do IPC request to kube-explorer

Dockerfile

@@ -1,10 +1,11 @@
+ARG KUBE_EXPLORER_VERSION=latest
+FROM cnrancher/kube-explorer:${KUBE_EXPLORER_VERSION} as kube-explorer
 FROM registry.suse.com/bci/bci-base:15.5
 ARG TARGETPLATFORM
 ARG TARGETARCH
 ARG TARGETOS
 
 ENV TARGETPLATFORM=${TARGETPLATFORM:-"linux/amd64"} ARCH=${TARGETARCH:-"amd64"} OS=${TARGETOS:-"linux"}
-ENV KUBE_EXPLORER_VERSION=v0.5.0
 ENV HELM_DASHBOARD_VERSION=1.3.3
 
 RUN zypper -n install curl ca-certificates tar gzip
@@ -14,8 +15,7 @@ RUN mkdir /home/shell && \
     echo 'source <(kubectl completion bash)' >> /home/shell/.bashrc && \
     echo 'PS1="> "' >> /home/shell/.bashrc
 
-RUN curl -sSL https://github.com/cnrancher/kube-explorer/releases/download/${KUBE_EXPLORER_VERSION}/kube-explorer-${OS}-${ARCH} > /usr/local/bin/kube-explorer && \
-    chmod +x /usr/local/bin/kube-explorer
+COPY --from=kube-explorer /usr/bin/kube-explorer /usr/local/bin/kube-explorer
 
 RUN curl -sLf https://github.com/komodorio/helm-dashboard/releases/download/v${HELM_DASHBOARD_VERSION}/helm-dashboard_${HELM_DASHBOARD_VERSION}_Linux_x86_64.tar.gz | tar xvzf - -C /usr/local/bin && \
     chmod +x /usr/local/bin/helm-dashboard

cmd/explorer.go

@@ -1,9 +1,14 @@
 package cmd
 
 import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+
 	"github.com/cnrancher/autok3s/pkg/common"
+	"github.com/cnrancher/autok3s/pkg/server/proxy"
 
-	k3dutil "github.com/k3d-io/k3d/v5/cmd/util"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -15,7 +20,7 @@ var (
 		Example: "autok3s explorer --context myk3s",
 	}
 	clusterID    = ""
-	explorerPort = 0
+	explorerPort = 8080
 )
 
 func init() {
@@ -31,18 +36,29 @@ func ExplorerCommand() *cobra.Command {
 		}
 		return nil
 	}
-	explorerCmd.Run = func(_ *cobra.Command, _ []string) {
+	explorerCmd.Run = func(cmd *cobra.Command, _ []string) {
 		if err := common.CheckCommandExist(common.KubeExplorerCommand); err != nil {
 			logrus.Fatalln(err)
 		}
-		if explorerPort == 0 {
-			port, err := k3dutil.GetFreePort()
-			if err != nil {
-				logrus.Fatalf("failed to get free port for kube-explorer: %v", err)
-			}
-			explorerPort = port
+
+		wait, err := common.StartKubeExplorer(cmd.Context(), clusterID)
+		if err != nil {
+			logrus.Fatalln(err)
+		}
+
+		server := http.Server{
+			Addr:    fmt.Sprintf(":%d", explorerPort),
+			Handler: proxy.DynamicPrefixProxy(clusterID),
+			BaseContext: func(_ net.Listener) context.Context {
+				return cmd.Context()
+			},
 		}
-		_ = common.StartKubeExplorer(explorerCmd.Context(), clusterID, explorerPort)
+		go func() {
+			logrus.Infof("autok3s serving kube-explorer on %s", server.Addr)
+			_ = server.ListenAndServe()
+		}()
+		<-wait
+		_ = server.Shutdown(context.Background())
 	}
 
 	return explorerCmd

go.mod

@@ -59,6 +59,7 @@ require (
 )
 
 require (
+	github.com/Microsoft/go-winio v0.6.2
 	github.com/moby/sys/signal v0.7.0
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00
 	sigs.k8s.io/yaml v1.4.0
@@ -71,7 +72,6 @@ require (
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -190,11 +190,9 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.16.1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
 	google.golang.org/grpc v1.59.0 // indirect

go.sum

@@ -22,8 +22,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
-github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
-github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
@@ -594,8 +594,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

hack/make-rules/autok3s.sh

@@ -66,9 +66,8 @@ function build() {
   go_generate
   autok3s::log::info "building autok3s(${GIT_VERSION},${GIT_COMMIT},${GIT_TREE_STATE},${BUILD_DATE})..."
   # TODO default k3s version in k3d should also get from k3d in
-  local K3D_VERSION=`go list -m all | grep k3d/v5 | awk '{print $2}'`
-  local K3S_TAG=`curl --silent --retry 3 "https://update.k3s.io/v1-release/channels/stable" | egrep -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/'`
-
+  local K3D_VERSION=$(go list -m github.com/k3d-io/k3d/v5 | awk '{print $2}')
+  local K3S_TAG=$(curl --silent --retry 3 'https://update.k3s.io/v1-release/channels/stable' | grep -E -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/')
   local version_flags="
     -X main.gitVersion=${GIT_VERSION}
     -X main.gitCommit=${GIT_COMMIT}
@@ -81,7 +80,7 @@ function build() {
     -X k8s.io/component-base/version.gitCommit=${GIT_COMMIT}
     -X k8s.io/component-base/version.gitTreeState=${GIT_TREE_STATE}
     -X k8s.io/component-base/version.buildDate=${BUILD_DATE}
-    -X github.com/k3d-io/k3d/v5/version.Version=${K3D_VERSION:-v5.4.4}
+    -X github.com/k3d-io/k3d/v5/version.Version=${K3D_VERSION:-v5.4.6}
     -X github.com/k3d-io/k3d/v5/version.K3sVersion=${K3S_TAG}"
   local flags="
     -w -s"

pkg/cluster/cluster.go

@@ -156,13 +156,10 @@ func (p *ProviderBase) InitK3sCluster(cluster *types.Cluster, deployCCM func() [
 	for plugin := range enabledPlugins {
 		if plugin == "explorer" {
 			// start kube-explorer
-			port, err := common.EnableExplorer(context.Background(), cluster.ContextName)
+			err := common.EnableExplorer(context.Background(), cluster.ContextName)
 			if err != nil {
 				p.Logger.Errorf("[%s] failed to start kube-explorer for cluster %s: %v", p.Provider, cluster.ContextName, err)
 			}
-			if port != 0 {
-				p.Logger.Infof("[%s] kube-explorer for cluster %s will listen on 127.0.0.1:%d...", p.Provider, cluster.Name, port)
-			}
 		}
 	}
 

pkg/common/explorer.go

@@ -6,8 +6,8 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"time"
 
-	k3dutil "github.com/k3d-io/k3d/v5/cmd/util"
 	"github.com/sirupsen/logrus"
 )
 
@@ -16,87 +16,73 @@ const (
 )
 
 // EnableExplorer will start kube-explorer with random port for specified K3s cluster
-func EnableExplorer(ctx context.Context, config string) (int, error) {
+func EnableExplorer(ctx context.Context, config string) error {
 	if _, ok := ExplorerWatchers[config]; ok {
-		return 0, fmt.Errorf("kube-explorer for cluster %s has already started", config)
+		return fmt.Errorf("kube-explorer for cluster %s has already started", config)
 	}
 	if err := CheckCommandExist(KubeExplorerCommand); err != nil {
-		return 0, err
+		return err
 	}
 
 	// command execution validate
 	if err := checkExplorerCmd(); err != nil {
-		return 0, err
+		return err
 	}
 
 	// save config for kube-explorer
 	exp, err := DefaultDB.GetExplorer(config)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
 	if exp == nil || !exp.Enabled {
-		var port int
-		if exp == nil {
-			port, err = k3dutil.GetFreePort()
-			if err != nil {
-				return 0, err
-			}
-		} else {
-			port = exp.Port
-		}
 		exp = &Explorer{
 			ContextName: config,
-			Port:        port,
 			Enabled:     true,
 		}
 		if err = DefaultDB.SaveExplorer(exp); err != nil {
-			return 0, err
+			return err
 		}
 	}
 
 	// start kube-explorer
 	explorerCtx, cancel := context.WithCancel(ctx)
 	ExplorerWatchers[config] = cancel
-	go func(ctx context.Context, config string, port int) {
-		_ = StartKubeExplorer(ctx, config, port)
-	}(explorerCtx, config, exp.Port)
-	return exp.Port, nil
+
+	if _, err := StartKubeExplorer(explorerCtx, config); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // DisableExplorer will stop kube-explorer server for specified K3s cluster
 func DisableExplorer(config string) error {
-	if _, ok := ExplorerWatchers[config]; !ok {
+	cancelFunc, ok := ExplorerWatchers[config]
+	if !ok {
 		return fmt.Errorf("cann't disable unactive kube-explorer for cluster %s", config)
 	}
+
 	// update kube-explorer settings
 	exp, err := DefaultDB.GetExplorer(config)
 	if err != nil {
 		return err
 	}
+	if exp != nil && exp.Enabled {
+		// stop kube-explorer
+		cancelFunc()
+		delete(ExplorerWatchers, config)
+	}
 	if exp == nil || exp.Enabled {
-		var port int
-		if exp == nil {
-			port, err = k3dutil.GetFreePort()
-			if err != nil {
-				return err
-			}
-		} else {
-			port = exp.Port
-		}
 		err = DefaultDB.SaveExplorer(&Explorer{
 			ContextName: config,
-			Port:        port,
 			Enabled:     false,
 		})
 		if err != nil {
 			return err
 		}
 	}
 
-	// stop kube-explorer
-	ExplorerWatchers[config]()
-	delete(ExplorerWatchers, config)
 	return nil
 }
 
@@ -110,26 +96,34 @@ func InitExplorer(ctx context.Context) {
 	for _, exp := range expList {
 		if exp.Enabled {
 			logrus.Infof("start kube-explorer for cluster %s", exp.ContextName)
-			go func(ctx context.Context, name string) {
-				if _, err = EnableExplorer(ctx, name); err != nil {
-					logrus.Errorf("failed to start kube-explorer for cluster %s: %v", name, err)
-				}
-			}(ctx, exp.ContextName)
+			if err = EnableExplorer(ctx, exp.ContextName); err != nil {
+				logrus.Errorf("failed to start kube-explorer for cluster %s: %v", exp.ContextName, err)
+			}
 		}
 	}
 }
 
 // StartKubeExplorer start kube-explorer server listen on specified port
-func StartKubeExplorer(ctx context.Context, config string, port int) error {
+func StartKubeExplorer(ctx context.Context, clusterID string) (chan int, error) {
+	socketName := GetSocketName(clusterID)
 	explorer := exec.CommandContext(ctx, KubeExplorerCommand, fmt.Sprintf("--kubeconfig=%s", filepath.Join(CfgPath, KubeCfgFile)),
-		fmt.Sprintf("--context=%s", config), fmt.Sprintf("--http-listen-port=%d", port), "--https-listen-port=0")
+		fmt.Sprintf("--context=%s", clusterID), fmt.Sprintf("--bind-address=%s", socketName))
 	explorer.Stdout = os.Stdout
 	explorer.Stderr = os.Stderr
+	explorer.Cancel = func() error {
+		return explorer.Process.Signal(os.Interrupt)
+	}
+	explorer.WaitDelay = 10 * time.Second
 	if err := explorer.Start(); err != nil {
-		logrus.Errorf("fail to start kube-explorer for cluster %s: %v", config, err)
+		logrus.Errorf("fail to start kube-explorer for cluster %s: %v", clusterID, err)
 	}
-	logrus.Infof("kube-explorer for %s K3s cluster will listen on 127.0.0.1:%d ...", config, port)
-	return explorer.Wait()
+	logrus.Infof("kube-explorer for %s K3s cluster will listen on %s ...", clusterID, socketName)
+	stopChan := make(chan int)
+	go func() {
+		_ = explorer.Wait()
+		close(stopChan)
+	}()
+	return stopChan, nil
 }
 
 func CheckCommandExist(cmd string) error {

pkg/common/explorer_unix.go

@@ -0,0 +1,29 @@
+//go:build unix
+// +build unix
+
+package common
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+	"path/filepath"
+)
+
+const (
+	socketFileName = "socket.sock"
+)
+
+func GetSocketName(clusterID string) string {
+	return fmt.Sprintf("unix://%s", filepath.Join(GetClusterContextPath(clusterID), socketFileName))
+}
+
+func GetSocketDialer() func(context.Context, string, string) (net.Conn, error) {
+	return func(ctx context.Context, _, addr string) (net.Conn, error) {
+		clusterID, _, _ := net.SplitHostPort(addr)
+		u, _ := url.Parse(GetSocketName(clusterID))
+		var d net.Dialer
+		return d.DialContext(ctx, "unix", u.Path)
+	}
+}

pkg/common/explorer_windows.go

@@ -0,0 +1,33 @@
+//go:build windows
+// +build windows
+
+package common
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"net/url"
+
+	"github.com/Microsoft/go-winio"
+)
+
+func GetSocketName(clusterID string) string {
+	return fmt.Sprintf("namedpipe:/\\.\\pipe\\autok3s-%s", md5hash(clusterID))
+}
+
+func GetSocketDialer() func(context.Context, string, string) (net.Conn, error) {
+	return func(ctx context.Context, _, addr string) (net.Conn, error) {
+		clusterID, _, _ := net.SplitHostPort(addr)
+		u, _ := url.Parse(GetSocketName(clusterID))
+		return winio.DialPipeContext(ctx, u.Path)
+	}
+}
+
+func md5hash(s string) string {
+	hash := md5.Sum([]byte(s))
+	hexStr := hex.EncodeToString(hash[:])
+	return hexStr[:16]
+}