diff --git a/bundle/manifests/sail-operator-metrics-service_v1_service.yaml b/bundle/manifests/sail-operator-metrics-service_v1_service.yaml index 48a94205d..301787350 100644 --- a/bundle/manifests/sail-operator-metrics-service_v1_service.yaml +++ b/bundle/manifests/sail-operator-metrics-service_v1_service.yaml @@ -3,11 +3,11 @@ kind: Service metadata: creationTimestamp: null labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: sail-operator app.kubernetes.io/created-by: sailoperator - app.kubernetes.io/instance: sail-operator-metrics-service + app.kubernetes.io/instance: sail-operator app.kubernetes.io/managed-by: helm - app.kubernetes.io/name: service + app.kubernetes.io/name: deployment app.kubernetes.io/part-of: sailoperator control-plane: sail-operator name: sail-operator-metrics-service @@ -17,10 +17,8 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: - app.kubernetes.io/created-by: sailoperator - app.kubernetes.io/part-of: sailoperator control-plane: sail-operator status: loadBalancer: {} diff --git a/bundle/manifests/sailoperator.clusterserviceversion.yaml b/bundle/manifests/sailoperator.clusterserviceversion.yaml index 888f50a84..bffe512bb 100644 --- a/bundle/manifests/sailoperator.clusterserviceversion.yaml +++ b/bundle/manifests/sailoperator.clusterserviceversion.yaml @@ -34,7 +34,7 @@ metadata: capabilities: Seamless Upgrades categories: OpenShift Optional, Integration & Delivery, Networking, Security containerImage: quay.io/maistra-dev/sail-operator:0.3-latest - createdAt: "2024-12-19T05:05:23Z" + createdAt: "2025-01-13T09:30:02Z" description: Experimental operator for installing Istio service mesh features.operators.openshift.io/cnf: "false" features.operators.openshift.io/cni: "true" @@ -714,32 +714,9 @@ spec: values: - linux containers: - - args: - - --secure-listen-address=:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --zap-log-level=info command: - /sail-operator diff --git a/chart/templates/auth_proxy_service.yaml b/chart/templates/auth_proxy_service.yaml index 0d298466f..e49a412e5 100644 --- a/chart/templates/auth_proxy_service.yaml +++ b/chart/templates/auth_proxy_service.yaml @@ -2,11 +2,11 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: sail-operator app.kubernetes.io/created-by: {{ .Values.name }} - app.kubernetes.io/instance: {{ .Values.deployment.name }}-metrics-service + app.kubernetes.io/instance: {{ .Values.deployment.name }} app.kubernetes.io/managed-by: helm - app.kubernetes.io/name: service + app.kubernetes.io/name: deployment app.kubernetes.io/part-of: {{ .Values.name }} control-plane: {{ .Values.deployment.name }} name: {{ .Values.deployment.name }}-metrics-service @@ -17,8 +17,6 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: - app.kubernetes.io/created-by: {{ .Values.name }} - app.kubernetes.io/part-of: {{ .Values.name }} control-plane: {{ .Values.deployment.name }} diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 1d54dde78..aa9502b79 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -48,40 +48,14 @@ spec: values: - linux containers: - - args: - - --secure-listen-address=:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: {{ .Values.proxy.image }} -{{- if .Values.proxy.imagePullPolicy }} - imagePullPolicy: {{ .Values.proxy.imagePullPolicy }} -{{- end }} - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: {{ .Values.proxy.resources.limits.cpu }} - memory: {{ .Values.proxy.resources.limits.memory }} - requests: - cpu: {{ .Values.proxy.resources.requests.cpu }} - memory: {{ .Values.proxy.resources.requests.memory }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --zap-log-level={{ .Values.operatorLogLevel }} command: - /sail-operator image: {{ .Values.image }} -{{- if .Values.proxy.imagePullPolicy }} +{{- if .Values.imagePullPolicy }} imagePullPolicy: {{ .Values.imagePullPolicy }} {{- end }} livenessProbe: diff --git a/chart/templates/rbac/auth_proxy_role_binding.yaml b/chart/templates/rbac/auth_proxy_role_binding.yaml index 7f5155305..96d619d1f 100644 --- a/chart/templates/rbac/auth_proxy_role_binding.yaml +++ b/chart/templates/rbac/auth_proxy_role_binding.yaml @@ -13,5 +13,5 @@ roleRef: name: {{ .Values.name }}-proxy-role subjects: - kind: ServiceAccount - name: {{ .Values.deployment.name }} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace }} diff --git a/chart/templates/rbac/leader_election_role_binding.yaml b/chart/templates/rbac/leader_election_role_binding.yaml index 7b59f4a79..1ee75b2e0 100644 --- a/chart/templates/rbac/leader_election_role_binding.yaml +++ b/chart/templates/rbac/leader_election_role_binding.yaml @@ -14,5 +14,5 @@ roleRef: name: leader-election-role subjects: - kind: ServiceAccount - name: {{ .Values.deployment.name }} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace }} diff --git a/chart/values.yaml b/chart/values.yaml index b7e608d05..148a12144 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -54,17 +54,6 @@ csv: image: quay.io/maistra-dev/sail-operator:0.3-latest # We're commenting out the imagePullPolicy to use k8s defaults # imagePullPolicy: Always -proxy: - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - # We're commenting out the imagePullPolicy to use k8s defaults - # imagePullPolicy: IfNotPresent - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi operator: resources: limits: diff --git a/cmd/main.go b/cmd/main.go index 740faecbb..bce7c9a4b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,7 @@ package main import ( + "crypto/tls" "flag" "fmt" "net/http" @@ -36,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) @@ -50,7 +52,7 @@ func main() { var leaderElectionEnabled bool var reconcilerCfg config.ReconcilerConfig - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.StringVar(&configFile, "config-file", "/etc/sail-operator/config.properties", "Location of the config file, propagated by k8s downward APIs") flag.StringVar(&reconcilerCfg.ResourceDirectory, "resource-directory", "/var/lib/sail-operator/resources", "Where to find resources (e.g. charts)") @@ -100,9 +102,32 @@ func main() { }) } + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + tlsOpts := []func(*tls.Config){ + // disable http/2 because of https://github.com/kubernetes/kubernetes/issues/121197 + disableHTTP2, + } + + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: true, + FilterProvider: filters.WithAuthenticationAndAuthorization, + TLSOpts: tlsOpts, + } + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, - Metrics: metricsserver.Options{BindAddress: metricsAddr}, + Metrics: metricsServerOptions, HealthProbeBindAddress: probeAddr, LeaderElection: leaderElectionEnabled, LeaderElectionID: "sail-operator-lock", diff --git a/go.mod b/go.mod index 0cc6605f8..1203506b7 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( ) require ( + cel.dev/expr v0.19.1 // indirect dario.cat/mergo v1.0.1 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect @@ -46,9 +47,11 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.3 // indirect github.com/containerd/containerd v1.7.23 // indirect @@ -70,6 +73,7 @@ require ( github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect @@ -85,6 +89,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.22.1 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect @@ -94,6 +99,7 @@ require ( github.com/gorilla/websocket v1.5.3 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.5.0 // indirect @@ -138,6 +144,7 @@ require ( github.com/spf13/cast v1.7.0 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -146,8 +153,12 @@ require ( go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk v1.33.0 // indirect go.opentelemetry.io/otel/trace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.31.0 // indirect @@ -175,6 +186,7 @@ require ( k8s.io/kubectl v0.32.0 // indirect k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect oras.land/oras-go v1.2.5 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1 // indirect sigs.k8s.io/controller-tools v0.15.0 // indirect sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/kustomize/api v0.18.0 // indirect diff --git a/go.sum b/go.sum index 13fa41fe8..ca22c51e6 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= @@ -32,6 +34,8 @@ github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/O github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -50,6 +54,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembj github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnTiM80= @@ -168,6 +174,8 @@ github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.22.1 h1:AfVXx3chM2qwoSbM7Da8g8hX8OVSkBFwX+rz2+PcK40= +github.com/google/cel-go v0.22.1/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -193,6 +201,8 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -349,6 +359,8 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -391,10 +403,20 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEj go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU= +go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q= go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -524,6 +546,8 @@ k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJ k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1 h1:uOuSLOMBWkJH0TWa9X6l+mj5nZdm6Ay6Bli8HL8rNfk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= sigs.k8s.io/controller-tools v0.15.0 h1:4dxdABXGDhIa68Fiwaif0vcu32xfwmgQ+w8p+5CxoAI= diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go index efd880949..3d30a04a5 100644 --- a/tests/e2e/operator/operator_install_test.go +++ b/tests/e2e/operator/operator_install_test.go @@ -17,6 +17,11 @@ package operator import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" "time" "github.com/istio-ecosystem/sail-operator/pkg/kube" @@ -95,7 +100,91 @@ var _ = Describe("Operator", Ordered, func() { It("starts successfully", func(ctx SpecContext) { Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). - Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Deployment status") + }) + + It("serves metrics securely", func(ctx SpecContext) { + metricsReaderRoleName := "metrics-reader" + metricsServiceName := deploymentName + "-metrics-service" + + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + err := k.CreateFromString(fmt.Sprintf(` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-reader-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: %s +subjects: +- kind: ServiceAccount + name: %s + namespace: %s +`, metricsReaderRoleName, deploymentName, namespace)) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd := exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + err = cmd.Run() + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + // By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + // cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + // err = cmd.Run() + // Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + k.GetPods() + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := cmd.Output() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", "deployment/"+deploymentName, "-n", namespace) + output, err := cmd.Output() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=quay.io/curl/curl:8.11.1", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + err = cmd.Run() + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := cmd.Output() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(string(output)).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) AfterAll(func() { @@ -133,3 +222,63 @@ func extractCRDNames(crdList *apiextensionsv1.CustomResourceDefinitionList) []st } return names } + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + out, err := cmd.Output() + metricsOutput := string(out) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/tests/e2e/operator/operator_suite_test.go b/tests/e2e/operator/operator_suite_test.go index 144cd885c..a7609691d 100644 --- a/tests/e2e/operator/operator_suite_test.go +++ b/tests/e2e/operator/operator_suite_test.go @@ -29,12 +29,13 @@ import ( ) var ( - cl client.Client - ocp = env.GetBool("OCP", false) - skipDeploy = env.GetBool("SKIP_DEPLOY", false) - namespace = common.OperatorNamespace - deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - multicluster = env.GetBool("MULTICLUSTER", false) + cl client.Client + ocp = env.GetBool("OCP", false) + skipDeploy = env.GetBool("SKIP_DEPLOY", false) + namespace = common.OperatorNamespace + deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") + serviceAccountName = deploymentName + multicluster = env.GetBool("MULTICLUSTER", false) k kubectl.Kubectl )