From 1ded538e3914450b71e1497cca7af256d3f53ac0 Mon Sep 17 00:00:00 2001 From: harshsharma071988 <125853522+harshsharma071988@users.noreply.github.com> Date: Tue, 28 Feb 2023 19:23:05 +0530 Subject: [PATCH] feat/csp_implementation (#1) * Add governor api call provision to send cluster telemetry data Signed-off-by: Nootan Singh * Added clusterId, url and apiToken as part of governor policy Signed-off-by: Nootan Singh * [BEGONIA-62] ADD CSP support in agent * [BEGONIA-71] Add CSP implementation --> Added code to get CSP api-token from secret --> Added code to send request to CSP and get access token for governor backend --> Added code to send token with every request to governor backend * refactored some code * removed unused file * removed unused file 1 * removed unused file 2 * refactored some code * removed unwanted changes * removed gitignore file changes * ran make manifests to generate manifests automatically * removed dependecy on csp gitlab library * Added UTs and addressed comments Signed-off-by: Nootan Singh * added UTs Signed-off-by: harshsharma071988 * setting correct context while update telemetry Signed-off-by: harshsharma071988 * fixed governor url config and governor api response status Signed-off-by: harshsharma071988 * Added server url of governor api Signed-off-by: Nootan Singh * changed accessSecret to API_TOKEN Signed-off-by: harshsharma071988 * handled empty namespace case and empty workload case * Added UTs for new files * Resolved minor comments * resolved minor comments Signed-off-by: harshsharma071988 * removed compilation issue with %w Signed-off-by: harshsharma071988 * CSP Refresh used using config maps + UTs added * removed fmt and used log Signed-off-by: harshsharma071988 * Used Errorf in place of Error for logging Signed-off-by: harshsharma071988 * changed comment Signed-off-by: harshsharma071988 * Used Secret in place of ConfigMap for storing access token of governor. --------- Signed-off-by: Nootan Singh Signed-off-by: harshsharma071988 Co-authored-by: Nootan Singh --- src/api/v1alpha1/inspectionpolicy_types.go | 4 +- src/cmd/inspector/main.go | 5 + ...oharbor.goharbor.io_assessmentreports.yaml | 4 +- ...harbor.goharbor.io_inspectionpolicies.yaml | 4 +- src/go.mod | 1 + src/go.sum | 1 + src/lib/cspauth/csp_auth.go | 115 +++++++++++++++ src/lib/cspauth/csp_auth_test.go | 139 ++++++++++++++++++ src/lib/cspauth/mock_token_manager.go | 30 ++++ src/lib/cspauth/mocks/mock_csp_auth.go | 55 +++++++ src/lib/cspauth/token_manager.go | 117 +++++++++++++++ src/lib/cspauth/token_manager_test.go | 70 +++++++++ src/lib/retry/retry.go | 108 ++++++++++++++ src/lib/retry/retry_test.go | 50 +++++++ src/pkg/data/consumers/governor/exporter.go | 37 ++++- .../data/consumers/governor/exporter_test.go | 89 +++++++++-- src/pkg/inspection/controller.go | 96 +++++++----- 17 files changed, 865 insertions(+), 60 deletions(-) create mode 100644 src/lib/cspauth/csp_auth.go create mode 100644 src/lib/cspauth/csp_auth_test.go create mode 100644 src/lib/cspauth/mock_token_manager.go create mode 100644 src/lib/cspauth/mocks/mock_csp_auth.go create mode 100644 src/lib/cspauth/token_manager.go create mode 100644 src/lib/cspauth/token_manager_test.go create mode 100644 src/lib/retry/retry.go create mode 100644 src/lib/retry/retry_test.go diff --git a/src/api/v1alpha1/inspectionpolicy_types.go b/src/api/v1alpha1/inspectionpolicy_types.go index 7a902f2b..893abb87 100644 --- a/src/api/v1alpha1/inspectionpolicy_types.go +++ b/src/api/v1alpha1/inspectionpolicy_types.go @@ -130,9 +130,9 @@ type Governor struct { // Api url to send telemetry data // +kubebuilder:validation:Optional URL string `json:"url"` - // Api token for user authentication + // Secret name where CSP api token is stored in cnsi-system namespace // +kubebuilder:validation:Optional - APIToken string `json:"apiToken"` + CspSecretName string `json:"cspSecretName"` } // FollowupAction defines what actions should be applied when security expectations are matched. diff --git a/src/cmd/inspector/main.go b/src/cmd/inspector/main.go index ae95c63b..b9e4880b 100644 --- a/src/cmd/inspector/main.go +++ b/src/cmd/inspector/main.go @@ -41,6 +41,7 @@ func main() { k8sClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{ Scheme: scheme, }) + if err != nil { log.Error(err, "unable to create k8s client") os.Exit(1) @@ -56,6 +57,10 @@ func main() { os.Exit(1) } + if inspectionPolicy.Spec.Inspection.Assessment.Governor.Enabled { + ctx = context.WithValue(ctx, "cspSecretName", inspectionPolicy.Spec.Inspection.Assessment.Governor.CspSecretName) + } + runner := inspection.NewController(). WithScheme(scheme). WithK8sClient(k8sClient). diff --git a/src/config/crd/bases/goharbor.goharbor.io_assessmentreports.yaml b/src/config/crd/bases/goharbor.goharbor.io_assessmentreports.yaml index c7eed68a..ca6296bf 100644 --- a/src/config/crd/bases/goharbor.goharbor.io_assessmentreports.yaml +++ b/src/config/crd/bases/goharbor.goharbor.io_assessmentreports.yaml @@ -143,8 +143,8 @@ spec: governor: description: Indicate whether to config of governor properties: - apiToken: - description: Api token for user authentication + cspSecretName: + description: Secret name where CSP api token is stored in cnsi-system namespace type: string clusterId: description: Unique identifier of the cluster diff --git a/src/config/crd/bases/goharbor.goharbor.io_inspectionpolicies.yaml b/src/config/crd/bases/goharbor.goharbor.io_inspectionpolicies.yaml index 760aab35..67397ee0 100644 --- a/src/config/crd/bases/goharbor.goharbor.io_inspectionpolicies.yaml +++ b/src/config/crd/bases/goharbor.goharbor.io_inspectionpolicies.yaml @@ -155,8 +155,8 @@ spec: governor: description: Indicate whether to config of governor properties: - apiToken: - description: Api token for user authentication + cspSecretName: + description: Secret name where CSP api token is stored in cnsi-system namespace type: string clusterId: description: Unique identifier of the cluster diff --git a/src/go.mod b/src/go.mod index d2543355..9868c843 100644 --- a/src/go.mod +++ b/src/go.mod @@ -53,6 +53,7 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect github.com/emicklei/go-restful/v3 v3.8.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect diff --git a/src/go.sum b/src/go.sum index 6c0a0854..e9b423ea 100644 --- a/src/go.sum +++ b/src/go.sum @@ -151,6 +151,7 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= diff --git a/src/lib/cspauth/csp_auth.go b/src/lib/cspauth/csp_auth.go new file mode 100644 index 00000000..f591e77e --- /dev/null +++ b/src/lib/cspauth/csp_auth.go @@ -0,0 +1,115 @@ +package cspauth + +import ( + "context" + "fmt" + "github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/log" + "github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/retry" + v12 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "math" + "time" +) + +const ( + tokenMaxAgeSeconds = 1700 + apiToken = "API_TOKEN" + accessTokenSecretName = "governor-accesstoken" + governorTokenExpiresIn = "governorAccessTokenExpiresIn" + governorAccessTokenKey = "governorAccessToken" + Retry = 3 +) + +var RetryDelay time.Duration = 5 + +// Provider is an interface to interact with an authorization service +type Provider interface { + // GetBearerToken retrieves a short-lived access token to use in a single HTTP request + GetBearerToken(kubernetes.Interface, context.Context, string, string) (string, error) +} + +type CspAuth struct { + CspClient CSPClient + + apiToken string +} + +func (a *CspAuth) GetBearerToken(clientSet kubernetes.Interface, ctx context.Context, cspSecretNamespace string, cspSecretName string) (string, error) { + accessSecret, err := getOrCreateSecretForAccessToken(clientSet, ctx, cspSecretNamespace) + if err != nil { + return "", err + } + + accessToken := string(accessSecret.Data[governorAccessTokenKey]) + expiresIn := string(accessSecret.Data[governorTokenExpiresIn]) + accessTokenExpiresIn, _ := time.Parse(time.Layout, expiresIn) + + if accessToken == "" || time.Now().After(accessTokenExpiresIn) { + apiToken, err := getCSPTokenFromSecret(clientSet, ctx, cspSecretNamespace, cspSecretName) + if err != nil { + return "", fmt.Errorf("Failed to fetch CSP api-token: %w", err) + } + a.apiToken = apiToken + if err := a.refreshToken(ctx, clientSet, cspSecretNamespace, accessSecret); err != nil { + return "", err + } + } + return string(accessSecret.Data[governorAccessTokenKey]), nil +} + +func (a *CspAuth) refreshToken(ctx context.Context, clientSet kubernetes.Interface, cspSecretNamespace string, accessTokenSecret *v12.Secret) error { + return retry.NewRetry( + retry.WithName("auth token refresh"), + retry.WithMaxAttempts(Retry), + retry.WithIncrementDelay(RetryDelay*time.Second, RetryDelay*time.Second), + ).Run(ctx, func() (bool, error) { + now := time.Now() + cspAuthResponse, err := a.CspClient.GetCspAuthorization(ctx, a.apiToken) + if err != nil { + log.Error(err, "We got an error back from CSP") + return false, nil + } + + expiresIn := time.Duration(math.Min(float64(cspAuthResponse.ExpiresIn), tokenMaxAgeSeconds)) * time.Second + formattedExpiration := now.Add(expiresIn).Format(time.Layout) + + log.Infof("Refreshed access token for governor: %s which expires in %s", cspAuthResponse.AccessToken, formattedExpiration) + accessTokenSecret.Data[governorAccessTokenKey] = []byte(cspAuthResponse.AccessToken) + accessTokenSecret.Data[governorTokenExpiresIn] = []byte(formattedExpiration) + _, err = clientSet.CoreV1().Secrets(cspSecretNamespace).Update(ctx, accessTokenSecret, v1.UpdateOptions{}) + if err != nil { + log.Error(err, "We got an error updating access token secret") + return false, nil + } + log.Infof("Obtained CSP access token, next refresh in %s\n", expiresIn) + return true, nil + }) +} + +func getCSPTokenFromSecret(clientSet kubernetes.Interface, ctx context.Context, ns string, secretName string) (string, error) { + secret, err := clientSet.CoreV1().Secrets(ns).Get(ctx, secretName, v1.GetOptions{}) + if err != nil { + log.Error(err, "Failed to fetch secret") + return "", err + } + cspApiToken := string(secret.Data[apiToken]) + return cspApiToken, err +} + +func getOrCreateSecretForAccessToken(clientSet kubernetes.Interface, ctx context.Context, ns string) (*v12.Secret, error) { + secret, err := clientSet.CoreV1().Secrets(ns).Get(ctx, accessTokenSecretName, v1.GetOptions{}) + if err != nil { + log.Warning(err, "Failed to fetch secret for access token, Now Trying to create new secret for same") + secret = &v12.Secret{} + secret.Name = accessTokenSecretName + secret.Namespace = ns + secret.Data = map[string][]byte{} + secret, err = clientSet.CoreV1().Secrets(ns).Create(ctx, secret, v1.CreateOptions{}) + if err != nil { + log.Error(err, "Failed to create secret for storing access token.") + return nil, err + } + } + return secret, err +} diff --git a/src/lib/cspauth/csp_auth_test.go b/src/lib/cspauth/csp_auth_test.go new file mode 100644 index 00000000..bb1001aa --- /dev/null +++ b/src/lib/cspauth/csp_auth_test.go @@ -0,0 +1,139 @@ +package cspauth + +import ( + "context" + v12 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "testing" +) + +const ( + ApiToken = "API_TOKEN" + GovernorAccessTokenKey = "governorAccessToken" +) + +func TestNewCSPAuthSuccessCase(t *testing.T) { + + RetryDelay = 1 + secret := &v12.Secret{} + secret.Name = "csp-secret" + secret.Namespace = "csp-namespace" + secret.Data = map[string][]byte{ApiToken: []byte("test-api-token")} + + errorSecret := &v12.Secret{} + errorSecret.Name = "csp-secret" + errorSecret.Namespace = "csp-namespace" + errorSecret.Data = map[string][]byte{ApiToken: []byte(SendError)} + + accessSecret := &v12.Secret{} + accessSecret.Name = "governor-accesstoken" + accessSecret.Namespace = "csp-namespace" + accessSecret.Data = map[string][]byte{GovernorAccessTokenKey: []byte("test-access-token")} + + tt := []struct { + name string + secretObject *v12.Secret + accessSecret *v12.Secret + wantErr bool + }{ + { + name: "Get CSP Auth should Pass", + secretObject: secret, + accessSecret: accessSecret, + wantErr: false, + }, + { + name: "Get CSP Auth should fail because no secret found for csp api-token", + secretObject: nil, + accessSecret: accessSecret, + wantErr: true, + }, + { + name: "Get CSP Auth should fail with giving up refresh retry(3times)", + secretObject: errorSecret, + accessSecret: accessSecret, + wantErr: true, + }, + { + name: "Get CSP Auth should pass with accessSecret not found", + secretObject: secret, + accessSecret: nil, + wantErr: false, + }, + } + + for i := range tt { + tc := tt[i] + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + objects := make([]runtime.Object, 0) + + if tc.secretObject != nil { + objects = append(objects, tc.secretObject) + } + if tc.accessSecret != nil { + objects = append(objects, tc.accessSecret) + } + clientSet := fake.NewSimpleClientset(objects...) + + tokenManager := NewMockCSPClient() + provider := &CspAuth{CspClient: tokenManager} + auth, err := provider.GetBearerToken(clientSet, context.Background(), secret.Namespace, secret.Name) + + if tc.wantErr && (auth != "" || err == nil) { + t.Fatal("NewCSPAuth call failed on tc: " + tc.name) + } + + if !tc.wantErr && (auth == "" || err != nil) { + t.Fatal("NewCSPAuth call failed on tc: " + tc.name) + } + }) + } + +} + +func TestGetBearerTokenSuccess(t *testing.T) { + secret := &v12.Secret{} + secret.Name = "csp-secret" + secret.Namespace = "csp-namespace" + secret.Data = map[string][]byte{ApiToken: []byte("test-api-token")} + + clientSet := fake.NewSimpleClientset(secret) + + tokenManager := NewMockCSPClient() + provider := &CspAuth{CspClient: tokenManager} + authToken, _ := provider.GetBearerToken(clientSet, context.Background(), secret.Namespace, secret.Name) + + if authToken != DummyAccessToken { + t.Fatal("GetBearer must not fail in this test case!") + } +} + +func TestGetBearerTokenReturnSameTokenSuccess(t *testing.T) { + secret := &v12.Secret{} + secret.Name = "csp-secret" + secret.Namespace = "csp-namespace" + secret.Data = map[string][]byte{ApiToken: []byte("test-api-token")} + + clientSet := fake.NewSimpleClientset(secret) + + tokenManager := NewMockCSPClient() + provider := &CspAuth{CspClient: tokenManager} + authToken, _ := provider.GetBearerToken(clientSet, context.Background(), secret.Namespace, secret.Name) + + if authToken != DummyAccessToken { + t.Fatal("GetBearer must not fail in this test case!") + } + + tokenPrev := DummyAccessToken + DummyAccessToken = "changed-dummy-access-token" + authToken1, _ := provider.GetBearerToken(clientSet, context.Background(), secret.Namespace, secret.Name) + + if authToken != authToken1 { + t.Fatal("GetBearer must return same token if called consequently, \nAuth1: " + authToken + "\n Auth2: " + authToken1) + } + DummyAccessToken = tokenPrev +} diff --git a/src/lib/cspauth/mock_token_manager.go b/src/lib/cspauth/mock_token_manager.go new file mode 100644 index 00000000..b2084970 --- /dev/null +++ b/src/lib/cspauth/mock_token_manager.go @@ -0,0 +1,30 @@ +package cspauth + +import ( + "context" + "github.com/pkg/errors" +) + +var ( + DummyAccessToken = "dummy-access-token" + SendError = "send-error" +) + +// MockCSPClient is a mock of the CSPClient interface +type MockCSPClient struct { +} + +// NewMockCSPClient creates a new mock instance +func NewMockCSPClient() *MockCSPClient { + return &MockCSPClient{} +} + +func (m *MockCSPClient) GetCspAuthorization(ctx context.Context, apiToken string) (*CSPAuthorizeResponse, error) { + if apiToken == SendError { + return nil, errors.New("Failed to get CSP Auth") + } + response := CSPAuthorizeResponse{} + response.AccessToken = DummyAccessToken + response.ExpiresIn = 1000 + return &response, nil +} diff --git a/src/lib/cspauth/mocks/mock_csp_auth.go b/src/lib/cspauth/mocks/mock_csp_auth.go new file mode 100644 index 00000000..4e862881 --- /dev/null +++ b/src/lib/cspauth/mocks/mock_csp_auth.go @@ -0,0 +1,55 @@ +// Code generated by mockery v2.20.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + kubernetes "k8s.io/client-go/kubernetes" + + mock "github.com/stretchr/testify/mock" +) + +// Provider is an autogenerated mock type for the Provider type +type Provider struct { + mock.Mock +} + +// GetBearerToken provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *Provider) GetBearerToken(_a0 kubernetes.Interface, _a1 context.Context, _a2 string, _a3 string) (string, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(kubernetes.Interface, context.Context, string, string) (string, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(kubernetes.Interface, context.Context, string, string) string); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(kubernetes.Interface, context.Context, string, string) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewProvider interface { + mock.TestingT + Cleanup(func()) +} + +// NewProvider creates a new instance of Provider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProvider(t mockConstructorTestingTNewProvider) *Provider { + mock := &Provider{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/src/lib/cspauth/token_manager.go b/src/lib/cspauth/token_manager.go new file mode 100644 index 00000000..fecb2aa8 --- /dev/null +++ b/src/lib/cspauth/token_manager.go @@ -0,0 +1,117 @@ +package cspauth + +import ( + "context" + "encoding/json" + "errors" + "github.com/goharbor/harbor/src/lib/log" + "io" + "net/http" + "net/url" + "strings" +) + +var ( + ErrorCspForbidden = errors.New("forbidden") + ErrorCspUnauthorized = errors.New("unauthorized") + ErrorCspBadRequest = errors.New("invalid api_token, it might be expired") + ErrorRefreshTokenNotValid = errors.New("refresh_token cannot be empty") + UnexpectedResponseStatusCode = errors.New("unexpected csp error") +) + +const ( + cspUrl = "https://console.cloud.vmware.com/csp/gateway/am/api/auth/api-tokens/authorize" +) + +// CSPClient represents a CSPClient place here for future mocking purposes +// Will contain all methods related to calls directly to CSP. +// Please note this is not usual since it would be TAC the one +// talking with CSP. +type CSPClient interface { + GetCspAuthorization(ctx context.Context, refreshToken string) (*CSPAuthorizeResponse, error) +} + +// CSPHttpClient is the client to perform talks to CSP. +type CSPHttpClient struct { + client *http.Client + host *url.URL +} + +// CSPAuthorizeResponse represents a response from CSP with information +// about the authorization. AccessToken is the value needed to +// perform valid calls to TAC backend. +type CSPAuthorizeResponse struct { + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` + AccessToken string `json:"access_token"` + Scope string `json:"scope"` + IDToken string `json:"id_token"` + TokenType string `json:"token_type"` +} + +// NewCspHTTPClient creates a new CSPHttpClient +func NewCspHTTPClient() (*CSPHttpClient, error) { + + parsedURL, err := url.Parse(cspUrl) + return &CSPHttpClient{ + client: http.DefaultClient, + host: parsedURL, + }, err +} + +// GetCspAuthorization connects to CSP to retrieve information regarding the given API token +func (c *CSPHttpClient) GetCspAuthorization(ctx context.Context, apiToken string) (*CSPAuthorizeResponse, error) { + if apiToken == "" { + return nil, ErrorRefreshTokenNotValid + } + values := url.Values{"grant_type": {"refresh_token"}, "refresh_token": {apiToken}} + + req, err := http.NewRequestWithContext(ctx, "POST", c.host.String(), strings.NewReader(values.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("accept-encoding", "application/json") + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + return + } + }(resp.Body) + + if err := c.checkCspAuthStatusCode(resp); err != nil { + log.Errorf("Found an error code: %v", err.Error()) + return nil, err + } + + var cspAuthResponse CSPAuthorizeResponse + return &cspAuthResponse, json.NewDecoder(resp.Body).Decode(&cspAuthResponse) +} + +// Checks the status code for the auth service. +// 400 status code is ambiguous because it can mean: +// - The api_token is wrong +// - The api_token is expired +// In the later case the user would need to generate a new from +// CSP console since, expired tokens are automatically removed from +// CSP. +func (c *CSPHttpClient) checkCspAuthStatusCode(resp *http.Response) error { + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusBadRequest: + // api_token can be expired + return ErrorCspBadRequest + case http.StatusUnauthorized: + return ErrorCspUnauthorized + case http.StatusForbidden: + return ErrorCspForbidden + default: + return UnexpectedResponseStatusCode + } +} diff --git a/src/lib/cspauth/token_manager_test.go b/src/lib/cspauth/token_manager_test.go new file mode 100644 index 00000000..d81d144e --- /dev/null +++ b/src/lib/cspauth/token_manager_test.go @@ -0,0 +1,70 @@ +package cspauth + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func TestGetCspAuthorization(t *testing.T) { + tt := []struct { + name string + token string + wantErr bool + }{ + { + name: "Get CSP Auth Success", + token: "dummy-api-token", + wantErr: false, + }, + { + name: "Get CSP Auth Failure", + token: "dummy-api-error-token", + wantErr: true, + }, + } + + for i := range tt { + tc := tt[i] + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + bodyStr := string(body) + if strings.Contains(bodyStr, "refresh_token=dummy-api-token") { + var cspAuthResponse CSPAuthorizeResponse + cspAuthResponse.AccessToken = "dummy-access-token" + successResponse, _ := json.Marshal(cspAuthResponse) + _, _ = w.Write(successResponse) + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusBadGateway) + } + })) + + defer server.Close() + + client, _ := NewCspHTTPClient() + client.client = http.DefaultClient + client.host, _ = url.Parse(server.URL) + apiToken := tc.token + authorization, err := client.GetCspAuthorization(context.Background(), apiToken) + + if tc.wantErr && (authorization != nil || err == nil) { + t.Fatalf("GetCspAuthorizationCase must fail but got success with token: %v", authorization) + } + + if !tc.wantErr && (authorization == nil || err != nil) { + t.Fatalf("GetCspAuthorizationCase should not fail but failed with error: %v", err) + } + }) + } + +} diff --git a/src/lib/retry/retry.go b/src/lib/retry/retry.go new file mode 100644 index 00000000..44da4ada --- /dev/null +++ b/src/lib/retry/retry.go @@ -0,0 +1,108 @@ +package retry + +import ( + "context" + "fmt" + "log" + "time" +) + +const ( + defaultMaxAttempts = 3 + defaultRetryStep = 3 * time.Second +) + +type delayFunc func(attempt int) time.Duration + +type retryConfig struct { + name string + maxAttempts int + delayFunc delayFunc +} + +type Option func(*retryConfig) + +// WithName allows configuring the name of the function in the error message +func WithName(name string) Option { + return func(o *retryConfig) { + o.name = name + } +} + +// WithMaxAttempts allows configuring the maximum tries for a given function +func WithMaxAttempts(n int) Option { + return func(o *retryConfig) { + o.maxAttempts = n + } +} + +// WithFixedDelay allows configuring a fixed-delay waiting strategy between retries +func WithFixedDelay(delay time.Duration) Option { + return func(o *retryConfig) { + o.delayFunc = func(_ int) time.Duration { return delay } + } +} + +// WithIncrementDelay allows configuring a waiting strategy with a custom base delay and increment +func WithIncrementDelay(baseDuration time.Duration, increment time.Duration) Option { + return func(o *retryConfig) { + o.delayFunc = func(n int) time.Duration { + stepIncrement := increment * time.Duration(n) + return baseDuration + stepIncrement + } + } +} + +type Retry struct { + retryConfig +} + +func NewRetry(opts ...Option) *Retry { + var c retryConfig + for _, o := range append([]Option{ + // Default values + WithName("retryable function"), + WithMaxAttempts(defaultMaxAttempts), + WithFixedDelay(defaultRetryStep), + }, opts...) { + o(&c) + } + return &Retry{c} +} + +func (r *Retry) Run(ctx context.Context, f func() (bool, error)) error { + timer := time.NewTimer(0) + defer timer.Stop() + var attempts int + for { + if success, err := f(); err != nil { + return fmt.Errorf("non retryable error running %q: %w", r.name, err) + } else if success { + return nil + } else { + log.Printf("running %q failed (%d/%d)\n", r.name, attempts+1, r.maxAttempts) + } + + delay := r.delayFunc(attempts) + attempts++ + if attempts == r.maxAttempts { + return fmt.Errorf("giving up retrying, max attempts %d reached", r.maxAttempts) + } + + timer.Reset(delay) + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + } +} + +// SetNextRetry allows configuring a custom duration only for the next retry calculated +func (r *Retry) SetNextRetry(duration time.Duration) { + orig := r.delayFunc + r.delayFunc = func(_ int) time.Duration { + r.delayFunc = orig // restore original function + return duration + } +} diff --git a/src/lib/retry/retry_test.go b/src/lib/retry/retry_test.go new file mode 100644 index 00000000..f5c5fa27 --- /dev/null +++ b/src/lib/retry/retry_test.go @@ -0,0 +1,50 @@ +package retry_test + +import ( + "context" + "fmt" + "github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/retry" + "testing" + "time" +) + +func TestSuccessCase(t *testing.T) { + var count int + err := retry.NewRetry().Run(context.Background(), func() (bool, error) { + count++ + return true, nil + }) + if err != nil { + t.Fatal(err) + } + if got, want := count, 1; got != want { + t.Errorf("unexpected executions count, got: %d, want: %d", got, want) + } +} + +func TestWithMaxAttempts(t *testing.T) { + testCases := []struct { + maxAttempts int + }{ + {maxAttempts: 1}, + {maxAttempts: 3}, + {maxAttempts: 10}, + } + for _, tc := range testCases { + t.Run(fmt.Sprintf("%d attempts", tc.maxAttempts), func(t *testing.T) { + var count int + if err := retry.NewRetry( + retry.WithMaxAttempts(tc.maxAttempts), + retry.WithFixedDelay(10*time.Millisecond), + ).Run(context.Background(), func() (bool, error) { + count++ + return count == tc.maxAttempts, nil + }); err != nil { + t.Fatal(err) + } + if got, want := count, tc.maxAttempts; got != want { + t.Errorf("expected function to be executed %d, got: %d", want, got) + } + }) + } +} diff --git a/src/pkg/data/consumers/governor/exporter.go b/src/pkg/data/consumers/governor/exporter.go index a7a1bb25..8a449ad0 100644 --- a/src/pkg/data/consumers/governor/exporter.go +++ b/src/pkg/data/consumers/governor/exporter.go @@ -5,25 +5,47 @@ import ( "errors" "fmt" api "github.com/vmware-tanzu/cloud-native-security-inspector/src/api/v1alpha1" + "github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/cspauth" "github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/log" openapi "github.com/vmware-tanzu/cloud-native-security-inspector/src/pkg/data/consumers/governor/go-client" + "k8s.io/client-go/kubernetes" "net/http" ) +const ( + cspSecretNamespace = "cnsi-system" +) + type GovernorExporter struct { - Report *api.AssessmentReport - ClusterID string - ApiToken string - ApiClient *openapi.APIClient + Report *api.AssessmentReport + ClusterID string + ApiClient *openapi.APIClient + CspProvider cspauth.Provider + KubeInterface kubernetes.Interface } // SendReportToGovernor is used to send report to governor url http end point. -func (g GovernorExporter) SendReportToGovernor() error { +func (g GovernorExporter) SendReportToGovernor(ctx context.Context) error { // Get governor api request model from assessment report. kubernetesCluster := g.getGovernorAPIPayload() + log.Info("Payload data for governor:") log.Info(kubernetesCluster) - apiSaveClusterRequest := g.ApiClient.ClustersApi.UpdateTelemetry(context.Background(), g.ClusterID).KubernetesTelemetryRequest(kubernetesCluster) + + cspSecretName := ctx.Value("cspSecretName") + if cspSecretName == nil { + log.Error("Error while retrieving access token !") + return errors.New("CSP secret name must be set to connect to Governor") + } + governorAccessToken, err := g.CspProvider.GetBearerToken(g.KubeInterface, ctx, cspSecretNamespace, cspSecretName.(string)) + if err != nil { + log.Error("Error while retrieving access token !") + return err + } + + ctx = context.WithValue(ctx, openapi.ContextAccessToken, governorAccessToken) + + apiSaveClusterRequest := g.ApiClient.ClustersApi.UpdateTelemetry(ctx, g.ClusterID).KubernetesTelemetryRequest(kubernetesCluster) // Call api cluster to send telemetry data and get response. response, err := g.ApiClient.ClustersApi.UpdateTelemetryExecute(apiSaveClusterRequest) @@ -46,7 +68,7 @@ func (g GovernorExporter) SendReportToGovernor() error { // getGovernorAPIPayload is used to map assessment report to client model. func (g GovernorExporter) getGovernorAPIPayload() openapi.KubernetesTelemetryRequest { kubernetesCluster := openapi.NewKubernetesTelemetryRequestWithDefaults() - + kubernetesCluster.Workloads = make([]openapi.KubernetesWorkload, 0) for _, nsa := range g.Report.Spec.NamespaceAssessments { for _, workloadAssessment := range nsa.WorkloadAssessments { kubernetesWorkloads := openapi.NewKubernetesWorkloadWithDefaults() @@ -54,6 +76,7 @@ func (g GovernorExporter) getGovernorAPIPayload() openapi.KubernetesTelemetryReq kubernetesWorkloads.Kind = workloadAssessment.Workload.Kind kubernetesWorkloads.Namespace = nsa.Namespace.Name kubernetesWorkloads.Replicas = workloadAssessment.Workload.Replicas + for _, pod := range workloadAssessment.Workload.Pods { containerData := openapi.NewContainerWithDefaults() for _, container := range pod.Containers { diff --git a/src/pkg/data/consumers/governor/exporter_test.go b/src/pkg/data/consumers/governor/exporter_test.go index 2566ca33..1b8f16ac 100644 --- a/src/pkg/data/consumers/governor/exporter_test.go +++ b/src/pkg/data/consumers/governor/exporter_test.go @@ -1,9 +1,12 @@ package consumers import ( + "context" + "errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" api "github.com/vmware-tanzu/cloud-native-security-inspector/src/api/v1alpha1" + "github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/cspauth/mocks" openapi "github.com/vmware-tanzu/cloud-native-security-inspector/src/pkg/data/consumers/governor/go-client" v1 "k8s.io/api/core/v1" "net/http" @@ -11,15 +14,17 @@ import ( ) var ( - clusterID = "testingId" - apiToken = "apiToken" - namespace = "testingNamespace" - name = "name" - image = "image" - imageID = "imageId" - replicaCount = 2 - testHeader = "testHeader" - testHeaderValue = "testHeaderValue" + clusterID = "testingId" + apiToken = "apiToken" + namespace = "testingNamespace" + name = "name" + image = "image" + imageID = "imageId" + replicaCount = 2 + testHeader = "testHeader" + testHeaderValue = "testHeaderValue" + testApiTokentoken = "test-access-token" + testCspSecretName = "cspSecretName" ) const ( @@ -38,6 +43,9 @@ func TestSendReportToGovernor(t *testing.T) { testClusterID string testAPIToken string testStatusCode int + testSecretName string + createCSPProvider bool + authToken string }{ { testCaseDescription: "Success: Happy flow end to end.", @@ -54,9 +62,12 @@ func TestSendReportToGovernor(t *testing.T) { Image: image, ImageID: imageID, }}}}}}}}}}}, - testClusterID: clusterID, - testAPIToken: apiToken, - testStatusCode: http.StatusNoContent, + testClusterID: clusterID, + testAPIToken: apiToken, + testStatusCode: http.StatusNoContent, + testSecretName: testCspSecretName, + createCSPProvider: true, + authToken: testApiTokentoken, }, { testCaseDescription: "Success: Empty payload, successful case", @@ -67,6 +78,9 @@ func TestSendReportToGovernor(t *testing.T) { testClusterID: clusterID, testAPIToken: apiToken, testStatusCode: http.StatusNoContent, + testSecretName: testCspSecretName, + createCSPProvider: true, + authToken: testApiTokentoken, }, { testCaseDescription: "Failure: Error from API call.", @@ -77,6 +91,9 @@ func TestSendReportToGovernor(t *testing.T) { testClusterID: clusterID, testAPIToken: apiToken, testStatusCode: http.StatusInternalServerError, + testSecretName: testCspSecretName, + createCSPProvider: true, + authToken: testApiTokentoken, }, { testCaseDescription: "Failure Invalid URL: Error from api.", @@ -87,6 +104,9 @@ func TestSendReportToGovernor(t *testing.T) { testClusterID: clusterID, testAPIToken: apiToken, testStatusCode: http.StatusBadRequest, + testSecretName: testCspSecretName, + createCSPProvider: true, + authToken: testApiTokentoken, }, { testCaseDescription: "Failure: Timeout to receive response from api.", @@ -97,6 +117,35 @@ func TestSendReportToGovernor(t *testing.T) { testClusterID: clusterID, testAPIToken: apiToken, testStatusCode: http.StatusRequestTimeout, + testSecretName: testCspSecretName, + createCSPProvider: true, + authToken: testApiTokentoken, + }, + { + testCaseDescription: "Failure: CSP Secret name not found", + testHost: testHost, + testHeader: testHeader, + testHeaderValue: testHeaderValue, + testReportData: &api.AssessmentReport{}, + testClusterID: clusterID, + testAPIToken: apiToken, + testStatusCode: http.StatusNotFound, + testSecretName: "", + createCSPProvider: false, + authToken: testApiTokentoken, + }, + { + testCaseDescription: "Failure: Access Token not available", + testHost: testHost, + testHeader: testHeader, + testHeaderValue: testHeaderValue, + testReportData: &api.AssessmentReport{}, + testClusterID: clusterID, + testAPIToken: apiToken, + testStatusCode: http.StatusNotFound, + testSecretName: testCspSecretName, + createCSPProvider: true, + authToken: "", }, } @@ -112,7 +161,6 @@ func TestSendReportToGovernor(t *testing.T) { g := GovernorExporter{ Report: tt.testReportData, ApiClient: clusterClient, - ApiToken: apiToken, ClusterID: tt.testClusterID, } mockAPIClient := new(ClustersApi) @@ -127,7 +175,20 @@ func TestSendReportToGovernor(t *testing.T) { StatusCode: tt.testStatusCode, }, nil) - errFromSendReportToGovernor := g.SendReportToGovernor() + ctx := context.Background() + if tt.testSecretName != "" { + ctx = context.WithValue(ctx, "cspSecretName", tt.testSecretName) + } + + provider := new(mocks.Provider) + if tt.authToken == "" { + provider.On("GetBearerToken", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.authToken, errors.New("Failed to fetch CSP auth token")) + } else { + provider.On("GetBearerToken", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.authToken, nil) + } + g.CspProvider = provider + + errFromSendReportToGovernor := g.SendReportToGovernor(ctx) if tt.testStatusCode != http.StatusNoContent { assert.Error(t, errFromSendReportToGovernor) } else { diff --git a/src/pkg/inspection/controller.go b/src/pkg/inspection/controller.go index 75299055..653aba70 100644 --- a/src/pkg/inspection/controller.go +++ b/src/pkg/inspection/controller.go @@ -5,6 +5,7 @@ package inspection import ( "context" "fmt" + "github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/cspauth" "github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/log" es "github.com/vmware-tanzu/cloud-native-security-inspector/src/pkg/data/consumers/es" governor "github.com/vmware-tanzu/cloud-native-security-inspector/src/pkg/data/consumers/governor" @@ -12,6 +13,7 @@ import ( osearch "github.com/vmware-tanzu/cloud-native-security-inspector/src/pkg/data/consumers/opensearch" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "time" "github.com/vmware-tanzu/cloud-native-security-inspector/src/pkg/policy/enforcement" @@ -141,6 +143,10 @@ func (c *controller) Run(ctx context.Context, policy *v1alpha1.InspectionPolicy) // Just in case. if len(nsl) == 0 { log.Info("no namespaces found") + err2 := c.checkAndSendReportToGovernor(ctx, policy, &v1alpha1.AssessmentReport{}) + if err2 != nil { + return err2 + } return nil } @@ -333,20 +339,10 @@ func (c *controller) Run(ctx context.Context, policy *v1alpha1.InspectionPolicy) } } - // Read config from InspectionPolicy, send assessment reports to Governor api if governor enabled. - if policy.Spec.Inspection.Assessment.Governor.Enabled { - governorConfig := policy.Spec.Inspection.Assessment.Governor - - if governorConfig.ClusterID == "" || governorConfig.URL == "" || governorConfig.APIToken == "" { - log.Error("Either ClusterID or URL or APIToken is empty") - return errors.New("Either ClusterID or URL or APIToken is empty") - } - - log.Info("Calling governor exporter") - if exporterErr := exportReportToGovernor(report, policy); exporterErr != nil { - log.Errorf("Error from exporter: %v", exporterErr) - return exporterErr - } + err2 := c.checkAndSendReportToGovernor(ctx, policy, report) + log.Info("Calling governor exporter") + if err2 != nil { + return err2 } // Create report CR if necessary. @@ -373,23 +369,57 @@ func (c *controller) Run(ctx context.Context, policy *v1alpha1.InspectionPolicy) return nil } -func exportReportToGovernor(report *v1alpha1.AssessmentReport, policy *v1alpha1.InspectionPolicy) error { +func (c *controller) checkAndSendReportToGovernor(ctx context.Context, policy *v1alpha1.InspectionPolicy, report *v1alpha1.AssessmentReport) error { + // Read config from InspectionPolicy, send assessment reports to Governor api if governor enabled. + if policy.Spec.Inspection.Assessment.Governor.Enabled { + governorConfig := policy.Spec.Inspection.Assessment.Governor + + if governorConfig.ClusterID == "" || governorConfig.URL == "" || governorConfig.CspSecretName == "" { + log.Error("Either ClusterID or URL or CSPSecretName is empty") + return errors.New("Either ClusterID or URL or CSPSecretName is empty") + } + + log.Info("Calling governor exporter") + if exporterErr := exportReportToGovernor(ctx, report, policy); exporterErr != nil { + log.Errorf("Error from exporter: %v", exporterErr) + return exporterErr + } + } + return nil +} + +func exportReportToGovernor(ctx context.Context, report *v1alpha1.AssessmentReport, policy *v1alpha1.InspectionPolicy) error { governorConfig := policy.Spec.Inspection.Assessment.Governor // Create api client to governor api. config := openapi.NewConfiguration() - //config.Host = governorConfig.URL - config.Servers = openapi.ServerConfigurations{{URL: governorConfig.URL}} + config.Servers = openapi.ServerConfigurations{{ + URL: governorConfig.URL, + }} apiClient := openapi.NewAPIClient(config) + cspClient, err := cspauth.NewCspHTTPClient() + if err != nil { + log.Errorf("Initializing CSP : %v", err) + return err + } + provider := &cspauth.CspAuth{CspClient: cspClient} + + clientSet, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) + if err != nil { + log.Error(err, "Failed to get kubernetes clientSet, check if kube config is correctly configured!") + return err + } + exporter := governor.GovernorExporter{ - Report: report, - ClusterID: governorConfig.ClusterID, - ApiToken: governorConfig.APIToken, - ApiClient: apiClient, + Report: report, + ClusterID: governorConfig.ClusterID, + ApiClient: apiClient, + CspProvider: provider, + KubeInterface: clientSet, } - if apiResponseErr := exporter.SendReportToGovernor(); apiResponseErr != nil { + if apiResponseErr := exporter.SendReportToGovernor(ctx); apiResponseErr != nil { log.Error("Err response from governor exporter", apiResponseErr) return apiResponseErr } @@ -398,15 +428,15 @@ func exportReportToGovernor(report *v1alpha1.AssessmentReport, policy *v1alpha1. } func exportReportToOpenSearch(report *v1alpha1.AssessmentReport, policy *v1alpha1.InspectionPolicy) error { - client := osearch.NewClient([]byte{}, + openSearchClient := osearch.NewClient([]byte{}, policy.Spec.Inspection.Assessment.OpenSearchAddr, policy.Spec.Inspection.Assessment.OpenSearchUser, policy.Spec.Inspection.Assessment.OpenSearchPasswd) - if client == nil { - log.Info("OpenSearch client is nil") + if openSearchClient == nil { + log.Info("OpenSearch openSearchClient is nil") } - exporter := osearch.OpenSearchExporter{Client: client} - err := exporter.NewExporter(client, "assessment_report") + exporter := osearch.OpenSearchExporter{Client: openSearchClient} + err := exporter.NewExporter(openSearchClient, "assessment_report") if err != nil { return err } @@ -434,17 +464,17 @@ func exportReportToES(report *v1alpha1.AssessmentReport, policy *v1alpha1.Inspec } log.Info("ES config: ", "addr", clientArgs.addr) log.Info("ES config: ", "clientArgs.username", clientArgs.username) - client := es.NewClient(clientArgs.cert, clientArgs.addr, clientArgs.username, clientArgs.passwd) - if client == nil { - log.Info("ES client is nil") + esClient := es.NewClient(clientArgs.cert, clientArgs.addr, clientArgs.username, clientArgs.passwd) + if esClient == nil { + log.Info("ES esClient is nil") } if err := es.TestClient(); err != nil { - log.Info("client test error") + log.Info("esClient test error") return err } - exporter := es.ElasticSearchExporter{Client: client} - err := exporter.NewExporter(client, "assessment_report") + exporter := es.ElasticSearchExporter{Client: esClient} + err := exporter.NewExporter(esClient, "assessment_report") if err != nil { return err }