Skip to content

Commit

Permalink
Merge pull request #170 from snootan/feat/add-governor-agent
Browse files Browse the repository at this point in the history
Add governor api call provision to send cluster telemetry data
  • Loading branch information
harshsharma071988 authored Mar 3, 2023
2 parents 3c3e226 + 38bd8c7 commit d7f4078
Show file tree
Hide file tree
Showing 37 changed files with 5,982 additions and 20 deletions.
19 changes: 19 additions & 0 deletions src/api/v1alpha1/inspectionpolicy_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,25 @@ type Assessment struct {
// ElasticSearch certificate for the client
// +kubebuilder:validation:Optional
OpenSearchCert string `json:"openSearchCert"`
// Indicate whether to config of governor
// +kubebuilder:validation:Optional
Governor Governor `json:"governor"`
}

// Governor contains policies for governor to send report
type Governor struct {
// Indicate whether to send the reports to governor
// +kubebuilder:default:=false
Enabled bool `json:"enabled"`
// Unique identifier of the cluster
// +kubebuilder:validation:Optional
ClusterID string `json:"clusterId"`
// Api url to send telemetry data
// +kubebuilder:validation:Optional
URL string `json:"url"`
// Secret name where CSP api token is stored in cnsi-system namespace
// +kubebuilder:validation:Optional
CspSecretName string `json:"cspSecretName"`
}

// FollowupAction defines what actions should be applied when security expectations are matched.
Expand Down
2 changes: 2 additions & 0 deletions src/api/v1alpha1/workload.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ var AllWorkloads = []string{
type Workload struct {
// For pure Pod, no object reference.
corev1.ObjectReference `json:"metadata"`
// Replicas of this workload.
Replicas int32 `json:"replicas"`
// Pods of this workload.
Pods []*Pod `json:"pods"`
}
Expand Down
5 changes: 5 additions & 0 deletions src/cmd/inspector/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func main() {
k8sClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{
Scheme: scheme,
})

if err != nil {
log.Error(err, "unable to create k8s client")
os.Exit(1)
Expand All @@ -56,6 +57,10 @@ func main() {
os.Exit(1)
}

if inspectionPolicy.Spec.Inspection.Assessment.Governor.Enabled {
ctx = context.WithValue(ctx, "cspSecretName", inspectionPolicy.Spec.Inspection.Assessment.Governor.CspSecretName)
}

runner := inspection.NewController().
WithScheme(scheme).
WithK8sClient(k8sClient).
Expand Down
24 changes: 24 additions & 0 deletions src/config/crd/bases/goharbor.goharbor.io_assessmentreports.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,25 @@ spec:
description: Generate indicates whether generate the assessment
report or not.
type: boolean
governor:
description: Indicate whether to config of governor
properties:
cspSecretName:
description: Secret name where CSP api token is stored in cnsi-system namespace
type: string
clusterId:
description: Unique identifier of the cluster
type: string
enabled:
default: false
description: Indicate whether to send the reports to governor
type: boolean
url:
description: Api url to send telemetry data
type: string
required:
- enabled
type: object
liveTime:
default: 86400
description: Live time of the generated report. Unit is second.
Expand Down Expand Up @@ -676,9 +695,14 @@ spec:
- metadata
type: object
type: array
replicas:
description: Replicas of this workload.
format: int32
type: integer
required:
- metadata
- pods
- replicas
type: object
required:
- passed
Expand Down
19 changes: 19 additions & 0 deletions src/config/crd/bases/goharbor.goharbor.io_inspectionpolicies.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,25 @@ spec:
description: Generate indicates whether generate the assessment
report or not.
type: boolean
governor:
description: Indicate whether to config of governor
properties:
cspSecretName:
description: Secret name where CSP api token is stored in cnsi-system namespace
type: string
clusterId:
description: Unique identifier of the cluster
type: string
enabled:
default: false
description: Indicate whether to send the reports to governor
type: boolean
url:
description: Api url to send telemetry data
type: string
required:
- enabled
type: object
liveTime:
default: 86400
description: Live time of the generated report. Unit is second.
Expand Down
5 changes: 5 additions & 0 deletions src/config/samples/policy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@ spec:
openSearchAddr: "https://opensearch-cluster-master.default:9200"
openSearchUser: "admin"
openSearchPasswd: "admin"
governor:
enabled: true
clusterId: "65a03970-c53a-4ba1-8d1f-42c9f95d2761"
url: "https://api.int.app-catalog.vmware.com/catalog-governor/v1"
cspSecretName: "csp-secret"
baselines:
- kind: "vulnerability"
baseline: "High"
Expand Down
3 changes: 3 additions & 0 deletions src/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/spf13/viper v1.14.0
github.com/stretchr/testify v1.8.1
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783
k8s.io/api v0.25.4
k8s.io/apimachinery v0.25.4
k8s.io/apiserver v0.25.4
Expand Down Expand Up @@ -52,6 +53,7 @@ require (
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.8.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/felixge/httpsnoop v1.0.2 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
Expand Down Expand Up @@ -119,6 +121,7 @@ require (
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.4.1 // indirect
github.com/ugorji/go/codec v1.2.7 // indirect
github.com/vladimirvivien/gexe v0.1.1 // indirect
Expand Down
1 change: 1 addition & 0 deletions src/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=
Expand Down
116 changes: 116 additions & 0 deletions src/lib/cspauth/csp_auth.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
package cspauth

import (
"context"
"fmt"
"github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/log"
"github.com/vmware-tanzu/cloud-native-security-inspector/src/lib/retry"
v12 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"math"
"time"
)

const (
tokenMaxAgeSeconds = 1700
apiToken = "API_TOKEN"
accessTokenSecretName = "governor-accesstoken"
governorTokenExpiresIn = "governorAccessTokenExpiresIn"
governorAccessTokenKey = "governorAccessToken"
Retry = 3
)

var RetryDelay time.Duration = 5

// Provider is an interface to interact with an authorization service
type Provider interface {
// GetBearerToken retrieves a short-lived access token to use in a single HTTP request
GetBearerToken(kubernetes.Interface, context.Context, string, string) (string, error)
}

type CspAuth struct {
CspClient CSPClient

apiToken string
}

func (a *CspAuth) GetBearerToken(clientSet kubernetes.Interface, ctx context.Context, cspSecretNamespace string, cspSecretName string) (string, error) {
accessSecret, err := getOrCreateSecretForAccessToken(clientSet, ctx, cspSecretNamespace)
if err != nil {
return "", err
}

accessToken := string(accessSecret.Data[governorAccessTokenKey])
expiresIn := string(accessSecret.Data[governorTokenExpiresIn])
accessTokenExpiresIn, _ := time.Parse(time.Layout, expiresIn)

if accessToken == "" || time.Now().After(accessTokenExpiresIn) {
apiToken, err := getCSPTokenFromSecret(clientSet, ctx, cspSecretNamespace, cspSecretName)
if err != nil {
return "", fmt.Errorf("Failed to fetch CSP api-token: %w", err)
}
a.apiToken = apiToken
if err := a.refreshToken(ctx, clientSet, cspSecretNamespace, accessSecret); err != nil {
return "", err
}
}
return string(accessSecret.Data[governorAccessTokenKey]), nil
}

func (a *CspAuth) refreshToken(ctx context.Context, clientSet kubernetes.Interface, cspSecretNamespace string, accessTokenSecret *v12.Secret) error {
return retry.NewRetry(
retry.WithName("auth token refresh"),
retry.WithMaxAttempts(Retry),
retry.WithIncrementDelay(RetryDelay*time.Second, RetryDelay*time.Second),
).Run(ctx, func() (bool, error) {
now := time.Now()
cspAuthResponse, err := a.CspClient.GetCspAuthorization(ctx, a.apiToken)
if err != nil {
log.Error(err, "We got an error back from CSP")
return false, nil
}

expiresIn := time.Duration(math.Min(float64(cspAuthResponse.ExpiresIn), tokenMaxAgeSeconds)) * time.Second
formattedExpiration := now.Add(expiresIn).Format(time.Layout)

log.Infof("Refreshed access token for governor which expires in %s", formattedExpiration)
accessTokenSecret.Data = make(map[string][]byte, 0)
accessTokenSecret.Data[governorAccessTokenKey] = []byte(cspAuthResponse.AccessToken)
accessTokenSecret.Data[governorTokenExpiresIn] = []byte(formattedExpiration)
_, err = clientSet.CoreV1().Secrets(cspSecretNamespace).Update(ctx, accessTokenSecret, v1.UpdateOptions{})
if err != nil {
log.Error(err, "We got an error updating access token secret")
return false, nil
}
log.Infof("Obtained CSP access token, next refresh in %s\n", expiresIn)
return true, nil
})
}

func getCSPTokenFromSecret(clientSet kubernetes.Interface, ctx context.Context, ns string, secretName string) (string, error) {
secret, err := clientSet.CoreV1().Secrets(ns).Get(ctx, secretName, v1.GetOptions{})
if err != nil {
log.Error(err, "Failed to fetch secret")
return "", err
}
cspApiToken := string(secret.Data[apiToken])
return cspApiToken, err
}

func getOrCreateSecretForAccessToken(clientSet kubernetes.Interface, ctx context.Context, ns string) (*v12.Secret, error) {
secret, err := clientSet.CoreV1().Secrets(ns).Get(ctx, accessTokenSecretName, v1.GetOptions{})
if err != nil {
log.Warning(err, "Failed to fetch secret for access token, Now Trying to create new secret for same")
secret = &v12.Secret{}
secret.Name = accessTokenSecretName
secret.Namespace = ns
secret.Data = make(map[string][]byte, 0)
secret, err = clientSet.CoreV1().Secrets(ns).Create(ctx, secret, v1.CreateOptions{})
if err != nil {
log.Error(err, "Failed to create secret for storing access token.")
return nil, err
}
}
return secret, err
}
Loading

0 comments on commit d7f4078

Please sign in to comment.