Skip to content

Commit

Permalink
featlinter): generate RuleSet from a Bundle. (#5)
Browse files Browse the repository at this point in the history
  • Loading branch information
Zenithar authored Feb 24, 2021
1 parent 4337f94 commit 428164c
Show file tree
Hide file tree
Showing 13 changed files with 631 additions and 185 deletions.
90 changes: 90 additions & 0 deletions cmd/harp-linter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,3 +143,93 @@ echo '{"app/qa/security/harp/v1.0.0/server/database/credentials":{}}' \
| harp-linter bundle lint --spec test/fixtures/ruleset/valid/database-secret-validator.yaml
{"level":"fatal","@timestamp":"2021-02-23T10:31:24.287Z","@caller":"[email protected]/command.go:856","@message":"unable to execute task","@appName":"harp-bundle-lint","@version":"","@revision":"8ebf40d","@appID":"7pflS7bCAAsDcAiPJWm36pypWY3nHhqOQwCc9Vp1ABCm8ZUWbmGinGL5zbP1EWvn","@fields":{"error":"unable to validate given bundle: package 'app/qa/security/harp/v1.0.0/server/database/credentials' doesn't validate rule 'HARP-SRV-0002'"}}
```

### Generate a ruleset from a bundle

It will use the input bundle structure to generate a `RuleSet`.

```sh
harp-linter ruleset from-bundle --in customer.bundle
```

```yaml
api_version: harp.elastic.co/linter/v1
kind: RuleSet
meta:
description: Generated from bundle content
name: vjz70BPFJuQhm_7quRGNt1ybocQU6DeXCn8h1o4aPm80CI4pM8lNwVBTDqH8SpW0W1r-8dXSVQK67pO-vtgS_Q
spec:
rules:
- constraints:
- p.has_secret("API_KEY")
name: LINT-vjz70B-1
path: app/production/customer1/ece/v1.0.0/adminconsole/authentication/otp/okta_api_key
- constraints:
- p.has_secret("host")
- p.has_secret("port")
- p.has_secret("options")
- p.has_secret("username")
- p.has_secret("password")
- p.has_secret("dbname")
name: LINT-vjz70B-2
path: app/production/customer1/ece/v1.0.0/adminconsole/database/usage_credentials
- constraints:
- p.has_secret("cookieEncryptionKey")
- p.has_secret("sessionSaltSeed")
- p.has_secret("jwtHmacKey")
name: LINT-vjz70B-3
path: app/production/customer1/ece/v1.0.0/adminconsole/http/session
- constraints:
- p.has_secret("API_KEY")
name: LINT-vjz70B-4
path: app/production/customer1/ece/v1.0.0/adminconsole/mailing/sender/mailgun_api_key
- constraints:
- p.has_secret("emailHashPepperSeedKey")
name: LINT-vjz70B-5
path: app/production/customer1/ece/v1.0.0/adminconsole/privacy/anonymizer
- constraints:
- p.has_secret("host")
- p.has_secret("port")
- p.has_secret("options")
- p.has_secret("username")
- p.has_secret("password")
- p.has_secret("dbname")
name: LINT-vjz70B-6
path: app/production/customer1/ece/v1.0.0/userconsole/database/usage_credentials
- constraints:
- p.has_secret("privateKey")
- p.has_secret("publicKey")
name: LINT-vjz70B-7
path: app/production/customer1/ece/v1.0.0/userconsole/http/certificate
- constraints:
- p.has_secret("cookieEncryptionKey")
- p.has_secret("sessionSaltSeed")
- p.has_secret("jwtHmacKey")
name: LINT-vjz70B-8
path: app/production/customer1/ece/v1.0.0/userconsole/http/session
- constraints:
- p.has_secret("user")
- p.has_secret("password")
name: LINT-vjz70B-9
path: infra/aws/essp-customer1/us-east-1/rds/adminconsole/accounts/root_credentials
- constraints:
- p.has_secret("API_KEY")
- p.has_secret("ca.pem")
name: LINT-vjz70B-10
path: platform/production/customer1/us-east-1/billing/recurly/vendor_api_key
- constraints:
- p.has_secret("username")
- p.has_secret("password")
name: LINT-vjz70B-11
path: platform/production/customer1/us-east-1/postgresql/admiconsole/admin_credentials
- constraints:
- p.has_secret("username")
- p.has_secret("password")
name: LINT-vjz70B-12
path: platform/production/customer1/us-east-1/zookeeper/accounts/admin_credentials
- constraints:
- p.has_secret("privateKey")
- p.has_secret("publicKey")
name: LINT-vjz70B-13
path: product/ece/v1.0.0/artifact/signature/key
```
2 changes: 2 additions & 0 deletions cmd/harp-linter/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2
github.com/elastic/harp v0.1.12
github.com/fatih/color v1.10.0
github.com/go-ozzo/ozzo-validation/v4 v4.3.0
github.com/gobwas/glob v0.2.3
github.com/golang/protobuf v1.4.3
github.com/google/cel-go v0.7.2
Expand All @@ -17,4 +18,5 @@ require (
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
google.golang.org/genproto v0.0.0-20210203152818-3206188e46ba
google.golang.org/protobuf v1.25.0
sigs.k8s.io/yaml v1.2.0
)
1 change: 1 addition & 0 deletions cmd/harp-linter/internal/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ var mainCmd = func() *cobra.Command {
cmd.AddCommand(configcmd.NewConfigCommand(conf, envPrefix))

cmd.AddCommand(bundleCmd())
cmd.AddCommand(rulesetCmd())

// Return command
return cmd
Expand Down
37 changes: 37 additions & 0 deletions cmd/harp-linter/internal/cmd/ruleset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package cmd

import (
"github.com/spf13/cobra"
)

// -----------------------------------------------------------------------------

var rulesetCmd = func() *cobra.Command {
cmd := &cobra.Command{
Use: "ruleset",
Aliases: []string{"rs"},
Short: "RuleSet commands",
}

// Bundle commands
cmd.AddCommand(rulesetFromBundle())

return cmd
}
63 changes: 63 additions & 0 deletions cmd/harp-linter/internal/cmd/ruleset_frombundle.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package cmd

import (
"github.com/spf13/cobra"
"go.uber.org/zap"

"github.com/elastic/harp-plugins/cmd/harp-linter/pkg/tasks/ruleset"
"github.com/elastic/harp/pkg/sdk/cmdutil"
"github.com/elastic/harp/pkg/sdk/log"
)

// -----------------------------------------------------------------------------

var rulesetFromBundle = func() *cobra.Command {
var (
inputPath string
outputPath string
)

cmd := &cobra.Command{
Use: "from-bundle",
Short: "Genereate a RuleSet descriptor from a Bundle",
Run: func(cmd *cobra.Command, args []string) {
// Initialize logger and context
ctx, cancel := cmdutil.Context(cmd.Context(), "harp-ruleset-from-bundle", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
defer cancel()

// Prepare task
t := &ruleset.FromBundleTask{
ContainerReader: cmdutil.FileReader(inputPath),
OutputWriter: cmdutil.FileWriter(outputPath),
}

// Run the task
if err := t.Run(ctx); err != nil {
log.For(ctx).Fatal("unable to execute task", zap.Error(err))
}
},
}

// Parameters
cmd.Flags().StringVar(&inputPath, "in", "-", "Container input ('-' for stdin or filename)")
cmd.Flags().StringVar(&outputPath, "out", "", "Output RuleSet specification path ('' for stdout or filename)")

return cmd
}
145 changes: 6 additions & 139 deletions cmd/harp-linter/pkg/bundle/linter/engine/cel/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,164 +22,31 @@ import (
"errors"
"fmt"

validation "github.com/go-ozzo/ozzo-validation/v4"
"github.com/go-ozzo/ozzo-validation/v4/is"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/checker/decls"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/ext"
"github.com/google/cel-go/interpreter/functions"
exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
celext "github.com/google/cel-go/ext"
"google.golang.org/protobuf/proto"

"github.com/elastic/harp-plugins/cmd/harp-linter/pkg/bundle/linter/engine"
"github.com/elastic/harp-plugins/cmd/harp-linter/pkg/bundle/linter/engine/cel/ext"
bundlev1 "github.com/elastic/harp/api/gen/go/harp/bundle/v1"
)

var (
harpPackageObjectType = decls.NewObjectType("harp.bundle.v1.Package")
harpKVObjectType = decls.NewObjectType("harp.bundle.v1.KV")
)

// -----------------------------------------------------------------------------

// New returns a Google CEL based linter engine.
func New(expressions []string) (engine.PackageLinter, error) {
// Prepare CEL Environment
env, err := cel.NewEnv(
cel.Types(&bundlev1.Bundle{}, &bundlev1.Package{}, &bundlev1.SecretChain{}, &bundlev1.KV{}),
cel.Declarations(
decls.NewVar("p", harpPackageObjectType),
decls.NewFunction("match_path",
decls.NewInstanceOverload("match_path",
[]*exprpb.Type{harpPackageObjectType, decls.String},
decls.Bool,
),
),
decls.NewFunction("has_secret",
decls.NewInstanceOverload("has_secret",
[]*exprpb.Type{harpPackageObjectType, decls.String},
decls.Bool,
),
),
decls.NewFunction("has_all_secrets",
decls.NewInstanceOverload("has_all_secrets",
[]*exprpb.Type{harpPackageObjectType, decls.NewListType(decls.String)},
decls.Bool,
),
),
decls.NewFunction("is_cso_compliant",
decls.NewInstanceOverload("is_cso_compliant",
[]*exprpb.Type{harpPackageObjectType},
decls.Bool,
),
),
decls.NewFunction("secret",
decls.NewInstanceOverload("secret",
[]*exprpb.Type{harpPackageObjectType, decls.String},
harpKVObjectType,
),
),
decls.NewFunction("is_base64",
decls.NewInstanceOverload("is_base64",
[]*exprpb.Type{harpKVObjectType},
decls.Bool,
),
),
decls.NewFunction("is_required",
decls.NewInstanceOverload("is_required",
[]*exprpb.Type{harpKVObjectType},
decls.Bool,
),
),
decls.NewFunction("is_url",
decls.NewInstanceOverload("is_url",
[]*exprpb.Type{harpKVObjectType},
decls.Bool,
),
),
decls.NewFunction("is_uuid",
decls.NewInstanceOverload("is_uuid",
[]*exprpb.Type{harpKVObjectType},
decls.Bool,
),
),
decls.NewFunction("is_email",
decls.NewInstanceOverload("is_email",
[]*exprpb.Type{harpKVObjectType},
decls.Bool,
),
),
decls.NewFunction("is_json",
decls.NewInstanceOverload("is_json",
[]*exprpb.Type{harpKVObjectType},
decls.Bool,
),
),
),
ext.Strings(),
ext.Encoders(),
ext.Packages(),
ext.Secrets(),
celext.Strings(),
)
if err != nil {
return nil, fmt.Errorf("unable to prepare CEL engine environment: %w", err)
}

// Registter types
reg, err := types.NewRegistry(
&bundlev1.KV{},
)
if err != nil {
return nil, fmt.Errorf("unable to register types: %w", err)
}

// Functions
funcs := cel.Functions(
&functions.Overload{
Operator: "match_path",
Binary: celPackageMatchPath,
},
&functions.Overload{
Operator: "has_secret",
Binary: celPackageHasSecret,
},
&functions.Overload{
Operator: "has_all_secrets",
Binary: celPackageHasAllSecrets,
},
&functions.Overload{
Operator: "is_cso_compliant",
Unary: celPackageIsCSOCompliant,
},
&functions.Overload{
Operator: "secret",
Binary: celPackageGetSecret(reg),
},
&functions.Overload{
Operator: "is_base64",
Unary: celValidatorBuilder(is.Base64),
},
&functions.Overload{
Operator: "is_required",
Unary: celValidatorBuilder(validation.Required),
},
&functions.Overload{
Operator: "is_url",
Unary: celValidatorBuilder(is.URL),
},
&functions.Overload{
Operator: "is_uuid",
Unary: celValidatorBuilder(is.UUID),
},
&functions.Overload{
Operator: "is_email",
Unary: celValidatorBuilder(is.EmailFormat),
},
&functions.Overload{
Operator: "is_json",
Unary: celValidatorBuilder(&jsonValidator{}),
},
)

// Assemble the complete ruleset
ruleset := make([]cel.Program, 0, len(expressions))
for _, exp := range expressions {
Expand All @@ -202,7 +69,7 @@ func New(expressions []string) (engine.PackageLinter, error) {
}

// Compile the program
p, err := env.Program(ast, funcs)
p, err := env.Program(ast)
if err != nil {
return nil, fmt.Errorf("error while creating CEL program: %w", err)
}
Expand Down
Loading

0 comments on commit 428164c

Please sign in to comment.