From 9c98e20b724b0ff5d87a19ce250854fd6e6fcdf5 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 13 Feb 2024 16:37:41 +0200 Subject: [PATCH 1/9] Add validate flag to root command --- cmd/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/main.go b/cmd/main.go index e9e0624a..802d324e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -30,6 +30,7 @@ const ( specialRulesFlagName = "add-special-rule" ignoreOnExitFlagName = "ignore-on-exit" maxTargetMegabytesFlagName = "max-target-megabytes" + validate = "validate" ) var ( @@ -40,6 +41,7 @@ var ( ignoreVar []string ignoreOnExitVar = ignoreOnExitNone secretsConfigVar secrets.SecretsConfig + validateVar bool ) var rootCmd = &cobra.Command{ @@ -89,6 +91,7 @@ func Execute() (int, error) { rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.SpecialList, specialRulesFlagName, []string{}, "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.") rootCmd.PersistentFlags().Var(&ignoreOnExitVar, ignoreOnExitFlagName, "defines which kind of non-zero exits code should be ignored\naccepts: all, results, errors, none\nexample: if 'results' is set, only engine errors will make 2ms exit code different from 0") rootCmd.PersistentFlags().IntVar(&secretsConfigVar.MaxTargetMegabytes, maxTargetMegabytesFlagName, 0, "files larger than this will be skipped.\nOmit or set to 0 to disable this check.") + rootCmd.PersistentFlags().BoolVar(&validateVar, validate, false, "Validate the secrets found") rootCmd.AddCommand(secrets.GetRulesCommand(&secretsConfigVar)) From 753c515f6643a3b3b5f56a900384b5499eeeda6f Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 13 Feb 2024 16:49:48 +0200 Subject: [PATCH 2/9] Refactor reporting package to use pointers for Secret struct --- cmd/main.go | 2 +- reporting/report.go | 8 ++++---- reporting/report_test.go | 6 +++--- reporting/sarif.go | 2 +- secrets/secrets.go | 6 +++--- secrets/secrets_test.go | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 802d324e..9edacb44 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -72,7 +72,7 @@ var channels = plugins.Channels{ } var report = reporting.Init() -var secretsChan = make(chan reporting.Secret) +var secretsChan = make(chan *reporting.Secret) func Execute() (int, error) { vConfig.SetEnvPrefix(envPrefix) diff --git a/reporting/report.go b/reporting/report.go index e2a75ff7..1be8437e 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -17,9 +17,9 @@ const ( ) type Report struct { - TotalItemsScanned int `json:"totalItemsScanned"` - TotalSecretsFound int `json:"totalSecretsFound"` - Results map[string][]Secret `json:"results"` + TotalItemsScanned int `json:"totalItemsScanned"` + TotalSecretsFound int `json:"totalSecretsFound"` + Results map[string][]*Secret `json:"results"` } type Secret struct { @@ -35,7 +35,7 @@ type Secret struct { func Init() *Report { return &Report{ - Results: make(map[string][]Secret), + Results: make(map[string][]*Secret), } } diff --git a/reporting/report_test.go b/reporting/report_test.go index 44a08423..27576b4d 100644 --- a/reporting/report_test.go +++ b/reporting/report_test.go @@ -23,9 +23,9 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT 4Xgur9w/aLZrLM3DSatR+kL+cVTyDTtgCt9Dc8k48Q== -----END RSA PRIVATE KEY-----`) - results := map[string][]Secret{} + results := map[string][]*Secret{} report := Report{len(results), 1, results} - secret := Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} + secret := &Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} source := "directory\\rawStringAsFile.txt" report.Results[source] = append(report.Results[source], secret) @@ -36,6 +36,6 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT } if !reflect.DeepEqual(report.Results, results) { - t.Errorf("got %q want %q", key, results) + t.Errorf("got %+v want %+v", key, results) } } diff --git a/reporting/sarif.go b/reporting/sarif.go index a26c3d60..db2f13f3 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -74,7 +74,7 @@ func getResults(report Report) []Results { return results } -func getLocation(secret Secret) []Locations { +func getLocation(secret *Secret) []Locations { return []Locations{ { PhysicalLocation: PhysicalLocation{ diff --git a/secrets/secrets.go b/secrets/secrets.go index 5809eb3c..2b6e71e2 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -58,7 +58,7 @@ func Init(secretsConfig SecretsConfig) (*Secrets, error) { }, nil } -func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret, wg *sync.WaitGroup, ignoredIds []string) { +func (s *Secrets) Detect(item plugins.Item, secretsChannel chan *reporting.Secret, wg *sync.WaitGroup, ignoredIds []string) { defer wg.Done() fragment := detect.Fragment{ @@ -66,7 +66,7 @@ func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret } for _, value := range s.detector.Detect(fragment) { itemId := getFindingId(item, value) - secret := reporting.Secret{ + secret := &reporting.Secret{ ID: itemId, Source: item.Source, RuleID: value.RuleID, @@ -76,7 +76,7 @@ func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret EndColumn: value.EndColumn, Value: value.Secret, } - if !isSecretIgnored(&secret, &ignoredIds) { + if !isSecretIgnored(secret, &ignoredIds) { secretsChannel <- secret } else { log.Debug().Msgf("Secret %s was ignored", secret.ID) diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index 44bdccb6..31ba5fdc 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -123,7 +123,7 @@ func TestSecrets(t *testing.T) { } t.Run(name, func(t *testing.T) { fmt.Printf("Start test %s", name) - secretsChan := make(chan reporting.Secret, 1) + secretsChan := make(chan *reporting.Secret, 1) wg := &sync.WaitGroup{} wg.Add(1) detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil) From 3bc1589ff71829a4c677b2c3422c80157d22f752 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 14 Feb 2024 11:28:59 +0200 Subject: [PATCH 3/9] Refactor code to move item and secret processing to separate functions --- cmd/main.go | 22 ++-------------------- cmd/workers.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 cmd/workers.go diff --git a/cmd/main.go b/cmd/main.go index 9edacb44..242f2cb9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -133,28 +133,10 @@ func preRun(cmd *cobra.Command, args []string) error { } channels.WaitGroup.Add(1) - go func() { - defer channels.WaitGroup.Done() - - wgItems := &sync.WaitGroup{} - for item := range channels.Items { - report.TotalItemsScanned++ - wgItems.Add(1) - go secrets.Detect(item, secretsChan, wgItems, ignoreVar) - } - wgItems.Wait() - close(secretsChan) - }() + go processItems(secrets) channels.WaitGroup.Add(1) - go func() { - defer channels.WaitGroup.Done() - for secret := range secretsChan { - report.TotalSecretsFound++ - report.Results[secret.ID] = append(report.Results[secret.ID], secret) - - } - }() + go processSecrets() return nil } diff --git a/cmd/workers.go b/cmd/workers.go new file mode 100644 index 00000000..f0947e01 --- /dev/null +++ b/cmd/workers.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "sync" + + "github.com/checkmarx/2ms/secrets" +) + +func processItems(detector *secrets.Secrets) { + defer channels.WaitGroup.Done() + + wgItems := &sync.WaitGroup{} + for item := range channels.Items { + report.TotalItemsScanned++ + wgItems.Add(1) + go detector.Detect(item, secretsChan, wgItems, ignoreVar) + } + wgItems.Wait() + close(secretsChan) +} + +func processSecrets() { + defer channels.WaitGroup.Done() + + for secret := range secretsChan { + report.TotalSecretsFound++ + if validateVar { + validationChan <- secret + } + report.Results[secret.ID] = append(report.Results[secret.ID], secret) + } + close(validationChan) +} + From 8f0343e70a630e27bac82aa57cb2a37867d0a8fb Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 14 Feb 2024 14:57:06 +0200 Subject: [PATCH 4/9] Move Secret into secrets package --- reporting/report.go | 20 +++++--------------- reporting/report_test.go | 6 ++++-- reporting/sarif.go | 3 ++- secrets/engine.go | 7 +++---- secrets/engine_test.go | 3 +-- secrets/secret.go | 12 ++++++++++++ 6 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 secrets/secret.go diff --git a/reporting/report.go b/reporting/report.go index 1be8437e..d57d42e3 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/checkmarx/2ms/config" + "github.com/checkmarx/2ms/secrets" "github.com/rs/zerolog/log" ) @@ -17,25 +18,14 @@ const ( ) type Report struct { - TotalItemsScanned int `json:"totalItemsScanned"` - TotalSecretsFound int `json:"totalSecretsFound"` - Results map[string][]*Secret `json:"results"` -} - -type Secret struct { - ID string `json:"id"` - Source string `json:"source"` - RuleID string `json:"ruleId"` - StartLine int `json:"startLine"` - EndLine int `json:"endLine"` - StartColumn int `json:"startColumn"` - EndColumn int `json:"endColumn"` - Value string `json:"value"` + TotalItemsScanned int `json:"totalItemsScanned"` + TotalSecretsFound int `json:"totalSecretsFound"` + Results map[string][]*secrets.Secret `json:"results"` } func Init() *Report { return &Report{ - Results: make(map[string][]*Secret), + Results: make(map[string][]*secrets.Secret), } } diff --git a/reporting/report_test.go b/reporting/report_test.go index 27576b4d..b028ed15 100644 --- a/reporting/report_test.go +++ b/reporting/report_test.go @@ -3,6 +3,8 @@ package reporting import ( "reflect" "testing" + + "github.com/checkmarx/2ms/secrets" ) func TestAddSecretToFile(t *testing.T) { @@ -23,9 +25,9 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT 4Xgur9w/aLZrLM3DSatR+kL+cVTyDTtgCt9Dc8k48Q== -----END RSA PRIVATE KEY-----`) - results := map[string][]*Secret{} + results := map[string][]*secrets.Secret{} report := Report{len(results), 1, results} - secret := &Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} + secret := &secrets.Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} source := "directory\\rawStringAsFile.txt" report.Results[source] = append(report.Results[source], secret) diff --git a/reporting/sarif.go b/reporting/sarif.go index db2f13f3..dc90f8a1 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/checkmarx/2ms/config" + "github.com/checkmarx/2ms/secrets" ) func writeSarif(report Report, cfg *config.Config) (string, error) { @@ -74,7 +75,7 @@ func getResults(report Report) []Results { return results } -func getLocation(secret *Secret) []Locations { +func getLocation(secret *secrets.Secret) []Locations { return []Locations{ { PhysicalLocation: PhysicalLocation{ diff --git a/secrets/engine.go b/secrets/engine.go index 4f2f05a2..14e3568e 100644 --- a/secrets/engine.go +++ b/secrets/engine.go @@ -10,7 +10,6 @@ import ( "text/tabwriter" "github.com/checkmarx/2ms/plugins" - "github.com/checkmarx/2ms/reporting" "github.com/checkmarx/2ms/secrets/rules" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -58,7 +57,7 @@ func Init(secretsConfig SecretsConfig) (*Engine, error) { }, nil } -func (s *Engine) Detect(item plugins.Item, secretsChannel chan *reporting.Secret, wg *sync.WaitGroup, ignoredIds []string) { +func (s *Engine) Detect(item plugins.Item, secretsChannel chan *Secret, wg *sync.WaitGroup, ignoredIds []string) { defer wg.Done() fragment := detect.Fragment{ @@ -66,7 +65,7 @@ func (s *Engine) Detect(item plugins.Item, secretsChannel chan *reporting.Secret } for _, value := range s.detector.Detect(fragment) { itemId := getFindingId(item, value) - secret := &reporting.Secret{ + secret := &Secret{ ID: itemId, Source: item.Source, RuleID: value.RuleID, @@ -107,7 +106,7 @@ func getFindingId(item plugins.Item, finding report.Finding) string { return fmt.Sprintf("%x", sha) } -func isSecretIgnored(secret *reporting.Secret, ignoredIds *[]string) bool { +func isSecretIgnored(secret *Secret, ignoredIds *[]string) bool { for _, ignoredId := range *ignoredIds { if secret.ID == ignoredId { return true diff --git a/secrets/engine_test.go b/secrets/engine_test.go index 3df10a8c..a77964c1 100644 --- a/secrets/engine_test.go +++ b/secrets/engine_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/checkmarx/2ms/plugins" - "github.com/checkmarx/2ms/reporting" "github.com/checkmarx/2ms/secrets/rules" ) @@ -123,7 +122,7 @@ func TestSecrets(t *testing.T) { } t.Run(name, func(t *testing.T) { fmt.Printf("Start test %s", name) - secretsChan := make(chan *reporting.Secret, 1) + secretsChan := make(chan *Secret, 1) wg := &sync.WaitGroup{} wg.Add(1) detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil) diff --git a/secrets/secret.go b/secrets/secret.go new file mode 100644 index 00000000..ea03a15f --- /dev/null +++ b/secrets/secret.go @@ -0,0 +1,12 @@ +package secrets + +type Secret struct { + ID string `json:"id"` + Source string `json:"source"` + RuleID string `json:"ruleId"` + StartLine int `json:"startLine"` + EndLine int `json:"endLine"` + StartColumn int `json:"startColumn"` + EndColumn int `json:"endColumn"` + Value string `json:"value"` +} From c62ba5e151a1644adefea7343316196ff02623d3 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 15 Feb 2024 09:07:53 +0200 Subject: [PATCH 5/9] use go-yaml/yaml v3 For #203 --- go.mod | 3 +-- go.sum | 2 -- reporting/yaml.go | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 5ce41fac..4274bb5c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/zricethezav/gitleaks/v8 v8.18.0 golang.org/x/time v0.1.0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -48,5 +48,4 @@ require ( golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 92e019ae..e885b7b6 100644 --- a/go.sum +++ b/go.sum @@ -533,8 +533,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/reporting/yaml.go b/reporting/yaml.go index e9335002..ad8e3d48 100644 --- a/reporting/yaml.go +++ b/reporting/yaml.go @@ -1,7 +1,7 @@ package reporting import ( - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func writeYaml(report Report) (string, error) { From c6554be9ed6dd2659e08e2e109f20cbabe4e087c Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 15 Feb 2024 10:59:01 +0200 Subject: [PATCH 6/9] infrastructure for validity check --- README.md | 15 +++++++++++++++ cmd/main.go | 8 +++++++- cmd/workers.go | 15 +++++++++++++++ secrets/secret.go | 46 ++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 62ea26fc..9275237c 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ Flags: --report-path strings path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif) --rule strings select rules by name or tag to apply to this scan --stdout-format string stdout output format, available formats are: json, yaml, sarif (default "yaml") + --validate Validate the secrets found -v, --version version for 2ms Use "2ms [command] --help" for more information about a command. @@ -158,6 +159,20 @@ Use "2ms [command] --help" for more information about a command. +## Validity Check + +From the help message: `--validate Validate the secrets found`. + +The `--validate` flag will check the validity of the secrets found. For example, if it is a Github token, it will check if the token is valid by making a request to the Github API. We will use the less intrusive method to check the validity of the secret. + +The result of the validation can be: + +- `valid` - The secret is valid +- `revoked` - The secret is revoked +- `unknown` - We are not checking the validity of the secret + +If the `--validate` flag is not provided, the validation field will be omitted from the output, or its value will be an empty string. + ## Special Rules Special rules are rules that are not part of the default ruleset, usually because they are too noisy or too specific. You can use the `--add-special-rule` flag to add special rules by rule ID. diff --git a/cmd/main.go b/cmd/main.go index 7fff5127..fc3fee1f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -72,7 +72,8 @@ var channels = plugins.Channels{ } var report = reporting.Init() -var secretsChan = make(chan *reporting.Secret) +var secretsChan = make(chan *secrets.Secret) +var validationChan = make(chan *secrets.Secret) func Execute() (int, error) { vConfig.SetEnvPrefix(envPrefix) @@ -138,6 +139,11 @@ func preRun(cmd *cobra.Command, args []string) error { channels.WaitGroup.Add(1) go processSecrets() + if validateVar { + channels.WaitGroup.Add(1) + go processValidation() + } + return nil } diff --git a/cmd/workers.go b/cmd/workers.go index 2246949b..2ebc5330 100644 --- a/cmd/workers.go +++ b/cmd/workers.go @@ -24,6 +24,21 @@ func processSecrets() { for secret := range secretsChan { report.TotalSecretsFound++ + if validateVar { + validationChan <- secret + } report.Results[secret.ID] = append(report.Results[secret.ID], secret) } + close(validationChan) +} + +func processValidation() { + defer channels.WaitGroup.Done() + + wgValidation := &sync.WaitGroup{} + for secret := range validationChan { + wgValidation.Add(1) + go secret.Validate(wgValidation) + } + wgValidation.Wait() } diff --git a/secrets/secret.go b/secrets/secret.go index ea03a15f..54959d0b 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -1,12 +1,42 @@ package secrets +import "sync" + +type ValidationResult string + +const ( + Valid ValidationResult = "Valid" + Revoked ValidationResult = "Revoked" + Unknown ValidationResult = "Unknown" +) + type Secret struct { - ID string `json:"id"` - Source string `json:"source"` - RuleID string `json:"ruleId"` - StartLine int `json:"startLine"` - EndLine int `json:"endLine"` - StartColumn int `json:"startColumn"` - EndColumn int `json:"endColumn"` - Value string `json:"value"` + ID string `json:"id"` + Source string `json:"source"` + RuleID string `json:"ruleId"` + StartLine int `json:"startLine"` + EndLine int `json:"endLine"` + StartColumn int `json:"startColumn"` + EndColumn int `json:"endColumn"` + Value string `json:"value"` + Validation ValidationResult `json:"validation,omitempty"` +} + +type validationFunc = func(*Secret) ValidationResult + +var ruleIDToFunction = map[string]validationFunc{ + "GitHub": validateGithub, +} + +func (s *Secret) Validate(wg *sync.WaitGroup) { + defer wg.Done() + if f, ok := ruleIDToFunction[s.RuleID]; ok { + s.Validation = f(s) + } else { + s.Validation = Unknown + } +} + +func validateGithub(s *Secret) ValidationResult { + return Unknown } From 5dbb9e9551cec2f0c0507a87f95d71e4ffc1cb41 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 15 Feb 2024 11:22:10 +0200 Subject: [PATCH 7/9] add logic to include fine-grained PAT validation for GitHub --- README.md | 2 +- secrets/secret.go | 32 +++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9275237c..96aa5e9e 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ The result of the validation can be: - `valid` - The secret is valid - `revoked` - The secret is revoked -- `unknown` - We are not checking the validity of the secret +- `unknown` - We failed to check, or we are not checking the validity of the secret at all If the `--validate` flag is not provided, the validation field will be omitted from the output, or its value will be an empty string. diff --git a/secrets/secret.go b/secrets/secret.go index 54959d0b..9aa01927 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -1,6 +1,12 @@ package secrets -import "sync" +import ( + "fmt" + "net/http" + "sync" + + "github.com/rs/zerolog/log" +) type ValidationResult string @@ -25,7 +31,8 @@ type Secret struct { type validationFunc = func(*Secret) ValidationResult var ruleIDToFunction = map[string]validationFunc{ - "GitHub": validateGithub, + "github-fine-grained-pat": validateGithub, + "github-pat": validateGithub, } func (s *Secret) Validate(wg *sync.WaitGroup) { @@ -38,5 +45,24 @@ func (s *Secret) Validate(wg *sync.WaitGroup) { } func validateGithub(s *Secret) ValidationResult { - return Unknown + const githubURL = "https://api.github.com/" + + req, err := http.NewRequest("GET", githubURL, nil) + if err != nil { + log.Warn().Err(err).Msg("Failed to validate secret") + return Unknown + } + req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value)) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Warn().Err(err).Msg("Failed to validate secret") + return Unknown + } + + if resp.StatusCode == http.StatusOK { + return Valid + } + return Revoked } From fb82f36469657787c012ef5681ba915d997eb1d6 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 15 Feb 2024 16:15:53 +0200 Subject: [PATCH 8/9] Update --validate flag description --- README.md | 4 ++-- cmd/main.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 96aa5e9e..6bceb546 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ Flags: --report-path strings path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif) --rule strings select rules by name or tag to apply to this scan --stdout-format string stdout output format, available formats are: json, yaml, sarif (default "yaml") - --validate Validate the secrets found + --validate trigger additional validation to check if discovered secrets are active or revoked -v, --version version for 2ms Use "2ms [command] --help" for more information about a command. @@ -161,7 +161,7 @@ Use "2ms [command] --help" for more information about a command. ## Validity Check -From the help message: `--validate Validate the secrets found`. +From the help message: `--validate trigger additional validation to check if discovered secrets are active or revoked`. The `--validate` flag will check the validity of the secrets found. For example, if it is a Github token, it will check if the token is valid by making a request to the Github API. We will use the less intrusive method to check the validity of the secret. diff --git a/cmd/main.go b/cmd/main.go index fc3fee1f..567fe85c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -92,7 +92,7 @@ func Execute() (int, error) { rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.SpecialList, specialRulesFlagName, []string{}, "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.") rootCmd.PersistentFlags().Var(&ignoreOnExitVar, ignoreOnExitFlagName, "defines which kind of non-zero exits code should be ignored\naccepts: all, results, errors, none\nexample: if 'results' is set, only engine errors will make 2ms exit code different from 0") rootCmd.PersistentFlags().IntVar(&secretsConfigVar.MaxTargetMegabytes, maxTargetMegabytesFlagName, 0, "files larger than this will be skipped.\nOmit or set to 0 to disable this check.") - rootCmd.PersistentFlags().BoolVar(&validateVar, validate, false, "Validate the secrets found") + rootCmd.PersistentFlags().BoolVar(&validateVar, validate, false, "trigger additional validation to check if discovered secrets are active or revoked") rootCmd.AddCommand(secrets.GetRulesCommand(&secretsConfigVar)) From c2985a6b439be68a9b18e1b0e73b7e7787792d24 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Mon, 19 Feb 2024 09:26:31 +0200 Subject: [PATCH 9/9] Update Secret struct field name for validation status --- secrets/secret.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/secrets/secret.go b/secrets/secret.go index 9aa01927..b012b992 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -17,15 +17,15 @@ const ( ) type Secret struct { - ID string `json:"id"` - Source string `json:"source"` - RuleID string `json:"ruleId"` - StartLine int `json:"startLine"` - EndLine int `json:"endLine"` - StartColumn int `json:"startColumn"` - EndColumn int `json:"endColumn"` - Value string `json:"value"` - Validation ValidationResult `json:"validation,omitempty"` + ID string `json:"id"` + Source string `json:"source"` + RuleID string `json:"ruleId"` + StartLine int `json:"startLine"` + EndLine int `json:"endLine"` + StartColumn int `json:"startColumn"` + EndColumn int `json:"endColumn"` + Value string `json:"value"` + ValidationStatus ValidationResult `json:"validationStatus,omitempty"` } type validationFunc = func(*Secret) ValidationResult @@ -38,9 +38,9 @@ var ruleIDToFunction = map[string]validationFunc{ func (s *Secret) Validate(wg *sync.WaitGroup) { defer wg.Done() if f, ok := ruleIDToFunction[s.RuleID]; ok { - s.Validation = f(s) + s.ValidationStatus = f(s) } else { - s.Validation = Unknown + s.ValidationStatus = Unknown } }