From 3311fde01988b34c623244041ec10e4375e8d66c Mon Sep 17 00:00:00 2001 From: Diogo-fj-rocha Date: Wed, 16 Oct 2024 17:26:47 +0100 Subject: [PATCH 1/8] Added 2 gitleaks rules locally. Added several private methods from gitleaks to allow new rule validation to work. --- .ci/check_new_rules.go | 2 +- engine/rules/plaid.go | 27 +++++++++++++++ engine/rules/rule.go | 12 +++++-- engine/rules/rules.go | 4 +-- engine/rules/utils.go | 75 ++++++++++++++++++++++++++++++++++++++++++ engine/rules/vault.go | 25 ++++++++++++++ 6 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 engine/rules/plaid.go create mode 100644 engine/rules/utils.go create mode 100644 engine/rules/vault.go diff --git a/.ci/check_new_rules.go b/.ci/check_new_rules.go index 9561c5ce..5abae2f3 100644 --- a/.ci/check_new_rules.go +++ b/.ci/check_new_rules.go @@ -12,7 +12,7 @@ import ( var ( regexGitleaksRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*rules\.([a-zA-Z0-9_]+)\(`) - regex2msRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*(?:// )?{Rule:\s*\*rules\.([a-zA-Z0-9_]+)\(\),`) + regex2msRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*(?:// )?{Rule:\s*\*(?:rules\.)?([a-zA-Z0-9_]+)\(\),`) ) func main() { diff --git a/engine/rules/plaid.go b/engine/rules/plaid.go new file mode 100644 index 00000000..4c6ba51e --- /dev/null +++ b/engine/rules/plaid.go @@ -0,0 +1,27 @@ +package rules + +import ( + "github.com/zricethezav/gitleaks/v8/cmd/generate/secrets" + "github.com/zricethezav/gitleaks/v8/config" +) + +// Using this local version because gitleaks has entropy as 3.5, which causes issues on this rule's validation +func PlaidAccessID() *config.Rule { + // define rule + r := config.Rule{ + RuleID: "plaid-client-id", + Description: "Uncovered a Plaid Client ID, which could lead to unauthorized financial service integrations and data breaches.", + Regex: generateSemiGenericRegex([]string{"plaid"}, alphaNumeric("24"), true), + + Entropy: 3.0, + Keywords: []string{ + "plaid", + }, + } + + // validate + tps := []string{ + generateSampleSecret("plaid", secrets.NewSecret(alphaNumeric("24"))), + } + return validate(r, tps, nil) +} diff --git a/engine/rules/rule.go b/engine/rules/rule.go index 5de08dd7..9b92dbbb 100644 --- a/engine/rules/rule.go +++ b/engine/rules/rule.go @@ -30,12 +30,20 @@ func validate(r config.Rule, truePositives []string, falsePositives []string) *c }) for _, tp := range truePositives { if len(d.DetectString(tp)) != 1 { - log.Fatal().Msgf("Failed to validate. For rule ID [%s], true positive [%s] was not detected by regexp [%s]", r.RuleID, tp, r.Regex) // lint:ignore This Fatal happens in a test + log.Fatal(). + Str("rule", r.RuleID). + Str("value", tp). + Str("regex", r.Regex.String()). + Msg("Failed to Validate. True positive was not detected by regex.") } } for _, fp := range falsePositives { if len(d.DetectString(fp)) != 0 { - log.Fatal().Msgf("Failed to validate. For rule ID [%s], false positive [%s] was detected by regexp [%s]", r.RuleID, fp, r.Regex) // lint:ignore This Fatal happens in a test + log.Fatal(). + Str("rule", r.RuleID). + Str("value", fp). + Str("regex", r.Regex.String()). + Msg("Failed to Validate. False positive was detected by regex.") } } return &r diff --git a/engine/rules/rules.go b/engine/rules/rules.go index ea4e4699..70b0f167 100644 --- a/engine/rules/rules.go +++ b/engine/rules/rules.go @@ -136,7 +136,7 @@ func getDefaultRules() *[]Rule { {Rule: *rules.NytimesAccessToken(), Tags: []string{TagAccessToken}}, {Rule: *rules.OktaAccessToken(), Tags: []string{TagAccessToken}}, {Rule: *rules.OpenAI(), Tags: []string{TagApiKey}}, - {Rule: *rules.PlaidAccessID(), Tags: []string{TagClientId}}, + {Rule: *PlaidAccessID(), Tags: []string{TagClientId}}, // {Rule: *rules.PlaidSecretKey(), Tags: []string{TagSecretKey}}, https://github.com/Checkmarx/2ms/issues/226 // {Rule: *rules.PlaidAccessToken(), Tags: []string{TagApiToken}}, https://github.com/Checkmarx/2ms/issues/226 {Rule: *rules.PlanetScalePassword(), Tags: []string{TagPassword}}, @@ -190,7 +190,7 @@ func getDefaultRules() *[]Rule { {Rule: *rules.TwitterBearerToken(), Tags: []string{TagApiToken}}, {Rule: *rules.Typeform(), Tags: []string{TagApiToken}}, {Rule: *rules.VaultBatchToken(), Tags: []string{TagApiToken}}, - {Rule: *rules.VaultServiceToken(), Tags: []string{TagApiToken}}, + {Rule: *VaultServiceToken(), Tags: []string{TagApiToken}}, {Rule: *rules.YandexAPIKey(), Tags: []string{TagApiKey}}, {Rule: *rules.YandexAWSAccessToken(), Tags: []string{TagAccessToken}}, {Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}}, diff --git a/engine/rules/utils.go b/engine/rules/utils.go new file mode 100644 index 00000000..5406ec7f --- /dev/null +++ b/engine/rules/utils.go @@ -0,0 +1,75 @@ +package rules + +import ( + "fmt" + "regexp" + "strings" +) + +const ( + // case insensitive prefix + caseInsensitive = `(?i)` + + // identifier prefix (just an ignore group) + identifierCaseInsensitivePrefix = `(?i:` + identifierCaseInsensitiveSuffix = `)` + identifierPrefix = `(?:` + identifierSuffix = `)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}` + + // commonly used assignment operators or function call + operator = `(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)` + + // boundaries for the secret + // \x60 = ` + secretPrefixUnique = `\b(` + secretPrefix = `(?:'|\"|\s|=|\x60){0,5}(` + secretSuffix = `)(?:['|\"|\n|\r|\s|\x60|;]|$)` +) + +func generateSemiGenericRegex(identifiers []string, secretRegex string, isCaseInsensitive bool) *regexp.Regexp { + var sb strings.Builder + // The identifiers should always be case-insensitive. + // This is inelegant but prevents an extraneous `(?i:)` from being added to the pattern; it could be removed. + if isCaseInsensitive { + sb.WriteString(caseInsensitive) + writeIdentifiers(&sb, identifiers) + } else { + sb.WriteString(identifierCaseInsensitivePrefix) + writeIdentifiers(&sb, identifiers) + sb.WriteString(identifierCaseInsensitiveSuffix) + } + sb.WriteString(operator) + sb.WriteString(secretPrefix) + sb.WriteString(secretRegex) + sb.WriteString(secretSuffix) + return regexp.MustCompile(sb.String()) +} + +func writeIdentifiers(sb *strings.Builder, identifiers []string) { + sb.WriteString(identifierPrefix) + sb.WriteString(strings.Join(identifiers, "|")) + sb.WriteString(identifierSuffix) +} + +func generateUniqueTokenRegex(secretRegex string, isCaseInsensitive bool) *regexp.Regexp { + var sb strings.Builder + if isCaseInsensitive { + sb.WriteString(caseInsensitive) + } + sb.WriteString(secretPrefixUnique) + sb.WriteString(secretRegex) + sb.WriteString(secretSuffix) + return regexp.MustCompile(sb.String()) +} + +func generateSampleSecret(identifier string, secret string) string { + return fmt.Sprintf("%s_api_token = \"%s\"", identifier, secret) +} + +func alphaNumeric(size string) string { + return fmt.Sprintf(`[a-z0-9]{%s}`, size) +} + +func alphaNumericExtendedShort(size string) string { + return fmt.Sprintf(`[a-z0-9_-]{%s}`, size) +} diff --git a/engine/rules/vault.go b/engine/rules/vault.go new file mode 100644 index 00000000..d6b73211 --- /dev/null +++ b/engine/rules/vault.go @@ -0,0 +1,25 @@ +package rules + +import ( + "github.com/zricethezav/gitleaks/v8/cmd/generate/secrets" + "github.com/zricethezav/gitleaks/v8/config" +) + +// Using this local version because newer versions of gitleaks have an entropy value, which was set as too high +// It's here as prevention in case a newer version of gitleaks starts getting used and causes issues on this rule +// If gitleaks is updated on 2ms and the new version of this rule has entropy, set it to 3.0 +func VaultServiceToken() *config.Rule { + // define rule + r := config.Rule{ + Description: "Identified a Vault Service Token, potentially compromising infrastructure security and access to sensitive credentials.", + RuleID: "vault-service-token", + Regex: generateUniqueTokenRegex(`hvs\.[a-z0-9_-]{90,100}`, true), + Keywords: []string{"hvs"}, + } + + // validate + tps := []string{ + generateSampleSecret("vault", "hvs."+secrets.NewSecret(alphaNumericExtendedShort("90"))), + } + return validate(r, tps, nil) +} From f21da887bcefaa315bbf9edc11a4c6edf8ab18fe Mon Sep 17 00:00:00 2001 From: Diogo-fj-rocha Date: Wed, 16 Oct 2024 18:08:38 +0100 Subject: [PATCH 2/8] Added v3 to go mod module --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b3e54405..95235c82 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/checkmarx/2ms +module github.com/checkmarx/2ms/v3 go 1.23.1 From db324faf206bf2bc01a84e9c8e634cfcf1cd1203 Mon Sep 17 00:00:00 2001 From: Diogo-fj-rocha Date: Wed, 16 Oct 2024 18:13:58 +0100 Subject: [PATCH 3/8] Reverting previous commit --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 95235c82..b3e54405 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/checkmarx/2ms/v3 +module github.com/checkmarx/2ms go 1.23.1 From 81da7c0394bb8c13f9928eaf92f057fb43ca50ad Mon Sep 17 00:00:00 2001 From: Diogo-fj-rocha Date: Thu, 17 Oct 2024 13:30:08 +0100 Subject: [PATCH 4/8] Added user to go mod file to fix bug with permissions when writing on file. Added previously failing autogen secret to plaid secret. Remove before opening PR --- engine/rules/plaid.go | 1 + go.mod | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/rules/plaid.go b/engine/rules/plaid.go index 4c6ba51e..8012eaf6 100644 --- a/engine/rules/plaid.go +++ b/engine/rules/plaid.go @@ -22,6 +22,7 @@ func PlaidAccessID() *config.Rule { // validate tps := []string{ generateSampleSecret("plaid", secrets.NewSecret(alphaNumeric("24"))), + "plaid_api_token = \"advm5ddxkh5h4xltv1h5dm5m\"", } return validate(r, tps, nil) } diff --git a/go.mod b/go.mod index b3e54405..95235c82 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/checkmarx/2ms +module github.com/checkmarx/2ms/v3 go 1.23.1 From c3326b38d84204bac9a7c60c7eb6adc1dba2cbb2 Mon Sep 17 00:00:00 2001 From: Diogo-fj-rocha Date: Thu, 17 Oct 2024 13:37:12 +0100 Subject: [PATCH 5/8] Reverted adding v3 in go mod module --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 95235c82..b3e54405 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/checkmarx/2ms/v3 +module github.com/checkmarx/2ms go 1.23.1 From 970ac65bddd1cab6084f0b973c9e741b6ce0aa81 Mon Sep 17 00:00:00 2001 From: Diogo-fj-rocha Date: Thu, 17 Oct 2024 15:27:42 +0100 Subject: [PATCH 6/8] Removed static sample secret used for tests --- engine/rules/plaid.go | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/rules/plaid.go b/engine/rules/plaid.go index 8012eaf6..4c6ba51e 100644 --- a/engine/rules/plaid.go +++ b/engine/rules/plaid.go @@ -22,7 +22,6 @@ func PlaidAccessID() *config.Rule { // validate tps := []string{ generateSampleSecret("plaid", secrets.NewSecret(alphaNumeric("24"))), - "plaid_api_token = \"advm5ddxkh5h4xltv1h5dm5m\"", } return validate(r, tps, nil) } From 7b9802e92bdbe7339aef5e7ce2b474820de556c6 Mon Sep 17 00:00:00 2001 From: Diogo-fj-rocha Date: Thu, 17 Oct 2024 15:48:07 +0100 Subject: [PATCH 7/8] Adding lint:ignore to fatal in validation --- engine/rules/rule.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/rules/rule.go b/engine/rules/rule.go index 9b92dbbb..2562d1e1 100644 --- a/engine/rules/rule.go +++ b/engine/rules/rule.go @@ -34,7 +34,7 @@ func validate(r config.Rule, truePositives []string, falsePositives []string) *c Str("rule", r.RuleID). Str("value", tp). Str("regex", r.Regex.String()). - Msg("Failed to Validate. True positive was not detected by regex.") + Msg("Failed to Validate. True positive was not detected by regex.") // lint:ignore This Fatal happens in a test } } for _, fp := range falsePositives { @@ -43,7 +43,7 @@ func validate(r config.Rule, truePositives []string, falsePositives []string) *c Str("rule", r.RuleID). Str("value", fp). Str("regex", r.Regex.String()). - Msg("Failed to Validate. False positive was detected by regex.") + Msg("Failed to Validate. False positive was detected by regex.") // lint:ignore This Fatal happens in a test } } return &r From cb3145470236f9ee19ba95a90fb8848a234480b9 Mon Sep 17 00:00:00 2001 From: Diogo-fj-rocha Date: Thu, 17 Oct 2024 16:19:53 +0100 Subject: [PATCH 8/8] Adding lint:ignore to fatal in validation --- engine/rules/rule.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/engine/rules/rule.go b/engine/rules/rule.go index 2562d1e1..0d704190 100644 --- a/engine/rules/rule.go +++ b/engine/rules/rule.go @@ -30,20 +30,20 @@ func validate(r config.Rule, truePositives []string, falsePositives []string) *c }) for _, tp := range truePositives { if len(d.DetectString(tp)) != 1 { - log.Fatal(). - Str("rule", r.RuleID). - Str("value", tp). - Str("regex", r.Regex.String()). - Msg("Failed to Validate. True positive was not detected by regex.") // lint:ignore This Fatal happens in a test + log.Fatal(). // lint:ignore This Fatal happens in a test + Str("rule", r.RuleID). + Str("value", tp). + Str("regex", r.Regex.String()). + Msg("Failed to Validate. True positive was not detected by regex.") } } for _, fp := range falsePositives { if len(d.DetectString(fp)) != 0 { - log.Fatal(). - Str("rule", r.RuleID). - Str("value", fp). - Str("regex", r.Regex.String()). - Msg("Failed to Validate. False positive was detected by regex.") // lint:ignore This Fatal happens in a test + log.Fatal(). // lint:ignore This Fatal happens in a test + Str("rule", r.RuleID). + Str("value", fp). + Str("regex", r.Regex.String()). + Msg("Failed to Validate. False positive was detected by regex.") } } return &r