From 4bf7a43e33f3f27bedc97a35e9d74dc95026cf28 Mon Sep 17 00:00:00 2001 From: LeonardoLordelloFontes Date: Fri, 17 Jan 2025 15:53:06 +0000 Subject: [PATCH] Fix line content and private key rule --- engine/engine.go | 3 +- engine/linecontent/linecontent.go | 93 +++++++++-- engine/linecontent/linecontent_test.go | 208 +++++++++++++++++++++++++ engine/rules/privateKey.go | 30 ++++ engine/rules/rules.go | 2 +- engine/score/score_test.go | 2 +- 6 files changed, 324 insertions(+), 14 deletions(-) create mode 100644 engine/linecontent/linecontent_test.go create mode 100644 engine/rules/privateKey.go diff --git a/engine/engine.go b/engine/engine.go index 31a6e89..785f9f4 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -94,6 +94,7 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S } else { startLine = value.StartLine endLine = value.EndLine + } secret := &secrets.Secret{ ID: itemId, @@ -104,7 +105,7 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S EndLine: endLine, EndColumn: value.EndColumn, Value: value.Secret, - LineContent: linecontent.GetLineContent(value.Line, value.StartColumn, value.EndColumn), + LineContent: linecontent.GetLineContent(value.Line, value.Secret), RuleDescription: value.Description, } if !isSecretIgnored(secret, &e.ignoredIds, &e.allowedValues) { diff --git a/engine/linecontent/linecontent.go b/engine/linecontent/linecontent.go index 5803415..cc7978d 100644 --- a/engine/linecontent/linecontent.go +++ b/engine/linecontent/linecontent.go @@ -1,22 +1,93 @@ package linecontent const ( - contextLeftSizeLimit = 250 - contextRightSizeLimit = 250 + lineContentMaxParseSize = 10000 + contextLeftSizeLimit = 250 + contextRightSizeLimit = 250 ) -func GetLineContent(lineContent string, startColumn, endColumn int) string { - lineContentSize := len(lineContent) +func GetLineContent(line, secret string) string { + lineSize := len(line) + if lineSize == 0 || len(secret) == 0 { + return "" + } + + // Truncate line to max parse size before converting to runes + shouldRemoveLastChars := false + if lineSize > lineContentMaxParseSize { + line = line[:lineContentMaxParseSize] + shouldRemoveLastChars = true // to prevent issues when truncating a multibyte character in the middle + } + + // Convert line and secret to runes + lineRunes, lineRunesSize := getLineRunes(line, shouldRemoveLastChars) + secretRunes := []rune(secret) + secretRunesSize := len(secretRunes) + + // Find the secret's position in the line (working with runes) + secretStartIndex := indexOf(lineRunes, secretRunes, lineRunesSize, secretRunesSize) + if secretStartIndex == -1 { + // Secret not found, return truncated content based on context limits + maxSize := contextLeftSizeLimit + contextRightSizeLimit + if lineRunesSize < maxSize { + return string(lineRunes) + } + return string(lineRunes[:maxSize]) + } + + // Calculate bounds for the result + secretEndIndex := secretStartIndex + secretRunesSize + start := maxIndex(secretStartIndex-contextLeftSizeLimit, 0) + end := minIndex(secretEndIndex+contextRightSizeLimit, lineRunesSize) + + return string(lineRunes[start:end]) +} - startIndex := startColumn - contextLeftSizeLimit - if startIndex < 0 { - startIndex = 0 +func getLineRunes(line string, shouldRemoveLastChars bool) ([]rune, int) { + lineRunes := []rune(line) + lineRunesSize := len(lineRunes) + if shouldRemoveLastChars { + // A single rune can be up to 4 bytes in UTF-8 encoding. + // If truncation occurs in the middle of a multibyte character, + // it will leave a partial byte sequence, potentially consisting of + // up to 3 bytes. Each of these remaining bytes will be treated + // as an invalid character, displayed as a replacement character (�). + // To prevent this, we adjust the rune count by removing the last + // 3 runes, ensuring no partial characters are included. + lineRunesSize -= 3 } + return lineRunes[:lineRunesSize], lineRunesSize +} - endIndex := endColumn + contextRightSizeLimit - if endIndex > lineContentSize { - endIndex = lineContentSize +func indexOf(line, secret []rune, lineSize, secretSize int) int { + for i := 0; i <= lineSize-secretSize; i++ { + if compareRunes(line[i:i+secretSize], secret) { + return i + } } + return -1 +} + +func compareRunes(a, b []rune) bool { + // a and b must have the same size. + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} - return lineContent[startIndex:endIndex] +func minIndex(a, b int) int { + if a < b { + return a + } + return b +} + +func maxIndex(a, b int) int { + if a > b { + return a + } + return b } diff --git a/engine/linecontent/linecontent_test.go b/engine/linecontent/linecontent_test.go new file mode 100644 index 0000000..76082d4 --- /dev/null +++ b/engine/linecontent/linecontent_test.go @@ -0,0 +1,208 @@ +package linecontent + +import ( + "strings" + "testing" +) + +const ( + dummySecret = "DummySecret" +) + +func TestGetLineContent(t *testing.T) { + tests := []struct { + name string + line string + secret string + expected string + }{ + { + name: "Empty line", + line: "", + secret: dummySecret, + expected: "", + }, + { + name: "Empty secret", + line: "line", + secret: "", + expected: "", + }, + { + name: "Secret not found with line size smaller than the parse limit", + line: "Dummy content line", + secret: dummySecret, + expected: "Dummy content line", + }, + { + name: "Secret not found with secret present and line size larger than the parse limit", + line: "This is the start of a big line content" + strings.Repeat("A", lineContentMaxParseSize) + dummySecret, + secret: dummySecret, + expected: "This is the start of a big line content" + strings.Repeat("A", contextLeftSizeLimit+contextRightSizeLimit-len("This is the start of a big line content")), + }, + { + name: "Secret larger than the line", + line: strings.Repeat("B", contextLeftSizeLimit) + strings.Repeat("A", contextRightSizeLimit), + secret: "large secret" + strings.Repeat("B", contextRightSizeLimit+contextLeftSizeLimit+100), + expected: strings.Repeat("B", contextLeftSizeLimit) + strings.Repeat("A", contextRightSizeLimit), + }, + { + name: "Secret at the beginning with line size smaller than the parse limit", + line: "start:" + dummySecret + strings.Repeat("A", lineContentMaxParseSize/2), + secret: dummySecret, + expected: "start:" + dummySecret + strings.Repeat("A", contextRightSizeLimit), + }, + { + name: "Secret found in middle with line size smaller than the parse limit", + line: "start" + strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit) + "end", + secret: dummySecret, + expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit), + }, + { + name: "Secret at the end with line size smaller than the parse limit", + line: strings.Repeat("A", lineContentMaxParseSize/2) + dummySecret + ":end", + secret: dummySecret, + expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + ":end", + }, + { + name: "Secret at the beginning with line size larger than the parse limit", + line: "start:" + dummySecret + strings.Repeat("A", lineContentMaxParseSize), + secret: dummySecret, + expected: "start:" + dummySecret + strings.Repeat("A", contextRightSizeLimit), + }, + { + name: "Secret found in middle with line size larger than the parse limit", + line: "start" + strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", lineContentMaxParseSize) + "end", + secret: dummySecret, + expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit), + }, + { + name: "Secret at the end with line size larger than the parse limit", + line: strings.Repeat("A", lineContentMaxParseSize-100) + dummySecret + strings.Repeat("A", lineContentMaxParseSize), + secret: dummySecret, + expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(100, 1, len(dummySecret))), + }, + { + name: "Secret at the beginning with line containing 2 byte chars and size smaller than the parse limit", + line: "start:" + dummySecret + strings.Repeat("é", lineContentMaxParseSize/4), + secret: dummySecret, + expected: "start:" + dummySecret + strings.Repeat("é", contextRightSizeLimit), + }, + { + name: "Secret found in middle with line containing 2 byte chars and size smaller than the parse limit", + line: "start" + strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", contextRightSizeLimit) + "end", + secret: dummySecret, + expected: strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", contextRightSizeLimit), + }, + { + name: "Secret at the end with line containing 2 byte chars and size smaller than the parse limit", + line: strings.Repeat("é", lineContentMaxParseSize/4) + dummySecret + ":end", + secret: dummySecret, + expected: strings.Repeat("é", contextLeftSizeLimit) + dummySecret + ":end", + }, + { + name: "Secret at the beginning with line containing 2 byte chars and size larger than the parse limit", + line: "start:" + dummySecret + strings.Repeat("é", lineContentMaxParseSize/2), + secret: dummySecret, + expected: "start:" + dummySecret + strings.Repeat("é", contextRightSizeLimit), + }, + { + name: "Secret found in middle with line containing 2 byte chars and size larger than the parse limit", + line: "start" + strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", lineContentMaxParseSize/2) + "end", + secret: dummySecret, + expected: strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", contextRightSizeLimit), + }, + { + name: "Secret at the end with line containing 2 byte chars and size larger than the parse limit", + line: strings.Repeat("é", lineContentMaxParseSize/2-100) + dummySecret + strings.Repeat("é", lineContentMaxParseSize/2), + secret: dummySecret, + expected: strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(100, 2, len(dummySecret))), + }, + { + name: "Secret at the beginning with line containing 3 byte chars and size smaller than the parse limit", + line: "start:" + dummySecret + strings.Repeat("ࠚ", lineContentMaxParseSize/6), + secret: dummySecret, + expected: "start:" + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit), + }, + { + name: "Secret found in middle with line containing 3 byte chars and size smaller than the parse limit", + line: "start" + strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit) + "end", + secret: dummySecret, + expected: strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit), + }, + { + name: "Secret at the end with line containing 3 byte chars and size smaller than the parse limit", + line: strings.Repeat("ࠚ", lineContentMaxParseSize/6) + dummySecret + ":end", + secret: dummySecret, + expected: strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + ":end", + }, + { + name: "Secret at the beginning with line containing 3 byte chars and size larger than the parse limit", + line: "start:" + dummySecret + strings.Repeat("ࠚ", lineContentMaxParseSize/3), + secret: dummySecret, + expected: "start:" + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit), + }, + { + name: "Secret found in middle with line containing 3 byte chars and size larger than the parse limit", + line: "start" + strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", lineContentMaxParseSize/3) + "end", + secret: dummySecret, + expected: strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit), + }, + { + name: "Secret at the end with line containing 3 byte chars and size larger than the parse limit", + line: strings.Repeat("ࠚ", lineContentMaxParseSize/3-100) + dummySecret + strings.Repeat("ࠚ", lineContentMaxParseSize/3), + secret: dummySecret, + expected: strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(100, 3, len(dummySecret))), + }, + { + name: "Secret at the beginning with line containing 4 byte chars and size smaller than the parse limit", + line: "start:" + dummySecret + strings.Repeat("𝄞", lineContentMaxParseSize/8), + secret: dummySecret, + expected: "start:" + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit), + }, + { + name: "Secret found in middle with line containing 4 byte chars and size smaller than the parse limit", + line: "start" + strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit) + "end", + secret: dummySecret, + expected: strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit), + }, + { + name: "Secret at the end with line containing 4 byte chars and size smaller than the parse limit", + line: strings.Repeat("𝄞", lineContentMaxParseSize/8) + dummySecret + ":end", + secret: dummySecret, + expected: strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + ":end", + }, + { + name: "Secret at the beginning with line containing 4 byte chars and size larger than the parse limit", + line: "start:" + dummySecret + strings.Repeat("𝄞", lineContentMaxParseSize/4), + secret: dummySecret, + expected: "start:" + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit), + }, + { + name: "Secret found in middle with line containing 4 byte chars and size larger than the parse limit", + line: "start" + strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", lineContentMaxParseSize/4) + "end", + secret: dummySecret, + expected: strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit), + }, + { + name: "Secret at the end with line containing 4 byte chars and size larger than the parse limit", + line: strings.Repeat("𝄞", lineContentMaxParseSize/4-100) + dummySecret + strings.Repeat("𝄞", lineContentMaxParseSize/4), + secret: dummySecret, + expected: strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(100, 4, len(dummySecret))), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetLineContent(tt.line, tt.secret) + if got != tt.expected { + t.Errorf("GetLineContent() = %v, want %v", got, tt.expected) + } + }) + } +} + +func calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(offset, bytes, secretLength int) int { + remainingSize := lineContentMaxParseSize - ((lineContentMaxParseSize/bytes - offset) * bytes) - secretLength + return (remainingSize - ((3 - ((remainingSize) % bytes)) * bytes)) / bytes +} diff --git a/engine/rules/privateKey.go b/engine/rules/privateKey.go new file mode 100644 index 0000000..66b956c --- /dev/null +++ b/engine/rules/privateKey.go @@ -0,0 +1,30 @@ +package rules + +import ( + "github.com/zricethezav/gitleaks/v8/config" + "regexp" +) + +func PrivateKey() *config.Rule { + // define rule + r := config.Rule{ + Description: "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption.", + RuleID: "private-key", + Regex: regexp.MustCompile(`(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]*?KEY(?: BLOCK)?-----`), + Keywords: []string{"-----BEGIN"}, + } + + // validate + tps := []string{`-----BEGIN PRIVATE KEY----- +anything +-----END PRIVATE KEY-----`, + `-----BEGIN RSA PRIVATE KEY----- +abcdefghijklmnopqrstuvwxyz +-----END RSA PRIVATE KEY----- +`, + `-----BEGIN PRIVATE KEY BLOCK----- +anything +-----END PRIVATE KEY BLOCK-----`, + } // gitleaks:allow + return validate(r, tps, nil) +} diff --git a/engine/rules/rules.go b/engine/rules/rules.go index b306057..a99b2e3 100644 --- a/engine/rules/rules.go +++ b/engine/rules/rules.go @@ -187,7 +187,7 @@ func getDefaultRules() *[]Rule { {Rule: *rules.PlanetScaleOAuthToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryDatabaseAsAService, RuleType: 4}}, {Rule: *rules.PostManAPI(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}}, {Rule: *rules.Prefect(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}}, - {Rule: *rules.PrivateKey(), Tags: []string{TagPrivateKey}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}}, + {Rule: *PrivateKey(), Tags: []string{TagPrivateKey}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}}, {Rule: *rules.PulumiAPIToken(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryCloudPlatform, RuleType: 4}}, {Rule: *rules.PyPiUploadToken(), Tags: []string{TagUploadToken}, ScoreParameters: ScoreParameters{Category: CategoryPackageManagement, RuleType: 4}}, {Rule: *rules.RapidAPIAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}}, diff --git a/engine/score/score_test.go b/engine/score/score_test.go index 164d452..4515a4a 100644 --- a/engine/score/score_test.go +++ b/engine/score/score_test.go @@ -135,7 +135,7 @@ func TestScore(t *testing.T) { ruleConfig.PlanetScaleOAuthToken().RuleID: {10, 5.2, 8.2}, ruleConfig.PostManAPI().RuleID: {10, 5.2, 8.2}, ruleConfig.Prefect().RuleID: {10, 5.2, 8.2}, - ruleConfig.PrivateKey().RuleID: {10, 5.2, 8.2}, + rules.PrivateKey().RuleID: {10, 5.2, 8.2}, ruleConfig.PulumiAPIToken().RuleID: {10, 5.2, 8.2}, ruleConfig.PyPiUploadToken().RuleID: {10, 5.2, 8.2}, ruleConfig.RapidAPIAccessToken().RuleID: {10, 5.2, 8.2},