Merge pull request #988 from k2io/develop

New feature and optimizations for security agent

v3/integrations/nrecho-v3/nrecho.go

@@ -105,7 +105,7 @@ func Middleware(app *newrelic.Application) func(echo.HandlerFunc) echo.HandlerFu
 					txn.SetWebResponse(nil).WriteHeader(http.StatusInternalServerError)
 				}
 				if newrelic.IsSecurityAgentPresent() {
-					newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", c.Response().Header())
+					newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", c.Response().Header(), txn.GetLinkingMetadata().TraceID)
 				}
 			}
 

v3/integrations/nrecho-v4/nrecho.go

@@ -128,7 +128,7 @@ func Middleware(app *newrelic.Application, opts ...ConfigOption) func(echo.Handl
 					txn.SetWebResponse(nil).WriteHeader(http.StatusInternalServerError)
 				}
 				if newrelic.IsSecurityAgentPresent() {
-					newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", c.Response().Header())
+					newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", c.Response().Header(), txn.GetLinkingMetadata().TraceID)
 				}
 			}
 

v3/integrations/nrgin/nrgin.go

@@ -165,6 +165,7 @@ func WrapRouter(engine *gin.Engine) {
 }
 func middleware(app *newrelic.Application, useNewNames bool) gin.HandlerFunc {
 	return func(c *gin.Context) {
+		traceID := ""
 		if app != nil {
 			name := c.Request.Method + " " + getName(c, useNewNames)
 
@@ -185,10 +186,11 @@ func middleware(app *newrelic.Application, useNewNames bool) gin.HandlerFunc {
 			defer repl.flushHeader()
 
 			c.Set(internal.GinTransactionContextKey, txn)
+			traceID = txn.GetLinkingMetadata().TraceID
 		}
 		c.Next()
 		if newrelic.IsSecurityAgentPresent() {
-			newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", c.Writer.Header())
+			newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", c.Writer.Header(), traceID)
 		}
 	}
 }

v3/integrations/nrgorilla/nrgorilla.go

@@ -124,7 +124,7 @@ func Middleware(app *newrelic.Application) mux.MiddlewareFunc {
 			r = newrelic.RequestWithTransactionContext(r, txn)
 			next.ServeHTTP(w, r)
 			if newrelic.IsSecurityAgentPresent() {
-				newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", w.Header())
+				newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", w.Header(), txn.GetLinkingMetadata().TraceID)
 			}
 		})
 	}

v3/integrations/nrgraphqlgo/nrgraphqlgo.go

@@ -42,7 +42,10 @@ type Extension struct{}
 var _ graphql.Extension = Extension{}
 
 // Init is used to help you initialize the extension - in this case, a noop
-func (Extension) Init(ctx context.Context, _ *graphql.Params) context.Context {
+func (Extension) Init(ctx context.Context, params *graphql.Params) context.Context {
+	if params != nil && newrelic.IsSecurityAgentPresent() {
+		newrelic.GetSecurityAgentInterface().SendEvent("GRAPHQL", params.RequestString != "", len(params.VariableValues) != 0)
+	}
 	return ctx
 }
 
@@ -79,6 +82,10 @@ func (Extension) ValidationDidStart(ctx context.Context) (context.Context, graph
 func (Extension) ExecutionDidStart(ctx context.Context) (context.Context, graphql.ExecutionFinishFunc) {
 	txn := newrelic.FromContext(ctx)
 	seg := txn.StartSegment("Execution")
+	if newrelic.IsSecurityAgentPresent() {
+		csecData := newrelic.GetSecurityAgentInterface().SendEvent("NEW_GOROUTINE", "")
+		txn.SetCsecAttributes("CSEC_DATA", csecData)
+	}
 	return ctx, func(res *graphql.Result) {
 		// noticing here also captures those during resolve
 		for _, err := range res.Errors {
@@ -91,7 +98,16 @@ func (Extension) ExecutionDidStart(ctx context.Context) (context.Context, graphq
 // ResolveFieldDidStart is called at the start of the resolving of a field
 func (Extension) ResolveFieldDidStart(ctx context.Context, i *graphql.ResolveInfo) (context.Context, graphql.ResolveFieldFinishFunc) {
 	seg := newrelic.FromContext(ctx).StartSegment("ResolveField:" + i.FieldName)
+	if newrelic.IsSecurityAgentPresent() {
+		txn := newrelic.FromContext(ctx)
+		csecData := txn.GetCsecAttributes()["CSEC_DATA"]
+		newrelic.GetSecurityAgentInterface().SendEvent("NEW_GOROUTINE_LINKER", csecData)
+	}
+
 	return ctx, func(interface{}, error) {
+		if newrelic.IsSecurityAgentPresent() {
+			newrelic.GetSecurityAgentInterface().SendEvent("NEW_GOROUTINE_END", "")
+		}
 		seg.End()
 	}
 }

v3/integrations/nrhttprouter/nrhttprouter.go

@@ -145,6 +145,7 @@ func (r *Router) HandlerFunc(method, path string, handler http.HandlerFunc) {
 
 // ServeHTTP replaces httprouter.Router.ServeHTTP.
 func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	traceID := ""
 	if nil != r.application {
 		h, _, _ := r.Router.Lookup(req.Method, req.URL.Path)
 		if nil == h {
@@ -155,11 +156,12 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 			txn.SetWebRequestHTTP(req)
 			w = txn.SetWebResponse(w)
+			traceID = txn.GetLinkingMetadata().TraceID
 		}
 	}
 
 	r.Router.ServeHTTP(w, req)
 	if newrelic.IsSecurityAgentPresent() {
-		newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", w.Header())
+		newrelic.GetSecurityAgentInterface().SendEvent("RESPONSE_HEADER", w.Header(), traceID)
 	}
 }

v3/integrations/nrsecurityagent/README.md

@@ -40,22 +40,85 @@ NEW_RELIC_SECURITY_CONFIG_PATH={YOUR_PATH}/myappsecurity.yaml
 
 The YAML file should have these contents (adjust as needed for your application):
 ```
+# Determines whether the security data is sent to New Relic or not. When this is disabled and agent.enabled is
+# true, the security module will run but data will not be sent. Default is false.
 enabled: true
 
- # NR security provides two modes IAST and RASP
- # Default is IAST
+# New Relic Security provides two modes: IAST and RASP
+# Default is IAST. Due to the invasive nature of IAST scanning, DO NOT enable this mode in either a
+# production environment or an environment where production data is processed.
 mode: IAST
 
- # New Relic’s SaaS connection URLs
+# New Relic Security's SaaS connection URL
 validator_service_url: wss://csec.nr-data.net
 
- # Following category of security events
- # can be disabled from generating.
+# These are the category of security events that can be detected. Set to false to disable detection of
+# individual event types. Default is true for each event type.
+# This config is deprecated,
 detection:
-  rxss:
-    enabled: true
-request:
-  body_limit:1
+   rci:
+      enabled: true
+   rxss:
+      enabled: true
+   deserialization:
+      enabled: true
+
+# Unique test identifier when runnning IAST with CI/CD
+iast_test_identifier: ""
+
+# IAST scan controllers to get more control over IAST analysis
+scan_controllers:
+   # maximum number of replay requests IAST Agent 
+   # can fire in a minute. Default is 3600. Minimum is 12 and maximum is 3600
+   iast_scan_request_rate_limit: 3600
+   # The number of application instances for a specific entity where IAST analysis is performed.
+   # Values are 0 or 1, 0 signifies run on all application instances
+   scan_instance_count: 0
+
+# The scan_schedule configuration allows to specify when IAST scans should be executed
+scan_schedule:
+   # The delay field specifies the delay in minutes before the IAST scan starts. 
+   # This allows to schedule the scan to start at a later time. In minutes, default is 0 min
+   delay: 0
+   # The duration field specifies the duration of the IAST scan in minutes. 
+   # This determines how long the scan will run. In minutes, default is forever
+   duration: 0
+   # The schedule field specifies a cron expression that defines when the IAST scan should start.
+   schedule: ""
+   # Allow continuously sample collection of IAST events regardless of scan schedule. Default is false
+   always_sample_traces: false
+
+# The exclude_from_iast_scan configuration allows to specify APIs, parameters, 
+# and categories that should not be scanned by Security Agents.
+exclude_from_iast_scan:
+   # The api field specifies list of APIs using regular expression (regex) patterns that follow the syntax of Perl 5.
+   # The regex pattern should provide a complete match for the URL without the endpoint.
+   api: []
+   # The http_request_parameters configuration allows users to specify headers, query parameters,
+   # and body keys that should be excluded from IAST scans.
+   http_request_parameters:
+      # A list of HTTP header keys. If a request includes any headers with these keys,
+      # the corresponding IAST scan will be skipped.
+      header: []
+      # A list of query parameter keys. The presence of these parameters in the request's query string
+      # will lead to skipping the IAST scan.
+      query: []
+      # A list of keys within the request body. If these keys are found in the body content,
+      # the IAST scan will be omitted.
+      body: []
+   # The iast_detection_category configuration allows to specify which categories
+   # of vulnerabilities should not be detected by Security Agents.
+   iast_detection_category:
+      insecure_settings: false
+      invalid_file_access: false
+      sql_injection: false
+      nosql_injection: false
+      ldap_injection: false
+      javascript_injection: false
+      command_injection: false
+      xpath_injection: false
+      ssrf: false
+      rxss: false
 ```
 
 * Based on additional packages imported by the user application, add suitable instrumentation package imports. 

v3/integrations/nrsecurityagent/go.mod

@@ -3,7 +3,7 @@ module github.com/newrelic/go-agent/v3/integrations/nrsecurityagent
 go 1.21
 
 require (
-	github.com/newrelic/csec-go-agent v1.5.0
+	github.com/newrelic/csec-go-agent v1.6.0
 	github.com/newrelic/go-agent/v3 v3.35.0
 	github.com/newrelic/go-agent/v3/integrations/nrsqlite3 v1.2.0
 	gopkg.in/yaml.v2 v2.4.0

v3/integrations/nrsecurityagent/nrsecurityagent.go

@@ -116,12 +116,14 @@ func ConfigSecurityFromYaml() ConfigOption {
 //	NEW_RELIC_SECURITY_MODE						scanning mode: "IAST" for now
 //	NEW_RELIC_SECURITY_AGENT_ENABLED			(boolean)
 //	NEW_RELIC_SECURITY_REQUEST_BODY_LIMIT		(integer) set limit on read request body in kb. By default, this is "300"
+//  NEW_RELIC_SECURITY_IAST_TEST_IDENTIFIER     (string) This configuration allows users to specify a unique test identifier when running IAST Scan with CI/CD
 //
 // NEW_RELIC_SECURITY_SCAN_SCHEDULE_DELAY      (integer) The delay field indicated time in minutes before the IAST scan starts after the application starts. By default is 0 min.
 // NEW_RELIC_SECURITY_SCAN_SCHEDULE_DURATION   (integer) The duration field specifies the duration of the IAST scan in minutes. This determines how long the scan will run. By default is forever.
 // NEW_RELIC_SECURITY_SCAN_SCHEDULE_SCHEDULE   (string) The schedule field specifies a cron expression that defines when the IAST scan should run.
 // NEW_RELIC_SECURITY_SCAN_SCHEDULE_ALWAYS_SAMPLE_TRACES      (boolean) always_sample_traces permits IAST to actively gather trace data in the background, and the collected data will be used by Security Agent to perform an IAST Scan at the scheduled time.
 // NEW_RELIC_SECURITY_SCAN_CONTROLLERS_IAST_SCAN_REQUEST_RATE_LIMIT       (integer) The IAST Scan Rate Limit settings limit the maximum number of analysis probes or requests that can be sent to the application in a minute, By default is 3600.
+// NEW_RELIC_SECURITY_SCAN_CONTROLLERS_SCAN_INSTANCE_COUNT  (integer) This configuration allows users to the number of application instances for a specific entity where IAST analysis is performed.
 //
 // NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_INSECURE_SETTINGS (boolean)
 // NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_INVALID_FILE_ACCESS (boolean)
@@ -167,12 +169,14 @@ func ConfigSecurityFromEnvironment() ConfigOption {
 		assignBool(&cfg.Security.Agent.Enabled, "NEW_RELIC_SECURITY_AGENT_ENABLED")
 		assignBool(&cfg.Security.Detection.Rxss.Enabled, "NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED")
 		assignInt(&cfg.Security.Request.BodyLimit, "NEW_RELIC_SECURITY_REQUEST_BODY_LIMIT")
+		assignString(&cfg.Security.IastTestIdentifier, "NEW_RELIC_SECURITY_IAST_TEST_IDENTIFIER")
 
 		assignInt(&cfg.Security.ScanSchedule.Delay, "NEW_RELIC_SECURITY_SCAN_SCHEDULE_DELAY")
 		assignInt(&cfg.Security.ScanSchedule.Duration, "NEW_RELIC_SECURITY_SCAN_SCHEDULE_DURATION")
 		assignString(&cfg.Security.ScanSchedule.Schedule, "NEW_RELIC_SECURITY_SCAN_SCHEDULE_SCHEDULE")
 		assignBool(&cfg.Security.ScanSchedule.AllowIastSampleCollection, "NEW_RELIC_SECURITY_SCAN_SCHEDULE_ALWAYS_SAMPLE_TRACES")
 		assignInt(&cfg.Security.ScanControllers.IastScanRequestRateLimit, "NEW_RELIC_SECURITY_SCAN_CONTROLLERS_IAST_SCAN_REQUEST_RATE_LIMIT")
+		assignInt(&cfg.Security.ScanControllers.ScanInstanceCount, "NEW_RELIC_SECURITY_SCAN_CONTROLLERS_SCAN_INSTANCE_COUNT")
 
 		assignBool(&cfg.Security.ExcludeFromIastScan.IastDetectionCategory.InsecureSettings, "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_INSECURE_SETTINGS")
 		assignBool(&cfg.Security.ExcludeFromIastScan.IastDetectionCategory.InvalidFileAccess, "NEW_RELIC_SECURITY_EXCLUDE_FROM_IAST_SCAN_IAST_DETECTION_CATEGORY_INVALID_FILE_ACCESS")
@@ -215,6 +219,14 @@ func ConfigSecurityValidatorServiceEndPointUrl(url string) ConfigOption {
 	}
 }
 
+// ConfigSecurityIastTestIdentifier sets the iast test identifier.
+// This configuration allows users to specify a unique test identifier when running IAST Scan with CI/CD.
+func ConfigSecurityIastTestIdentifier(testIdentifier string) ConfigOption {
+	return func(cfg *SecurityConfig) {
+		cfg.Security.IastTestIdentifier = testIdentifier
+	}
+}
+
 // ConfigSecurityDetectionDisableRxss is used to enable or disable RXSS validation.
 func ConfigSecurityDetectionDisableRxss(isDisable bool) ConfigOption {
 	return func(cfg *SecurityConfig) {
@@ -275,3 +287,11 @@ func ConfigIastScanRequestRateLimit(limit int) ConfigOption {
 		cfg.Security.ScanControllers.IastScanRequestRateLimit = limit
 	}
 }
+
+// ConfigScanIstanceCount is used to set scan instance count.
+// This configuration allows users to the number of application instances for a specific entity where IAST analysis is performed.
+func ConfigScanInstanceCount(limit int) ConfigOption {
+	return func(cfg *SecurityConfig) {
+		cfg.Security.ScanControllers.ScanInstanceCount = limit
+	}
+}

v3/newrelic/instrumentation.go

@@ -85,7 +85,7 @@ func WrapHandle(app *Application, pattern string, handler http.Handler, options
 
 		handler.ServeHTTP(w, r)
 		if IsSecurityAgentPresent() {
-			secureAgent.SendEvent("RESPONSE_HEADER", w.Header())
+			secureAgent.SendEvent("RESPONSE_HEADER", w.Header(), txn.GetLinkingMetadata().TraceID)
 		}
 	})
 }

v3/newrelic/internal_response_writer.go

@@ -29,9 +29,8 @@ func (rw *replacementResponseWriter) Write(b []byte) (n int, err error) {
 	n, err = rw.original.Write(b)
 
 	headersJustWritten(rw.thd, http.StatusOK, hdr)
-
 	if IsSecurityAgentPresent() {
-		secureAgent.SendEvent("INBOUND_WRITE", string(b), hdr)
+		secureAgent.SendEvent("INBOUND_WRITE", string(b), hdr, rw.thd.GetLinkingMetadata().TraceID)
 	}
 	return
 }

v3/newrelic/internal_txn.go

@@ -1385,13 +1385,16 @@ func (txn *txn) setCsecData() {
 	}
 }
 
-func (txn *txn) getCsecAttributes() any {
+func (txn *txn) getCsecAttributes() map[string]any {
 	txn.Lock()
 	defer txn.Unlock()
+	if txn.csecAttributes == nil {
+		return map[string]any{}
+	}
 	return txn.csecAttributes
 }
 
-func (txn *txn) setCsecAttributes(key, value string) {
+func (txn *txn) setCsecAttributes(key string, value any) {
 	txn.Lock()
 	defer txn.Unlock()
 	if txn.csecAttributes == nil {

v3/newrelic/transaction.go

@@ -43,7 +43,7 @@ func (txn *Transaction) End() {
 		}
 	}
 	if txn.thread.IsWeb && IsSecurityAgentPresent() {
-		secureAgent.SendEvent("INBOUND_END", "")
+		secureAgent.SendEvent("INBOUND_END", txn.GetLinkingMetadata().TraceID)
 	}
 	txn.thread.logAPIError(txn.thread.End(r), "end transaction", nil)
 }
@@ -545,14 +545,14 @@ func (txn *Transaction) IsSampled() bool {
 	return txn.thread.IsSampled()
 }
 
-func (txn *Transaction) GetCsecAttributes() any {
+func (txn *Transaction) GetCsecAttributes() map[string]any {
 	if txn == nil || txn.thread == nil {
 		return nil
 	}
 	return txn.thread.getCsecAttributes()
 }
 
-func (txn *Transaction) SetCsecAttributes(key, value string) {
+func (txn *Transaction) SetCsecAttributes(key string, value any) {
 	if txn == nil || txn.thread == nil {
 		return
 	}