fix: increase relay latency alarm threshold (#391)

reown-com · Feb 29, 2024 · 3a5550a · 3a5550a
1 parent f1b4950
commit 3a5550a
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 2 deletions.
diff --git a/terraform/monitoring/panels/app/http_request_latency.libsonnet b/terraform/monitoring/panels/app/http_request_latency.libsonnet
@@ -21,6 +21,25 @@ local targets   = grafana.targets;
       message       = '%(env)s - HTTP request latency too high' % { env: vars.environment },
       notifications = vars.notifications,
       noDataState   = 'no_data',
+      conditions    = [
+        grafana.alertCondition.new(
+          evaluatorParams = [ 10000 ],
+          evaluatorType   = 'gt',
+          operatorType    = 'or',
+          queryRefId      = 'HttpRequestLatency',
+          queryTimeStart  = '5m',
+          queryTimeEnd    = 'now',
+          reducerType     = grafana.alert_reducers.Avg
+        ),
+      ],
+    ))
+
+    .setAlert(vars.environment, grafana.alert.new(
+      namespace     = vars.namespace,
+      name          = '%(env)s - HTTP (filtered) request latency too high' % { env: vars.environment },
+      message       = '%(env)s - HTTP (filtered) request latency too high' % { env: vars.environment },
+      notifications = vars.notifications,
+      noDataState   = 'no_data',
       conditions    = [
         grafana.alertCondition.new(
           evaluatorParams = [ 2000 ],
@@ -44,7 +63,7 @@ local targets   = grafana.targets;
 
     .addTarget(targets.prometheus(
       datasource    = ds.prometheus,
-      expr          = 'sum by (aws_ecs_task_revision, method, endpoint) (rate(http_request_latency_sum{endpoint!="/:project_id/subscribers"}[$__rate_interval])) / sum by (aws_ecs_task_revision, method, endpoint) (rate(http_request_latency_count{endpoint!="/:project_id/subscribers"}[$__rate_interval]))',
+      expr          = 'sum by (aws_ecs_task_revision, method, endpoint) (rate(http_request_latency_sum{endpoint!~"^(/:project_id/subscribers|/v1/relay-webhook)"}[$__rate_interval])) / sum by (aws_ecs_task_revision, method, endpoint) (rate(http_request_latency_count{endpoint!="/:project_id/subscribers"}[$__rate_interval]))',
       legendFormat  = '{{method}} {{endpoint}} r{{aws_ecs_task_revision}}',
       exemplar      = false,
       refId         = 'FilteredHttpRequestLatency',

diff --git a/terraform/monitoring/panels/app/relay_incoming_message_latency.libsonnet b/terraform/monitoring/panels/app/relay_incoming_message_latency.libsonnet
@@ -23,7 +23,7 @@ local targets   = grafana.targets;
       noDataState   = 'no_data',
       conditions    = [
         grafana.alertCondition.new(
-          evaluatorParams = [ 5000 ],
+          evaluatorParams = [ 10000 ],
           evaluatorType   = 'gt',
           operatorType    = 'or',
           queryRefId      = 'RelayIncomingMessageLatency',