From c553c852e7486a69fc10d5042ab9526711c0b211 Mon Sep 17 00:00:00 2001
From: Chris Smith <1979423+chris13524@users.noreply.github.com>
Date: Sun, 21 Jan 2024 11:07:30 -0500
Subject: [PATCH] fix: don't alarm on get subscribers endpoint (#310)

---
 .../panels/app/http_request_latency.libsonnet       | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/terraform/monitoring/panels/app/http_request_latency.libsonnet b/terraform/monitoring/panels/app/http_request_latency.libsonnet
index a9dde4c1..7d578426 100644
--- a/terraform/monitoring/panels/app/http_request_latency.libsonnet
+++ b/terraform/monitoring/panels/app/http_request_latency.libsonnet
@@ -23,10 +23,10 @@ local targets   = grafana.targets;
       noDataState   = 'no_data',
       conditions    = [
         grafana.alertCondition.new(
-          evaluatorParams = [ 3000 ],
+          evaluatorParams = [ 2000 ],
           evaluatorType   = 'gt',
           operatorType    = 'or',
-          queryRefId      = 'HttpRequestLatency',
+          queryRefId      = 'FilteredHttpRequestLatency',
           queryTimeStart  = '5m',
           queryTimeEnd    = 'now',
           reducerType     = grafana.alert_reducers.Avg
@@ -41,4 +41,13 @@ local targets   = grafana.targets;
       exemplar      = false,
       refId         = 'HttpRequestLatency',
     ))
+
+    .addTarget(targets.prometheus(
+      datasource    = ds.prometheus,
+      expr          = 'sum by (aws_ecs_task_revision, method, endpoint) (rate(http_request_latency_sum{endpoint!="/:project_id/subscribers"}[$__rate_interval])) / sum by (aws_ecs_task_revision, method, endpoint) (rate(http_request_latency_count{endpoint!="/:project_id/subscribers"}[$__rate_interval]))',
+      legendFormat  = '{{method}} {{endpoint}} r{{aws_ecs_task_revision}}',
+      exemplar      = false,
+      refId         = 'FilteredHttpRequestLatency',
+      hide          = true,
+    ))
 }