Merge branch '43-default-buckets-for-pacehttpduration-are-off' into '…

…master'

Resolve "Default buckets for paceHTTPDuration are off"

Closes #43

See merge request pace/go-microservice!34

backend/postgres/postgres.go

@@ -31,21 +31,36 @@ var (
 			Name: "pace_postgres_query_total",
 			Help: "Collects stats about the number of postgres queries made",
 		},
-		[]string{"query", "database", "addr"},
+		[]string{"database"},
 	)
 	pacePostgresQueryFailed = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "pace_postgres_query_failed",
 			Help: "Collects stats about the number of postgres queries failed",
 		},
-		[]string{"query", "database", "addr"},
+		[]string{"database"},
 	)
 	pacePostgresQueryDurationSeconds = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name: "pace_postgres_query_duration_seconds",
-			Help: "Collect performance metrics for each postgres query",
+			Name:    "pace_postgres_query_duration_seconds",
+			Help:    "Collect performance metrics for each postgres query",
+			Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 60},
 		},
-		[]string{"query", "database", "addr"},
+		[]string{"database"},
+	)
+	pacePostgresQueryRowsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "pace_postgres_query_rows_total",
+			Help: "Collects stats about the number of rows returned by a postgres query",
+		},
+		[]string{"database"},
+	)
+	pacePostgresQueryAffectedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "pace_postgres_query_affected_total",
+			Help: "Collects stats about the number of rows affected by a postgres query",
+		},
+		[]string{"database"},
 	)
 )
 
@@ -55,6 +70,8 @@ func init() {
 	prometheus.MustRegister(pacePostgresQueryTotal)
 	prometheus.MustRegister(pacePostgresQueryFailed)
 	prometheus.MustRegister(pacePostgresQueryDurationSeconds)
+	prometheus.MustRegister(pacePostgresQueryRowsTotal)
+	prometheus.MustRegister(pacePostgresQueryAffectedTotal)
 
 	// parse log config
 	err := env.Parse(&cfg)
@@ -162,21 +179,18 @@ func openTracingAdapter(event *pg.QueryProcessedEvent) {
 
 func metricsAdapter(event *pg.QueryProcessedEvent, opts *pg.Options) {
 	dur := float64(time.Since(event.StartTime)) / float64(time.Millisecond)
-	q, qe := event.UnformattedQuery()
-	if qe != nil {
-		// this is only a display issue not a "real" issue
-		q = qe.Error()
-	}
 	labels := prometheus.Labels{
-		"query":    q,
-		"database": opts.Database,
-		"addr":     opts.Addr,
+		"database": opts.Addr + "/" + opts.Database,
 	}
 
 	pacePostgresQueryTotal.With(labels).Inc()
 
 	if event.Error != nil {
 		pacePostgresQueryFailed.With(labels).Inc()
+	} else {
+		r := event.Result
+		pacePostgresQueryRowsTotal.With(labels).Add(float64(r.RowsReturned()))
+		pacePostgresQueryAffectedTotal.With(labels).Add(float64(r.RowsAffected()))
 	}
 
 	pacePostgresQueryDurationSeconds.With(labels).Observe(dur)

backend/redis/redis.go

@@ -53,8 +53,9 @@ var (
 	)
 	paceRedisCmdDurationSeconds = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name: "pace_redis_cmd_duration_seconds",
-			Help: "Collect performance metrics for each method",
+			Name:    "pace_redis_cmd_duration_seconds",
+			Help:    "Collect performance metrics for each method",
+			Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 60},
 		},
 		[]string{"method"},
 	)

http/metrics.go

@@ -8,6 +8,11 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+const (
+	kb = 1024
+	mb = kb * kb
+)
+
 var (
 	paceHTTPInFlightGauge = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "pace_http_in_flight_requests",
@@ -28,7 +33,7 @@ var (
 		prometheus.HistogramOpts{
 			Name:    "pace_http_request_duration_milliseconds",
 			Help:    "A histogram of latencies for requests.",
-			Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10},
+			Buckets: []float64{10, 50, 100, 300, 600, 1000, 2500, 5000, 10000, 60000},
 		},
 		[]string{"code", "method"},
 	)
@@ -37,9 +42,13 @@ var (
 	// ObserverVec.
 	paceHTTPResponseSize = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name:    "pace_http_request_size_bytes",
-			Help:    "A histogram of response sizes for requests.",
-			Buckets: []float64{100, 200, 500, 900, 1500},
+			Name: "pace_http_request_size_bytes",
+			Help: "A histogram of response sizes for requests.",
+			Buckets: []float64{
+				100,
+				kb, 10 * kb, 100 * kb,
+				1 * mb, 5 * mb, 10 * mb, 100 * mb,
+			},
 		},
 		[]string{"code", "method"},
 	)
@@ -53,6 +62,7 @@ func init() {
 func metricsMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		paceHTTPInFlightGauge.Inc()
+		defer paceHTTPInFlightGauge.Dec()
 		startTime := time.Now()
 		srw := statusWriter{ResponseWriter: w}
 		next.ServeHTTP(&srw, r)
@@ -64,7 +74,6 @@ func metricsMiddleware(next http.Handler) http.Handler {
 		paceHTTPCounter.With(labels).Inc()
 		paceHTTPDuration.With(labels).Observe(dur)
 		paceHTTPResponseSize.With(labels).Observe(float64(srw.length))
-		paceHTTPInFlightGauge.Dec()
 	})
 }