-
Notifications
You must be signed in to change notification settings - Fork 21
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #209 from pace/add-queue
Add queueing system via rmq
- Loading branch information
Showing
101 changed files
with
4,884 additions
and
153 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package queue | ||
|
|
||
| import ( | ||
| "log" | ||
| "time" | ||
|
|
||
| "github.com/caarlos0/env" | ||
| ) | ||
|
|
||
| type config struct { | ||
| HealthCheckResultTTL time.Duration `env:"RMQ_HEALTH_CHECK_RESULT_TTL" envDefault:"10s"` | ||
| MetricsRefreshInterval time.Duration `env:"RMQ_METRICS_REFRESH_INTERVAL" envDefault:"10s"` | ||
| } | ||
|
|
||
| var cfg config | ||
|
|
||
| func init() { | ||
| err := env.Parse(&cfg) | ||
| if err != nil { | ||
| log.Fatalf("Failed to parse queue environment: %v", err) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package queue | ||
|
|
||
| import ( | ||
| "context" | ||
| "time" | ||
|
|
||
| "github.com/adjust/rmq/v2" | ||
| "github.com/pace/bricks/maintenance/log" | ||
| "github.com/pace/bricks/pkg/routine" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| ) | ||
|
|
||
| type queueStatsGauges struct { | ||
| readyGauge *prometheus.GaugeVec | ||
| rejectedGauge *prometheus.GaugeVec | ||
| connectionGauge *prometheus.GaugeVec | ||
| consumerGauge *prometheus.GaugeVec | ||
| unackedGauge *prometheus.GaugeVec | ||
| } | ||
|
|
||
| func gatherMetrics(connection rmq.Connection) { | ||
| gauges := registerConnection(connection) | ||
| ctx := log.WithContext(context.Background()) | ||
|
|
||
| routine.Run(ctx, func(_ context.Context) { | ||
| stats := connection.CollectStats(connection.GetOpenQueues()) | ||
| for queue, queueStats := range stats.QueueStats { | ||
| labels := prometheus.Labels{ | ||
| "queue": queue, | ||
| } | ||
| gauges.readyGauge.With(labels).Set(float64(queueStats.ReadyCount)) | ||
| gauges.rejectedGauge.With(labels).Set(float64(queueStats.RejectedCount)) | ||
| gauges.connectionGauge.With(labels).Set(float64(queueStats.ConnectionCount())) | ||
| gauges.consumerGauge.With(labels).Set(float64(queueStats.ConsumerCount())) | ||
| gauges.unackedGauge.With(labels).Set(float64(queueStats.UnackedCount())) | ||
| } | ||
| time.Sleep(cfg.MetricsRefreshInterval) | ||
| }) | ||
| } | ||
|
|
||
| func registerConnection(connection rmq.Connection) queueStatsGauges { | ||
| gauges := queueStatsGauges{ | ||
| readyGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Namespace: "rmq", | ||
| Name: "ready", | ||
| Help: "Number of ready messages on queue", | ||
| }, []string{"queue"}), | ||
| rejectedGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Namespace: "rmq", | ||
| Name: "rejected", | ||
| Help: "Number of rejected messages on queue", | ||
| }, []string{"queue"}), | ||
| connectionGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Namespace: "rmq", | ||
| Name: "connection", | ||
| Help: "Number of connections consuming a queue", | ||
| }, []string{"queue"}), | ||
| consumerGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Namespace: "rmq", | ||
| Name: "consumer", | ||
| Help: "Number of consumers consuming messages for a queue", | ||
| }, []string{"queue"}), | ||
| unackedGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Namespace: "rmq", | ||
| Name: "unacked", | ||
| Help: "Number of unacked messages on a consumer", | ||
| }, []string{"queue"}), | ||
| } | ||
|
|
||
| prometheus.MustRegister(gauges.readyGauge) | ||
| prometheus.MustRegister(gauges.rejectedGauge) | ||
| prometheus.MustRegister(gauges.connectionGauge) | ||
| prometheus.MustRegister(gauges.consumerGauge) | ||
| prometheus.MustRegister(gauges.unackedGauge) | ||
|
|
||
| return gauges | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package queue | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/adjust/rmq/v2" | ||
| "github.com/pace/bricks/backend/redis" | ||
| "github.com/pace/bricks/maintenance/health/servicehealthcheck" | ||
| ) | ||
|
|
||
| var ( | ||
| rmqConnection rmq.Connection | ||
| queueHealthLimits sync.Map | ||
| ) | ||
|
|
||
| func init() { | ||
| rmqConnection = rmq.OpenConnectionWithRedisClient("default", redis.Client()) | ||
| gatherMetrics(rmqConnection) | ||
| servicehealthcheck.RegisterHealthCheck("rmq", &HealthCheck{}) | ||
| } | ||
|
|
||
| // NewQueue creates a new rmq.Queue and initializes health checks for this queue | ||
| // Whenever the number of items in the queue exceeds the healthyLimit | ||
| // The queue will be reported as unhealthy | ||
| // If the queue has already been opened, it will just be returned. Limits will not | ||
| // be updated | ||
| func NewQueue(name string, healthyLimit int) rmq.Queue { | ||
| queue := rmqConnection.OpenQueue(name) | ||
| if _, ok := queueHealthLimits.Load(name); ok { | ||
| return queue | ||
| } | ||
| queueHealthLimits.Store(name, healthyLimit) | ||
| return queue | ||
| } | ||
|
|
||
| type HealthCheck struct { | ||
| state servicehealthcheck.ConnectionState | ||
| // IgnoreInterval is a switch used for testing, just to allow multiple | ||
| // functional queries of HealthCheck in rapid bursts | ||
| IgnoreInterval bool | ||
| } | ||
|
|
||
| // HealthCheck checks if the queues are healthy, i.e. whether the number of | ||
| // items accumulated is below the healthyLimit defined when opening the queue | ||
| func (h *HealthCheck) HealthCheck(ctx context.Context) servicehealthcheck.HealthCheckResult { | ||
| if !h.IgnoreInterval && time.Since(h.state.LastChecked()) <= cfg.HealthCheckResultTTL { | ||
| return h.state.GetState() | ||
| } | ||
|
|
||
| stats := rmqConnection.CollectStats(rmqConnection.GetOpenQueues()) | ||
| queueHealthLimits.Range(func(k, v interface{}) bool { | ||
| name := k.(string) | ||
| healthLimit := v.(int) | ||
| stat := stats.QueueStats[name] | ||
| if stat.ReadyCount > healthLimit { | ||
| h.state.SetErrorState(fmt.Errorf("Queue '%s' exceeded safe health limit of '%d'", name, healthLimit)) | ||
| return false | ||
| } | ||
| h.state.SetHealthy() | ||
| return true | ||
| }) | ||
| return h.state.GetState() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package queue_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
|
|
||
| "github.com/pace/bricks/backend/queue" | ||
| "github.com/pace/bricks/maintenance/log" | ||
| ) | ||
|
|
||
| func TestIntegrationHealthCheck(t *testing.T) { | ||
| if testing.Short() { | ||
| t.SkipNow() | ||
| } | ||
| ctx := log.WithContext(context.Background()) | ||
| q1 := queue.NewQueue("integrationTestTasks", 1) | ||
| q1.Publish("nothing here") | ||
|
|
||
| check := &queue.HealthCheck{IgnoreInterval: true} | ||
| res := check.HealthCheck(ctx) | ||
| if res.State != "OK" { | ||
| t.Errorf("Expected health check to be OK for a non-full queue") | ||
| } | ||
|
|
||
| q1.Publish("nothing here either") | ||
|
|
||
| res = check.HealthCheck(ctx) | ||
| if res.State == "OK" { | ||
| t.Errorf("Expected health check to be ERR for a full queue") | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Oops, something went wrong.