From 90b4286166b7a75b412e807981f0e3ccf5afaa07 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Fri, 12 Jan 2024 16:25:23 -0600 Subject: [PATCH] reorg metrics (#132) * Basic logging around reorgs * Add rolled back records * fmt/lint * Update to use the optionals for the new fields * Add reorg metrics * Update to 0.5.3 --- cmd/root.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- internal/metrics/fullnode.go | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index dcd7b1c..54b5aba 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -35,7 +35,7 @@ func init() { logLevel string requestTimeout time.Duration disableCentralHarvesterCollection bool - logBlockTimes bool + logBlockTimes bool ) cobra.OnInitialize(initConfig) diff --git a/go.mod b/go.mod index 63dc8bf..32ad41a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter go 1.21 require ( - github.com/chia-network/go-chia-libs v0.5.2 + github.com/chia-network/go-chia-libs v0.5.3 github.com/chia-network/go-modules v0.0.4 github.com/oschwald/maxminddb-golang v1.12.0 github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index a89824e..468f271 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chia-network/go-chia-libs v0.5.2 h1:knK/LAWhaRw2OG1iiK/gK9gKRmNNdB6BKptOdu9Hams= -github.com/chia-network/go-chia-libs v0.5.2/go.mod h1:zQ2cAB+soETmCQBBfcd352FWJRTzXrSqifWuQ2/Gx9Q= +github.com/chia-network/go-chia-libs v0.5.3 h1:+7/Myjp1pi7bJ+tFM7zRgww2YmbAvx8SbUkeR7vgLSM= +github.com/chia-network/go-chia-libs v0.5.3/go.mod h1:zQ2cAB+soETmCQBBfcd352FWJRTzXrSqifWuQ2/Gx9Q= github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI= github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/internal/metrics/fullnode.go b/internal/metrics/fullnode.go index 4864a5f..e1bb144 100644 --- a/internal/metrics/fullnode.go +++ b/internal/metrics/fullnode.go @@ -78,6 +78,12 @@ type FullNodeServiceMetrics struct { heightToHash *wrappedPrometheus.LazyGauge subEpochSummaries *wrappedPrometheus.LazyGauge + // Reorg Metrics + lastBlockReorgDepth *wrappedPrometheus.LazyGauge + lastReorgReorgDepth *wrappedPrometheus.LazyGauge + lastBlockRolledBackRecords *wrappedPrometheus.LazyGauge + lastReorgRolledBackRecords *wrappedPrometheus.LazyGauge + // Debug Metric debug *prometheus.GaugeVec @@ -130,6 +136,12 @@ func (s *FullNodeServiceMetrics) InitMetrics() { s.heightToHash = s.metrics.newGauge(chiaServiceFullNode, "height_to_hash_filesize", "Size of height_to_hash file") s.subEpochSummaries = s.metrics.newGauge(chiaServiceFullNode, "sub_epoch_summaries_filesize", "Size of sub_epoch_summaries file") + // Reorg Related + s.lastBlockReorgDepth = s.metrics.newGauge(chiaServiceFullNode, "last_block_reorg_depth", "For the last block, the reorg depth. Generally expected to be zero, indicating no reorg happened for this block") + s.lastReorgReorgDepth = s.metrics.newGauge(chiaServiceFullNode, "last_reorg_reorg_depth", "For the last reorg that was seen, the reorg depth. This does not get set to a new value until the next reorg is seen") + s.lastBlockRolledBackRecords = s.metrics.newGauge(chiaServiceFullNode, "last_block_rolled_back_records", "For the last block, the number of records that were rolled back. Generally expected to be zero, indicating no reorg happened for this block") + s.lastReorgRolledBackRecords = s.metrics.newGauge(chiaServiceFullNode, "last_reorg_rolled_back_records", "For the last reorg that was seen, the number of records that were rolled back. This does not get set to a new value until the next reorg is seen") + // Debug Metric s.debug = s.metrics.newGaugeVec(chiaServiceFullNode, "debug_metrics", "misc debugging metrics distinguished by labels", []string{"key"}) } @@ -183,6 +195,11 @@ func (s *FullNodeServiceMetrics) Disconnected() { s.totalSignagePoints.Unregister() s.signagePointsSubSlot.Unregister() s.currentSignagePoint.Unregister() + + s.lastBlockReorgDepth.Unregister() + s.lastReorgReorgDepth.Unregister() + s.lastBlockRolledBackRecords.Unregister() + s.lastReorgRolledBackRecords.Unregister() } // Reconnected is called when the service is reconnected after the websocket was disconnected @@ -313,6 +330,21 @@ func (s *FullNodeServiceMetrics) Block(resp *types.WebsocketResponse) { return } + if block.ForkHeight.IsPresent() && block.RolledBackRecords.IsPresent() { + forkHeight := block.ForkHeight.MustGet() + rolledBackRecords := block.RolledBackRecords.MustGet() + // Normal new block is "1", so remove the 1 so that a standard block add is just 0 + reorgDepth := block.Height - forkHeight - 1 + log.Debugf("Fork height is: %d, Block height is %d, Reorg depth is: %d, Rolled Back Records: %d\n", forkHeight, block.Height, reorgDepth, rolledBackRecords) + s.lastBlockReorgDepth.Set(float64(reorgDepth)) + s.lastBlockRolledBackRecords.Set(float64(rolledBackRecords)) + + if reorgDepth > 0 || rolledBackRecords > 0 { + s.lastReorgReorgDepth.Set(float64(reorgDepth)) + s.lastReorgRolledBackRecords.Set(float64(rolledBackRecords)) + } + } + s.kSize.WithLabelValues(fmt.Sprintf("%d", block.KSize)).Inc() s.preValidationTime.Set(block.PreValidationTime) s.validationTime.Set(block.ValidationTime)