reorg metrics (#132)

* Basic logging around reorgs * Add rolled back records * fmt/lint * Update to use the optionals for the new fields * Add reorg metrics * Update to 0.5.3
Chia-Network · Jan 12, 2024 · 90b4286 · 90b4286
1 parent c921985
commit 90b4286
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 4 deletions.
diff --git a/cmd/root.go b/cmd/root.go
@@ -35,7 +35,7 @@ func init() {
 		logLevel                          string
 		requestTimeout                    time.Duration
 		disableCentralHarvesterCollection bool
-		logBlockTimes bool
+		logBlockTimes                     bool
 	)
 
 	cobra.OnInitialize(initConfig)

diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter
 go 1.21
 
 require (
-	github.com/chia-network/go-chia-libs v0.5.2
+	github.com/chia-network/go-chia-libs v0.5.3
 	github.com/chia-network/go-modules v0.0.4
 	github.com/oschwald/maxminddb-golang v1.12.0
 	github.com/prometheus/client_golang v1.18.0

diff --git a/go.sum b/go.sum
@@ -2,8 +2,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chia-network/go-chia-libs v0.5.2 h1:knK/LAWhaRw2OG1iiK/gK9gKRmNNdB6BKptOdu9Hams=
-github.com/chia-network/go-chia-libs v0.5.2/go.mod h1:zQ2cAB+soETmCQBBfcd352FWJRTzXrSqifWuQ2/Gx9Q=
+github.com/chia-network/go-chia-libs v0.5.3 h1:+7/Myjp1pi7bJ+tFM7zRgww2YmbAvx8SbUkeR7vgLSM=
+github.com/chia-network/go-chia-libs v0.5.3/go.mod h1:zQ2cAB+soETmCQBBfcd352FWJRTzXrSqifWuQ2/Gx9Q=
 github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI=
 github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

diff --git a/internal/metrics/fullnode.go b/internal/metrics/fullnode.go
@@ -78,6 +78,12 @@ type FullNodeServiceMetrics struct {
 	heightToHash      *wrappedPrometheus.LazyGauge
 	subEpochSummaries *wrappedPrometheus.LazyGauge
 
+	// Reorg Metrics
+	lastBlockReorgDepth        *wrappedPrometheus.LazyGauge
+	lastReorgReorgDepth        *wrappedPrometheus.LazyGauge
+	lastBlockRolledBackRecords *wrappedPrometheus.LazyGauge
+	lastReorgRolledBackRecords *wrappedPrometheus.LazyGauge
+
 	// Debug Metric
 	debug *prometheus.GaugeVec
 
@@ -130,6 +136,12 @@ func (s *FullNodeServiceMetrics) InitMetrics() {
 	s.heightToHash = s.metrics.newGauge(chiaServiceFullNode, "height_to_hash_filesize", "Size of height_to_hash file")
 	s.subEpochSummaries = s.metrics.newGauge(chiaServiceFullNode, "sub_epoch_summaries_filesize", "Size of sub_epoch_summaries file")
 
+	// Reorg Related
+	s.lastBlockReorgDepth = s.metrics.newGauge(chiaServiceFullNode, "last_block_reorg_depth", "For the last block, the reorg depth. Generally expected to be zero, indicating no reorg happened for this block")
+	s.lastReorgReorgDepth = s.metrics.newGauge(chiaServiceFullNode, "last_reorg_reorg_depth", "For the last reorg that was seen, the reorg depth. This does not get set to a new value until the next reorg is seen")
+	s.lastBlockRolledBackRecords = s.metrics.newGauge(chiaServiceFullNode, "last_block_rolled_back_records", "For the last block, the number of records that were rolled back. Generally expected to be zero, indicating no reorg happened for this block")
+	s.lastReorgRolledBackRecords = s.metrics.newGauge(chiaServiceFullNode, "last_reorg_rolled_back_records", "For the last reorg that was seen, the number of records that were rolled back. This does not get set to a new value until the next reorg is seen")
+
 	// Debug Metric
 	s.debug = s.metrics.newGaugeVec(chiaServiceFullNode, "debug_metrics", "misc debugging metrics distinguished by labels", []string{"key"})
 }
@@ -183,6 +195,11 @@ func (s *FullNodeServiceMetrics) Disconnected() {
 	s.totalSignagePoints.Unregister()
 	s.signagePointsSubSlot.Unregister()
 	s.currentSignagePoint.Unregister()
+
+	s.lastBlockReorgDepth.Unregister()
+	s.lastReorgReorgDepth.Unregister()
+	s.lastBlockRolledBackRecords.Unregister()
+	s.lastReorgRolledBackRecords.Unregister()
 }
 
 // Reconnected is called when the service is reconnected after the websocket was disconnected
@@ -313,6 +330,21 @@ func (s *FullNodeServiceMetrics) Block(resp *types.WebsocketResponse) {
 		return
 	}
 
+	if block.ForkHeight.IsPresent() && block.RolledBackRecords.IsPresent() {
+		forkHeight := block.ForkHeight.MustGet()
+		rolledBackRecords := block.RolledBackRecords.MustGet()
+		// Normal new block is "1", so remove the 1 so that a standard block add is just 0
+		reorgDepth := block.Height - forkHeight - 1
+		log.Debugf("Fork height is: %d, Block height is %d, Reorg depth is: %d, Rolled Back Records: %d\n", forkHeight, block.Height, reorgDepth, rolledBackRecords)
+		s.lastBlockReorgDepth.Set(float64(reorgDepth))
+		s.lastBlockRolledBackRecords.Set(float64(rolledBackRecords))
+
+		if reorgDepth > 0 || rolledBackRecords > 0 {
+			s.lastReorgReorgDepth.Set(float64(reorgDepth))
+			s.lastReorgRolledBackRecords.Set(float64(rolledBackRecords))
+		}
+	}
+
 	s.kSize.WithLabelValues(fmt.Sprintf("%d", block.KSize)).Inc()
 	s.preValidationTime.Set(block.PreValidationTime)
 	s.validationTime.Set(block.ValidationTime)