Add new compactor_objects_combined metric and test (#339)

* Add new compactor_traces_combined metric and test * Update changelog * Fix lint error, check err return in test * Add module reference * Rename metric from traces_combined to objects_combined * Add new tempodb_compaction_objects_combined metric to operational dashboard
grafana · Nov 13, 2020 · 077b806 · 077b806
1 parent fd43f2d
commit 077b806
Show file tree

Hide file tree

Showing 6 changed files with 139 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,7 @@
 ## master / unreleased
 
 * [CHANGE] From path.Join to filepath.Join [#338](https://github.com/grafana/tempo/pull/338)
+* [ENHANCEMENT] Add tempodb_compaction_objects_combined metric. [#339](https://github.com/grafana/tempo/pull/339)
 * [BUGFIX] Frequent errors logged by compactor regarding meta not found [#327](https://github.com/grafana/tempo/pull/327)
 
 ## v0.3.0

diff --git a/go.mod b/go.mod
@@ -27,6 +27,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.11.1
 	github.com/prometheus/prometheus v1.8.2-0.20200722151933-4a8531a64b32
 	github.com/sirupsen/logrus v1.6.0

diff --git a/operations/tempo-mixin/tempo-operational.json b/operations/tempo-mixin/tempo-operational.json
@@ -4492,6 +4492,117 @@
         ],
         "title": "Vulture",
         "type": "row"
+      },
+      {
+        "collapsed": false,
+        "datasource": null,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 76
+        },
+        "id": 81,
+        "panels": [],
+        "title": "Compactor",
+        "type": "row"
+      },
+      {
+        "aliasColors": {},
+        "bars": false,
+        "dashLength": 10,
+        "dashes": false,
+        "datasource": "$ds",
+        "fieldConfig": {
+          "defaults": {
+            "custom": {}
+          },
+          "overrides": []
+        },
+        "fill": 1,
+        "fillGradient": 0,
+        "gridPos": {
+          "h": 8,
+          "w": 7,
+          "x": 0,
+          "y": 77
+        },
+        "hiddenSeries": false,
+        "id": 83,
+        "legend": {
+          "avg": false,
+          "current": false,
+          "max": false,
+          "min": false,
+          "show": false,
+          "total": false,
+          "values": false
+        },
+        "lines": true,
+        "linewidth": 1,
+        "nullPointMode": "null",
+        "options": {
+          "alertThreshold": true
+        },
+        "percentage": false,
+        "pluginVersion": "7.3.0-beta1",
+        "pointradius": 2,
+        "points": false,
+        "renderer": "flot",
+        "seriesOverrides": [],
+        "spaceLength": 10,
+        "stack": false,
+        "steppedLine": false,
+        "targets": [
+          {
+            "expr": "increase(tempodb_compaction_objects_combined_total{job=~\"$namespace/compactor\"}[$__rate_interval])",
+            "interval": "",
+            "legendFormat": "",
+            "refId": "A"
+          }
+        ],
+        "thresholds": [],
+        "timeFrom": null,
+        "timeRegions": [],
+        "timeShift": null,
+        "title": "Objects Combined",
+        "tooltip": {
+          "shared": true,
+          "sort": 0,
+          "value_type": "individual"
+        },
+        "type": "graph",
+        "xaxis": {
+          "buckets": null,
+          "mode": "time",
+          "name": null,
+          "show": true,
+          "values": []
+        },
+        "yaxes": [
+          {
+            "$$hashKey": "object:150",
+            "format": "short",
+            "label": null,
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true
+          },
+          {
+            "$$hashKey": "object:151",
+            "format": "short",
+            "label": null,
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true
+          }
+        ],
+        "yaxis": {
+          "align": false,
+          "alignLevel": null
+        }        
       }
     ],
     "refresh": "30s",

diff --git a/tempodb/compactor.go b/tempodb/compactor.go
@@ -32,6 +32,11 @@ var (
 		Name:      "compaction_errors_total",
 		Help:      "Total number of errors occurring during compaction.",
 	})
+	metricCompactionObjectsCombined = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "tempodb",
+		Name:      "compaction_objects_combined_total",
+		Help:      "Total number of objects combined during compaction.",
+	})
 )
 
 const (
@@ -154,6 +159,7 @@ func (rw *readerWriter) compact(blockMetas []*encoding.BlockMeta, tenantID strin
 			if bytes.Equal(currentID, lowestID) {
 				lowestObject = rw.compactorSharder.Combine(currentObject, lowestObject)
 				b.clear()
+				metricCompactionObjectsCombined.Inc()
 			} else if len(lowestID) == 0 || bytes.Compare(currentID, lowestID) == -1 {
 				lowestID = currentID
 				lowestObject = currentObject

diff --git a/tempodb/compactor_test.go b/tempodb/compactor_test.go
@@ -20,6 +20,9 @@ import (
 	"github.com/grafana/tempo/tempodb/encoding"
 	"github.com/grafana/tempo/tempodb/pool"
 	"github.com/grafana/tempo/tempodb/wal"
+	"github.com/prometheus/client_golang/prometheus"
+
+	dto "github.com/prometheus/client_model/go"
 )
 
 type mockSharder struct {
@@ -219,6 +222,9 @@ func TestSameIDCompaction(t *testing.T) {
 
 	rw := r.(*readerWriter)
 
+	combinedStart, err := GetCounterValue(metricCompactionObjectsCombined)
+	assert.NoError(t, err)
+
 	// poll
 	checkBlocklists(t, uuid.Nil, blockCount, 0, rw)
 
@@ -239,6 +245,19 @@ func TestSameIDCompaction(t *testing.T) {
 		records += meta.TotalObjects
 	}
 	assert.Equal(t, blockCount-blocksPerCompaction, records)
+
+	combinedFinish, err := GetCounterValue(metricCompactionObjectsCombined)
+	assert.NoError(t, err)
+	assert.Equal(t, float64(1), combinedFinish-combinedStart)
+}
+
+func GetCounterValue(metric prometheus.Counter) (float64, error) {
+	var m = &dto.Metric{}
+	err := metric.Write(m)
+	if err != nil {
+		return 0, err
+	}
+	return m.Counter.GetValue(), nil
 }
 
 func TestCompactionUpdatesBlocklist(t *testing.T) {

diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -877,6 +877,7 @@ github.com/prometheus/client_golang/prometheus/promauto
 github.com/prometheus/client_golang/prometheus/promhttp
 github.com/prometheus/client_golang/prometheus/push
 # github.com/prometheus/client_model v0.2.0
+## explicit
 github.com/prometheus/client_model/go
 # github.com/prometheus/common v0.11.1
 ## explicit