Delete the index for empty tenants (#2678)

* Delete the index for empty tenants * Test the writer calls to Delete on RawWriter * Run make fmt * Spell correctly
grafana · Jul 20, 2023 · c869872 · c869872
1 parent 6f1c659
commit c869872
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 11 deletions.
diff --git a/tempodb/backend/mocks.go b/tempodb/backend/mocks.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"strings"
 
 	tempo_io "github.com/grafana/tempo/pkg/io"
 
@@ -55,6 +56,7 @@ type MockRawWriter struct {
 	writeBuffer       []byte
 	appendBuffer      []byte
 	closeAppendCalled bool
+	deleteCalls       map[string]map[string]int
 }
 
 func (m *MockRawWriter) Write(_ context.Context, _ string, _ KeyPath, data io.Reader, size int64, _ bool) error {
@@ -73,7 +75,20 @@ func (m *MockRawWriter) CloseAppend(context.Context, AppendTracker) error {
 	return nil
 }
 
-func (m *MockRawWriter) Delete(context.Context, string, KeyPath) error {
+func (m *MockRawWriter) Delete(_ context.Context, name string, keypath KeyPath) error {
+	if m.deleteCalls == nil {
+		m.deleteCalls = make(map[string]map[string]int)
+	}
+
+	if _, ok := m.deleteCalls[name]; !ok {
+		m.deleteCalls[name] = make(map[string]int)
+	}
+
+	if _, ok := m.deleteCalls[name][strings.Join(keypath, "/")]; !ok {
+		m.deleteCalls[name][strings.Join(keypath, "/")] = 0
+	}
+
+	m.deleteCalls[name][strings.Join(keypath, "/")]++
 	return nil
 }
 

diff --git a/tempodb/backend/raw.go b/tempodb/backend/raw.go
@@ -89,6 +89,11 @@ func (w *writer) CloseAppend(ctx context.Context, tracker AppendTracker) error {
 }
 
 func (w *writer) WriteTenantIndex(ctx context.Context, tenantID string, meta []*BlockMeta, compactedMeta []*CompactedBlockMeta) error {
+	// If meta and compactedMeta are empty, call delete the tenant index.
+	if len(meta) == 0 && len(compactedMeta) == 0 {
+		return w.w.Delete(ctx, TenantIndexName, KeyPath([]string{tenantID}))
+	}
+
 	b := newTenantIndex(meta, compactedMeta)
 
 	indexBytes, err := b.marshal()

diff --git a/tempodb/backend/raw_test.go b/tempodb/backend/raw_test.go
@@ -50,6 +50,15 @@ func TestWriter(t *testing.T) {
 
 	assert.True(t, cmp.Equal([]*BlockMeta{meta}, idx.Meta))                  // using cmp.Equal to compare json datetimes
 	assert.True(t, cmp.Equal([]*CompactedBlockMeta(nil), idx.CompactedMeta)) // using cmp.Equal to compare json datetimes
+
+	// When there are no blocks, the tenant index should be deleted
+	assert.Equal(t, map[string]map[string]int(nil), w.(*writer).w.(*MockRawWriter).deleteCalls)
+
+	err = w.WriteTenantIndex(ctx, "test", nil, nil)
+	assert.NoError(t, err)
+
+	expectedDeleteMap := map[string]map[string]int{TenantIndexName: {"test": 1}}
+	assert.Equal(t, expectedDeleteMap, w.(*writer).w.(*MockRawWriter).deleteCalls)
 }
 
 func TestReader(t *testing.T) {

diff --git a/tempodb/blocklist/poller.go b/tempodb/blocklist/poller.go
@@ -11,12 +11,13 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/google/uuid"
-	"github.com/grafana/tempo/pkg/boundedwaitgroup"
-	"github.com/grafana/tempo/tempodb/backend"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"go.uber.org/atomic"
+
+	"github.com/grafana/tempo/pkg/boundedwaitgroup"
+	"github.com/grafana/tempo/tempodb/backend"
 )
 
 const (
@@ -154,16 +155,23 @@ func (p *Poller) Do() (PerTenant, PerTenantCompacted, error) {
 		}
 
 		consecutiveErrors = 0
-		metricBlocklistLength.WithLabelValues(tenantID).Set(float64(len(newBlockList)))
+		if len(newBlockList) > 0 || len(newCompactedBlockList) > 0 {
+			blocklist[tenantID] = newBlockList
+			compactedBlocklist[tenantID] = newCompactedBlockList
 
-		blocklist[tenantID] = newBlockList
-		compactedBlocklist[tenantID] = newCompactedBlockList
+			metricBlocklistLength.WithLabelValues(tenantID).Set(float64(len(newBlockList)))
 
-		backendMetaMetrics := sumTotalBackendMetaMetrics(newBlockList, newCompactedBlockList)
-		metricBackendObjects.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalObjects))
-		metricBackendObjects.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalObjects))
-		metricBackendBytes.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalBytes))
-		metricBackendBytes.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalBytes))
+			backendMetaMetrics := sumTotalBackendMetaMetrics(newBlockList, newCompactedBlockList)
+			metricBackendObjects.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalObjects))
+			metricBackendObjects.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalObjects))
+			metricBackendBytes.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalBytes))
+			metricBackendBytes.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalBytes))
+			continue
+		}
+		metricBlocklistLength.DeleteLabelValues(tenantID)
+		metricBackendObjects.DeleteLabelValues(tenantID)
+		metricBackendObjects.DeleteLabelValues(tenantID)
+		metricBackendBytes.DeleteLabelValues(tenantID)
 	}
 
 	return blocklist, compactedBlocklist, nil