From c8698728fc2297e5c458caa727f1e9146250ab14 Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Thu, 20 Jul 2023 20:02:05 +0000 Subject: [PATCH] Delete the index for empty tenants (#2678) * Delete the index for empty tenants * Test the writer calls to Delete on RawWriter * Run make fmt * Spell correctly --- tempodb/backend/mocks.go | 17 ++++++++++++++++- tempodb/backend/raw.go | 5 +++++ tempodb/backend/raw_test.go | 9 +++++++++ tempodb/blocklist/poller.go | 28 ++++++++++++++++++---------- 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/tempodb/backend/mocks.go b/tempodb/backend/mocks.go index 2bd4e3ff564..69ac96bdc3f 100644 --- a/tempodb/backend/mocks.go +++ b/tempodb/backend/mocks.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io" + "strings" tempo_io "github.com/grafana/tempo/pkg/io" @@ -55,6 +56,7 @@ type MockRawWriter struct { writeBuffer []byte appendBuffer []byte closeAppendCalled bool + deleteCalls map[string]map[string]int } func (m *MockRawWriter) Write(_ context.Context, _ string, _ KeyPath, data io.Reader, size int64, _ bool) error { @@ -73,7 +75,20 @@ func (m *MockRawWriter) CloseAppend(context.Context, AppendTracker) error { return nil } -func (m *MockRawWriter) Delete(context.Context, string, KeyPath) error { +func (m *MockRawWriter) Delete(_ context.Context, name string, keypath KeyPath) error { + if m.deleteCalls == nil { + m.deleteCalls = make(map[string]map[string]int) + } + + if _, ok := m.deleteCalls[name]; !ok { + m.deleteCalls[name] = make(map[string]int) + } + + if _, ok := m.deleteCalls[name][strings.Join(keypath, "/")]; !ok { + m.deleteCalls[name][strings.Join(keypath, "/")] = 0 + } + + m.deleteCalls[name][strings.Join(keypath, "/")]++ return nil } diff --git a/tempodb/backend/raw.go b/tempodb/backend/raw.go index 925ff0d122c..13a18a0222b 100644 --- a/tempodb/backend/raw.go +++ b/tempodb/backend/raw.go @@ -89,6 +89,11 @@ func (w *writer) CloseAppend(ctx context.Context, tracker AppendTracker) error { } func (w *writer) WriteTenantIndex(ctx context.Context, tenantID string, meta []*BlockMeta, compactedMeta []*CompactedBlockMeta) error { + // If meta and compactedMeta are empty, call delete the tenant index. + if len(meta) == 0 && len(compactedMeta) == 0 { + return w.w.Delete(ctx, TenantIndexName, KeyPath([]string{tenantID})) + } + b := newTenantIndex(meta, compactedMeta) indexBytes, err := b.marshal() diff --git a/tempodb/backend/raw_test.go b/tempodb/backend/raw_test.go index edf19f90c7d..b47f6c765dc 100644 --- a/tempodb/backend/raw_test.go +++ b/tempodb/backend/raw_test.go @@ -50,6 +50,15 @@ func TestWriter(t *testing.T) { assert.True(t, cmp.Equal([]*BlockMeta{meta}, idx.Meta)) // using cmp.Equal to compare json datetimes assert.True(t, cmp.Equal([]*CompactedBlockMeta(nil), idx.CompactedMeta)) // using cmp.Equal to compare json datetimes + + // When there are no blocks, the tenant index should be deleted + assert.Equal(t, map[string]map[string]int(nil), w.(*writer).w.(*MockRawWriter).deleteCalls) + + err = w.WriteTenantIndex(ctx, "test", nil, nil) + assert.NoError(t, err) + + expectedDeleteMap := map[string]map[string]int{TenantIndexName: {"test": 1}} + assert.Equal(t, expectedDeleteMap, w.(*writer).w.(*MockRawWriter).deleteCalls) } func TestReader(t *testing.T) { diff --git a/tempodb/blocklist/poller.go b/tempodb/blocklist/poller.go index 873493d4a04..e6fcc61b4a3 100644 --- a/tempodb/blocklist/poller.go +++ b/tempodb/blocklist/poller.go @@ -11,12 +11,13 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/google/uuid" - "github.com/grafana/tempo/pkg/boundedwaitgroup" - "github.com/grafana/tempo/tempodb/backend" opentracing "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "go.uber.org/atomic" + + "github.com/grafana/tempo/pkg/boundedwaitgroup" + "github.com/grafana/tempo/tempodb/backend" ) const ( @@ -154,16 +155,23 @@ func (p *Poller) Do() (PerTenant, PerTenantCompacted, error) { } consecutiveErrors = 0 - metricBlocklistLength.WithLabelValues(tenantID).Set(float64(len(newBlockList))) + if len(newBlockList) > 0 || len(newCompactedBlockList) > 0 { + blocklist[tenantID] = newBlockList + compactedBlocklist[tenantID] = newCompactedBlockList - blocklist[tenantID] = newBlockList - compactedBlocklist[tenantID] = newCompactedBlockList + metricBlocklistLength.WithLabelValues(tenantID).Set(float64(len(newBlockList))) - backendMetaMetrics := sumTotalBackendMetaMetrics(newBlockList, newCompactedBlockList) - metricBackendObjects.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalObjects)) - metricBackendObjects.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalObjects)) - metricBackendBytes.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalBytes)) - metricBackendBytes.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalBytes)) + backendMetaMetrics := sumTotalBackendMetaMetrics(newBlockList, newCompactedBlockList) + metricBackendObjects.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalObjects)) + metricBackendObjects.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalObjects)) + metricBackendBytes.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalBytes)) + metricBackendBytes.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalBytes)) + continue + } + metricBlocklistLength.DeleteLabelValues(tenantID) + metricBackendObjects.DeleteLabelValues(tenantID) + metricBackendObjects.DeleteLabelValues(tenantID) + metricBackendBytes.DeleteLabelValues(tenantID) } return blocklist, compactedBlocklist, nil