Handle prefixes when listing blocks from S3 and GCS

CHANGELOG.md

@@ -6,6 +6,7 @@
 * [BUGFIX] Fix metrics query results when series contain empty strings or nil values [#3429](https://github.com/grafana/tempo/issues/3429) (@mdisibio)
 * [BUGFIX] Return unfiltered results when a bad TraceQL query is provided in autocomplete. [#3426](https://github.com/grafana/tempo/pull/3426) (@mapno)
 * [BUGFIX] Correctly handle 429s in GRPC search streaming. [#3469](https://github.com/grafana/tempo/pull/3469) (@joe-ellitot)
+* [BUGFIX] Fix compaction/retention in AWS S3 and GCS when a prefix is configured. [#3465](https://github.com/grafana/tempo/issues/3465) (@bpfoster)
 
 ## v2.4.0
 

tempodb/backend/azure/azure_test.go

@@ -252,6 +252,150 @@ func TestObjectWithPrefix(t *testing.T) {
 	}
 }
 
+func TestListBlocksWithPrefix(t *testing.T) {
+	tests := []struct {
+		name        string
+		prefix      string
+		objectName  string
+		keyPath     backend.KeyPath
+		httpHandler func(t *testing.T) http.HandlerFunc
+	}{
+		{
+			name:    "with prefix",
+			prefix:  "a/b/c/",
+			keyPath: backend.KeyPath{"test"},
+			httpHandler: func(t *testing.T) http.HandlerFunc {
+				return func(w http.ResponseWriter, r *http.Request) {
+					if r.Method == "GET" {
+						_, _ = w.Write([]byte(`
+						<?xml version="1.0" encoding="utf-8"?>
+						<EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/"  ContainerName="mycontainer">
+						  <Prefix>a/b/c/</Prefix>
+						  <MaxResults>100</MaxResults>
+						  <Blobs>
+							<Blob>
+							  <Name>a/b/c/single-tenant/00000000-0000-0000-0000-000000000000/meta.json</Name>
+							  <Url>https://myaccount.blob.core.windows.net/mycontainer/a/b/c/single-tenant/00000000-0000-0000-0000-000000000000/meta.json</Url>
+							  <Properties>
+								<Last-Modified>Fri, 01 Mar 2024 00:00:00 GMT</Last-Modified>
+								<Etag>0x8CBFF45D8A29A19</Etag>
+								<Content-Length>100</Content-Length>
+								<Content-Type>text/html</Content-Type>
+								<Content-Encoding />
+								<Content-Language>en-US</Content-Language>
+								<Content-MD5 />
+								<Cache-Control>no-cache</Cache-Control>
+								<BlobType>BlockBlob</BlobType>
+								<LeaseStatus>unlocked</LeaseStatus>
+							  </Properties>
+							</Blob>
+
+							<Blob>
+							  <Name>a/b/c/single-tenant/00000000-0000-0000-0000-000000000001/meta.compacted.json</Name>
+							  <Url>https://myaccount.blob.core.windows.net/mycontainer/a/b/c/single-tenant/00000000-0000-0000-0000-000000000001/meta.compacted.json</Url>
+							  <Properties>
+								<Last-Modified>Fri, 01 Mar 2024 00:00:00 GMT</Last-Modified>
+								<Etag>0x8CBFF45D8A29A19</Etag>
+								<Content-Length>100</Content-Length>
+								<Content-Type>text/html</Content-Type>
+								<Content-Encoding />
+								<Content-Language>en-US</Content-Language>
+								<Content-MD5 />
+								<Cache-Control>no-cache</Cache-Control>
+								<BlobType>BlockBlob</BlobType>
+								<LeaseStatus>unlocked</LeaseStatus>
+							  </Properties>
+							</Blob>
+						  </Blobs>
+						  <NextMarker />
+						</EnumerationResults>
+						`))
+						return
+					}
+				}
+			},
+		},
+		{
+			name:    "without prefix",
+			prefix:  "",
+			keyPath: backend.KeyPath{"test"},
+			httpHandler: func(t *testing.T) http.HandlerFunc {
+				return func(w http.ResponseWriter, r *http.Request) {
+					if r.Method == "GET" {
+						_, _ = w.Write([]byte(`
+						<?xml version="1.0" encoding="utf-8"?>
+						<EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/"  ContainerName="mycontainer">
+						  <Prefix></Prefix>
+						  <MaxResults>100</MaxResults>
+						  <Blobs>
+							<Blob>
+							  <Name>single-tenant/00000000-0000-0000-0000-000000000000/meta.json</Name>
+							  <Url>https://myaccount.blob.core.windows.net/mycontainer/single-tenant/00000000-0000-0000-0000-000000000000/meta.json</Url>
+							  <Properties>
+								<Last-Modified>Fri, 01 Mar 2024 00:00:00 GMT</Last-Modified>
+								<Etag>0x8CBFF45D8A29A19</Etag>
+								<Content-Length>100</Content-Length>
+								<Content-Type>text/html</Content-Type>
+								<Content-Encoding />
+								<Content-Language>en-US</Content-Language>
+								<Content-MD5 />
+								<Cache-Control>no-cache</Cache-Control>
+								<BlobType>BlockBlob</BlobType>
+								<LeaseStatus>unlocked</LeaseStatus>
+							  </Properties>
+							</Blob>
+
+							<Blob>
+							  <Name>single-tenant/00000000-0000-0000-0000-000000000001/meta.compacted.json</Name>
+							  <Url>https://myaccount.blob.core.windows.net/mycontainer/single-tenant/00000000-0000-0000-0000-000000000001/meta.compacted.json</Url>
+							  <Properties>
+								<Last-Modified>Fri, 01 Mar 2024 00:00:00 GMT</Last-Modified>
+								<Etag>0x8CBFF45D8A29A19</Etag>
+								<Content-Length>100</Content-Length>
+								<Content-Type>text/html</Content-Type>
+								<Content-Encoding />
+								<Content-Language>en-US</Content-Language>
+								<Content-MD5 />
+								<Cache-Control>no-cache</Cache-Control>
+								<BlobType>BlockBlob</BlobType>
+								<LeaseStatus>unlocked</LeaseStatus>
+							  </Properties>
+							</Blob>
+						  </Blobs>
+                          <NextMarker />
+						</EnumerationResults>
+						`))
+						return
+					}
+				}
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			server := testServer(t, tc.httpHandler(t))
+			r, _, _, err := New(&config.Config{
+				StorageAccountName: "testing_account",
+				StorageAccountKey:  flagext.SecretWithValue("YQo="),
+				MaxBuffers:         3,
+				BufferSize:         1000,
+				ContainerName:      "blerg",
+				Prefix:             tc.prefix,
+				Endpoint:           server.URL[7:], // [7:] -> strip http://,
+			})
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			blockIDs, compactedBlockIDs, err2 := r.ListBlocks(ctx, "single-tenant")
+			assert.NoError(t, err2)
+
+			assert.Equal(t, 1, len(blockIDs))
+			assert.Equal(t, 1, len(compactedBlockIDs))
+		})
+	}
+}
+
 func testServer(t *testing.T, httpHandler http.HandlerFunc) *httptest.Server {
 	t.Helper()
 	assert.NotNil(t, httpHandler)

tempodb/backend/gcs/gcs.go

@@ -253,7 +253,7 @@ func (rw *readerWriter) ListBlocks(ctx context.Context, tenant string) ([]uuid.U
 					return
 				}
 
-				parts = strings.Split(attrs.Name, "/")
+				parts = strings.Split(strings.TrimPrefix(attrs.Name, rw.cfg.Prefix), "/")
 				// ie: <tenant>/<blockID>/meta.json
 				if len(parts) != 3 {
 					continue

tempodb/backend/gcs/gcs_test.go

@@ -299,6 +299,110 @@ func TestObjectWithPrefix(t *testing.T) {
 	}
 }
 
+func TestListBlocksWithPrefix(t *testing.T) {
+	tests := []struct {
+		name        string
+		prefix      string
+		objectName  string
+		keyPath     backend.KeyPath
+		httpHandler func(t *testing.T) http.HandlerFunc
+	}{
+		{
+			name:    "with prefix",
+			prefix:  "a/b/c/",
+			keyPath: backend.KeyPath{"test"},
+			httpHandler: func(t *testing.T) http.HandlerFunc {
+				return func(w http.ResponseWriter, r *http.Request) {
+					if r.Method == "GET" {
+						_, _ = w.Write([]byte(`
+						{
+							"kind": "storage#objects",
+							"items": [{
+								"kind": "storage#object",
+								"id": "1",
+								"name": "a/b/c/single-tenant/00000000-0000-0000-0000-000000000000/meta.json",
+								"bucket": "blerg",
+								"storageClass": "STANDARD",
+								"size": "1024",
+								"timeCreated": "2024-03-01T00:00:00.000Z",
+								"updated": "2024-03-01T00:00:00.000Z"
+							}, {
+								"kind": "storage#object",
+								"id": "2",
+								"name": "a/b/c/single-tenant/00000000-0000-0000-0000-000000000000/meta.compacted.json",
+								"bucket": "blerg",
+								"storageClass": "STANDARD",
+								"size": "1024",
+								"timeCreated": "2024-03-01T00:00:00.000Z",
+								"updated": "2024-03-01T00:00:00.000Z"
+							}]
+						}
+						`))
+						return
+					}
+				}
+			},
+		},
+		{
+			name:    "without prefix",
+			prefix:  "",
+			keyPath: backend.KeyPath{"test"},
+			httpHandler: func(t *testing.T) http.HandlerFunc {
+				return func(w http.ResponseWriter, r *http.Request) {
+					if r.Method == "GET" {
+						_, _ = w.Write([]byte(`
+						{
+							"kind": "storage#objects",
+							"items": [{
+								"kind": "storage#object",
+								"id": "1",
+								"name": "single-tenant/00000000-0000-0000-0000-000000000000/meta.json",
+								"bucket": "blerg",
+								"storageClass": "STANDARD",
+								"size": "1024",
+								"timeCreated": "2024-03-01T00:00:00.000Z",
+								"updated": "2024-03-01T00:00:00.000Z"
+							}, {
+								"kind": "storage#object",
+								"id": "2",
+								"name": "single-tenant/00000000-0000-0000-0000-000000000000/meta.compacted.json",
+								"bucket": "blerg",
+								"storageClass": "STANDARD",
+								"size": "1024",
+								"timeCreated": "2024-03-01T00:00:00.000Z",
+								"updated": "2024-03-01T00:00:00.000Z"
+							}]
+						}
+						`))
+						return
+					}
+				}
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			server := testServer(t, tc.httpHandler(t))
+			r, _, _, err := New(&Config{
+				BucketName:            "blerg",
+				Endpoint:              server.URL,
+				Insecure:              true,
+				Prefix:                tc.prefix,
+				ListBlocksConcurrency: 1,
+			})
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			blockIDs, compactedBlockIDs, err := r.ListBlocks(ctx, "single-tenant")
+			assert.NoError(t, err)
+
+			assert.Equal(t, 1, len(blockIDs))
+			assert.Equal(t, 1, len(compactedBlockIDs))
+		})
+	}
+}
+
 func testServer(t *testing.T, httpHandler http.HandlerFunc) *httptest.Server {
 	t.Helper()
 	assert.NotNil(t, httpHandler)

tempodb/backend/s3/s3.go

@@ -334,7 +334,7 @@ func (rw *readerWriter) ListBlocks(
 
 				for _, c := range res.Contents {
 					// i.e: <tenantID/<blockID>/meta
-					parts := strings.Split(c.Key, "/")
+					parts := strings.Split(strings.TrimPrefix(c.Key, rw.cfg.Prefix), "/")
 					if len(parts) != 3 {
 						continue
 					}