metrics-generator disable x-scope-orgid header append

CHANGELOG.md

@@ -29,6 +29,7 @@
 * [ENHANCEMENT] Add `target_info_excluded_dimensions` to user-config api [#2945](https://github.com/grafana/tempo/pull/2945) (@ie-pham)
 * [ENHANCEMENT] User-configurable overrides: add scope query parameter to return merged overrides for tenant [#2915](https://github.com/grafana/tempo/pull/2915) (@kvrhdn)
 * [ENHANCEMENT] Add histogram buckets to metrics-generator config in user-configurable overrides [#2928](https://github.com/grafana/tempo/pull/2928) (@mar4uk)
+* [ENHANCEMENT] added a metrics generator config option to enable/disable X-Scope-OrgID headers on remote write. [#2974](https://github.com/grafana/tempo/pull/2974) (@vineetjp)
 * [BUGFIX] Fix panic in metrics summary api [#2738](https://github.com/grafana/tempo/pull/2738) (@mdisibio)
 * [BUGFIX] Fix rare deadlock when uploading blocks to Azure Blob Storage [#2129](https://github.com/grafana/tempo/issues/2129) (@LasseHels)
 * [BUGFIX] Only search ingester blocks that fall within the request time range. [#2783](https://github.com/grafana/tempo/pull/2783) (@joe-elliott)

docs/sources/tempo/configuration/_index.md

@@ -376,6 +376,9 @@ metrics_generator:
         # How long to wait when flushing samples on shutdown
         [remote_write_flush_deadline: <duration> | default = 1m]
 
+        # Whether to add X-Scope-OrgID header in remote write requests
+        [remote_write_add_org_id_header: <bool> | default = true]
+
         # A list of remote write endpoints.
         # https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
         remote_write:

docs/sources/tempo/metrics-generator/_index.md

@@ -57,4 +57,4 @@ The `remote_write` endpoint is configurable and can be any [Prometheus-compatibl
 To learn more about the endpoint configuration, refer to the [Metrics-generator]({{< relref "../configuration#metrics-generator" >}}) section of the Tempo Configuration documentation.
 Writing interval can be controlled via `metrics_generator.registry.collection_interval`.
 
-When multi-tenancy is enabled, the metrics-generator forwards the `X-Scope-OrgID` header of the original request to the `remote_write` endpoint.
+When multi-tenancy is enabled, the metrics-generator forwards the `X-Scope-OrgID` header of the original request to the `remote_write` endpoint. This feature can be disabled by setting `remote_write_add_org_id_header` to false.

modules/generator/storage/config.go

@@ -17,6 +17,9 @@ type Config struct {
 	// How long to wait when flushing sample on shutdown
 	RemoteWriteFlushDeadline time.Duration `yaml:"remote_write_flush_deadline"`
 
+	// Add X-Scope-OrgID header in remote write requests
+	RemoteWriteAddOrgIDHeader bool `yaml:"remote_write_add_org_id_header,omitempty"`
+
 	// Prometheus remote write config
 	// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
 	RemoteWrite []prometheus_config.RemoteWriteConfig `yaml:"remote_write,omitempty"`
@@ -26,6 +29,8 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) {
 	cfg.Wal = agentDefaultOptions()
 
 	cfg.RemoteWriteFlushDeadline = time.Minute
+
+	cfg.RemoteWriteAddOrgIDHeader = true
 }
 
 // agentOptions is a copy of agent.Options but with yaml struct tags. Refer to agent.Options for

modules/generator/storage/config_test.go

@@ -41,9 +41,10 @@ remote_write:
 	}
 
 	expectedCfg := Config{
-		Path:                     "/var/wal/tempo",
-		Wal:                      walCfg,
-		RemoteWriteFlushDeadline: 5 * time.Minute,
+		Path:                      "/var/wal/tempo",
+		Wal:                       walCfg,
+		RemoteWriteFlushDeadline:  5 * time.Minute,
+		RemoteWriteAddOrgIDHeader: true,
 		RemoteWrite: []prometheus_config.RemoteWriteConfig{
 			remoteWriteConfig,
 		},

modules/generator/storage/config_util.go

@@ -12,16 +12,16 @@ import (
 )
 
 // generateTenantRemoteWriteConfigs creates a copy of the remote write configurations with the
-// X-Scope-OrgID header present for the given tenant, unless Tempo is run in single tenant mode.
-func generateTenantRemoteWriteConfigs(originalCfgs []prometheus_config.RemoteWriteConfig, tenant string, logger log.Logger) []*prometheus_config.RemoteWriteConfig {
+// X-Scope-OrgID header present for the given tenant, unless Tempo is run in single tenant mode or instructed not to add X-Scope-OrgID header.
+func generateTenantRemoteWriteConfigs(originalCfgs []prometheus_config.RemoteWriteConfig, tenant string, addOrgIDHeader bool, logger log.Logger) []*prometheus_config.RemoteWriteConfig {
 	var cloneCfgs []*prometheus_config.RemoteWriteConfig
 
 	for _, originalCfg := range originalCfgs {
 		cloneCfg := &prometheus_config.RemoteWriteConfig{}
 		*cloneCfg = originalCfg
 
 		// Inject/overwrite X-Scope-OrgID header in multi-tenant setups
-		if tenant != util.FakeTenantID {
+		if tenant != util.FakeTenantID && addOrgIDHeader {
 			// Copy headers so we can modify them
 			cloneCfg.Headers = copyMap(cloneCfg.Headers)
 

modules/generator/storage/config_util_test.go

@@ -30,7 +30,9 @@ func Test_generateTenantRemoteWriteConfigs(t *testing.T) {
 		},
 	}
 
-	result := generateTenantRemoteWriteConfigs(original, "my-tenant", logger)
+	addOrgIDHeader := true
+
+	result := generateTenantRemoteWriteConfigs(original, "my-tenant", addOrgIDHeader, logger)
 
 	assert.Equal(t, original[0].URL, result[0].URL)
 	assert.Equal(t, map[string]string{}, original[0].Headers, "Original headers have been modified")
@@ -57,7 +59,9 @@ func Test_generateTenantRemoteWriteConfigs_singleTenant(t *testing.T) {
 		},
 	}
 
-	result := generateTenantRemoteWriteConfigs(original, util.FakeTenantID, logger)
+	addOrgIDHeader := true
+
+	result := generateTenantRemoteWriteConfigs(original, util.FakeTenantID, addOrgIDHeader, logger)
 
 	assert.Equal(t, original[0].URL, result[0].URL)
 
@@ -72,6 +76,34 @@ func Test_generateTenantRemoteWriteConfigs_singleTenant(t *testing.T) {
 	assert.Equal(t, map[string]string{"x-scope-orgid": "my-custom-tenant-id"}, result[1].Headers)
 }
 
+func Test_generateTenantRemoteWriteConfigs_addOrgIDHeader(t *testing.T) {
+	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout))
+
+	original := []prometheus_config.RemoteWriteConfig{
+		{
+			URL:     &prometheus_common_config.URL{URL: urlMustParse("http://prometheus-1/api/prom/push")},
+			Headers: map[string]string{},
+		},
+		{
+			URL: &prometheus_common_config.URL{URL: urlMustParse("http://prometheus-2/api/prom/push")},
+			Headers: map[string]string{
+				"foo":           "bar",
+				"x-scope-orgid": "fake-tenant",
+			},
+		},
+	}
+
+	addOrgIDHeader := false
+
+	result := generateTenantRemoteWriteConfigs(original, "my-tenant", addOrgIDHeader, logger)
+
+	assert.Equal(t, original[0].URL, result[0].URL)
+	assert.Empty(t, original[0].Headers, "X-Scope-OrgID header is not added")
+
+	assert.Equal(t, original[1].URL, result[1].URL)
+	assert.Equal(t, map[string]string{"foo": "bar", "x-scope-orgid": "fake-tenant"}, result[1].Headers, "Original headers not modified")
+}
+
 func Test_copyMap(t *testing.T) {
 	original := map[string]string{
 		"k1": "v1",

modules/generator/storage/instance.go

@@ -59,7 +59,7 @@ func New(cfg *Config, tenant string, reg prometheus.Registerer, logger log.Logge
 	remoteStorage := remote.NewStorage(log.With(logger, "component", "remote"), reg, startTimeCallback, walDir, cfg.RemoteWriteFlushDeadline, &noopScrapeManager{})
 
 	remoteStorageConfig := &prometheus_config.Config{
-		RemoteWriteConfigs: generateTenantRemoteWriteConfigs(cfg.RemoteWrite, tenant, logger),
+		RemoteWriteConfigs: generateTenantRemoteWriteConfigs(cfg.RemoteWrite, tenant, cfg.RemoteWriteAddOrgIDHeader, logger),
 	}
 
 	err = remoteStorage.ApplyConfig(remoteStorageConfig)