diff --git a/CHANGELOG.md b/CHANGELOG.md index 5afce48ecef..d9375f17d4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,13 @@ Additionally, default label `span_status` is renamed to `status_code`. * [CHANGE] Update to Go 1.18 [#1504](https://github.com/grafana/tempo/pull/1504) (@annanay25) * [CHANGE] Change tag/value lookups to return partial results when reaching response size limit instead of failing [#1517](https://github.com/grafana/tempo/pull/1517) (@mdisibio) * [CHANGE] Change search to be case-sensitive [#1547](https://github.com/grafana/tempo/issues/1547) (@mdisibio) +* [CHANGE] Relax Hedged request defaults for external endpoints. [#1566](https://github.com/grafana/tempo/pull/1566) (@joe-elliott) + ``` + querier: + search: + external_hedge_requests_at: 4s -> 8s + external_hedge_requests_up_to: 3 -> 2 + ``` * [FEATURE] metrics-generator: support per-tenant processor configuration [#1434](https://github.com/grafana/tempo/pull/1434) (@kvrhdn) * [FEATURE] Include rollout dashboard [#1456](https://github.com/grafana/tempo/pull/1456) (@zalegrala) * [FEATURE] Add SentinelPassword configuration for Redis [#1463](https://github.com/grafana/tempo/pull/1463) (@zalegrala) diff --git a/docs/tempo/website/configuration/_index.md b/docs/tempo/website/configuration/_index.md index 840e888daf5..0b98eb964ce 100644 --- a/docs/tempo/website/configuration/_index.md +++ b/docs/tempo/website/configuration/_index.md @@ -375,11 +375,11 @@ querier: # If set to a non-zero value a second request will be issued at the provided duration. Recommended to # be set to p99 of external search requests to reduce long tail latency. - # (default: 4s) + # (default: 8s) [external_hedge_requests_at: ] # The maximum number of requests to execute when hedging. Requires hedge_requests_at to be set. - # (default: 3) + # (default: 2) [external_hedge_requests_up_to: ] # config of the worker that connects to the query frontend diff --git a/docs/tempo/website/configuration/manifest.md b/docs/tempo/website/configuration/manifest.md index 93070f04d2f..79c462ed89f 100644 --- a/docs/tempo/website/configuration/manifest.md +++ b/docs/tempo/website/configuration/manifest.md @@ -156,8 +156,8 @@ querier: query_timeout: 30s prefer_self: 2 external_endpoints: [] - external_hedge_requests_at: 4s - external_hedge_requests_up_to: 3 + external_hedge_requests_at: 8s + external_hedge_requests_up_to: 2 query_timeout: 10s max_concurrent_queries: 5 frontend_worker: diff --git a/docs/tempo/website/operations/backend_search.md b/docs/tempo/website/operations/backend_search.md index a9f4853ab59..1f77dac31cf 100644 --- a/docs/tempo/website/operations/backend_search.md +++ b/docs/tempo/website/operations/backend_search.md @@ -53,10 +53,10 @@ querier: # If set to a non-zero value a second request will be issued at the provided duration. Recommended to # be set to p99 of search requests to reduce long tail latency. - external_hedge_requests_at: 5s + external_hedge_requests_at: 8s # The maximum number of requests to execute when hedging. Requires hedge_requests_at to be set. - external_hedge_requests_up_to: 3 + external_hedge_requests_up_to: 2 ``` ### Query frontend diff --git a/modules/querier/config.go b/modules/querier/config.go index e6f73b32144..d9af470569c 100644 --- a/modules/querier/config.go +++ b/modules/querier/config.go @@ -35,8 +35,8 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) cfg.ExtraQueryDelay = 0 cfg.MaxConcurrentQueries = 5 cfg.Search.PreferSelf = 2 - cfg.Search.HedgeRequestsAt = 4 * time.Second - cfg.Search.HedgeRequestsUpTo = 3 + cfg.Search.HedgeRequestsAt = 8 * time.Second + cfg.Search.HedgeRequestsUpTo = 2 cfg.Search.QueryTimeout = 30 * time.Second cfg.Worker = worker.Config{ MatchMaxConcurrency: true,