Relax External Endpoint Hedged Defaults (#1566)

* Change defaults Signed-off-by: Joe Elliott <number101010@gmail.com> * changelog Signed-off-by: Joe Elliott <number101010@gmail.com>
grafana · Jul 12, 2022 · b932d51 · b932d51
1 parent b9b4f03
commit b932d51
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,13 @@ Additionally, default label `span_status` is renamed to `status_code`.
 * [CHANGE] Update to Go 1.18 [#1504](https://github.com/grafana/tempo/pull/1504) (@annanay25)
 * [CHANGE] Change tag/value lookups to return partial results when reaching response size limit instead of failing [#1517](https://github.com/grafana/tempo/pull/1517) (@mdisibio)
 * [CHANGE] Change search to be case-sensitive [#1547](https://github.com/grafana/tempo/issues/1547) (@mdisibio)
+* [CHANGE] Relax Hedged request defaults for external endpoints. [#1566](https://github.com/grafana/tempo/pull/1566) (@joe-elliott)
+  ```
+  querier:
+    search:
+      external_hedge_requests_at: 4s    -> 8s
+      external_hedge_requests_up_to: 3  -> 2
+  ```
 * [FEATURE] metrics-generator: support per-tenant processor configuration [#1434](https://github.com/grafana/tempo/pull/1434) (@kvrhdn)
 * [FEATURE] Include rollout dashboard [#1456](https://github.com/grafana/tempo/pull/1456) (@zalegrala)
 * [FEATURE] Add SentinelPassword configuration for Redis [#1463](https://github.com/grafana/tempo/pull/1463) (@zalegrala)

diff --git a/docs/tempo/website/configuration/_index.md b/docs/tempo/website/configuration/_index.md
@@ -375,11 +375,11 @@ querier:
 
         # If set to a non-zero value a second request will be issued at the provided duration. Recommended to
         # be set to p99 of external search requests to reduce long tail latency.
-        # (default: 4s)
+        # (default: 8s)
         [external_hedge_requests_at: <duration>]
 
         # The maximum number of requests to execute when hedging. Requires hedge_requests_at to be set.
-        # (default: 3)
+        # (default: 2)
         [external_hedge_requests_up_to: <int>]
 
     # config of the worker that connects to the query frontend

diff --git a/docs/tempo/website/configuration/manifest.md b/docs/tempo/website/configuration/manifest.md
@@ -156,8 +156,8 @@ querier:
     query_timeout: 30s
     prefer_self: 2
     external_endpoints: []
-    external_hedge_requests_at: 4s
-    external_hedge_requests_up_to: 3
+    external_hedge_requests_at: 8s
+    external_hedge_requests_up_to: 2
   query_timeout: 10s
   max_concurrent_queries: 5
   frontend_worker:

diff --git a/docs/tempo/website/operations/backend_search.md b/docs/tempo/website/operations/backend_search.md
@@ -53,10 +53,10 @@ querier:
 
     # If set to a non-zero value a second request will be issued at the provided duration. Recommended to
     # be set to p99 of search requests to reduce long tail latency.
-    external_hedge_requests_at: 5s
+    external_hedge_requests_at: 8s
 
     # The maximum number of requests to execute when hedging. Requires hedge_requests_at to be set.
-    external_hedge_requests_up_to: 3
+    external_hedge_requests_up_to: 2
 ```
 
 ### Query frontend

diff --git a/modules/querier/config.go b/modules/querier/config.go
@@ -35,8 +35,8 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	cfg.ExtraQueryDelay = 0
 	cfg.MaxConcurrentQueries = 5
 	cfg.Search.PreferSelf = 2
-	cfg.Search.HedgeRequestsAt = 4 * time.Second
-	cfg.Search.HedgeRequestsUpTo = 3
+	cfg.Search.HedgeRequestsAt = 8 * time.Second
+	cfg.Search.HedgeRequestsUpTo = 2
 	cfg.Search.QueryTimeout = 30 * time.Second
 	cfg.Worker = worker.Config{
 		MatchMaxConcurrency:   true,