add zone aware replication for ingesters

grafana · Dec 5, 2022 · 7687060 · 7687060
1 parent bf2bf6b
commit 7687060
Show file tree

Hide file tree

Showing 18 changed files with 333 additions and 95 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,12 @@
 ## main / unreleased
+* [ENHANCEMENT] Add zone awareness replication for ingesters. [#1936](https://github.com/grafana/tempo/pull/1936) (@manohar-koukuntla)
+```
+# use the following fields in _config field of jsonnet config, to enable zone aware ingester
+    multi_zone_ingester_enabled: false,
+    multi_zone_ingester_migration_enabled: false,
+    multi_zone_ingester_replicas: 0,
+    multi_zone_ingester_max_unavailable: 25,
+```
 * [BUGFIX] Stop distributors on Otel receiver fatal error[#1887](https://github.com/grafana/tempo/pull/1887) (@rdooley)
 * [CHANGE] **BREAKING CHANGE** Use snake case on Azure Storage config [#1879](https://github.com/grafana/tempo/issues/1879) (@faustodavid)
 Example of using snake case on Azure Storage config:

diff --git a/operations/jsonnet-compiled/Deployment-compactor.yaml b/operations/jsonnet-compiled/Deployment-compactor.yaml
@@ -25,9 +25,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=compactor
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=compactor
         env:
         - name: GOMEMLIMIT
           value: 5GiB

diff --git a/operations/jsonnet-compiled/Deployment-distributor.yaml b/operations/jsonnet-compiled/Deployment-distributor.yaml
@@ -27,9 +27,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=distributor
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=distributor
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: distributor

diff --git a/operations/jsonnet-compiled/Deployment-metrics-generator.yaml b/operations/jsonnet-compiled/Deployment-metrics-generator.yaml
@@ -27,9 +27,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=metrics-generator
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=metrics-generator
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: metrics-generator

diff --git a/operations/jsonnet-compiled/Deployment-querier.yaml b/operations/jsonnet-compiled/Deployment-querier.yaml
@@ -27,9 +27,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=querier
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=querier
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: querier

diff --git a/operations/jsonnet-compiled/Deployment-query-frontend.yaml b/operations/jsonnet-compiled/Deployment-query-frontend.yaml
@@ -25,9 +25,9 @@ spec:
     spec:
       containers:
       - args:
-        - -target=query-frontend
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=query-frontend
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: query-frontend

diff --git a/operations/jsonnet-compiled/StatefulSet-ingester.yaml b/operations/jsonnet-compiled/StatefulSet-ingester.yaml
@@ -30,9 +30,9 @@ spec:
             topologyKey: kubernetes.io/hostname
       containers:
       - args:
-        - -target=ingester
         - -config.file=/conf/tempo.yaml
         - -mem-ballast-size-mbs=1024
+        - -target=ingester
         image: grafana/tempo:latest
         imagePullPolicy: IfNotPresent
         name: ingester
@@ -59,6 +59,7 @@ spec:
           name: ingester-data
         - mountPath: /overrides
           name: overrides
+      terminationGracePeriodSeconds: 1200
       volumes:
       - configMap:
           name: tempo-ingester

diff --git a/operations/jsonnet-compiled/util/jsonnetfile.lock.json b/operations/jsonnet-compiled/util/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
           "subdir": "ksonnet-util"
         }
       },
-      "version": "32101ab7cf57915901a75af9c2cf2442416677f6",
+      "version": "3fa03d69faac3d57516b8f4dd8671bfa16de7cf2",
       "sum": "/pkNOLhRqvQoPA0yYdUuJvpPHqhkCLauAUMD2ZHMIkE="
     },
     {
@@ -18,7 +18,7 @@
           "subdir": "memcached"
         }
       },
-      "version": "32101ab7cf57915901a75af9c2cf2442416677f6",
+      "version": "3fa03d69faac3d57516b8f4dd8671bfa16de7cf2",
       "sum": "SWywAq4U0MRPMbASU0Ez8O9ArRNeoZzb75sEuReueow="
     },
     {

diff --git a/operations/jsonnet/microservices/compactor.libsonnet b/operations/jsonnet/microservices/compactor.libsonnet
@@ -12,24 +12,25 @@
   local tempo_data_volume = 'tempo-data',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_compactor_ports:: [ containerPort.new('prom-metrics', $._config.port)],
+  tempo_compactor_args:: {
+                             target: target_name,
+                             'config.file': '/conf/tempo.yaml',
+                             'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+                            },
+
   tempo_compactor_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_compactor_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_compactor_args)) +
     (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
       volumeMount.new(tempo_overrides_config_volume, '/overrides'),
     ]) +
     $.util.withResources($._config.compactor.resources) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}) +
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}) +
     (if $._config.compactor.resources.limits.memory != null then container.withEnvMixin([envVar.new('GOMEMLIMIT', $._config.compactor.resources.limits.memory + 'B')]) else {}),
 
   tempo_compactor_deployment:

diff --git a/operations/jsonnet/microservices/config.libsonnet b/operations/jsonnet/microservices/config.libsonnet
@@ -3,14 +3,21 @@
     tempo: 'grafana/tempo:latest',
     tempo_query: 'grafana/tempo-query:latest',
     tempo_vulture: 'grafana/tempo-vulture:latest',
+    rollout_operator: 'grafana/rollout-operator:v0.1.1',
     memcached: 'memcached:1.6.9-alpine',
     memcachedExporter: 'prom/memcached-exporter:v0.6.0',
   },
 
   _config+:: {
     gossip_member_label: 'tempo-gossip-member',
+    // Labels that service selectors should not use
+    service_ignored_labels:: [self.gossip_member_label],
+
     variables_expansion: false,
     variables_expansion_env_mixin: null,
+    node_selector: null,
+    ingester_allow_multiple_replicas_on_same_node: false,
+
     compactor: {
       replicas: 1,
       resources: {

diff --git a/operations/jsonnet/microservices/distributor.libsonnet b/operations/jsonnet/microservices/distributor.libsonnet
@@ -11,24 +11,25 @@
   local tempo_config_volume = 'tempo-conf',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_distributor_ports:: [ containerPort.new('prom-metrics', $._config.port)],
+  tempo_distributor_args:: {
+                              target: target_name,
+                              'config.file': '/conf/tempo.yaml',
+                              'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+                             },
+
   tempo_distributor_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_distributor_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_distributor_args)) +
     (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
       volumeMount.new(tempo_overrides_config_volume, '/overrides'),
     ]) +
     $.util.withResources($._config.distributor.resources) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}),
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}),
 
   tempo_distributor_deployment:
     deployment.new(target_name,

diff --git a/operations/jsonnet/microservices/frontend.libsonnet b/operations/jsonnet/microservices/frontend.libsonnet
@@ -14,24 +14,25 @@
   local tempo_data_volume = 'tempo-data',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_query_frontend_ports:: [ containerPort.new('prom-metrics', $._config.port)],
+  tempo_query_frontend_args:: {
+                                target: target_name,
+                                'config.file': '/conf/tempo.yaml',
+                                'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+                               },
+
   tempo_query_frontend_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_query_frontend_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_query_frontend_args)) +
     (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
       volumeMount.new(tempo_overrides_config_volume, '/overrides'),
     ]) +
     $.util.withResources($._config.query_frontend.resources) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}),
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}),
 
   tempo_query_container::
     container.new('tempo-query', $._images.tempo_query) +

diff --git a/operations/jsonnet/microservices/generator.libsonnet b/operations/jsonnet/microservices/generator.libsonnet
@@ -12,16 +12,17 @@
   local tempo_generator_wal_volume = 'metrics-generator-wal-data',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_metrics_generator_ports:: [ containerPort.new('prom-metrics', $._config.port)],
+  tempo_metrics_generator_args:: {
+                                  target: target_name,
+                                  'config.file': '/conf/tempo.yaml',
+                                  'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+                                 },
+
   tempo_metrics_generator_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_metrics_generator_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_metrics_generator_args)) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
       volumeMount.new(tempo_generator_wal_volume, $.tempo_metrics_generator_config.metrics_generator.storage.path),
@@ -32,7 +33,7 @@
     container.mixin.resources.withRequestsMixin({ 'ephemeral-storage': $._config.metrics_generator.ephemeral_storage_request_size }) +
     container.mixin.resources.withLimitsMixin({ 'ephemeral-storage': $._config.metrics_generator.ephemeral_storage_limit_size }) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}),
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}),
 
   tempo_metrics_generator_deployment:
     deployment.new(

diff --git a/operations/jsonnet/microservices/ingester.libsonnet b/operations/jsonnet/microservices/ingester.libsonnet
@@ -14,6 +14,13 @@
   local tempo_data_volume = 'ingester-data',
   local tempo_overrides_config_volume = 'overrides',
 
+  tempo_ingester_ports:: [ containerPort.new('prom-metrics', $._config.port)],
+  tempo_ingester_args:: {
+                             target: target_name,
+                             'config.file': '/conf/tempo.yaml',
+                             'mem-ballast-size-mbs': $._config.ballast_size_mbs,
+                            },
+
   tempo_ingester_pvc::
     pvc.new()
     + pvc.mixin.metadata.withName(tempo_data_volume)
@@ -25,14 +32,8 @@
 
   tempo_ingester_container::
     container.new(target_name, $._images.tempo) +
-    container.withPorts([
-      containerPort.new('prom-metrics', $._config.port),
-    ]) +
-    container.withArgs([
-      '-target=' + target_name,
-      '-config.file=/conf/tempo.yaml',
-      '-mem-ballast-size-mbs=' + $._config.ballast_size_mbs,
-    ]) +
+    container.withPorts($.tempo_ingester_ports) +
+    container.withArgs($.util.mapToFlags($.tempo_ingester_args)) +
     (if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) +
     container.withVolumeMounts([
       volumeMount.new(tempo_config_volume, '/conf'),
@@ -41,45 +42,38 @@
     ]) +
     $.util.withResources($._config.ingester.resources) +
     $.util.readinessProbe +
-    (if $._config.variables_expansion then container.withArgsMixin(['--config.expand-env=true']) else {}),
+    (if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}),
 
-  tempo_ingester_statefulset:
-    statefulset.new(
-      target_name,
-      $._config.ingester.replicas,
-      self.tempo_ingester_container,
-      self.tempo_ingester_pvc,
-      {
-        app: target_name,
-        [$._config.gossip_member_label]: 'true',
-      },
-    )
-    + k.util.antiAffinityStatefulSet
-    + statefulset.mixin.spec.withServiceName(target_name)
-    + statefulset.mixin.spec.template.metadata.withAnnotations({
-      config_hash: std.md5(std.toString($.tempo_ingester_configmap.data['tempo.yaml'])),
-    })
-    + statefulset.mixin.spec.template.spec.withVolumes([
-      volume.fromConfigMap(tempo_config_volume, $.tempo_ingester_configmap.metadata.name),
-      volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name),
-    ])
-    + statefulset.mixin.spec.withPodManagementPolicy('Parallel'),
+  newIngesterStatefulSet(name, container, with_anti_affinity=true)::
+   statefulset.new(
+         name,
+         3,
+         container,
+         self.tempo_ingester_pvc,
+         {
+           app: target_name,
+           [$._config.gossip_member_label]: 'true',
+         },
+       )
+       + k.util.antiAffinityStatefulSet
+       + statefulset.mixin.spec.withServiceName(target_name)
+       + statefulset.mixin.spec.template.metadata.withAnnotations({
+         config_hash: std.md5(std.toString($.tempo_ingester_configmap.data['tempo.yaml'])),
+       })
+       + statefulset.mixin.spec.template.spec.withVolumes([
+         volume.fromConfigMap(tempo_config_volume, $.tempo_ingester_configmap.metadata.name),
+         volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name),
+       ]) +
+       statefulset.mixin.spec.withPodManagementPolicy('Parallel')+
+       statefulset.mixin.spec.template.spec.withTerminationGracePeriodSeconds(1200) +
+       $.util.podPriority('high') +
+       (if with_anti_affinity then $.util.antiAffinity else {}),
+
+  tempo_ingester_statefulset: $.newIngesterStatefulSet(target_name, self.tempo_ingester_container) + statefulset.mixin.spec.withReplicas($._config.ingester.replicas),
 
   tempo_ingester_service:
     k.util.serviceFor($.tempo_ingester_statefulset),
 
-  gossip_ring_service:
-    service.new(
-      'gossip-ring',  // name
-      {
-        [$._config.gossip_member_label]: 'true',
-      },
-      [
-        servicePort.newNamed('gossip-ring', $._config.gossip_ring_port, $._config.gossip_ring_port) +
-        servicePort.withProtocol('TCP'),
-      ],
-    ) + service.mixin.spec.withClusterIp('None'),  // headless service
-
   local podDisruptionBudget = k.policy.v1beta1.podDisruptionBudget,
   ingester_pdb:
     podDisruptionBudget.new() +

diff --git a/operations/jsonnet/microservices/memberlist.libsonnet b/operations/jsonnet/microservices/memberlist.libsonnet
@@ -0,0 +1,18 @@
+{
+  local k = import 'ksonnet-util/kausal.libsonnet',
+  local service = $.core.v1.service,
+  local servicePort = $.core.v1.servicePort,
+
+  // Headless service (= no assigned IP, DNS returns all targets instead) pointing to gossip network members.
+  gossip_ring_service:
+    service.new(
+      'gossip-ring',  // name
+      {
+        [$._config.gossip_member_label]: 'true',
+      },
+      [
+        servicePort.newNamed('gossip-ring', $._config.gossip_ring_port, $._config.gossip_ring_port) +
+        servicePort.withProtocol('TCP'),
+      ],
+    ) + service.mixin.spec.withClusterIp('None')
+}