From b96040be4f18f508a34980abce5bfe17d17cf96b Mon Sep 17 00:00:00 2001 From: Suraj Nath <9503187+electron0zero@users.noreply.github.com> Date: Thu, 30 Mar 2023 23:13:58 +0530 Subject: [PATCH 1/5] Add tag and link to Tenants dashboard --- operations/tempo-mixin/dashboards/tempo-tenants.libsonnet | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/operations/tempo-mixin/dashboards/tempo-tenants.libsonnet b/operations/tempo-mixin/dashboards/tempo-tenants.libsonnet index b9c855557bb..80979e68e83 100644 --- a/operations/tempo-mixin/dashboards/tempo-tenants.libsonnet +++ b/operations/tempo-mixin/dashboards/tempo-tenants.libsonnet @@ -22,8 +22,7 @@ dashboard_utils { grafanaDashboards+: { 'tempo-tenants.json': $.dashboard('Tempo / Tenants') - .addTemplate('cluster', 'tempo_build_info', $._config.per_cluster_label) - .addTemplate('namespace', 'tempo_build_info{' + $._config.per_cluster_label + "=~'$cluster'}", 'namespace') + .addClusterSelectorTemplates() .addTemplate('tenant', 'tempodb_blocklist_length{%s}' % $.jobMatcher($._config.jobs.compactor), 'tenant') .addRow( g.row('Tenant info') From 4646ee454ce4a1c0dc8e11e284bbce1d3022f2cb Mon Sep 17 00:00:00 2001 From: Suraj Nath <9503187+electron0zero@users.noreply.github.com> Date: Thu, 30 Mar 2023 23:21:09 +0530 Subject: [PATCH 2/5] Add CPU and Memory request in Resources Dashboard --- .../dashboards/dashboard-utils.libsonnet | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/operations/tempo-mixin/dashboards/dashboard-utils.libsonnet b/operations/tempo-mixin/dashboards/dashboard-utils.libsonnet index 6a4fd5b2a7a..0dc403409c1 100644 --- a/operations/tempo-mixin/dashboards/dashboard-utils.libsonnet +++ b/operations/tempo-mixin/dashboards/dashboard-utils.libsonnet @@ -104,7 +104,8 @@ grafana { $.queryPanel([ 'sum by(pod) (rate(container_cpu_usage_seconds_total{%s,container=~"%s"}[$__interval]))' % [$.namespaceMatcher(), containerName], 'min(container_spec_cpu_quota{%s,container=~"%s"} / container_spec_cpu_period{%s,container=~"%s"})' % [$.namespaceMatcher(), containerName, $.namespaceMatcher(), containerName], - ], ['{{pod}}', 'limit']) + + 'min(kube_pod_container_resource_requests{%s,container=~"%s", resource="cpu"} > 0)' % [$.namespaceMatcher(), containerName], + ], ['{{pod}}', 'limit', 'request']) + { seriesOverrides: [ { @@ -112,6 +113,11 @@ grafana { color: '#E02F44', fill: 0, }, + { + alias: 'request', + color: '#FCE300', + fill: 0, + }, ], }, @@ -120,7 +126,8 @@ grafana { $.queryPanel([ 'sum by(pod) (container_memory_working_set_bytes{%s,container=~"%s"})' % [$.namespaceMatcher(), containerName], 'min(container_spec_memory_limit_bytes{%s,container=~"%s"} > 0)' % [$.namespaceMatcher(), containerName], - ], ['{{pod}}', 'limit']) + + 'min(kube_pod_container_resource_requests{%s,container=~"%s", resource="memory"} > 0)' % [$.namespaceMatcher(), containerName], + ], ['{{pod}}', 'limit', 'request']) + { seriesOverrides: [ { @@ -128,6 +135,11 @@ grafana { color: '#E02F44', fill: 0, }, + { + alias: 'request', + color: '#FCE300', + fill: 0, + }, ], yaxes: $.yaxes('bytes'), }, From d24a0065e036e91b49b3783cb941f1005e4e0bdd Mon Sep 17 00:00:00 2001 From: Suraj Nath <9503187+electron0zero@users.noreply.github.com> Date: Thu, 30 Mar 2023 23:28:23 +0530 Subject: [PATCH 3/5] compile tempo-mixin --- .../dashboards/tempo-resources.json | 196 ++++++++++++++++++ .../dashboards/tempo-rollout-progress.json | 6 +- .../dashboards/tempo-tenants.json | 33 ++- 3 files changed, 222 insertions(+), 13 deletions(-) diff --git a/operations/tempo-mixin-compiled/dashboards/tempo-resources.json b/operations/tempo-mixin-compiled/dashboards/tempo-resources.json index 7ddfc23ab5a..157a00e0f23 100644 --- a/operations/tempo-mixin-compiled/dashboards/tempo-resources.json +++ b/operations/tempo-mixin-compiled/dashboards/tempo-resources.json @@ -62,6 +62,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -86,6 +91,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -162,6 +176,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -186,6 +205,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw(-internal)?\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -361,6 +389,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -385,6 +418,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -461,6 +503,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -485,6 +532,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -660,6 +716,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -684,6 +745,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -760,6 +830,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -784,6 +859,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -959,6 +1043,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -983,6 +1072,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -1059,6 +1157,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -1083,6 +1186,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"metrics-generator\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -1258,6 +1370,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -1282,6 +1399,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -1358,6 +1484,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -1382,6 +1513,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -1557,6 +1697,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -1581,6 +1726,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -1657,6 +1811,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -1681,6 +1840,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -1856,6 +2024,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -1880,6 +2053,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\", resource=\"cpu\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -1956,6 +2138,11 @@ "alias": "limit", "color": "#E02F44", "fill": 0 + }, + { + "alias": "request", + "color": "#FCE300", + "fill": 0 } ], "spaceLength": 10, @@ -1980,6 +2167,15 @@ "legendFormat": "limit", "legendLink": null, "step": 10 + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\", resource=\"memory\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "request", + "legendLink": null, + "step": 10 } ], "thresholds": [ diff --git a/operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json b/operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json index 04f3695dd67..54af04bd13b 100644 --- a/operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json +++ b/operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json @@ -304,7 +304,7 @@ }, { "color": "orange", - "value": 0.20000000000000001 + "value": 0.2 }, { "color": "red", @@ -548,7 +548,7 @@ }, { "color": "orange", - "value": 0.20000000000000001 + "value": 0.2 }, { "color": "red", @@ -792,7 +792,7 @@ }, { "color": "red", - "value": 0.050000000000000003 + "value": 0.05 } ] }, diff --git a/operations/tempo-mixin-compiled/dashboards/tempo-tenants.json b/operations/tempo-mixin-compiled/dashboards/tempo-tenants.json index 231d5f15274..7e401ef2ba8 100644 --- a/operations/tempo-mixin-compiled/dashboards/tempo-tenants.json +++ b/operations/tempo-mixin-compiled/dashboards/tempo-tenants.json @@ -9,7 +9,18 @@ "graphTooltip": 0, "hideControls": false, "links": [ - + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "tempo" + ], + "targetBlank": false, + "title": "Tempo Dashboards", + "type": "dashboards" + } ], "refresh": "10s", "rows": [ @@ -1029,7 +1040,7 @@ "schemaVersion": 14, "style": "dark", "tags": [ - + "tempo" ], "templating": { "list": [ @@ -1052,14 +1063,15 @@ { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ @@ -1079,14 +1091,15 @@ { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "namespace", - "multi": false, + "multi": true, "name": "namespace", "options": [ From 1a10f1352748ac225fe4cae845cbd7887c565111 Mon Sep 17 00:00:00 2001 From: Suraj Nath <9503187+electron0zero@users.noreply.github.com> Date: Thu, 30 Mar 2023 23:33:45 +0530 Subject: [PATCH 4/5] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 714b7ba90b2..a047524cdfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## main / unreleased +* [CHANGE] Update tempo-mixin to show request in Resources dashboard [#2281](https://github.com/grafana/tempo/pull/2008) (@electron0zero) * [FEATURE] Add support for Azure Workload Identity authentication [#2195](https://github.com/grafana/tempo/pull/2195) (@LambArchie) * [ENHANCEMENT] Add Throughput and SLO Metrics with SLOConfig in Query Frontend [#2008](https://github.com/grafana/tempo/pull/2008) (@electron0zero) - **BREAKING CHANGE** `query_frontend_result_metrics_inspected_bytes` metric removed in favour of `query_frontend_bytes_processed_per_second` From 5e4a621b5757744867bff2f51f970acb7e39f295 Mon Sep 17 00:00:00 2001 From: Suraj Nath <9503187+electron0zero@users.noreply.github.com> Date: Thu, 30 Mar 2023 23:39:46 +0530 Subject: [PATCH 5/5] make jsonnet happy in CI --- .../dashboards/tempo-rollout-progress.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json b/operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json index 54af04bd13b..04f3695dd67 100644 --- a/operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json +++ b/operations/tempo-mixin-compiled/dashboards/tempo-rollout-progress.json @@ -304,7 +304,7 @@ }, { "color": "orange", - "value": 0.2 + "value": 0.20000000000000001 }, { "color": "red", @@ -548,7 +548,7 @@ }, { "color": "orange", - "value": 0.2 + "value": 0.20000000000000001 }, { "color": "red", @@ -792,7 +792,7 @@ }, { "color": "red", - "value": 0.05 + "value": 0.050000000000000003 } ] },