From 2134f79c5136a38fe2f0b7e9e6cc8ea30dbdf963 Mon Sep 17 00:00:00 2001 From: Kemp Po Date: Wed, 12 Jun 2024 02:32:12 +0800 Subject: [PATCH] Add tests for JMX and Prometheus --- charts/trino/Chart.yaml | 2 +- charts/trino/README.md | 54 ++++++++++--- charts/trino/templates/configmap-jmx.yaml | 28 ------- .../templates/deployment-coordinator.yaml | 2 +- charts/trino/templates/podmonitor.yaml | 22 ------ charts/trino/templates/servicemonitor.yaml | 1 - .../templates/tests/test-connection.yaml | 2 +- charts/trino/templates/tests/test-jmx.yaml | 32 ++++++++ charts/trino/values.yaml | 2 - test-values.yaml | 44 ++++------- test.sh | 75 ++----------------- 11 files changed, 100 insertions(+), 164 deletions(-) delete mode 100644 charts/trino/templates/configmap-jmx.yaml delete mode 100644 charts/trino/templates/podmonitor.yaml create mode 100644 charts/trino/templates/tests/test-jmx.yaml diff --git a/charts/trino/Chart.yaml b/charts/trino/Chart.yaml index 40a74b28..c3436615 100644 --- a/charts/trino/Chart.yaml +++ b/charts/trino/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.23.1 +version: 0.24.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/trino/README.md b/charts/trino/README.md index 3aa1a42b..c850dbf2 100644 --- a/charts/trino/README.md +++ b/charts/trino/README.md @@ -1,6 +1,6 @@ # trino -![Version: 0.23.1](https://img.shields.io/badge/Version-0.23.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 449](https://img.shields.io/badge/AppVersion-449-informational?style=flat-square) +![Version: 0.24.0](https://img.shields.io/badge/Version-0.24.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 449](https://img.shields.io/badge/AppVersion-449-informational?style=flat-square) Fast distributed SQL query engine for big data analytics that helps you explore your data universe @@ -605,18 +605,50 @@ Fast distributed SQL query engine for big data analytics that helps you explore } } ``` -* `jmx.enabled` - bool, default: `false` +* `jmx.enabled` - bool, default: `false` + + Set to true to enable the RMI server to expose Trino's [JMX metrics](https://trino.io/docs/current/admin/jmx.html). * `jmx.registryPort` - int, default: `9080` * `jmx.serverPort` - int, default: `9081` -* `jmxExporter.enabled` - bool, default: `false` -* `jmxExporter.image` - string, default: `"bitnami/jmx-exporter:latest"` -* `jmxExporter.pullPolicy` - string, default: `"Always"` -* `jmxExporter.port` - int, default: `5556` -* `jmxExporter.configProperties` - list, default: `[]` -* `podMonitor.enabled` - bool, default: `false` -* `podMonitor.selector.prometheus` - string, default: `"kube-prometheus"` -* `podMonitor.path` - string, default: `"/admin/metrics"` -* `podMonitor.interval` - string, default: `"30s"` +* `jmx.exporter.enabled` - bool, default: `false` + + Set to true to export JMX Metrics via HTTP for [Prometheus](https://github.com/prometheus/jmx_exporter) consumption +* `jmx.exporter.image` - string, default: `"bitnami/jmx-exporter:latest"` +* `jmx.exporter.pullPolicy` - string, default: `"Always"` +* `jmx.exporter.port` - int, default: `5556` +* `jmx.exporter.configProperties` - list, default: `[]` + + JMX Config Properties is mounted to /etc/jmx-exporter/jmx-exporter-config.yaml + Example: + ```yaml + configProperties: |- + startDelaySeconds: 0 + ssl: false + lowercaseOutputName: false + lowercaseOutputLabelNames: false + includeObjectNames: ["java.lang:type=Threading"] + autoExcludeObjectNameAttributes: true + excludeObjectNameAttributes: + "java.lang:type=OperatingSystem": + - "ObjectName" + "java.lang:type=Runtime": + - "ClassPath" + - "SystemProperties" + rules: + - pattern: 'java\.lang<(.*)>ThreadCount: (.*)' + name: java_lang_Threading_ThreadCount + value: '$2' + help: 'ThreadCount (java.lang<>ThreadCount)' + type: UNTYPED +* `serviceMonitor.enabled` - bool, default: `false` + + Set to true to create resources for the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator). +* `serviceMonitor.selector` - object, default: `{"prometheus":"kube-prometheus"}` + + Labels for serviceMonitor, so that Prometheus can select it +* `serviceMonitor.interval` - string, default: `"30s"` + + The serviceMonitor web endpoint interval * `commonLabels` - object, default: `{}` Labels that get applied to every resource's metadata diff --git a/charts/trino/templates/configmap-jmx.yaml b/charts/trino/templates/configmap-jmx.yaml deleted file mode 100644 index 40365902..00000000 --- a/charts/trino/templates/configmap-jmx.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.jmx.enabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "trino.fullname" }}-jmx-config - namespace: {{ .Release.Namespace }} - labels: - {{- include "trino.labels" . | nindent 4 }} - app.kubernetes.io/component: jmx -data: - jmx-config: | - {{- range $configValue := .Values.jmx.configProperties }} - {{- $configValue }} - {{- end }} - -# jmx-config.yaml: | -# startDelaySeconds: 0 -# hostPort: 127.0.0.1:9080 -# rules: -# - pattern: 'java.lang(\w+): (.*)' -# name: jmx_jvm_memory_HeapMemoryUsed_$1 -# help: "JVM heap memory $1" -# value: $2 -# type: GAUGE -# - pattern: 'trino.memory*' -# - pattern: 'trino.execution*' -# - pattern: 'iceberg*' -{{- end }} diff --git a/charts/trino/templates/deployment-coordinator.yaml b/charts/trino/templates/deployment-coordinator.yaml index 3decfcd7..ad53ade2 100644 --- a/charts/trino/templates/deployment-coordinator.yaml +++ b/charts/trino/templates/deployment-coordinator.yaml @@ -168,7 +168,7 @@ spec: containerPort: {{ .Values.jmx.serverPort }} protocol: TCP {{- end }} - {{- if .Values.podMonitor.enabled }} + {{- if .Values.serviceMonitor.enabled }} - name: jmx-exporter containerPort: {{ .Values.jmx.exporter.port }} protocol: TCP diff --git a/charts/trino/templates/podmonitor.yaml b/charts/trino/templates/podmonitor.yaml deleted file mode 100644 index e50433b9..00000000 --- a/charts/trino/templates/podmonitor.yaml +++ /dev/null @@ -1,22 +0,0 @@ - -{{- if .Values.podMonitor.enabled -}} -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: {{ template "trino.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - release: prometheus-operator - {{- include "trino.labels" . | nindent 4 }} - {{- if .Values.podMonitor.selector }} - {{- toYaml .Values.podMonitor.selector | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- include "trino.selectorLabels" . | nindent 6 }} - endpoints: - - port: jmx-exporter - path: {{ .Values.serviceMonitor.path }} - interval: {{ .Values.serviceMonitor.interval }} -{{- end }} diff --git a/charts/trino/templates/servicemonitor.yaml b/charts/trino/templates/servicemonitor.yaml index bccab7df..aa15a96b 100644 --- a/charts/trino/templates/servicemonitor.yaml +++ b/charts/trino/templates/servicemonitor.yaml @@ -16,6 +16,5 @@ spec: {{- include "trino.selectorLabels" . | nindent 6 }} endpoints: - port: jmx-exporter - path: {{ .Values.serviceMonitor.path }} interval: {{ .Values.serviceMonitor.interval }} {{- end }} diff --git a/charts/trino/templates/tests/test-connection.yaml b/charts/trino/templates/tests/test-connection.yaml index c7f2b7f0..6131474e 100644 --- a/charts/trino/templates/tests/test-connection.yaml +++ b/charts/trino/templates/tests/test-connection.yaml @@ -24,8 +24,8 @@ spec: - --password {{- end }} - --debug - - --network-logging=BODY - --execute=SELECT 1 + - --no-progress {{- if eq .Values.server.config.authenticationType "PASSWORD" }} env: - name: TRINO_PASSWORD diff --git a/charts/trino/templates/tests/test-jmx.yaml b/charts/trino/templates/tests/test-jmx.yaml new file mode 100644 index 00000000..ae839c69 --- /dev/null +++ b/charts/trino/templates/tests/test-jmx.yaml @@ -0,0 +1,32 @@ +{{- if .Values.jmx.exporter.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "trino.fullname" . }}-test-jmx + labels: + {{- include "trino.labels" . | nindent 4 }} + app.kubernetes.io/component: test + test: jmx + annotations: + "helm.sh/hook": test-success +spec: + containers: + {{- if .Values.jmx.exporter.enabled }} + - name: trino-jmx + image: {{ include "trino.image" . }} + command: ["/bin/bash", "-c"] + args: + - curl -s {{ include "trino.fullname" . }}.{{ .Release.Namespace }}:5556 | grep -q trino + {{- end }} + {{- if .Values.serviceMonitor.enabled }} + - name: podmonitor + image: {{ include "trino.image" . }} + command: ["/bin/bash", "-c"] + args: + - sleep 60 + - | + output=$(curl -s "prometheus-operator-kube-p-prometheus.default:9090/api/v1/targets?scrapePool=serviceMonitor/{{ .Release.Namespace }}/{{ include "trino.fullname" . }}/0&state=active" | + python -c 'import sys, json; print(json.load(sys.stdin)["data"]["activeTargets"][0]["discoveredLabels"]["__meta_kubernetes_service_name"])') && [ "$output" == "{{ include "trino.fullname" . }}" ] + {{- end }} + restartPolicy: Never +{{- end }} diff --git a/charts/trino/values.yaml b/charts/trino/values.yaml index 4b3d3f60..8958d897 100644 --- a/charts/trino/values.yaml +++ b/charts/trino/values.yaml @@ -769,8 +769,6 @@ serviceMonitor: # serviceMonitor.selector -- Labels for serviceMonitor, so that Prometheus can select it selector: prometheus: kube-prometheus - # serviceMonitor.path -- The serviceMonitor web endpoint path - path: /admin/metrics # serviceMonitor.interval -- The serviceMonitor web endpoint interval interval: "30s" diff --git a/test-values.yaml b/test-values.yaml index f10d4c77..410c7ce7 100644 --- a/test-values.yaml +++ b/test-values.yaml @@ -144,38 +144,22 @@ jmx: enabled: true registryPort: 9080 serverPort: 9081 - -jmxExporter: - enabled: true - image: bitnami/jmx-exporter:latest - pullPolicy: Always - port: 5556 - configProperties: | - startDelaySeconds: 0 - hostPort: 127.0.0.1:9080 - ssl: false - lowercaseOutputName: false - lowercaseOutputLabelNames: false - includeObjectNames: ["java.lang:type=Threading"] - autoExcludeObjectNameAttributes: true - excludeObjectNameAttributes: - "java.lang:type=OperatingSystem": - - "ObjectName" - "java.lang:type=Runtime": - - "ClassPath" - - "SystemProperties" - rules: - - pattern: 'java\.lang<(.*)>ThreadCount: (.*)' - name: java_lang_Threading_ThreadCount - value: '$2' - help: 'ThreadCount (java.lang<>ThreadCount)' - type: UNTYPED - -podMonitor: + exporter: + enabled: true + image: bitnami/jmx-exporter:latest + pullPolicy: Always + port: 5556 + configProperties: | + startDelaySeconds: 0 + hostPort: 127.0.0.1:9080 + rules: + - pattern: 'trino.memory*' + - pattern: 'trino.execution*' + +serviceMonitor: enabled: true selector: - prometheus: kube-prometheus - path: /admin/metrics + prometheus: default interval: "30s" ingress: diff --git a/test.sh b/test.sh index 33c730b9..f13d11eb 100755 --- a/test.sh +++ b/test.sh @@ -16,34 +16,6 @@ function join_by { fi } -function uninstall_helm_repo() { - repo_name=$1 - # clusterrole being pesky here so just uninstall in all namespaces - # TODO: find a way to only install non-cluster level resources - if helm list -a -A | grep -q "$repo_name"; then - namespaces=$(helm list -a -A --filter "$repo_name" | awk 'NR>1 {print $2}' | uniq) - for namespace in $namespaces - do - echo 1>&2 "Uninstalling $repo_name in $namespace" - helm uninstall "$repo_name" -n "$namespace" - done - fi -} - -function add_helm_repo() { - repo_name=$1 - repo_url=$2 - echo 1>&2 "Adding Helm Repo $repo_name" - helm repo add "$repo_name" "$repo_url" -} - -function install_helm_repo() { - repo_name=$1 - chart=$2 - version=$3 - echo 1>&2 "Installing $repo_name" - helm install "$repo_name" "$chart" -n "$NAMESPACE" --version "$version" -} # default to randomly generated namespace, same as chart-testing would do, but we need to load secrets into the same namespace NAMESPACE=trino-$(LC_ALL=C tr -dc 'a-z0-9' &2 "❌ Test $test_name failed" echo 1>&2 "Test logs:" - # Get list of failed test pods - failed_tests=$(kubectl get pods --namespace "$NAMESPACE" \ - --field-selector=status.phase=Failed \ - --selector app.kubernetes.io/component=test \ - -o jsonpath='{.items[*].metadata.name}') - - # Iterate through each failed test pod - for pod in $failed_tests; do - # Get list of failed containers in the pod - containers=$(kubectl get pod "$pod" --namespace "$NAMESPACE" \ - -o jsonpath="{range .status.containerStatuses[?(@.state.terminated.exitCode != 0)]}{.name}{' '}{end}") - - # Check if there are any failed containers - if [ -n "$containers" ]; then - # Iterate through each failed container - for container in $containers; do - echo 1>&2 "$pod $container" - # Get logs for the failed container - kubectl logs -n "$NAMESPACE" "$pod" "$container" --tail=-1 - done - fi - done - + kubectl --namespace "$NAMESPACE" logs --tail=-1 --selector app.kubernetes.io/component=test --all-containers=true result=1 else echo 1>&2 "✅ Test $test_name completed" @@ -169,6 +108,8 @@ done if [ "$CLEANUP_NAMESPACE" == "true" ]; then kubectl delete namespace "$NAMESPACE" + helm uninstall "prometheus-operator" --ignore-not-found + # TODO: delete CRDs https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#uninstall-helm-chart fi exit $result