Add tests for JMX and Prometheus

charts/trino/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.23.1
+version: 0.24.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/trino/README.md

@@ -1,6 +1,6 @@
 # trino
 
-![Version: 0.23.1](https://img.shields.io/badge/Version-0.23.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 449](https://img.shields.io/badge/AppVersion-449-informational?style=flat-square)
+![Version: 0.24.0](https://img.shields.io/badge/Version-0.24.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 449](https://img.shields.io/badge/AppVersion-449-informational?style=flat-square)
 
 Fast distributed SQL query engine for big data analytics that helps you explore your data universe
 
@@ -605,18 +605,50 @@ Fast distributed SQL query engine for big data analytics that helps you explore
        }
      }
   ```
-* `jmx.enabled` - bool, default: `false`
+* `jmx.enabled` - bool, default: `false`  
+
+  Set to true to enable the RMI server to expose Trino's [JMX metrics](https://trino.io/docs/current/admin/jmx.html).
 * `jmx.registryPort` - int, default: `9080`
 * `jmx.serverPort` - int, default: `9081`
-* `jmxExporter.enabled` - bool, default: `false`
-* `jmxExporter.image` - string, default: `"bitnami/jmx-exporter:latest"`
-* `jmxExporter.pullPolicy` - string, default: `"Always"`
-* `jmxExporter.port` - int, default: `5556`
-* `jmxExporter.configProperties` - list, default: `[]`
-* `podMonitor.enabled` - bool, default: `false`
-* `podMonitor.selector.prometheus` - string, default: `"kube-prometheus"`
-* `podMonitor.path` - string, default: `"/admin/metrics"`
-* `podMonitor.interval` - string, default: `"30s"`
+* `jmx.exporter.enabled` - bool, default: `false`  
+
+  Set to true to export JMX Metrics via HTTP for [Prometheus](https://github.com/prometheus/jmx_exporter) consumption
+* `jmx.exporter.image` - string, default: `"bitnami/jmx-exporter:latest"`
+* `jmx.exporter.pullPolicy` - string, default: `"Always"`
+* `jmx.exporter.port` - int, default: `5556`
+* `jmx.exporter.configProperties` - list, default: `[]`  
+
+  JMX Config Properties is mounted to /etc/jmx-exporter/jmx-exporter-config.yaml
+  Example:
+  ```yaml
+   configProperties: |-
+      startDelaySeconds: 0
+      ssl: false
+      lowercaseOutputName: false
+      lowercaseOutputLabelNames: false
+      includeObjectNames: ["java.lang:type=Threading"]
+      autoExcludeObjectNameAttributes: true
+      excludeObjectNameAttributes:
+        "java.lang:type=OperatingSystem":
+          - "ObjectName"
+        "java.lang:type=Runtime":
+          - "ClassPath"
+          - "SystemProperties"
+      rules:
+        - pattern: 'java\.lang<type=Threading><(.*)>ThreadCount: (.*)'
+          name: java_lang_Threading_ThreadCount
+          value: '$2'
+          help: 'ThreadCount (java.lang<type=Threading><>ThreadCount)'
+          type: UNTYPED
+* `serviceMonitor.enabled` - bool, default: `false`  
+
+  Set to true to create resources for the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator).
+* `serviceMonitor.selector` - object, default: `{"prometheus":"kube-prometheus"}`  
+
+  Labels for serviceMonitor, so that Prometheus can select it
+* `serviceMonitor.interval` - string, default: `"30s"`  
+
+  The serviceMonitor web endpoint interval
 * `commonLabels` - object, default: `{}`  
 
   Labels that get applied to every resource's metadata

charts/trino/templates/deployment-coordinator.yaml

@@ -168,7 +168,7 @@ spec:
               containerPort: {{ .Values.jmx.serverPort }}
               protocol: TCP
             {{- end }}
-            {{- if .Values.podMonitor.enabled }}
+            {{- if .Values.serviceMonitor.enabled }}
             - name: jmx-exporter
               containerPort: {{ .Values.jmx.exporter.port }}
               protocol: TCP

charts/trino/templates/servicemonitor.yaml

@@ -16,6 +16,5 @@ spec:
       {{- include "trino.selectorLabels" . | nindent 6 }}
   endpoints:
     - port: jmx-exporter
-      path: {{ .Values.serviceMonitor.path }}
       interval: {{ .Values.serviceMonitor.interval }}
 {{- end }}

charts/trino/templates/tests/test-connection.yaml

@@ -24,8 +24,8 @@ spec:
       - --password
     {{- end }}
       - --debug
-      - --network-logging=BODY
       - --execute=SELECT 1
+      - --no-progress
     {{- if eq .Values.server.config.authenticationType "PASSWORD" }}
       env:
         - name: TRINO_PASSWORD

charts/trino/templates/tests/test-jmx.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.jmx.exporter.enabled }}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ include "trino.fullname" . }}-test-jmx
+  labels:
+    {{- include "trino.labels" . | nindent 4 }}
+    app.kubernetes.io/component: test
+    test: jmx
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+  containers:
+    {{- if .Values.jmx.exporter.enabled }}
+    - name: trino-jmx
+      image: {{ include "trino.image" . }}
+      command: ["/bin/bash", "-c"]
+      args:
+      - curl -s {{ include "trino.fullname" . }}.{{ .Release.Namespace }}:5556 | grep -q trino
+    {{- end }}
+    {{- if .Values.serviceMonitor.enabled }}
+    - name: podmonitor
+      image: {{ include "trino.image" . }}
+      command: ["/bin/bash", "-c"]
+      args:
+      - sleep 60
+      - |
+        output=$(curl -s "prometheus-operator-kube-p-prometheus.default:9090/api/v1/targets?scrapePool=serviceMonitor/{{ .Release.Namespace }}/{{ include "trino.fullname" . }}/0&state=active" |
+        python -c 'import sys, json; print(json.load(sys.stdin)["data"]["activeTargets"][0]["discoveredLabels"]["__meta_kubernetes_service_name"])') && [ "$output" == "{{ include "trino.fullname" . }}" ]
+    {{- end }}
+  restartPolicy: Never
+{{- end }}

charts/trino/values.yaml

@@ -769,8 +769,6 @@ serviceMonitor:
   # serviceMonitor.selector -- Labels for serviceMonitor, so that Prometheus can select it
   selector:
     prometheus: kube-prometheus
-  # serviceMonitor.path -- The serviceMonitor web endpoint path
-  path: /admin/metrics
   # serviceMonitor.interval -- The serviceMonitor web endpoint interval
   interval: "30s"
 

test-values.yaml

@@ -144,38 +144,22 @@ jmx:
   enabled: true
   registryPort: 9080
   serverPort: 9081
-
-jmxExporter:
-  enabled: true
-  image: bitnami/jmx-exporter:latest
-  pullPolicy: Always
-  port: 5556
-  configProperties: |
-    startDelaySeconds: 0
-    hostPort: 127.0.0.1:9080
-    ssl: false
-    lowercaseOutputName: false
-    lowercaseOutputLabelNames: false
-    includeObjectNames: ["java.lang:type=Threading"]
-    autoExcludeObjectNameAttributes: true
-    excludeObjectNameAttributes:
-      "java.lang:type=OperatingSystem":
-        - "ObjectName"
-      "java.lang:type=Runtime":
-        - "ClassPath"
-        - "SystemProperties"
-    rules:
-      - pattern: 'java\.lang<type=Threading><(.*)>ThreadCount: (.*)'
-        name: java_lang_Threading_ThreadCount
-        value: '$2'
-        help: 'ThreadCount (java.lang<type=Threading><>ThreadCount)'
-        type: UNTYPED
-
-podMonitor:
+  exporter:
+    enabled: true
+    image: bitnami/jmx-exporter:latest
+    pullPolicy: Always
+    port: 5556
+    configProperties: |
+      startDelaySeconds: 0
+      hostPort: 127.0.0.1:9080
+      rules:
+        - pattern: 'trino.memory*'
+        - pattern: 'trino.execution<name=QueryManager>*'
+
+serviceMonitor:
   enabled: true
   selector:
-    prometheus: kube-prometheus
-  path: /admin/metrics
+    prometheus: default
   interval: "30s"
 
 ingress:

test.sh

@@ -16,34 +16,6 @@ function join_by {
     fi
 }
 
-function uninstall_helm_repo() {
-    repo_name=$1
-    # clusterrole being pesky here so just uninstall in all namespaces
-    # TODO: find a way to only install non-cluster level resources
-    if helm list -a -A | grep -q "$repo_name"; then
-        namespaces=$(helm list -a -A --filter "$repo_name" | awk 'NR>1 {print $2}' | uniq)
-        for namespace in $namespaces
-        do
-            echo 1>&2 "Uninstalling $repo_name in $namespace"
-            helm uninstall "$repo_name" -n "$namespace"
-        done
-    fi
-}
-
-function add_helm_repo() {
-    repo_name=$1
-    repo_url=$2
-    echo 1>&2 "Adding Helm Repo $repo_name"
-    helm repo add "$repo_name" "$repo_url"
-}
-
-function install_helm_repo() {
-    repo_name=$1
-    chart=$2
-    version=$3
-    echo 1>&2 "Installing $repo_name"
-    helm install "$repo_name" "$chart" -n "$NAMESPACE" --version "$version"
-}
 
 # default to randomly generated namespace, same as chart-testing would do, but we need to load secrets into the same namespace
 NAMESPACE=trino-$(LC_ALL=C tr -dc 'a-z0-9' </dev/urandom | head -c 6 || true)
@@ -104,22 +76,11 @@ kubectl create namespace "$NAMESPACE"
 kubectl -n "$NAMESPACE" create secret tls certificates --cert=cert.crt --key=cert.key
 
 # only install prometheus helm chart for these tests
-PROMETHEUS_TESTS=(complete_values)
-for test_name in "${PROMETHEUS_TESTS[@]}"; do
-    found=false
-    for item in "${TEST_NAMES[@]}"; do
-        if [[ $item == "$test_name" ]]; then
-            found=true
-            break
-        fi
-    done
-    if $found; then
-        uninstall_helm_repo "prometheus-operator"
-        add_helm_repo "prometheus-community" "https://prometheus-community.github.io/helm-charts"
-        install_helm_repo "prometheus-operator" "prometheus-community/kube-prometheus-stack" "60.0.2"
-        kubectl rollout status --watch deployments -l release=prometheus-operator -n "$NAMESPACE"
-    fi
-done
+if printf '%s\0' "${TEST_NAMES[@]}" | grep -qwz complete_values; then
+    helm repo add "prometheus-community" "https://prometheus-community.github.io/helm-charts"
+    helm upgrade --install "prometheus-operator" "prometheus-community/kube-prometheus-stack" -n default --version "60.0.2"
+    kubectl rollout status --watch deployments -l release=prometheus-operator -n default
+fi
 
 
 CT_ARGS+=(--namespace "$NAMESPACE")
@@ -132,29 +93,7 @@ for test_name in "${TEST_NAMES[@]}"; do
     if ! time ct install "${CT_ARGS[@]}" --helm-extra-set-args "$HELM_EXTRA_SET_ARGS ${testCases[$test_name]}"; then
         echo 1>&2 "❌ Test $test_name failed"
         echo 1>&2 "Test logs:"
-        # Get list of failed test pods
-        failed_tests=$(kubectl get pods --namespace "$NAMESPACE" \
-                    --field-selector=status.phase=Failed \
-                    --selector app.kubernetes.io/component=test \
-                    -o jsonpath='{.items[*].metadata.name}')
-
-        # Iterate through each failed test pod
-        for pod in $failed_tests; do
-            # Get list of failed containers in the pod
-            containers=$(kubectl get pod "$pod" --namespace "$NAMESPACE" \
-                -o jsonpath="{range .status.containerStatuses[?(@.state.terminated.exitCode != 0)]}{.name}{' '}{end}")
-
-            # Check if there are any failed containers
-            if [ -n "$containers" ]; then
-                # Iterate through each failed container
-                for container in $containers; do
-                    echo 1>&2 "$pod $container"
-                    # Get logs for the failed container
-                    kubectl logs -n "$NAMESPACE" "$pod" "$container" --tail=-1
-                done
-            fi
-        done
-
+        kubectl --namespace "$NAMESPACE" logs --tail=-1 --selector app.kubernetes.io/component=test --all-containers=true
         result=1
     else
         echo 1>&2 "✅ Test $test_name completed"
@@ -169,6 +108,8 @@ done
 
 if [ "$CLEANUP_NAMESPACE" == "true" ]; then
     kubectl delete namespace "$NAMESPACE"
+    helm uninstall "prometheus-operator" --ignore-not-found
+    # TODO: delete CRDs https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#uninstall-helm-chart
 fi
 
 exit $result