From 25c38513325172f1d3f1ceb486d2093ef855f5db Mon Sep 17 00:00:00 2001 From: yardenc Date: Sun, 1 Sep 2024 14:25:10 +0300 Subject: [PATCH] Added condition for exchange manager properties file creation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jan Waś --- charts/trino/README.md | 21 +++++++++-- .../templates/configmap-coordinator.yaml | 4 +-- charts/trino/templates/configmap-worker.yaml | 4 +-- charts/trino/values.yaml | 24 +++++++++++-- test-exchange-manager-values.yaml | 35 +++++++++++++++++++ test.sh | 15 +++++++- 6 files changed, 93 insertions(+), 10 deletions(-) create mode 100644 test-exchange-manager-values.yaml diff --git a/charts/trino/README.md b/charts/trino/README.md index 86d9c99a..56c3829f 100644 --- a/charts/trino/README.md +++ b/charts/trino/README.md @@ -54,8 +54,25 @@ Fast distributed SQL query engine for big data analytics that helps you explore Trino supports multiple [authentication types](https://trino.io/docs/current/security/authentication-types.html): PASSWORD, CERTIFICATE, OAUTH2, JWT, KERBEROS. * `server.config.query.maxMemory` - string, default: `"4GB"` -* `server.exchangeManager.name` - string, default: `"filesystem"` -* `server.exchangeManager.baseDir` - string, default: `"/tmp/trino-local-file-system-exchange-manager"` +* `server.exchangeManager` - object, default: `{}` + + Mandatory [exchange manager configuration](https://trino.io/docs/current/admin/fault-tolerant-execution.html#id1). + Used to set the name and location(s) of the spooling storage destination. + * To enable fault-tolerant execution, you must set the `retry-policy` property in `additionalConfigProperties`. + * Additional exchange manager configurations can be added to `additionalExchangeManagerProperties`. + Example: + ```yaml + server: + exchangeManager: + name: "filesystem" + baseDir: "/tmp/trino-local-file-system-exchange-manager" + additionalConfigProperties: + - retry-policy=TASK + additionalExchangeManagerProperties: + - exchange.sink-buffer-pool-min-size=10 + - exchange.sink-buffers-per-partition=2 + - exchange.source-concurrent-readers=4 + ``` * `server.workerExtraConfig` - string, default: `""` * `server.coordinatorExtraConfig` - string, default: `""` * `server.autoscaling.enabled` - bool, default: `false` diff --git a/charts/trino/templates/configmap-coordinator.yaml b/charts/trino/templates/configmap-coordinator.yaml index 39e018ff..667ab0e3 100644 --- a/charts/trino/templates/configmap-coordinator.yaml +++ b/charts/trino/templates/configmap-coordinator.yaml @@ -106,14 +106,14 @@ data: resource-groups.config-file={{ .Values.server.config.path }}/resource-groups/resource-groups.json {{- end }} +{{- if .Values.server.exchangeManager }} exchange-manager.properties: | exchange-manager.name={{ .Values.server.exchangeManager.name }} - {{ if eq .Values.server.exchangeManager.name "filesystem" }} exchange.base-directories={{ .Values.server.exchangeManager.baseDir }} - {{- end }} {{- range $configValue := .Values.additionalExchangeManagerProperties }} {{ $configValue }} {{- end }} +{{- end }} log.properties: | io.trino={{ .Values.server.log.trino.level }} diff --git a/charts/trino/templates/configmap-worker.yaml b/charts/trino/templates/configmap-worker.yaml index e209f223..23fb7432 100644 --- a/charts/trino/templates/configmap-worker.yaml +++ b/charts/trino/templates/configmap-worker.yaml @@ -61,14 +61,14 @@ data: {{- .Values.server.workerExtraConfig | nindent 4 }} {{- end }} +{{- if .Values.server.exchangeManager }} exchange-manager.properties: | exchange-manager.name={{ .Values.server.exchangeManager.name }} - {{ if eq .Values.server.exchangeManager.name "filesystem" }} exchange.base-directories={{ .Values.server.exchangeManager.baseDir }} - {{- end }} {{- range $configValue := .Values.additionalExchangeManagerProperties }} {{ $configValue }} {{- end }} +{{- end }} log.properties: | io.trino={{ .Values.server.log.trino.level }} diff --git a/charts/trino/values.yaml b/charts/trino/values.yaml index 74c742d5..745ff3e2 100644 --- a/charts/trino/values.yaml +++ b/charts/trino/values.yaml @@ -55,9 +55,27 @@ server: authenticationType: "" query: maxMemory: "4GB" - exchangeManager: - name: "filesystem" - baseDir: "/tmp/trino-local-file-system-exchange-manager" + exchangeManager: {} + # server.exchangeManager -- Mandatory [exchange manager + # configuration](https://trino.io/docs/current/admin/fault-tolerant-execution.html#id1). + # @raw + # Used to set the name and location(s) of the spooling storage destination. + # * To enable fault-tolerant execution, you must set the `retry-policy` property in `additionalConfigProperties`. + # * Additional exchange manager configurations can be added to `additionalExchangeManagerProperties`. + # Example: + # ```yaml + # server: + # exchangeManager: + # name: "filesystem" + # baseDir: "/tmp/trino-local-file-system-exchange-manager" + # additionalConfigProperties: + # - retry-policy=TASK + # additionalExchangeManagerProperties: + # - exchange.sink-buffer-pool-min-size=10 + # - exchange.sink-buffers-per-partition=2 + # - exchange.source-concurrent-readers=4 + # ``` + workerExtraConfig: "" coordinatorExtraConfig: "" autoscaling: diff --git a/test-exchange-manager-values.yaml b/test-exchange-manager-values.yaml new file mode 100644 index 00000000..87fa13bb --- /dev/null +++ b/test-exchange-manager-values.yaml @@ -0,0 +1,35 @@ +# Exchange Manager values to test. +# This is a YAML-formatted file. + +coordinator: + additionalVolumes: + - name: exchange-volume + persistentVolumeClaim: + claimName: exchange-manager-pvc + + additionalVolumeMounts: + - name: exchange-volume + mountPath: "/tmp/trino-local-file-system-exchange-manager" + +worker: + additionalVolumes: + - name: exchange-volume + persistentVolumeClaim: + claimName: exchange-manager-pvc + + additionalVolumeMounts: + - name: exchange-volume + mountPath: "/tmp/trino-local-file-system-exchange-manager" + +server: + exchangeManager: + name: "filesystem" + baseDir: "/tmp/trino-local-file-system-exchange-manager" + +additionalConfigProperties: + - retry-policy=TASK + +additionalExchangeManagerProperties: + - exchange.sink-buffer-pool-min-size=10 + - exchange.sink-buffers-per-partition=2 + - exchange.source-concurrent-readers=4 diff --git a/test.sh b/test.sh index 25447b6f..6f039413 100755 --- a/test.sh +++ b/test.sh @@ -8,6 +8,7 @@ declare -A testCases=( [complete_values]="--values test-values.yaml" [overrides]="--set coordinatorNameOverride=coordinator-overridden,workerNameOverride=worker-overridden,nameOverride=overridden" [access_control_properties_values]="--values test-access-control-properties-values.yaml" + [exchange_manager_values]="--values test-exchange-manager-values.yaml" ) function join_by { @@ -22,7 +23,7 @@ NAMESPACE=trino-$(LC_ALL=C tr -dc 'a-z0-9' &2 @@ -74,6 +75,18 @@ openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \ kubectl create namespace "$NAMESPACE" --dry-run=client --output yaml | kubectl apply --filename - kubectl -n "$NAMESPACE" create secret tls certificates --cert=cert.crt --key=cert.key --dry-run=client --output yaml | kubectl apply --filename - +cat <