From f6e5c5b2db7d18a3eb6769ed09b1e6f81df3642e Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 05:58:52 +0200 Subject: [PATCH 01/18] Allows multiple bases and variants in this repo --- 10conf-d.yml => base/conf-d.yaml | 0 base/kustomization.yaml | 9 +++++++++ .../mariadb-ready-service.yaml | 0 20mariadb-service.yml => base/mariadb-service.yaml | 0 50mariadb.yml => base/mariadb-statefulset.yaml | 0 30mysql-service.yml => base/mysql-service.yaml | 0 kustomization.yaml | 6 ------ 7 files changed, 9 insertions(+), 6 deletions(-) rename 10conf-d.yml => base/conf-d.yaml (100%) create mode 100644 base/kustomization.yaml rename 21mariadb-ready-service.yml => base/mariadb-ready-service.yaml (100%) rename 20mariadb-service.yml => base/mariadb-service.yaml (100%) rename 50mariadb.yml => base/mariadb-statefulset.yaml (100%) rename 30mysql-service.yml => base/mysql-service.yaml (100%) delete mode 100644 kustomization.yaml diff --git a/10conf-d.yml b/base/conf-d.yaml similarity index 100% rename from 10conf-d.yml rename to base/conf-d.yaml diff --git a/base/kustomization.yaml b/base/kustomization.yaml new file mode 100644 index 0000000..f6dbab6 --- /dev/null +++ b/base/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- conf-d.yaml +- mariadb-service.yaml +- mariadb-ready-service.yaml +- mysql-service.yaml +- mariadb-statefulset.yaml diff --git a/21mariadb-ready-service.yml b/base/mariadb-ready-service.yaml similarity index 100% rename from 21mariadb-ready-service.yml rename to base/mariadb-ready-service.yaml diff --git a/20mariadb-service.yml b/base/mariadb-service.yaml similarity index 100% rename from 20mariadb-service.yml rename to base/mariadb-service.yaml diff --git a/50mariadb.yml b/base/mariadb-statefulset.yaml similarity index 100% rename from 50mariadb.yml rename to base/mariadb-statefulset.yaml diff --git a/30mysql-service.yml b/base/mysql-service.yaml similarity index 100% rename from 30mysql-service.yml rename to base/mysql-service.yaml diff --git a/kustomization.yaml b/kustomization.yaml deleted file mode 100644 index 176a01a..0000000 --- a/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- 10conf-d.yml -- 20mariadb-service.yml -- 21mariadb-ready-service.yml -- 30mysql-service.yml -- 50mariadb.yml From 13277a0b93e386c141f31f5632a57dc3d42956e7 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 05:59:17 +0200 Subject: [PATCH 02/18] With Kustomize version>2 patches would fail to find a resource if the namespace was overridden --- base/conf-d.yaml | 1 - base/mariadb-ready-service.yaml | 1 - base/mariadb-service.yaml | 1 - base/mariadb-statefulset.yaml | 1 - base/mysql-service.yaml | 1 - 5 files changed, 5 deletions(-) diff --git a/base/conf-d.yaml b/base/conf-d.yaml index 39a2700..85f7a6c 100644 --- a/base/conf-d.yaml +++ b/base/conf-d.yaml @@ -1,7 +1,6 @@ kind: ConfigMap metadata: name: conf-d - namespace: mysql apiVersion: v1 data: datadir.cnf: | diff --git a/base/mariadb-ready-service.yaml b/base/mariadb-ready-service.yaml index 6895dbd..718f66a 100644 --- a/base/mariadb-ready-service.yaml +++ b/base/mariadb-ready-service.yaml @@ -4,7 +4,6 @@ apiVersion: v1 kind: Service metadata: name: mariadb-ready - namespace: mysql annotations: service.alpha.kubernetes.io/tolerate-unready-endpoints: "false" spec: diff --git a/base/mariadb-service.yaml b/base/mariadb-service.yaml index c9d5f94..a0bb7f6 100644 --- a/base/mariadb-service.yaml +++ b/base/mariadb-service.yaml @@ -4,7 +4,6 @@ apiVersion: v1 kind: Service metadata: name: mariadb - namespace: mysql annotations: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" spec: diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 9a1843d..47fe686 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: mariadb - namespace: mysql spec: selector: matchLabels: diff --git a/base/mysql-service.yaml b/base/mysql-service.yaml index dee62dc..dcef547 100644 --- a/base/mysql-service.yaml +++ b/base/mysql-service.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: Service metadata: name: mysql - namespace: mysql spec: ports: - port: 3306 From 47edea5e5581a2eed0e6148561ec38f705550006 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 06:00:39 +0200 Subject: [PATCH 03/18] Current MariaDB --- base/mariadb-statefulset.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 47fe686..4e0df20 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -50,10 +50,10 @@ spec: mountPath: /etc/mysql/conf.d - name: initdb mountPath: /docker-entrypoint-initdb.d - image: mariadb:10.2.36-bionic@sha256:b7be3ade3d5441c79b5c8a9cf2c2269f14bf420876a06def7d50e1763f042238 + image: mariadb:10.5.11-focal@sha256:228d87496cc141569f5f692594823fb1d62a361b6f8848165eb4f953af5e3c3a containers: - name: mariadb - image: mariadb:10.2.36-bionic@sha256:b7be3ade3d5441c79b5c8a9cf2c2269f14bf420876a06def7d50e1763f042238 + image: mariadb:10.5.11-focal@sha256:228d87496cc141569f5f692594823fb1d62a361b6f8848165eb4f953af5e3c3a ports: - containerPort: 3306 name: mysql @@ -96,7 +96,7 @@ spec: - name: initdb mountPath: /docker-entrypoint-initdb.d - name: metrics - image: prom/mysqld-exporter:v0.12.1@sha256:9fe9938c4ac9216cc24005144338f14fac4f604f139b481cc541bead008db3c1 + image: prom/mysqld-exporter:v0.13.0@sha256:a8af600c3ef1c8df179b736b94d04dc5ec209be88407a4c1c1bd0fc6394f56e8 env: - name: DATA_SOURCE_NAME value: root@(localhost:3306)/ From 3e2ca23d386a0bb556bf5d38897af79c2a31f539 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 06:01:15 +0200 Subject: [PATCH 04/18] Names the metrics port for use with PodMonitor --- base/mariadb-statefulset.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 4e0df20..edbbe43 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -101,7 +101,8 @@ spec: - name: DATA_SOURCE_NAME value: root@(localhost:3306)/ ports: - - containerPort: 9104 + - name: metrics + containerPort: 9104 volumes: - name: conf emptyDir: {} From b3fa8719aefb0c6afc526e7c812541dbda024bbb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 06:04:56 +0200 Subject: [PATCH 05/18] Allows rollout undo on config changes, and simplifies edit --- base-defaultconfig/datadir.cnf | 2 + base-defaultconfig/galera.cnf | 25 ++++ base-defaultconfig/init.sh | 96 ++++++++++++++++ base-defaultconfig/kustomization.yaml | 13 +++ base-defaultconfig/utf8.cnf | 25 ++++ base/conf-d.yaml | 157 -------------------------- base/kustomization.yaml | 1 - 7 files changed, 161 insertions(+), 158 deletions(-) create mode 100644 base-defaultconfig/datadir.cnf create mode 100644 base-defaultconfig/galera.cnf create mode 100644 base-defaultconfig/init.sh create mode 100644 base-defaultconfig/kustomization.yaml create mode 100644 base-defaultconfig/utf8.cnf delete mode 100644 base/conf-d.yaml diff --git a/base-defaultconfig/datadir.cnf b/base-defaultconfig/datadir.cnf new file mode 100644 index 0000000..19af51e --- /dev/null +++ b/base-defaultconfig/datadir.cnf @@ -0,0 +1,2 @@ +[mysqld] +datadir=/data/db diff --git a/base-defaultconfig/galera.cnf b/base-defaultconfig/galera.cnf new file mode 100644 index 0000000..abf3742 --- /dev/null +++ b/base-defaultconfig/galera.cnf @@ -0,0 +1,25 @@ +# +# * Galera-related settings +# +# https://mariadb.com/kb/en/mariadb/galera-cluster-system-variables/ +# +[galera] +# Mandatory settings +wsrep_on=ON +wsrep_provider="/usr/lib/galera/libgalera_smm.so" +#init-new-cluster#wsrep_new_cluster=TRUE +#init-recover#wsrep_recover=TRUE +binlog_format=ROW +default_storage_engine=InnoDB +innodb_autoinc_lock_mode=2 +#init-wsrep#wsrep_cluster_address="gcomm://mariadb" +wsrep-sst-method=rsync + +# +# Allow server to accept connections on all interfaces. +# +bind-address=0.0.0.0 +# +# Optional setting +#wsrep_slave_threads=1 +#innodb_flush_log_at_trx_commit=0 diff --git a/base-defaultconfig/init.sh b/base-defaultconfig/init.sh new file mode 100644 index 0000000..00b1b00 --- /dev/null +++ b/base-defaultconfig/init.sh @@ -0,0 +1,96 @@ +#!/bin/bash +set -x +[ "$(pwd)" != "/etc/mysql/conf.d" ] && cp * /etc/mysql/conf.d/ + +HOST_ID=${HOSTNAME##*-} + +STATEFULSET_SERVICE=$(dnsdomainname -d) +POD_FQDN=$(dnsdomainname -A) + +echo "This is pod $HOST_ID ($POD_FQDN) for statefulset $STATEFULSET_SERVICE" + +[ -z "$WSREP_CLUSTER_ADDRESS" ] && echo "Missing WSREP_CLUSTER_ADDRESS env" && exit 1 +sed -i "s|^#init-wsrep#.*|wsrep_cluster_address=$WSREP_CLUSTER_ADDRESS|" /etc/mysql/conf.d/galera.cnf + +[ -z "$DATADIR" ] && exit "Missing DATADIR variable" && exit 1 + +SUGGEST_EXEC_COMMAND="kubectl --namespace=$POD_NAMESPACE exec -c init-config $POD_NAME --" + +function wsrepNewCluster { + sed -i 's|^#init-new-cluster#||' /etc/mysql/conf.d/galera.cnf +} + +function wsrepRecover { + sed -i 's|^#init-recover#||' /etc/mysql/conf.d/galera.cnf +} + +function wsrepForceBootstrap { + sed -i 's|safe_to_bootstrap: 0|safe_to_bootstrap: 1|' /data/db/grastate.dat +} + +[[ $STATEFULSET_SERVICE = mariadb.* ]] || echo "WARNING: unexpected service name $STATEFULSET_SERVICE, Peer detection below may fail falsely." + +if [ $HOST_ID -eq 0 ]; then + echo "This is the 1st statefulset pod. Checking if the statefulset is down ..." + getent hosts mariadb-ready + [ $? -eq 2 ] && { + # https://github.com/docker-library/mariadb/commit/f76084f0f9dc13f29cce48c727440eb79b4e92fa#diff-b0fa4b30392406b32de6b8ffe36e290dR80 + if [ ! -d "$DATADIR/mysql" ]; then + echo "No database in $DATADIR; configuring $POD_NAME for initial start" + wsrepNewCluster + else + set +x + echo "----- ACTION REQUIRED -----" + echo "No peers found, but data exists. To start in wsrep_new_cluster mode, run:" + echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-new-cluster" + echo "Or to start in recovery mode, to see replication state, run:" + echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-recover" + echo "Or to force bootstrap on this node, potentially losing writes, run:" + echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-force-bootstrap" + #echo " NOTE This bypasses the following warning from new cluster mode:" + #echo " It may not be safe to bootstrap the cluster from this node. It was not the last one to leave the cluster and may not contain all the updates. To force cluster bootstrap with this node, edit the grastate.dat file manually and set safe_to_bootstrap to 1 ." + echo "Or to try a regular start (for example after recovery + manual intervention), run:" + echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-resume" + if [ ! -z "$AUTO_RECOVERY_MODE" ]; then + echo "The AUTO_RECOVERY_MODE env was set to $AUTO_RECOVERY_MODE, will trigger that choice" + touch /tmp/$AUTO_RECOVERY_MODE + else + echo "Waiting for response ..." + fi + while [ ! -f /tmp/confirm-resume ]; do + if [ "$AUTO_NEW_CLUSTER" = "true" ]; then + echo "The AUTO_NEW_CLUSTER env was set to $AUTO_NEW_CLUSTER, will proceed without confirmation" + echo "NOTE this env is deprecated, use AUTO_RECOVERY_MODE instead" + wsrepNewCluster + touch /tmp/confirm-resume + elif [ -f /tmp/confirm-new-cluster ]; then + echo "Confirmation received. Resuming new cluster start ..." + wsrepNewCluster + touch /tmp/confirm-resume + elif [ -f /tmp/confirm-force-bootstrap ]; then + echo "Forcing bootstrap on this node ..." + wsrepForceBootstrap + touch /tmp/confirm-new-cluster + elif [ -f /tmp/confirm-recover ]; then + echo "Confirmation received. Resuming in recovery mode." + echo "Note: to start the other pods you need to edit OrderedReady and add a command: --wsrep-recover" + wsrepRecover + touch /tmp/confirm-resume + fi + sleep 1 + done + rm /tmp/confirm-* + set -x + fi + } +else + getent hosts mariadb-ready + [ $? -eq 2 ] && { + echo "This is NOT the 1st statefulset pod. Must not go up as primary." + echo "Found no ready pods. Will exit to trigger a crash loop back off." + exit 1 + } +fi + +# https://github.com/docker-library/mariadb/blob/master/10.2/docker-entrypoint.sh#L62 +mysqld --verbose --help --log-bin-index="$(mktemp -u)" | tee /tmp/mariadb-start-config | grep -e ^version -e ^datadir -e ^wsrep -e ^binlog -e ^character-set -e ^collation diff --git a/base-defaultconfig/kustomization.yaml b/base-defaultconfig/kustomization.yaml new file mode 100644 index 0000000..b0f7206 --- /dev/null +++ b/base-defaultconfig/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../base + +configMapGenerator: +- name: conf-d + files: + - init.sh + - datadir.cnf + - galera.cnf + - utf8.cnf diff --git a/base-defaultconfig/utf8.cnf b/base-defaultconfig/utf8.cnf new file mode 100644 index 0000000..88d7995 --- /dev/null +++ b/base-defaultconfig/utf8.cnf @@ -0,0 +1,25 @@ +[client-server] + +# This will be passed to all mysql clients +[client] +default-character-set = utf8mb4 + +# The MySQL server +[mysqld] +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci +init-connect='SET NAMES utf8mb4' + +[mysql] +default-character-set = utf8mb4 + +[mysqldump] + +# This group is only read by MariaDB servers, not by MySQL. +# If you use the same .cnf file for MySQL and MariaDB, +# you can put MariaDB-only options here +[mariadb] + +[mariadb-10.1] + +[mariadb-10.2] diff --git a/base/conf-d.yaml b/base/conf-d.yaml deleted file mode 100644 index 85f7a6c..0000000 --- a/base/conf-d.yaml +++ /dev/null @@ -1,157 +0,0 @@ -kind: ConfigMap -metadata: - name: conf-d -apiVersion: v1 -data: - datadir.cnf: | - [mysqld] - datadir=/data/db - galera.cnf: | - # - # * Galera-related settings - # - # https://mariadb.com/kb/en/mariadb/galera-cluster-system-variables/ - # - [galera] - # Mandatory settings - wsrep_on=ON - wsrep_provider="/usr/lib/galera/libgalera_smm.so" - #init-new-cluster#wsrep_new_cluster=TRUE - #init-recover#wsrep_recover=TRUE - binlog_format=ROW - default_storage_engine=InnoDB - innodb_autoinc_lock_mode=2 - #init-wsrep#wsrep_cluster_address="gcomm://mariadb-0.mariadb,mariadb-1.mariadb,mariadb-2.mariadb" - wsrep-sst-method=rsync - - # - # Allow server to accept connections on all interfaces. - # - bind-address=0.0.0.0 - # - # Optional setting - #wsrep_slave_threads=1 - #innodb_flush_log_at_trx_commit=0 - utf8.cnf: | - [client-server] - - # This will be passed to all mysql clients - [client] - default-character-set = utf8mb4 - - # The MySQL server - [mysqld] - character-set-server = utf8mb4 - collation-server = utf8mb4_unicode_ci - init-connect='SET NAMES utf8mb4' - - [mysql] - default-character-set = utf8mb4 - - [mysqldump] - - # This group is only read by MariaDB servers, not by MySQL. - # If you use the same .cnf file for MySQL and MariaDB, - # you can put MariaDB-only options here - [mariadb] - - [mariadb-10.1] - - [mariadb-10.2] - init.sh: | - #!/bin/bash - set -x - [ "$(pwd)" != "/etc/mysql/conf.d" ] && cp * /etc/mysql/conf.d/ - - HOST_ID=${HOSTNAME##*-} - - STATEFULSET_SERVICE=$(dnsdomainname -d) - POD_FQDN=$(dnsdomainname -A) - - echo "This is pod $HOST_ID ($POD_FQDN) for statefulset $STATEFULSET_SERVICE" - - [ -z "$WSREP_CLUSTER_ADDRESS" ] && echo "Missing WSREP_CLUSTER_ADDRESS env" && exit 1 - sed -i "s|^#init-wsrep#.*|wsrep_cluster_address=$WSREP_CLUSTER_ADDRESS|" /etc/mysql/conf.d/galera.cnf - - [ -z "$DATADIR" ] && exit "Missing DATADIR variable" && exit 1 - - SUGGEST_EXEC_COMMAND="kubectl --namespace=$POD_NAMESPACE exec -c init-config $POD_NAME --" - - function wsrepNewCluster { - sed -i 's|^#init-new-cluster#||' /etc/mysql/conf.d/galera.cnf - } - - function wsrepRecover { - sed -i 's|^#init-recover#||' /etc/mysql/conf.d/galera.cnf - } - - function wsrepForceBootstrap { - sed -i 's|safe_to_bootstrap: 0|safe_to_bootstrap: 1|' /data/db/grastate.dat - } - - [[ $STATEFULSET_SERVICE = mariadb.* ]] || echo "WARNING: unexpected service name $STATEFULSET_SERVICE, Peer detection below may fail falsely." - - if [ $HOST_ID -eq 0 ]; then - echo "This is the 1st statefulset pod. Checking if the statefulset is down ..." - getent hosts mariadb-ready - [ $? -eq 2 ] && { - # https://github.com/docker-library/mariadb/commit/f76084f0f9dc13f29cce48c727440eb79b4e92fa#diff-b0fa4b30392406b32de6b8ffe36e290dR80 - if [ ! -d "$DATADIR/mysql" ]; then - echo "No database in $DATADIR; configuring $POD_NAME for initial start" - wsrepNewCluster - else - set +x - echo "----- ACTION REQUIRED -----" - echo "No peers found, but data exists. To start in wsrep_new_cluster mode, run:" - echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-new-cluster" - echo "Or to start in recovery mode, to see replication state, run:" - echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-recover" - echo "Or to force bootstrap on this node, potentially losing writes, run:" - echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-force-bootstrap" - #echo " NOTE This bypasses the following warning from new cluster mode:" - #echo " It may not be safe to bootstrap the cluster from this node. It was not the last one to leave the cluster and may not contain all the updates. To force cluster bootstrap with this node, edit the grastate.dat file manually and set safe_to_bootstrap to 1 ." - echo "Or to try a regular start (for example after recovery + manual intervention), run:" - echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-resume" - if [ ! -z "$AUTO_RECOVERY_MODE" ]; then - echo "The AUTO_RECOVERY_MODE env was set to $AUTO_RECOVERY_MODE, will trigger that choice" - touch /tmp/$AUTO_RECOVERY_MODE - else - echo "Waiting for response ..." - fi - while [ ! -f /tmp/confirm-resume ]; do - if [ "$AUTO_NEW_CLUSTER" = "true" ]; then - echo "The AUTO_NEW_CLUSTER env was set to $AUTO_NEW_CLUSTER, will proceed without confirmation" - echo "NOTE this env is deprecated, use AUTO_RECOVERY_MODE instead" - wsrepNewCluster - touch /tmp/confirm-resume - elif [ -f /tmp/confirm-new-cluster ]; then - echo "Confirmation received. Resuming new cluster start ..." - wsrepNewCluster - touch /tmp/confirm-resume - elif [ -f /tmp/confirm-force-bootstrap ]; then - echo "Forcing bootstrap on this node ..." - wsrepForceBootstrap - touch /tmp/confirm-new-cluster - elif [ -f /tmp/confirm-recover ]; then - echo "Confirmation received. Resuming in recovery mode." - echo "Note: to start the other pods you need to edit OrderedReady and add a command: --wsrep-recover" - wsrepRecover - touch /tmp/confirm-resume - fi - sleep 1 - done - rm /tmp/confirm-* - set -x - fi - } - else - getent hosts mariadb-ready - [ $? -eq 2 ] && { - echo "This is NOT the 1st statefulset pod. Must not go up as primary." - echo "Found no ready pods. Will exit to trigger a crash loop back off." - exit 1 - } - fi - - # https://github.com/docker-library/mariadb/blob/master/10.2/docker-entrypoint.sh#L62 - mysqld --verbose --help --log-bin-index="$(mktemp -u)" | tee /tmp/mariadb-start-config | grep -e ^version -e ^datadir -e ^wsrep -e ^binlog -e ^character-set -e ^collation diff --git a/base/kustomization.yaml b/base/kustomization.yaml index f6dbab6..387cd35 100644 --- a/base/kustomization.yaml +++ b/base/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- conf-d.yaml - mariadb-service.yaml - mariadb-ready-service.yaml - mysql-service.yaml From ac5ffea58dc6296cadc76ee4098e4f5598103006 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 06:05:13 +0200 Subject: [PATCH 06/18] From what I understand this is the default behavior with 10.4+ See https://mariadb.com/kb/en/authentication-from-mariadb-104/ --- base/mariadb-statefulset.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index edbbe43..8bf6537 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -67,10 +67,6 @@ spec: - containerPort: 4568 name: ist env: - - name: MYSQL_ROOT_HOST - value: "localhost" - - name: MYSQL_ALLOW_EMPTY_PASSWORD - value: "yes" - name: MYSQL_INITDB_SKIP_TZINFO value: "yes" readinessProbe: From fa7dcbda36d5be5191978ec62db46daaca225601 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 06:51:45 +0200 Subject: [PATCH 07/18] Metrics gets "Access denied for user" but is now disabled by default We use a pod monitor as below, and password by env override with for example CREATE USER 'exporter'@'127.0.0.1' IDENTIFIED BY 'exporter' WITH MAX_USER_CONNECTIONS 3; GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'127.0.0.1'; An alternative would be to use socket+root, but the exporter runs as nonroot properly and I'd recommend against making it root. apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: mariadb labels: prometheus: now spec: jobLabel: app.kubernetes.io/name namespaceSelector: any: false selector: matchLabels: app: mariadb podMetricsEndpoints: - port: metrics --- base/mariadb-statefulset.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 8bf6537..fd72efc 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -13,9 +13,6 @@ spec: metadata: labels: app: mariadb - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9104" spec: terminationGracePeriodSeconds: 30 initContainers: @@ -95,7 +92,8 @@ spec: image: prom/mysqld-exporter:v0.13.0@sha256:a8af600c3ef1c8df179b736b94d04dc5ec209be88407a4c1c1bd0fc6394f56e8 env: - name: DATA_SOURCE_NAME - value: root@(localhost:3306)/ + # https://github.com/prometheus/mysqld_exporter#required-grants + value: exporter:exporter@tcp(127.0.0.1:3306)/ ports: - name: metrics containerPort: 9104 From 18cb65da92e9dfc0084eafc4285a12c826a0a67a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 10:01:11 +0200 Subject: [PATCH 08/18] GKE's limit for pods on preemptible nodes https://cloud.google.com/kubernetes-engine/docs/how-to/preemptible-vms#kubernetes_preemptible_nodes --- base/mariadb-statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index fd72efc..c0bd066 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -14,7 +14,7 @@ spec: labels: app: mariadb spec: - terminationGracePeriodSeconds: 30 + terminationGracePeriodSeconds: 25 initContainers: - name: init-config command: ['/bin/bash', '/etc/mysql/conf.d-configmap/init.sh'] From 16ab70f9e45f4f34566ac28f3960ad44c6a235e8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 Jun 2021 10:58:46 +0200 Subject: [PATCH 09/18] With the kustomize base in a subfolder we can use relative refs Also let's remove obsolete variants. Note that scale-2 is not recommended as Galera cluster size. --- ...-storageclass-ssd-regional-europewest1cd.yaml | 11 ----------- variants/gke/gke-storageclass-ssd.yml | 9 --------- variants/gke/kustomization.yaml | 7 ------- variants/gke/volume-claims.yaml | 16 ---------------- variants/namespace/kustomization.yaml | 3 --- variants/scale-1-ephemeral/kustomization.yaml | 5 ++++- variants/scale-1/kustomization.yaml | 8 ++++++-- variants/scale-1/replicas-1.yaml | 1 - variants/scale-2/kustomization.yaml | 8 ++++++-- 9 files changed, 16 insertions(+), 52 deletions(-) delete mode 100644 variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml delete mode 100644 variants/gke/gke-storageclass-ssd.yml delete mode 100644 variants/gke/kustomization.yaml delete mode 100644 variants/gke/volume-claims.yaml delete mode 100644 variants/namespace/kustomization.yaml diff --git a/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml b/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml deleted file mode 100644 index c5bd899..0000000 --- a/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: mysql-data -provisioner: kubernetes.io/gce-pd -reclaimPolicy: Retain -allowVolumeExpansion: true -parameters: - type: pd-ssd - replication-type: regional-pd - zones: europe-west1-c, europe-west1-d diff --git a/variants/gke/gke-storageclass-ssd.yml b/variants/gke/gke-storageclass-ssd.yml deleted file mode 100644 index cd4852f..0000000 --- a/variants/gke/gke-storageclass-ssd.yml +++ /dev/null @@ -1,9 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: mysql-data -provisioner: kubernetes.io/gce-pd -reclaimPolicy: Retain -allowVolumeExpansion: true -parameters: - type: pd-ssd diff --git a/variants/gke/kustomization.yaml b/variants/gke/kustomization.yaml deleted file mode 100644 index 50b758f..0000000 --- a/variants/gke/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -bases: -- github.com/Yolean/kubernetes-mysql-cluster?ref=009cad0 -resources: -- gke-storageclass-ssd.yml -#- gke-storageclass-ssd-regional-europewest1cd.yaml -patchesStrategicMerge: -- volume-claims.yaml diff --git a/variants/gke/volume-claims.yaml b/variants/gke/volume-claims.yaml deleted file mode 100644 index 0de97f6..0000000 --- a/variants/gke/volume-claims.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mariadb - namespace: mysql -spec: - volumeClaimTemplates: - - metadata: - name: mysql - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: mysql-data - resources: - requests: - storage: 1Gi diff --git a/variants/namespace/kustomization.yaml b/variants/namespace/kustomization.yaml deleted file mode 100644 index 16d767e..0000000 --- a/variants/namespace/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -bases: -- github.com/Yolean/kubernetes-mysql-cluster?ref=549e804 -namespace: analytics diff --git a/variants/scale-1-ephemeral/kustomization.yaml b/variants/scale-1-ephemeral/kustomization.yaml index 5498756..fccc152 100644 --- a/variants/scale-1-ephemeral/kustomization.yaml +++ b/variants/scale-1-ephemeral/kustomization.yaml @@ -1,4 +1,7 @@ -bases: +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: - ../scale-1 patchesStrategicMerge: - ephemeral.yaml diff --git a/variants/scale-1/kustomization.yaml b/variants/scale-1/kustomization.yaml index 6e1fa9a..d32ef46 100644 --- a/variants/scale-1/kustomization.yaml +++ b/variants/scale-1/kustomization.yaml @@ -1,5 +1,9 @@ -bases: -- github.com/Yolean/kubernetes-mysql-cluster?ref=a9e6956000a31c973d183d8c318d6828cd73c26c +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base-defaultconfig + patchesStrategicMerge: - replicas-1.yaml - wsrep-1.yaml diff --git a/variants/scale-1/replicas-1.yaml b/variants/scale-1/replicas-1.yaml index 2d5a351..212208f 100644 --- a/variants/scale-1/replicas-1.yaml +++ b/variants/scale-1/replicas-1.yaml @@ -2,6 +2,5 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: mariadb - namespace: mysql spec: replicas: 1 diff --git a/variants/scale-2/kustomization.yaml b/variants/scale-2/kustomization.yaml index 2f7540f..bc0afbd 100644 --- a/variants/scale-2/kustomization.yaml +++ b/variants/scale-2/kustomization.yaml @@ -1,5 +1,9 @@ -bases: -- github.com/Yolean/kubernetes-mysql-cluster?ref=8c4439f +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base-defaultconfig + patchesStrategicMerge: - replicas-2.yaml - wsrep-2.yaml From 95d2ff9737be2610085616f555dc1e7ab8981a37 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 27 Jun 2021 10:55:46 +0200 Subject: [PATCH 10/18] Fixes "no matches for IdId apps_v1_StatefulSet|~X|mariadb" when using an overlay with "namespace:" --- variants/scale-1-ephemeral/ephemeral.yaml | 1 - variants/scale-2/replicas-2.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/variants/scale-1-ephemeral/ephemeral.yaml b/variants/scale-1-ephemeral/ephemeral.yaml index c2f015b..57b576c 100644 --- a/variants/scale-1-ephemeral/ephemeral.yaml +++ b/variants/scale-1-ephemeral/ephemeral.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: mariadb - namespace: mysql spec: template: spec: diff --git a/variants/scale-2/replicas-2.yaml b/variants/scale-2/replicas-2.yaml index e57c624..41fee73 100644 --- a/variants/scale-2/replicas-2.yaml +++ b/variants/scale-2/replicas-2.yaml @@ -2,6 +2,5 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: mariadb - namespace: mysql spec: replicas: 2 From 8d3cb626800c15303a20d13aa819a80ac6f62ecb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 27 Jun 2021 11:16:06 +0200 Subject: [PATCH 11/18] Revert "From what I understand this is the default behavior with 10.4+" I still think it's the default behavior, but maybe the entrypoint script hasn't been updated to reflect the invalid-by-default root pwd. Without these envs you get: [ERROR] [Entrypoint]: Database is uninitialized and password option is not specified You need to specify one of MARIADB_ROOT_PASSWORD, MARIADB_ALLOW_EMPTY_ROOT_PASSWORD and MARIADB_RANDOM_ROOT_PASSWORD This reverts commit ac5ffea58dc6296cadc76ee4098e4f5598103006. --- base/mariadb-statefulset.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index c0bd066..067f0fa 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -64,6 +64,10 @@ spec: - containerPort: 4568 name: ist env: + - name: MYSQL_ROOT_HOST + value: "localhost" + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "yes" - name: MYSQL_INITDB_SKIP_TZINFO value: "yes" readinessProbe: From c5d29f30f66dfb5a7671e07a41528dfbeb05a44f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 20 Sep 2021 20:17:15 +0200 Subject: [PATCH 12/18] We have no indication that we've ever had use for this liveness probe and as the comment suggested it actually caused downtime for a large state transfer --- base/mariadb-statefulset.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 067f0fa..06114f9 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -77,14 +77,6 @@ spec: - -ec - test ON = "$(mysql -e "SHOW STATUS LIKE 'wsrep_ready';" -N -B | sed 's/wsrep_ready\t//')" initialDelaySeconds: 30 - livenessProbe: - exec: - command: - - mysql - - -e - - "SHOW DATABASES;" - # might need to be tweaked for large initial state transfers - initialDelaySeconds: 60 volumeMounts: - name: mysql mountPath: /data From 0343acf27ed1e3c699b2cc1ebfbdd8614a362c79 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 20 Sep 2021 20:24:54 +0200 Subject: [PATCH 13/18] Current mariadb 10.5 --- base/mariadb-statefulset.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 06114f9..4d072f1 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -47,10 +47,10 @@ spec: mountPath: /etc/mysql/conf.d - name: initdb mountPath: /docker-entrypoint-initdb.d - image: mariadb:10.5.11-focal@sha256:228d87496cc141569f5f692594823fb1d62a361b6f8848165eb4f953af5e3c3a + image: mariadb:10.5.12-focal@sha256:dfcba5641bdbfd7cbf5b07eeed707e6a3672f46823695a0d3aba2e49bbd9b1dd containers: - name: mariadb - image: mariadb:10.5.11-focal@sha256:228d87496cc141569f5f692594823fb1d62a361b6f8848165eb4f953af5e3c3a + image: mariadb:10.5.12-focal@sha256:dfcba5641bdbfd7cbf5b07eeed707e6a3672f46823695a0d3aba2e49bbd9b1dd ports: - containerPort: 3306 name: mysql From 93c4de217080d258889c43c92235994af577dd22 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 20 Nov 2021 16:41:51 +0100 Subject: [PATCH 14/18] Current MariaDB 10.5.x --- base/mariadb-statefulset.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 4d072f1..0109642 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -47,10 +47,10 @@ spec: mountPath: /etc/mysql/conf.d - name: initdb mountPath: /docker-entrypoint-initdb.d - image: mariadb:10.5.12-focal@sha256:dfcba5641bdbfd7cbf5b07eeed707e6a3672f46823695a0d3aba2e49bbd9b1dd + image: mariadb:10.5.13-focal@sha256:6062d06836b8a108cffa424f659a564cb6a82ff18a4927ee5625d0e41fced2bf containers: - name: mariadb - image: mariadb:10.5.12-focal@sha256:dfcba5641bdbfd7cbf5b07eeed707e6a3672f46823695a0d3aba2e49bbd9b1dd + image: mariadb:10.5.13-focal@sha256:6062d06836b8a108cffa424f659a564cb6a82ff18a4927ee5625d0e41fced2bf ports: - containerPort: 3306 name: mysql From 194a4485403c32a2a3faa640c1cb3a89314292da Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 24 Nov 2021 07:38:06 +0100 Subject: [PATCH 15/18] Simplifies scale-1 runtime by disabling galera --- variants/scale-1/galera.cnf | 2 ++ variants/scale-1/kustomization.yaml | 8 +++++++- variants/scale-1/wsrep-1.yaml | 12 ------------ variants/scale-1/wsrep-off.yaml | 20 ++++++++++++++++++++ 4 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 variants/scale-1/galera.cnf delete mode 100644 variants/scale-1/wsrep-1.yaml create mode 100644 variants/scale-1/wsrep-off.yaml diff --git a/variants/scale-1/galera.cnf b/variants/scale-1/galera.cnf new file mode 100644 index 0000000..56b788c --- /dev/null +++ b/variants/scale-1/galera.cnf @@ -0,0 +1,2 @@ +[galera] +wsrep_on=OFF diff --git a/variants/scale-1/kustomization.yaml b/variants/scale-1/kustomization.yaml index d32ef46..d206f89 100644 --- a/variants/scale-1/kustomization.yaml +++ b/variants/scale-1/kustomization.yaml @@ -6,4 +6,10 @@ resources: patchesStrategicMerge: - replicas-1.yaml -- wsrep-1.yaml +- wsrep-off.yaml + +configMapGenerator: +- name: conf-d + behavior: merge + files: + - galera.cnf diff --git a/variants/scale-1/wsrep-1.yaml b/variants/scale-1/wsrep-1.yaml deleted file mode 100644 index 95a61ef..0000000 --- a/variants/scale-1/wsrep-1.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mariadb -spec: - template: - spec: - initContainers: - - name: init-config - env: - - name: WSREP_CLUSTER_ADDRESS - value: "gcomm://mariadb-0.mariadb" diff --git a/variants/scale-1/wsrep-off.yaml b/variants/scale-1/wsrep-off.yaml new file mode 100644 index 0000000..6870398 --- /dev/null +++ b/variants/scale-1/wsrep-off.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mariadb +spec: + template: + spec: + containers: + - name: mariadb + readinessProbe: + exec: + command: + - mysqladmin + - status + initContainers: + - name: init-config + env: + # So the init script can configure galera unconditionally; galera.cnf is overridden to disable wsrep + - name: WSREP_CLUSTER_ADDRESS + value: "gcomm://mariadb-0.mariadb" From 0fbbaf9ac73776b39fdb9a38c6975abf6c77646d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Mar 2022 21:30:57 +0100 Subject: [PATCH 16/18] Current MariaDB 10.5.x multiarch --- base/mariadb-statefulset.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 0109642..0e60cb2 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -47,10 +47,10 @@ spec: mountPath: /etc/mysql/conf.d - name: initdb mountPath: /docker-entrypoint-initdb.d - image: mariadb:10.5.13-focal@sha256:6062d06836b8a108cffa424f659a564cb6a82ff18a4927ee5625d0e41fced2bf + image: mariadb:10.5.15-focal@sha256:c911279a1005ed40435962466920eafa5e55b1f5c51182086e76ccf7262b45cd containers: - name: mariadb - image: mariadb:10.5.13-focal@sha256:6062d06836b8a108cffa424f659a564cb6a82ff18a4927ee5625d0e41fced2bf + image: mariadb:10.5.15-focal@sha256:c911279a1005ed40435962466920eafa5e55b1f5c51182086e76ccf7262b45cd ports: - containerPort: 3306 name: mysql From 8b5c19a6df883621168e3423dd3ad4849e035af2 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Mar 2022 21:32:00 +0100 Subject: [PATCH 17/18] Current mysqld-exporter multiarch --- base/mariadb-statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 0e60cb2..4b3d4e1 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -85,7 +85,7 @@ spec: - name: initdb mountPath: /docker-entrypoint-initdb.d - name: metrics - image: prom/mysqld-exporter:v0.13.0@sha256:a8af600c3ef1c8df179b736b94d04dc5ec209be88407a4c1c1bd0fc6394f56e8 + image: prom/mysqld-exporter:v0.14.0@sha256:eb6fe170738bf9181c51f5bc89f93adb26672ec49ffdcb22f55c24834003b45d env: - name: DATA_SOURCE_NAME # https://github.com/prometheus/mysqld_exporter#required-grants From cdc1ff514685a27f51e2596456e478c3e908dab9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 19 Feb 2023 09:50:10 +0100 Subject: [PATCH 18/18] Current MariaDB 10.5.x --- base/mariadb-statefulset.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/mariadb-statefulset.yaml b/base/mariadb-statefulset.yaml index 4b3d4e1..58ad66b 100644 --- a/base/mariadb-statefulset.yaml +++ b/base/mariadb-statefulset.yaml @@ -47,10 +47,10 @@ spec: mountPath: /etc/mysql/conf.d - name: initdb mountPath: /docker-entrypoint-initdb.d - image: mariadb:10.5.15-focal@sha256:c911279a1005ed40435962466920eafa5e55b1f5c51182086e76ccf7262b45cd + image: mariadb:10.5.19-focal@sha256:ce75a36261b12948b411f6debb71254d92889c8a06e2ebc34c761f90416b1024 containers: - name: mariadb - image: mariadb:10.5.15-focal@sha256:c911279a1005ed40435962466920eafa5e55b1f5c51182086e76ccf7262b45cd + image: mariadb:10.5.19-focal@sha256:ce75a36261b12948b411f6debb71254d92889c8a06e2ebc34c761f90416b1024 ports: - containerPort: 3306 name: mysql