diff --git a/.gitignore b/.gitignore index ca60915..ed6baff 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ cluster-lock.json .DS_Store data/ .idea -.charon +.charon* prometheus/prometheus.yml +clusters/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..71a3322 --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +.PHONY: multi-cluster-setup multi-cluster-add-cluster multi-cluster-delete-cluster multi-cluster-start-cluster multi-cluster-stop-cluster multi-cluster-start-base multi-cluster-stop-base + +check_defined = \ + $(strip $(foreach 1,$1, \ + $(call __check_defined,$1,$(strip $(value 2))))) +__check_defined = \ + $(if $(value $1),, \ + $(error $1$(if $2, ($2)) is not set. Set it by running `make $1$(if $2, ($2))=`)) + +multi-cluster-setup: + $(call check_defined, name) + ./multi_cluster/setup.sh $(name) + +multi-cluster-add-cluster: + $(call check_defined, name) + ./multi_cluster/cluster.sh add $(name) + +multi-cluster-delete-cluster: + $(call check_defined, name) + ./multi_cluster/cluster.sh delete $(name) + +multi-cluster-start-cluster: + $(call check_defined, name) + ./multi_cluster/cluster.sh start $(name) + +multi-cluster-stop-cluster: + $(call check_defined, name) + ./multi_cluster/cluster.sh stop $(name) + +multi-cluster-start-base: + ./multi_cluster/base.sh start + +multi-cluster-stop-base: + ./multi_cluster/base.sh stop diff --git a/README.md b/README.md index 75ea3ba..43b2bbb 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,85 @@ docker compose -f examples/nethermind_teku_lighthouse.yml up # FAQs Check the Obol docs for frequent [errors and resolutions](https://docs.obol.tech/docs/faq/errors) + + +# Multi cluster setup + +There is an option to run multiple Charon clusters using the same Execution Layer Client (EL), Consensus Layer Client (CL) and Grafana. This way you can operate multiple clusters for different purposes, without putting much more pressure on your system. + +The way this is achieved is by separating the EL, CL and Grafana from the Charon node, Validator Client (VC) and Prometheus. Instead of having `.charon/` folder in the root directory it is moved to `clusters/{CLUSTER_NAME}/.charon`. Moreover, the VC and Prometheus data is now per cluster as well, moved from `data/lodestar` and `data/prometheus` to `clusters/{CLUSTER_NAME}/data/lodestar` and `clusters/{CLUSTER_NAME}/data/prometheus`, respectively. `docker-compose.yml` and `.env` are also used per cluster. There are also supporting scripts for the Charon node and the VC. + +## Setup + +If you already have running validator node in Docker, the Docker containers will be moved to the new multi cluster setup. + +```bash +./multi_cluster/setup.sh {CLUSTER_NAME} +``` + +You can inspect what you have in the `./clusters/` directory. Each subfolder is a cluster with the following structure: + +```directory +clusters +└───{CLUSTER_NAME} # cluster name +│ │ .charon # folder including secret material used by charon +│ │ data # data from the validator client and prometheus +│ │ lodestar # scripts used by lodestar +│ │ prometheus # scripts and configs used by prometheus +│ │ .env # environment variables used by the cluster +│ │ docker-compose.yml # docker compose used by the cluster +│ # N.B.: only services with profile "cluster" are ran +└───{CLUSTER_NAME_2} +└───{CLUSTER_NAME_...} +└───{CLUSTER_NAME_N} +``` + +Note that those folders and files are copied from the root directory. Meaning all configurations and setup you have already done, will be copied to this first cluster of the multi cluster setup. + +## Manage cluster + +Manage the Charon + Validator Client + Prometheus containers of each cluster found in `./clusters/`. + +### Add cluster + +```bash +./multi_cluster/cluster.sh add {CLUSTER_NAME} +``` + +Note that only the `.env`, `lodestar/`, `prometheus/` and `docker-compose.yml` files and directories are coiped from the root directory to the new cluster. `.charon/` and `data/` folders are expected to be from a brand new cluster that you will setup in the `./clusters/{CLUSTER_NAME}` directory. + +### Start cluster + +It is expected that you have already done the regular procedure from cluster setup and you have `./clusters/{CLUSTER_NAME}/.charon/` folder. + +```bash +./multi_cluster/cluster.sh start {CLUSTER_NAME} +``` + +### Stop cluster + +```bash +./multi_cluster/cluster.sh stop {CLUSTER_NAME} +``` + +### Delete cluster + +```bash +./multi_cluster/cluster.sh delete {CLUSTER_NAME} +``` + +## Manage base node + +Manage the EL + CL + Grafana containers. + +### Start base node + +```bash +./multi_cluster/base.sh start +``` + +### Stop base node + +```bash +./multi_cluster/base.sh stop +``` diff --git a/docker-compose.yml b/docker-compose.yml index 4a47426..c975e4b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: # | | | | __/ |_| | | | __/ | | | | | | | | | | | (_| | # |_| |_|\___|\__|_| |_|\___|_| |_| |_| |_|_|_| |_|\__,_| nethermind: + profiles: ["base", ""] image: nethermind/nethermind:${NETHERMIND_VERSION:-1.29.0} restart: unless-stopped ports: @@ -44,6 +45,7 @@ services: # |___/ lighthouse: + profiles: ["base", ""] image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v5.3.0} ports: - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/tcp # P2P TCP @@ -77,22 +79,21 @@ services: # \___|_| |_|\__,_|_| \___/|_| |_| charon: + profiles: ["cluster", ""] image: obolnetwork/charon:${CHARON_VERSION:-v1.2.0} environment: - - CHARON_BEACON_NODE_ENDPOINTS=${CHARON_BEACON_NODE_ENDPOINTS:-http://lighthouse:5052} - - CHARON_BEACON_NODE_TIMEOUT=${CHARON_BEACON_NODE_TIMEOUT:-3s} - - CHARON_BEACON_NODE_SUBMIT_TIMEOUT=${CHARON_BEACON_NODE_SUBMIT_TIMEOUT:-4s} - - CHARON_LOG_LEVEL=${CHARON_LOG_LEVEL:-info} - - CHARON_LOG_FORMAT=${CHARON_LOG_FORMAT:-console} - - CHARON_P2P_RELAYS=${CHARON_P2P_RELAYS:-https://0.relay.obol.tech,https://1.relay.obol.tech/} - - CHARON_P2P_EXTERNAL_HOSTNAME=${CHARON_P2P_EXTERNAL_HOSTNAME:-} # Empty default required to avoid warnings. - - CHARON_P2P_TCP_ADDRESS=0.0.0.0:${CHARON_PORT_P2P_TCP:-3610} - - CHARON_VALIDATOR_API_ADDRESS=0.0.0.0:3600 - - CHARON_MONITORING_ADDRESS=0.0.0.0:3620 - - CHARON_BUILDER_API=${BUILDER_API_ENABLED:-false} - - CHARON_FEATURE_SET_ENABLE=${CHARON_FEATURE_SET_ENABLE:-} - - CHARON_LOKI_ADDRESSES=${CHARON_LOKI_ADDRESSES:-http://loki:3100/loki/api/v1/push} - - CHARON_LOKI_SERVICE=charon + CHARON_BEACON_NODE_ENDPOINTS: ${CHARON_BEACON_NODE_ENDPOINTS:-http://lighthouse:5052} + CHARON_LOG_LEVEL: ${CHARON_LOG_LEVEL:-info} + CHARON_LOG_FORMAT: ${CHARON_LOG_FORMAT:-console} + CHARON_P2P_RELAYS: ${CHARON_P2P_RELAYS:-https://0.relay.obol.tech,https://1.relay.obol.tech/} + CHARON_P2P_EXTERNAL_HOSTNAME: ${CHARON_P2P_EXTERNAL_HOSTNAME:-} # Empty default required to avoid warnings. + CHARON_P2P_TCP_ADDRESS: 0.0.0.0:${CHARON_PORT_P2P_TCP:-3610} + CHARON_VALIDATOR_API_ADDRESS: 0.0.0.0:3600 + CHARON_MONITORING_ADDRESS: 0.0.0.0:3620 + CHARON_BUILDER_API: ${BUILDER_API_ENABLED:-false} + CHARON_FEATURE_SET_ENABLE: ${CHARON_FEATURE_SET_ENABLE:-} + CHARON_LOKI_ADDRESSES: ${CHARON_LOKI_ADDRESSES:-http://loki:3100/loki/api/v1/push} + CHARON_LOKI_SERVICE: charon ports: - ${CHARON_PORT_P2P_TCP:-3610}:${CHARON_PORT_P2P_TCP:-3610}/tcp # P2P TCP libp2p networks: [dvnode] @@ -109,6 +110,7 @@ services: # |_|\___/ \__,_|\___||___/\__\__,_|_| lodestar: + profiles: ["cluster", ""] image: chainsafe/lodestar:${LODESTAR_VERSION:-v1.23.0} depends_on: [charon] entrypoint: /opt/lodestar/run.sh @@ -130,6 +132,7 @@ services: # | | | | | | __/\ V /_____| |_) | (_) | (_) \__ \ |_ # |_| |_| |_|\___| \_/ |_.__/ \___/ \___/|___/\__| mev-boost: + profiles: ["base", ""] image: ${MEVBOOST_IMAGE:-flashbots/mev-boost}:${MEVBOOST_VERSION:-1.8.1} command: | -${NETWORK} @@ -147,6 +150,7 @@ services: # |_| |_| |_|\___/|_| |_|_|\__\___/|_| |_|_| |_|\__, | # |___/ prometheus: + profiles: ["cluster", ""] image: prom/prometheus:${PROMETHEUS_VERSION:-v2.50.1} user: ":" networks: [dvnode] @@ -160,6 +164,7 @@ services: restart: unless-stopped grafana: + profiles: ["base", ""] image: grafana/grafana:${GRAFANA_VERSION:-10.4.2} user: ":" ports: diff --git a/multi_cluster/base.sh b/multi_cluster/base.sh new file mode 100755 index 0000000..bec085c --- /dev/null +++ b/multi_cluster/base.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 [OPTIONS] COMMAND" + echo "" + echo " Manage the base ethereum node (EL, CL, MEV boost, Grafana), without interfering with any validator." + echo "" + echo "Commands:" + echo " start Start an ethereum node, MEV-boost and Grafana." + echo " stop Stop an ethereum node, MEV-boost and Grafana." + echo "" + echo "Options:" + echo " -h Display this help message." +} + +usage_start() { + echo "Usage: $0 start [OPTIONS]" + echo "" + echo " Start the base ethereum node." + echo "" + echo "Options:" + echo " -h Display this help message." + echo "" + echo "Example:" + echo " $0 start" +} + +usage_stop() { + echo "Usage: $0 stop [OPTIONS]" + echo "" + echo " Stop the base ethereum node." + echo "" + echo "Options:" + echo " -h Display this help message." + echo "" + echo "Example:" + echo " $0 stop" +} + +start() { + docker compose --profile base up -d +} + +stop() { + docker compose --profile base stop +} + +while getopts ":h" opt; do + case $opt in + h) + usage + exit 0 + ;; + \?) + usage + exit 1 + ;; + :) + usage + exit 1 + ;; + esac +done + +shift $((OPTIND - 1)) + +subcommand=$1 +shift +case "$subcommand" in +# Parse options to the install sub command +start) + while getopts ":h" opt; do + case $opt in + h) + usage_start + exit 0 + ;; + ?) # Invalid option + usage_start + exit 1 + ;; + esac + done + shift $((OPTIND - 1)) + start + ;; +stop) + while getopts ":h" opt; do + case $opt in + h) + usage_stop + exit 0 + ;; + ?) # Invalid option + usage_stop + exit 1 + ;; + esac + done + shift $((OPTIND - 1)) + stop + ;; +*) + usage + exit 1 + ;; + +esac diff --git a/multi_cluster/cluster.sh b/multi_cluster/cluster.sh new file mode 100755 index 0000000..2bfa76c --- /dev/null +++ b/multi_cluster/cluster.sh @@ -0,0 +1,410 @@ +#!/bin/bash + +# shellcheck disable=SC1090,SC1091 + +unset -v cluster_name +skip_port_free_check= +p2p_default_port=3610 + +usage_base() { + echo "Usage: $0 [OPTIONS] COMMAND" + echo "" + echo " Manage a validator cluster (Charon + VC + Prometheus), found in ./clusters directory." + echo "" + echo "Commands:" + echo " add string Add a validator cluster to the ./clusters directory." + echo " delete string Delete a validator cluster from the ./clusters directory." + echo " start string Start a validator cluster, found in the ./clusters directory." + echo " stop string Stop a validator cluster, found in the ./clusters directory." + echo "" + echo "Options:" + echo " -h Display this help message." +} + +usage_add() { + echo "Usage: $0 add [OPTIONS] NAME" + echo "" + echo " Add a new cluster with specified name." + echo "" + echo "Options:" + echo " -h Display this help message." + echo " -s Skip free port checking with netstat/ss." + echo " -p integer Override the default port (3610) from which to start the search of a free port." + echo "" + echo "Example:" + echo " $0 add second-cluster" + echo " $0 add -s third-cluster-without-free-port-check" + echo " $0 add -p 3615 fourth-cluster-with-custom-port" +} + +usage_delete() { + echo "Usage: $0 delete [OPTIONS] NAME" + echo "" + echo " Delete an existing cluster with the specified name. A cluster name is a folder in ./clusters dir." + echo "" + echo "Options:" + echo " -h Display this help message." + echo "" + echo "Example:" + echo " $0 delete second-cluster" +} + +usage_start() { + echo "Usage: $0 start [OPTIONS] NAME" + echo "" + echo " Start an existing cluster with the specified name. A cluster name is a folder in ./clusters dir." + echo "" + echo "Options:" + echo " -h Display this help message." + echo "" + echo "Example:" + echo " $0 start second-cluster" +} + +usage_stop() { + echo "Usage: $0 stop [OPTIONS] NAME" + echo "" + echo " Stop an existing cluster with the specified name. A cluster name is a folder in ./clusters dir." + echo "" + echo "Options:" + echo " -h Display this help message." + echo "" + echo "Example:" + echo " $0 stop second-cluster" +} + +# Check if cluster_name variable is set. +check_missing_cluster_name() { + if [ -z "$cluster_name" ]; then + echo 'Missing cluster name argument.' >&2 + exit 1 + fi +} + +# Check if ./clusters directory exists. +check_clusters_dir_does_not_exist() { + if test ! -d ./clusters; then + echo "./clusters directory does not exist. Run setup.sh first." + exit 1 + fi +} + +# Check if cluster with the specified cluster_name already exists. +check_cluster_already_exists() { + if test -d "./clusters/${cluster_name}"; then + echo "./clusters/${cluster_name} directory already exists." + exit 1 + fi +} + +# Check if cluster with the specified cluster_name does not exist. +check_cluster_does_not_exist() { + if test ! -d "./clusters/${cluster_name}"; then + echo "./clusters/$cluster_name directory does not exist." + exit 1 + fi +} + +# Add cluster to the ./clusters/{cluster_name} directory. +add() { + # Try to find free and unallocated to another cluster ports. + # Port number from which to start the search of free port, default is 3610. + port=$p2p_default_port + + is_occupied=1 + # Run loop until is_occupied is empty. + while [[ -n "$is_occupied" ]]; do + # Check if TCP port is free, if it is, is_occupied is set to empty, otherwise increment the port by 1 and continue the loop. + if [ -z ${skip_port_free_check} ]; then + if [ -x "$(command -v netstat)" ]; then + if is_occupied=$(netstat -taln | grep -w "$port"); then + port=$((port + 1)) + continue + fi + elif [ -x "$(command -v ss)" ]; then + if is_occupied=$(ss -taln | grep -w "$port"); then + port=$((port + 1)) + continue + fi + else + echo "Neither netstat or ss commands found. Please install either of those to check for free ports or add the -p flag to skip port check." + exit 1 + fi + else + # Assume port is not occupied if no netstat/ss check. + is_occupied= + fi + # Check if TCP port is used by another cluster from the ./clusters directory. + for cluster in ./clusters/*; do + # Check if it is used by the p2p TCP port of this cluster. + p2p_cluster_port=$( + . "./${cluster}/.env" + printf '%s' "$CHARON_PORT_P2P_TCP" + ) + # If the free port is the same as the port in the cluster, mark as occupied and break the loop. + if [ "$port" -eq "$p2p_cluster_port" ]; then + is_occupied=1 + break + fi + done + # If the port was occupied by any cluster, increment the port by 1 and continue the loop. + if [ -n "$is_occupied" ]; then + port=$((port + 1)) + continue + fi + + # Check if TCP port is used by the base. + + # Fetch the NETHERMIND_PORT_P2P from the base .env file. + nethermind_p2p_port=$( + . ./.env + printf '%s' "${NETHERMIND_PORT_P2P}" + ) + # If the NETHERMIND_PORT_P2P is not set and the free port is the same as the default one, increment the port by 1 and continue the loop. + if [ -z "$nethermind_p2p_port" ]; then + if [ "$port" -eq "30303" ]; then + port=$((port + 1)) + continue + fi + # If the NETHERMIND_PORT_P2P is set and the free port is the same, increment the port by 1 and continue the loop. + elif [ "$port" -eq "$nethermind_p2p_port" ]; then + port=$((port + 1)) + continue + fi + + # Fetch the NETHERMIND_PORT_HTTP from the base .env file. + nethermind_http_port=$( + . ./.env + printf '%s' "${NETHERMIND_PORT_HTTP}" + ) + # If the NETHERMIND_PORT_HTTP is not set and the free port is the same as the default one, increment the port by 1 and continue the loop. + if [ -z "$nethermind_http_port" ]; then + if [ "$port" -eq "8545" ]; then + port=$((port + 1)) + continue + fi + # If the NETHERMIND_PORT_HTTP is set and the free port is the same, increment the port by 1 and continue the loop. + elif [ "$port" -eq "$nethermind_http_port" ]; then + port=$((port + 1)) + continue + fi + + # Fetch the NETHERMIND_PORT_ENGINE from the base .env file. + nethermind_engine_port=$( + . ./.env + printf '%s' "$NETHERMIND_PORT_ENGINE" + ) + # If the NETHERMIND_PORT_ENGINE is not set and the free port is the same as the default one, increment the port by 1 and continue the loop. + if [ -z "$nethermind_engine_port" ]; then + if [ "$port" -eq "8551" ]; then + port=$((port + 1)) + continue + fi + # If the NETHERMIND_PORT_ENGINE is set and the free port is the same, increment the port by 1 and continue the loop. + elif [ "$port" -eq "$nethermind_engine_port" ]; then + port=$((port + 1)) + continue + fi + + # Fetch the LIGHTHOUSE_PORT_P2P from the base .env file. + lighthouse_p2p_port=$( + . ./.env + printf '%s' "$LIGHTHOUSE_PORT_P2P" + ) + # If the LIGHTHOUSE_PORT_P2P is not set and the free port is the same as the default one, increment the port by 1 and continue the loop. + if [ -z "$lighthouse_p2p_port" ]; then + if [ "$port" -eq "9000" ]; then + port=$((port + 1)) + continue + fi + # If the LIGHTHOUSE_PORT_P2P is set and the free port is the same, increment the port by 1 and continue the loop. + elif [ "$port" -eq "$lighthouse_p2p_port" ]; then + port=$((port + 1)) + continue + fi + done + + # Create dir for the cluster. + mkdir -p "./clusters/${cluster_name}" + cluster_dir="./clusters/${cluster_name}" + + # Copy .env from root dir to cluster's dir (if it exists). + if test -f ./.env; then + cp .env "${cluster_dir}/" + fi + + # Copy docker-compose.yml from root dir to cluster's dir (if it exists). + if test -f ./docker-compose.yml; then + cp ./docker-compose.yml "$cluster_dir"/ + fi + + # Write the found free port in the .env file. + if grep -xq "CHARON_PORT_P2P_TCP=.*" ./.env; then + echo "CHARON_PORT_P2P_TCP already set, overwriting it with port $port" + sed "s|CHARON_PORT_P2P_TCP=|CHARON_PORT_P2P_TCP=$port|" "${cluster_dir}/.env" >"${cluster_dir}/.env.tmp" + else + sed "s|#CHARON_PORT_P2P_TCP=|CHARON_PORT_P2P_TCP=$port|" "${cluster_dir}/.env" >"${cluster_dir}/.env.tmp" + fi + mv "${cluster_dir}/.env.tmp" "${cluster_dir}/.env" + + # Create data dir. + mkdir "${cluster_dir}/data" + + # Copy prometheus files and data. + cp -r ./prometheus "${cluster_dir}/" + if test -d ./data/prometheus; then + cp -r ./data/prometheus "${cluster_dir}/data/" + fi + + # Copy lodestar files. + cp -r ./lodestar "${cluster_dir}/" + + # Add the base network on which EL + CL + MEV-boost + Grafana run. + sed "s| dvnode:| dvnode:\n shared-node:\n external:\n name: charon-distributed-validator-node_dvnode|" "${cluster_dir}/docker-compose.yml" >"${cluster_dir}/docker-compose.yml.tmp" + mv "${cluster_dir}/docker-compose.yml.tmp" "${cluster_dir}/docker-compose.yml" + + # Include the base network in the cluster-specific services' network config. + sed "s| networks: \[dvnode\]| networks: [dvnode,shared-node]|" "${cluster_dir}/docker-compose.yml" >"${cluster_dir}/docker-compose.yml.tmp" + mv "${cluster_dir}/docker-compose.yml.tmp" "${cluster_dir}/docker-compose.yml" + + echo "Added new cluster $cluster_name with the following cluster-specific config:" + echo "CHARON_PORT_P2P_TCP: $port" + echo "" + echo "You can start it by running $0 start $cluster_name" +} + +delete() { + read -r -p "Are you sure you want to delete the cluster? This will delete your private keys, which will be unrecoverable if you do not have backup! [y/N] " response + if [[ "$response" =~ ^([yY][eE][sS]|[yY])$ ]]; then + rm -rf "./clusters/$cluster_name" + echo "Delete cluster $cluster_name." + fi +} + +start() { + docker compose --profile cluster -f "./clusters/${cluster_name}/docker-compose.yml" up -d + echo "Started cluster $cluster_name" + echo "You can stop it by running $0 stop $cluster_name" +} + +stop() { + docker compose --profile cluster -f "./clusters/${cluster_name}/docker-compose.yml" down + echo "Stopped cluster $cluster_name" + echo "You can start it again by running $0 start $cluster_name" +} + +# Match global flags +while getopts ":h" opt; do + case $opt in + h) + usage_base + exit 0 + ;; + \?) # unknown flag + usage_base + exit 1 + ;; + esac +done + +# Capture the subcommand passed. +shift "$((OPTIND - 1))" +subcommand=$1 +shift +# Execute subcommand. +case "$subcommand" in +add) + while getopts ":hsp:" opt; do + case $opt in + h) + usage_add + exit 0 + ;; + s) + skip_port_free_check=true + ;; + p) + p2p_default_port=${OPTARG} + ;; + ?) # Invalid option + usage_add + exit 1 + ;; + esac + done + shift "$((OPTIND - 1))" + cluster_name=$1 + check_missing_cluster_name + check_clusters_dir_does_not_exist + check_cluster_already_exists + add + exit 0 + ;; +delete) + while getopts ":h" opt; do + case $opt in + h) + usage_delete + exit 0 + ;; + ?) # Invalid option + usage_delete + exit 1 + ;; + esac + done + shift $((OPTIND - 1)) + cluster_name=$1 + check_missing_cluster_name + check_clusters_dir_does_not_exist + check_cluster_does_not_exist + delete + exit 0 + ;; +start) + while getopts ":h" opt; do + case $opt in + h) + usage_start + exit 0 + ;; + ?) # Invalid option + usage_start + exit 1 + ;; + esac + done + shift $((OPTIND - 1)) + cluster_name=$1 + check_missing_cluster_name + check_clusters_dir_does_not_exist + check_cluster_does_not_exist + start + exit 0 + ;; +stop) + while getopts ":h" opt; do + case $opt in + h) + usage_stop + exit 0 + ;; + ?) # Invalid option + usage_stop + exit 1 + ;; + esac + done + shift $((OPTIND - 1)) + cluster_name=$1 + check_missing_cluster_name + check_clusters_dir_does_not_exist + check_cluster_does_not_exist + stop + exit 0 + ;; +*) + usage_base + exit 1 + ;; +esac diff --git a/multi_cluster/setup.sh b/multi_cluster/setup.sh new file mode 100755 index 0000000..b7d8831 --- /dev/null +++ b/multi_cluster/setup.sh @@ -0,0 +1,217 @@ +#!/bin/bash + +# shellcheck disable=SC1090,SC1091,SC2012 + +cluster_already_set= + +usage() { + echo "Usage: $0 [OPTIONS] NAME" + echo "" + echo " Create a multi cluster setup from a traditional single cluster setup. Name of the first cluster should be specified." + echo "" + echo "Options:" + echo " -h Display this help message." + echo "" + echo "Example:" + echo " $0 initial-cluster" +} + +while getopts "h:" opt; do + case $opt in + h) + usage + exit 0 + ;; + \?) + usage + exit 1 + ;; + esac +done +shift "$((OPTIND - 1))" +cluster_name=$1 + +if [ -z "$cluster_name" ]; then + echo 'Missing cluster name argument.' >&2 + usage + exit 1 +fi + +cluster_dir=./clusters/${cluster_name} + +# Check if clusters directory already exists. +if test -d ./clusters; then + echo "./clusters directory already exists. Cannot setup already set multi cluster CDVN." + exit 1 +fi + +# Create clusters directory. +mkdir -p "$cluster_dir" + +# Delete ./clusters dir if the script exits with non-zero code. +cleanupClusterDir() { + if [ "$1" != "0" ]; then + rm -rf ./clusters + fi +} +trap 'cleanupClusterDir $?' EXIT + +# Copy .charon folder to clusters directory (if it exists). +if test -d ./.charon; then + owner="$(ls -ld ".charon" | awk '{print $3}')" + if [ "$owner" = "$USER" ]; then + cp -r .charon "$cluster_dir"/ + cluster_already_set=1 + else + echo "current user ${USER} is not owner of .charon/" + exit 1 + fi +fi + +# Copy .env file to clusters directory (if it exists). +if test -f ./.env; then + owner="$(ls -ld ".env" | awk '{print $3}')" + if [ "${owner}" = "${USER}" ]; then + cp .env "$cluster_dir"/ + else + echo "current user ${USER} is not owner of .env" + exit 1 + fi +fi + +# Copy docker-compose.yml to clusters directory (if it exists). +if test -f ./docker-compose.yml; then + owner="$(ls -ld "docker-compose.yml" | awk '{print $3}')" + if [ "${owner}" = "${USER}" ]; then + cp ./docker-compose.yml "$cluster_dir"/ + else + echo "current user ${USER} is not owner of docker-compose.yml" + exit 1 + fi +fi + +# Write default charon ports in .env file if they are not set. +if grep -xq "CHARON_PORT_VALIDATOR_API=.*" ./.env; then + echo "CHARON_PORT_VALIDATOR_API already set, using the set port instead of the default 3600" +else + sed 's|#CHARON_PORT_VALIDATOR_API=|CHARON_PORT_VALIDATOR_API=3600|' "${cluster_dir}/.env" >"${cluster_dir}/.env~" + mv "${cluster_dir}/.env~" "${cluster_dir}/.env" +fi + +if grep -xq "CHARON_PORT_MONITORING=.*" ./.env; then + echo "CHARON_PORT_MONITORING already set, using the set port instead of the default 3620" +else + sed 's|#CHARON_PORT_MONITORING=|CHARON_PORT_MONITORING=3620|' "${cluster_dir}/.env" >"${cluster_dir}/.env~" + mv "${cluster_dir}/.env~" "${cluster_dir}/.env" +fi + +if grep -xq "CHARON_PORT_P2P_TCP=.*" ./.env; then + echo "CHARON_PORT_P2P_TCP already set, using the set port instead of the default 3610" +else + sed 's|#CHARON_PORT_P2P_TCP=|CHARON_PORT_P2P_TCP=3610|' "${cluster_dir}/.env" >"${cluster_dir}/.env~" + mv "${cluster_dir}/.env~" "${cluster_dir}/.env" +fi + +# Create data dir. +mkdir "${cluster_dir}/data" + +# Copy lodestar files. +owner="$(ls -ld "lodestar" | awk '{print $3}')" +if [ "${owner}" = "${USER}" ]; then + cp -r ./lodestar "${cluster_dir}/" +else + echo "current user ${USER} is not owner of lodestar/" + exit 1 +fi + +# Copy lodestar data, if it exists. +if test -d ./data/lodestar; then + owner="$(ls -ld "data/lodestar" | awk '{print $3}')" + if [ "${owner}" = "${USER}" ]; then + cp -r ./data/lodestar "${cluster_dir}/data/" + else + echo "current user ${USER} is not owner of data/lodestar/" + exit 1 + fi +fi + +# Copy prometheus files. +owner="$(ls -ld "prometheus" | awk '{print $3}')" +if [ "${owner}" = "${USER}" ]; then + cp -r ./prometheus "${cluster_dir}/" +else + echo "current user ${USER} is not owner of prometheus/" + exit 1 +fi + +# Copy prometheus data, if it exists. +if test -d ./data/prometheus; then + owner="$(ls -ld "data/prometheus" | awk '{print $3}')" + if [ "${owner}" = "${USER}" ]; then + cp -r ./data/prometheus "${cluster_dir}/data/" + else + echo "current user ${USER} is not owner of data/prometheus/" + exit 1 + fi +fi + +# Add the base network on which EL + CL + MEV-boost + Grafana run. +sed "s| dvnode:| dvnode:\n shared-node:\n external:\n name: charon-distributed-validator-node_dvnode|" "${cluster_dir}/docker-compose.yml" >"${cluster_dir}/docker-compose.yml~" +mv "${cluster_dir}/docker-compose.yml~" "${cluster_dir}/docker-compose.yml" + +# Include the base network in the cluster-specific services' network config. +sed "s| networks: \[dvnode\]| networks: [dvnode,shared-node]|" "${cluster_dir}/docker-compose.yml" >"${cluster_dir}/docker-compose.yml~" +mv "${cluster_dir}/docker-compose.yml~" "${cluster_dir}/docker-compose.yml" + +if ! docker info >/dev/null 2>&1; then + echo "Docker daemon is not running, please start Docker first." + exit 1 +fi + +# If containers were already started, restart the cluster with the new setup. +if [[ $(docker compose ps -aq) ]]; then + echo "Restarting the cluster-specific containers from the new multi cluster directory ${cluster_dir}" + # Stop the cluster-specific containers that are running in root directory - Charon, Lodestar, Prometheus. + docker compose --profile cluster down + # Start the base containers in the root directory. + docker compose --profile base up -d + # Start the cluster-specific containers in cluster-specific directory (i.e.: charon, VC). + docker compose --profile cluster -f "${cluster_dir}/docker-compose.yml" up -d +fi + +migrated_readme() { + cat >"$1" <