*: multi cluster setup

.gitignore

@@ -10,5 +10,6 @@ cluster-lock.json
 .DS_Store
 data/
 .idea
-.charon
+.charon*
 prometheus/prometheus.yml
+clusters/

Makefile

@@ -0,0 +1,34 @@
+.PHONY: multi-cluster-setup multi-cluster-add-cluster multi-cluster-delete-cluster multi-cluster-start-cluster multi-cluster-stop-cluster multi-cluster-start-base multi-cluster-stop-base
+
+check_defined = \
+    $(strip $(foreach 1,$1, \
+        $(call __check_defined,$1,$(strip $(value 2)))))
+__check_defined = \
+    $(if $(value $1),, \
+      $(error $1$(if $2, ($2)) is not set. Set it by running `make $1$(if $2, ($2))=<value>`))
+
+multi-cluster-setup:
+	$(call check_defined, name)
+	./multi_cluster/setup.sh $(name)
+
+multi-cluster-add-cluster:
+	$(call check_defined, name)
+	./multi_cluster/cluster.sh add $(name)
+
+multi-cluster-delete-cluster:
+	$(call check_defined, name)
+	./multi_cluster/cluster.sh delete $(name)
+
+multi-cluster-start-cluster:
+	$(call check_defined, name)
+	./multi_cluster/cluster.sh start $(name)
+
+multi-cluster-stop-cluster:
+	$(call check_defined, name)
+	./multi_cluster/cluster.sh stop $(name)
+
+multi-cluster-start-base:
+	./multi_cluster/base.sh start
+
+multi-cluster-stop-base:
+	./multi_cluster/base.sh stop

README.md

@@ -43,3 +43,85 @@ docker compose -f examples/nethermind_teku_lighthouse.yml up
 # FAQs
 
 Check the Obol docs for frequent [errors and resolutions](https://docs.obol.tech/docs/faq/errors)
+
+<!-- TODO: move this guide to the docs -->
+# Multi cluster setup
+
+There is an option to run multiple Charon clusters using the same Execution Layer Client (EL), Consensus Layer Client (CL) and Grafana. This way you can operate multiple clusters for different purposes, without putting much more pressure on your system.
+
+The way this is achieved is by separating the EL, CL and Grafana from the Charon node, Validator Client (VC) and Prometheus. Instead of having `.charon/` folder in the root directory it is moved to `clusters/{CLUSTER_NAME}/.charon`. Moreover, the VC and Prometheus data is now per cluster as well, moved from `data/lodestar` and `data/prometheus` to `clusters/{CLUSTER_NAME}/data/lodestar` and `clusters/{CLUSTER_NAME}/data/prometheus`, respectively. `docker-compose.yml` and `.env` are also used per cluster. There are also supporting scripts for the Charon node and the VC.
+
+## Setup
+
+If you already have running validator node in Docker, the Docker containers will be moved to the new multi cluster setup.
+
+```bash
+./multi_cluster/setup.sh {CLUSTER_NAME}
+```
+
+You can inspect what you have in the `./clusters/` directory. Each subfolder is a cluster with the following structure:
+
+```directory
+clusters
+└───{CLUSTER_NAME}     # cluster name
+│   │   .charon             # folder including secret material used by charon
+│   │   data                # data from the validator client and prometheus
+│   │   lodestar            # scripts used by lodestar
+│   │   prometheus          # scripts and configs used by prometheus
+│   │   .env                # environment variables used by the cluster
+│   │   docker-compose.yml  # docker compose used by the cluster
+│                           # N.B.: only services with profile "cluster" are ran
+└───{CLUSTER_NAME_2}
+└───{CLUSTER_NAME_...}
+└───{CLUSTER_NAME_N}
+```
+
+Note that those folders and files are copied from the root directory. Meaning all configurations and setup you have already done, will be copied to this first cluster of the multi cluster setup.
+
+## Manage cluster
+
+Manage the Charon + Validator Client + Prometheus containers of each cluster found in `./clusters/`.
+
+### Add cluster
+
+```bash
+./multi_cluster/cluster.sh add {CLUSTER_NAME}
+```
+
+Note that only the `.env`, `lodestar/`, `prometheus/` and `docker-compose.yml` files and directories are coiped from the root directory to the new cluster. `.charon/` and `data/` folders are expected to be from a brand new cluster that you will setup in the `./clusters/{CLUSTER_NAME}` directory.
+
+### Start cluster
+
+It is expected that you have already done the regular procedure from cluster setup and you have `./clusters/{CLUSTER_NAME}/.charon/` folder.
+
+```bash
+./multi_cluster/cluster.sh start {CLUSTER_NAME}
+```
+
+### Stop cluster
+
+```bash
+./multi_cluster/cluster.sh stop {CLUSTER_NAME}
+```
+
+### Delete cluster
+
+```bash
+./multi_cluster/cluster.sh delete {CLUSTER_NAME}
+```
+
+## Manage base node
+
+Manage the EL + CL + Grafana containers.
+
+### Start base node
+
+```bash
+./multi_cluster/base.sh start
+```
+
+### Stop base node
+
+```bash
+./multi_cluster/base.sh stop
+```

docker-compose.yml

@@ -9,6 +9,7 @@ services:
   # | | | |  __/ |_| | | |  __/ |  | | | | | | | | | | (_| |
   # |_| |_|\___|\__|_| |_|\___|_|  |_| |_| |_|_|_| |_|\__,_|
   nethermind:
+    profiles: ["base", ""]
     image: nethermind/nethermind:${NETHERMIND_VERSION:-1.29.0}
     restart: unless-stopped
     ports:
@@ -44,6 +45,7 @@ services:
   #      |___/
 
   lighthouse:
+    profiles: ["base", ""]
     image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v5.3.0}
     ports:
       - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/tcp # P2P TCP
@@ -77,22 +79,21 @@ services:
   #  \___|_| |_|\__,_|_|  \___/|_| |_|
 
   charon:
+    profiles: ["cluster", ""]
     image: obolnetwork/charon:${CHARON_VERSION:-v1.2.0}
     environment:
-      - CHARON_BEACON_NODE_ENDPOINTS=${CHARON_BEACON_NODE_ENDPOINTS:-http://lighthouse:5052}
-      - CHARON_BEACON_NODE_TIMEOUT=${CHARON_BEACON_NODE_TIMEOUT:-3s}
-      - CHARON_BEACON_NODE_SUBMIT_TIMEOUT=${CHARON_BEACON_NODE_SUBMIT_TIMEOUT:-4s}
-      - CHARON_LOG_LEVEL=${CHARON_LOG_LEVEL:-info}
-      - CHARON_LOG_FORMAT=${CHARON_LOG_FORMAT:-console}
-      - CHARON_P2P_RELAYS=${CHARON_P2P_RELAYS:-https://0.relay.obol.tech,https://1.relay.obol.tech/}
-      - CHARON_P2P_EXTERNAL_HOSTNAME=${CHARON_P2P_EXTERNAL_HOSTNAME:-} # Empty default required to avoid warnings.
-      - CHARON_P2P_TCP_ADDRESS=0.0.0.0:${CHARON_PORT_P2P_TCP:-3610}
-      - CHARON_VALIDATOR_API_ADDRESS=0.0.0.0:3600
-      - CHARON_MONITORING_ADDRESS=0.0.0.0:3620
-      - CHARON_BUILDER_API=${BUILDER_API_ENABLED:-false}
-      - CHARON_FEATURE_SET_ENABLE=${CHARON_FEATURE_SET_ENABLE:-}
-      - CHARON_LOKI_ADDRESSES=${CHARON_LOKI_ADDRESSES:-http://loki:3100/loki/api/v1/push}
-      - CHARON_LOKI_SERVICE=charon
+      CHARON_BEACON_NODE_ENDPOINTS: ${CHARON_BEACON_NODE_ENDPOINTS:-http://lighthouse:5052}
+      CHARON_LOG_LEVEL: ${CHARON_LOG_LEVEL:-info}
+      CHARON_LOG_FORMAT: ${CHARON_LOG_FORMAT:-console}
+      CHARON_P2P_RELAYS: ${CHARON_P2P_RELAYS:-https://0.relay.obol.tech,https://1.relay.obol.tech/}
+      CHARON_P2P_EXTERNAL_HOSTNAME: ${CHARON_P2P_EXTERNAL_HOSTNAME:-} # Empty default required to avoid warnings.
+      CHARON_P2P_TCP_ADDRESS: 0.0.0.0:${CHARON_PORT_P2P_TCP:-3610}
+      CHARON_VALIDATOR_API_ADDRESS: 0.0.0.0:3600
+      CHARON_MONITORING_ADDRESS: 0.0.0.0:3620
+      CHARON_BUILDER_API: ${BUILDER_API_ENABLED:-false}
+      CHARON_FEATURE_SET_ENABLE: ${CHARON_FEATURE_SET_ENABLE:-}
+      CHARON_LOKI_ADDRESSES: ${CHARON_LOKI_ADDRESSES:-http://loki:3100/loki/api/v1/push}
+      CHARON_LOKI_SERVICE: charon
     ports:
       - ${CHARON_PORT_P2P_TCP:-3610}:${CHARON_PORT_P2P_TCP:-3610}/tcp # P2P TCP libp2p
     networks: [dvnode]
@@ -109,6 +110,7 @@ services:
   # |_|\___/ \__,_|\___||___/\__\__,_|_|
 
   lodestar:
+    profiles: ["cluster", ""]
     image: chainsafe/lodestar:${LODESTAR_VERSION:-v1.23.0}
     depends_on: [charon]
     entrypoint: /opt/lodestar/run.sh
@@ -130,6 +132,7 @@ services:
   #  | | | | | |  __/\ V /_____| |_) | (_) | (_) \__ \ |_
   #  |_| |_| |_|\___| \_/      |_.__/ \___/ \___/|___/\__|
   mev-boost:
+    profiles: ["base", ""]
     image: ${MEVBOOST_IMAGE:-flashbots/mev-boost}:${MEVBOOST_VERSION:-1.8.1}
     command: |
       -${NETWORK}
@@ -147,6 +150,7 @@ services:
   # |_| |_| |_|\___/|_| |_|_|\__\___/|_|  |_|_| |_|\__, |
   #                                                |___/
   prometheus:
+    profiles: ["cluster", ""]
     image: prom/prometheus:${PROMETHEUS_VERSION:-v2.50.1}
     user: ":"
     networks: [dvnode]
@@ -160,6 +164,7 @@ services:
     restart: unless-stopped
 
   grafana:
+    profiles: ["base", ""]
     image: grafana/grafana:${GRAFANA_VERSION:-10.4.2}
     user: ":"
     ports:

multi_cluster/base.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 [OPTIONS] COMMAND"
+  echo ""
+  echo "    Manage the base ethereum node (EL, CL, MEV boost, Grafana), without interfering with any validator."
+  echo ""
+  echo "Commands:"
+  echo "    start   Start an ethereum node, MEV-boost and Grafana."
+  echo "    stop    Stop an ethereum node, MEV-boost and Grafana."
+  echo ""
+  echo "Options:"
+  echo "    -h          Display this help message."
+}
+
+usage_start() {
+  echo "Usage: $0 start [OPTIONS]"
+  echo ""
+  echo "    Start the base ethereum node."
+  echo ""
+  echo "Options:"
+  echo "    -h          Display this help message."
+  echo ""
+  echo "Example:"
+  echo "  $0 start"
+}
+
+usage_stop() {
+  echo "Usage: $0 stop [OPTIONS]"
+  echo ""
+  echo "    Stop the base ethereum node."
+  echo ""
+  echo "Options:"
+  echo "    -h          Display this help message."
+  echo ""
+  echo "Example:"
+  echo "  $0 stop"
+}
+
+start() {
+  docker compose --profile base up -d
+}
+
+stop() {
+  docker compose --profile base stop
+}
+
+while getopts ":h" opt; do
+  case $opt in
+  h)
+    usage
+    exit 0
+    ;;
+  \?)
+    usage
+    exit 1
+    ;;
+  :)
+    usage
+    exit 1
+    ;;
+  esac
+done
+
+shift $((OPTIND - 1))
+
+subcommand=$1
+shift
+case "$subcommand" in
+# Parse options to the install sub command
+start)
+  while getopts ":h" opt; do
+    case $opt in
+    h)
+      usage_start
+      exit 0
+      ;;
+    ?) # Invalid option
+      usage_start
+      exit 1
+      ;;
+    esac
+  done
+  shift $((OPTIND - 1))
+  start
+  ;;
+stop)
+  while getopts ":h" opt; do
+    case $opt in
+    h)
+      usage_stop
+      exit 0
+      ;;
+    ?) # Invalid option
+      usage_stop
+      exit 1
+      ;;
+    esac
+  done
+  shift $((OPTIND - 1))
+  stop
+  ;;
+*)
+  usage
+  exit 1
+  ;;
+
+esac