Testnet: Kzgrs config (#797)

* Remove out of place wait_online test param

* Cfgsync crate for testnet startup

* Config dispersal to node hosts

* Remove etcd dep and obsolete bash scripts

* Kzgrs test params in testnet

* Nodes listen on all interfaces

.env.example

@@ -1,9 +1,2 @@
 # Environment variables for compose.yml file config.
 DOCKER_COMPOSE_LIBP2P_REPLICAS=1
-DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK=2000000000000000000000000000000000000000000000000000000000000000
-DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD=1
-DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379
-DOCKER_COMPOSE_ETCDCTL_API=3
-DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000
-DOCKER_COMPOSE_OVERLAY_NODES=$DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY
-DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/udp/3000/quic-v1

.env.testnet

@@ -1,12 +1,2 @@
 # Environment variables for compose.yml file config.
 DOCKER_COMPOSE_LIBP2P_REPLICAS=3
-DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK=2000000000000000000000000000000000000000000000000000000000000000
-DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD=1
-DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379
-DOCKER_COMPOSE_ETCDCTL_API=3
-DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000
-DOCKER_COMPOSE_OVERLAY_NODES=1000000000000000000000000000000000000000000000000000000000000000
-DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/udp/3000/quic-v1
-
-GRAYLOG_PASSWORD_SECRET="Jcjw7g22kJw3aSjjnCQ7DiQvlSJJ38WZ2MvuIyZ4RTILUoxGEQb5EsmAAdcp3lnnlwdSKaZTDFcvh4Xq2h4aTsa4HLx3SZxM"
-GRAYLOG_ROOT_PASSWORD_SHA2="7092a9ec7c94ba6c452a3937a380b9cfdac8e2d0b342c034ea9e306d41ce6d89"

Cargo.toml

@@ -35,6 +35,7 @@ members = [
     "ledger/cryptarchia-ledger",
     "cl/cl",
     "proof_of_leadership/proof_statements",
+    "testnet/cfgsync",
     "tests",
 ]
 exclude = ["proof_of_leadership/risc0/risc0_proofs"]

compose.static.yml

@@ -1,8 +1,19 @@
 version: '3.8'
 
 services:
-  bootstrap:
-    container_name: bootstrap
+
+  cfgsync:
+    container_name: cfgsync
+    build:
+      context: .
+      dockerfile: testnet/Dockerfile
+    image: nomos:latest
+    volumes:
+      - ./testnet:/etc/nomos
+    entrypoint: /etc/nomos/scripts/run_cfgsync.sh
+
+  nomos-node-0:
+    container_name: nomos_node_0
     build:
       context: .
       dockerfile: testnet/Dockerfile
@@ -12,110 +23,63 @@ services:
       - "18080:18080/tcp"
     volumes:
       - ./testnet:/etc/nomos
+      - ./tests/kzgrs/kzgrs_test_params:/kzgrs_test_params:z
     depends_on:
+      - cfgsync
       - graylog
-    environment:
-      - BOOTSTRAP_NODE_KEY=${DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY:-1000000000000000000000000000000000000000000000000000000000000000}
-      - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
-      - LIBP2P_REPLICAS=3
-      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
-    entrypoint: /etc/nomos/scripts/run_bootstrap_node.sh
+    entrypoint: /etc/nomos/scripts/run_nomos_node.sh
 
-  libp2p-node-1:
-    container_name: libp2p_node_1
+  nomos-node-1:
+    container_name: nomos_node_1
     build:
       context: .
       dockerfile: testnet/Dockerfile
     image: nomos:latest
     volumes:
       - ./testnet:/etc/nomos
+      - ./tests/kzgrs/kzgrs_test_params:/kzgrs_test_params:z
     depends_on:
-      - bootstrap
-      - etcd
+      - cfgsync
       - graylog
     ports:
       - "3001:3000/udp"
       - "18081:18080/tcp"
-    environment:
-      - LIBP2P_REPLICAS=3
-      - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379}
-      - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3}
-      - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1}
     entrypoint: /etc/nomos/scripts/run_nomos_node.sh
 
-  libp2p-node-2:
-    container_name: libp2p_node_2
+  nomos-node-2:
+    container_name: nomos_node_2
     build:
       context: .
       dockerfile: testnet/Dockerfile
     image: nomos:latest
     volumes:
       - ./testnet:/etc/nomos
+      - ./tests/kzgrs/kzgrs_test_params:/kzgrs_test_params:z
     depends_on:
-      - bootstrap
-      - etcd
+      - cfgsync
       - graylog
     ports:
       - "3002:3000/udp"
       - "18082:18080/tcp"
-    environment:
-      - LIBP2P_REPLICAS=3
-      - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379}
-      - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3}
-      - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1}
     entrypoint: /etc/nomos/scripts/run_nomos_node.sh
 
-  libp2p-node-3:
-    container_name: libp2p_node_3
+  nomos-node-3:
+    container_name: nomos_node_3
     build:
       context: .
       dockerfile: testnet/Dockerfile
     image: nomos:latest
     volumes:
       - ./testnet:/etc/nomos
+      - ./tests/kzgrs/kzgrs_test_params:/kzgrs_test_params:z
     depends_on:
-      - bootstrap
-      - etcd
+      - cfgsync
       - graylog
     ports:
       - "3003:3000/udp"
       - "18083:18080/tcp"
-    environment:
-      - LIBP2P_REPLICAS=3
-      - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379}
-      - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3}
-      - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1}
     entrypoint: /etc/nomos/scripts/run_nomos_node.sh
 
-  chatbot:
-    container_name: chatbot
-    build:
-      context: .
-      dockerfile: testnet/Dockerfile
-    image: nomos:latest
-    volumes:
-      - ./testnet:/etc/nomos
-    entrypoint: /etc/nomos/scripts/run_nomos_bot.sh
-
-  etcd:
-    container_name: etcd
-    image: quay.io/coreos/etcd:v3.4.15
-    ports:
-      - "2379:2379/tcp"
-    command:
-      - /usr/local/bin/etcd
-      - --advertise-client-urls=http://etcd:2379
-      - --listen-client-urls=http://0.0.0.0:2379
-
   prometheus:
     container_name: prometheus
     image: prom/prometheus:latest

compose.yml

@@ -1,22 +1,16 @@
 services:
 
-  bootstrap:
+  cfgsync:
+    container_name: cfgsync
     build:
       context: .
       dockerfile: testnet/Dockerfile
-    ports:
-      - "3000:3000/udp"
-      - "18080:18080/tcp"
+    image: nomos:latest
     volumes:
       - ./testnet:/etc/nomos
-    environment:
-      - BOOTSTRAP_NODE_KEY=${DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY:-1000000000000000000000000000000000000000000000000000000000000000}
-      - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
-      - LIBP2P_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1}
-      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
-    entrypoint: /etc/nomos/scripts/run_bootstrap_node.sh
+    entrypoint: /etc/nomos/scripts/run_cfgsync.sh
 
-  libp2p-node:
+  nomos-node:
     build:
       context: .
       dockerfile: testnet/Dockerfile
@@ -25,23 +19,5 @@ services:
     deploy:
       replicas: ${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1} 
     depends_on:
-      - bootstrap
-      - etcd
-    environment:
-      - LIBP2P_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1}
-      - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379}
-      - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3}
-      - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
-      - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1}
+      - cfgsync
     entrypoint: /etc/nomos/scripts/run_nomos_node.sh
-
-  etcd:
-    image: quay.io/coreos/etcd:v3.4.15
-    ports:
-      - "2379:2379/tcp"
-    command:
-      - /usr/local/bin/etcd
-      - --advertise-client-urls=http://etcd:2379
-      - --listen-client-urls=http://0.0.0.0:2379

nodes/nomos-node/src/config.rs

@@ -122,7 +122,6 @@ pub struct Config {
     pub http: <NomosApiService as ServiceData>::Settings,
     pub cryptarchia: <crate::Cryptarchia as ServiceData>::Settings,
     pub storage: <crate::StorageService<RocksBackend<Wire>> as ServiceData>::Settings,
-    pub wait_online_secs: u64,
 }
 
 impl Config {

testnet/Dockerfile

@@ -1,13 +1,13 @@
 # BUILD IMAGE ---------------------------------------------------------
 
-FROM rust:1.80.0-slim-bookworm AS builder
+FROM rust:1.81.0-slim-bookworm AS builder
 
 WORKDIR /nomos
 COPY . . 
 
 # Install dependencies needed for building RocksDB and etcd.
 RUN apt-get update && apt-get install -yq \
-    git gcc g++ clang etcd-client libssl-dev \
+    git gcc g++ clang libssl-dev \
     pkg-config protobuf-compiler
 
 RUN cargo install cargo-binstall
@@ -18,18 +18,20 @@ RUN cargo build --release --all --features metrics
 
 # NODE IMAGE ----------------------------------------------------------
 
-FROM bitnami/minideb:latest
+FROM bitnami/minideb:bookworm
 
 LABEL maintainer="augustinas@status.im" \
       source="https://github.com/logos-co/nomos-node" \
       description="Nomos testnet image"
 
 # nomos default ports
-EXPOSE 3000 8080 9000 60000	
+EXPOSE 3000 8080 9000 60000
+
+RUN apt-get update && apt-get install -y libssl3
 
 COPY --from=builder /nomos/target/release/nomos-node /usr/bin/nomos-node
 COPY --from=builder /nomos/target/release/nomos-cli /usr/bin/nomos-cli
-COPY --from=builder /usr/bin/etcdctl /usr/bin/etcdctl
-COPY nodes/nomos-node/config.yaml /etc/nomos/config.yaml
+COPY --from=builder /nomos/target/release/cfgsync-server /usr/bin/cfgsync-server
+COPY --from=builder /nomos/target/release/cfgsync-client /usr/bin/cfgsync-client
 
 ENTRYPOINT ["/usr/bin/nomos-node"]

testnet/README.md

@@ -2,9 +2,7 @@
 
 The Nomos Docker Compose Testnet contains four distinct service types:
 
-- **Bootstrap Node Service**: A singular Nomos node with its own service and a deterministic DNS address. Other nodes utilize this as their initial peer.
-- **Libp2p Node Services**: Multiple dynamically spawned Nomos nodes that announce their existence through etcd.
-- **Etcd Service**: A container running an etcd instance.
+- **Nomos Node Services**: Multiple dynamically spawned Nomos nodes that synchronizes their configuration via cfgsync utility.
 
 ## Building
 
@@ -41,15 +39,15 @@ docker compose up -d
 Followed by:
 
 ```bash
-docker compose logs -f {bootstrap,libp2p-node,etcd}
+docker compose logs -f nomos-node
 ```
 
 ## Using testnet
 
-Bootstrap node is accessible from the host via `3000` and `18080` ports. To expose other nomos nodes, please update `libp2p-node` service in the `compose.yml` file with this configuration:
+Bootstrap node is accessible from the host via `3000` and `18080` ports. To expose other nomos nodes, please update `nomos-node` service in the `compose.yml` file with this configuration:
 
 ```bash
-  libp2p-node:
+  nomos-node-0:
     ports:
     - "3001-3010:3000" # Use range depending on the number of nomos node replicas.
     - "18081-18190:18080"

testnet/cfgsync.yaml

@@ -0,0 +1,16 @@
+port: 4400
+n_hosts: 4
+timeout: 10
+
+# ConsensusConfig related parameters
+security_param: 10
+active_slot_coeff: 0.9
+
+# DaConfig related parameters
+subnetwork_size: 2
+dispersal_factor: 2
+num_samples: 1
+num_subnets: 2
+old_blobs_check_interval_secs: 5
+blobs_validity_duration_secs: 60
+global_params_path: "/kzgrs_test_params"