From c8886828a1a3cdeddb041dcb9080b85e60f6ee75 Mon Sep 17 00:00:00 2001 From: Ulrich Petri Date: Mon, 25 Sep 2023 11:31:27 +0200 Subject: [PATCH 1/3] Remove sentry nodes from docker setup Currently all validators connect to validator-0 instead of bootstrap (see `02-init-chain.sh:56`) --- docker/02-init-chain.sh | 45 ++++++-------- docker/04-bootstrap.sh | 2 +- docker/05-test.sh | 11 +++- docker/config.example/keyper-0.toml | 2 +- docker/config.example/keyper-1.toml | 2 +- docker/config.example/keyper-2.toml | 2 +- docker/config.example/keyper-3.toml | 2 +- docker/docker-compose.yml | 92 ++++------------------------- 8 files changed, 42 insertions(+), 116 deletions(-) diff --git a/docker/02-init-chain.sh b/docker/02-init-chain.sh index 4b2c4d1d..9b3a3a6b 100755 --- a/docker/02-init-chain.sh +++ b/docker/02-init-chain.sh @@ -4,13 +4,13 @@ source ./common.sh $DC stop geth $DC rm -f geth -$DC stop chain-{0..3}-{validator,sentry} chain-seed -$DC rm -f chain-{0..3}-{validator,sentry} chain-seed +$DC stop chain-{0..3}-validator chain-seed +$DC rm -f chain-{0..3}-validator chain-seed ${BB} rm -rf data/geth -${BB} rm -rf data/chain-{0..3}-{validator,sentry} data/chain-seed -${BB} mkdir -p data/chain-{0..3}-{validator,sentry}/config data/chain-seed/config -${BB} chmod -R a+rwX data/chain-{0..3}-{validator,sentry}/config data/chain-seed/config +${BB} rm -rf data/chain-{0..3}-validator data/chain-seed +${BB} mkdir -p data/chain-{0..3}-validator/config data/chain-seed/config +${BB} chmod -R a+rwX data/chain-{0..3}-validator/config data/chain-seed/config ${BB} rm -rf data/deployments # has geth as dependency @@ -19,56 +19,45 @@ $DC up deploy-contracts $DC run --rm --no-deps chain-seed init \ --root /chain \ --blocktime 1 \ - --listen-address tcp://127.0.0.1:${TM_RPC_PORT} \ + --listen-address tcp://0.0.0.0:${TM_RPC_PORT} \ --role seed for num in 0 1 2 3; do validator_cmd=chain-$num-validator - sentry_cmd=chain-$num-sentry - $DC run --rm --no-deps ${sentry_cmd} init \ - --root /chain \ - --blocktime 1 \ - --listen-address tcp://0.0.0.0:${TM_RPC_PORT} \ - --role sentry - - # TODO: check if validator can have listen-address tcp://127.0.0.1... $DC run --rm --no-deps ${validator_cmd} init \ --root /chain \ --genesis-keyper 0x440Dc6F164e9241F04d282215ceF2780cd0B755e \ --blocktime 1 \ - --listen-address tcp://127.0.0.1:${TM_RPC_PORT} \ + --listen-address tcp://0.0.0.0:${TM_RPC_PORT} \ --role validator ${BB} sed -i "/ValidatorPublicKey/c\ValidatorPublicKey = \"$(cat data/${validator_cmd}/config/priv_validator_pubkey.hex)\"" /config/keyper-${num}.toml if [ $num -eq 0 ]; then - for destination in data/chain-seed/config/ data/chain-{1..3}-validator/config/ data/chain-{0..3}-sentry/config/; do + for destination in data/chain-seed/config/ data/chain-{1..3}-validator/config/ ; do ${BB} cp -v data/chain-0-validator/config/genesis.json "${destination}" done fi done seed_node=$(cat data/chain-seed/config/node_key.json.id)@chain-seed:${TM_P2P_PORT} +validator_0_node=$(cat data/chain-0-validator/config/node_key.json.id)@chain-0-validator:${TM_P2P_PORT} -for num in 0 1 2 3; do - sentry_cmd=chain-$num-sentry +${BB} sed -i "/^moniker/c\moniker = \"chain-seed\"" data/chain-seed/config/config.toml + +for num in {0..3}; do validator_cmd=chain-$num-validator validator_id=$(cat data/${validator_cmd}/config/node_key.json.id) validator_node=${validator_id}@${validator_cmd}:${TM_P2P_PORT} - sentry_node=$(cat data/${sentry_cmd}/config/node_key.json.id)@${sentry_cmd}:${TM_P2P_PORT} - # set seed node for sentry - ${BB} sed -i "/^persistent_peers =/c\persistent_peers = \"${seed_node}\"" data/${sentry_cmd}/config/config.toml - # set validator node for sentry - ${BB} sed -i "/^private_peer_ids =/c\private_peer_ids = \"${validator_id}\"" data/${sentry_cmd}/config/config.toml - ${BB} sed -i "/^unconditional_peer_ids =/c\unconditional_peer_ids = \"${validator_id}\"" data/${sentry_cmd}/config/config.toml - ${BB} sed -i "/^external_address =/c\external_address = \"${sentry_cmd}:${TM_P2P_PORT}\"" data/${sentry_cmd}/config/config.toml - - # set sentry node for validator - ${BB} sed -i "/^persistent_peers =/c\persistent_peers = \"${sentry_node}\"" data/${validator_cmd}/config/config.toml + # set seed node for validator + ${BB} sed -i "/^bootstrap_peers =/c\bootstrap_peers = \"${validator_0_node}\"" data/${validator_cmd}/config/config.toml ${BB} sed -i "/^external_address =/c\external_address = \"${validator_cmd}:${TM_P2P_PORT}\"" data/${validator_cmd}/config/config.toml + ${BB} sed -i "/^pex/c\pex = true" data/${validator_cmd}/config/config.toml + ${BB} sed -i "/^moniker/c\moniker = \"${validator_cmd}\"" data/${validator_cmd}/config/config.toml + done $DC stop -t 30 diff --git a/docker/04-bootstrap.sh b/docker/04-bootstrap.sh index 59ab9788..fb6481f9 100755 --- a/docker/04-bootstrap.sh +++ b/docker/04-bootstrap.sh @@ -7,5 +7,5 @@ echo "Submitting bootstrap transaction" $DC run --rm --no-deps --entrypoint /rolling-shutter chain-0-validator bootstrap \ --deployment-dir /deployments/dockerGeth \ --ethereum-url http://geth:8545 \ - --shuttermint-url http://chain-0-sentry:${TM_RPC_PORT} \ + --shuttermint-url http://chain-0-validator:${TM_RPC_PORT} \ --signing-key 479968ffa5ee4c84514a477a8f15f3db0413964fd4c20b08a55fed9fed790fad diff --git a/docker/05-test.sh b/docker/05-test.sh index 60ca2e64..72e00a73 100755 --- a/docker/05-test.sh +++ b/docker/05-test.sh @@ -3,11 +3,16 @@ source ./common.sh set +ex -echo "Testing decryption key generation" EPOCH_ID=$(LC_ALL=C tr -dc 'a-f0-9' Date: Mon, 25 Sep 2023 15:21:06 +0200 Subject: [PATCH 2/3] Change `chain init` roles This changes the `validator` role to create a configuration for a network exposed validator. The previous setting, where `--role validator` set the parameters for an isolated validator behind a `sentry` mode can now be configured by using `--role isolated-validator`. This is now more in line with the setup used in https://github.com/shutter-network/snapshot-keyper which means less configuration changes are needed during the setup. It also allows for a simpler local docker compose test setup. --- rolling-shutter/cmd/chain/init.go | 61 +++++++++++-------- .../docs/rolling-shutter_chain_init.md | 2 +- rolling-shutter/go.mod | 2 +- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/rolling-shutter/cmd/chain/init.go b/rolling-shutter/cmd/chain/init.go index 3e1df699..99412c9a 100644 --- a/rolling-shutter/cmd/chain/init.go +++ b/rolling-shutter/cmd/chain/init.go @@ -23,11 +23,17 @@ import ( "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/types" + "golang.org/x/exp/slices" "github.com/shutter-network/rolling-shutter/rolling-shutter/app" ) -const VALIDATOR = "validator" +const ( + VALIDATOR = "validator" + ISOLATEDVALIDATOR = "isolated-validator" + SENTRY = "sentry" + SEED = "seed" +) type Config struct { RootDir string `mapstructure:"root"` @@ -65,7 +71,7 @@ func initCmd() *cobra.Command { cmd.PersistentFlags().Float64("blocktime", 1.0, "block time in seconds") cmd.PersistentFlags().StringSlice("genesis-keyper", nil, "genesis keyper address") cmd.PersistentFlags().String("listen-address", "tcp://127.0.0.1:26657", "tendermint RPC listen address") - cmd.PersistentFlags().String("role", "validator", "tendermint node role (validator, sentry, seed)") + cmd.PersistentFlags().String("role", "validator", "tendermint node role (validator, isolated-validator, sentry, seed)") cmd.PersistentFlags().Uint64("initial-eon", 0, "initial eon") return cmd } @@ -95,7 +101,7 @@ func getArgFromViper[T interface{}](getter func(string) T, name string, required func initFiles(_ *cobra.Command, config *Config, _ []string) error { keypers := []common.Address{} - if config.Role == VALIDATOR { + if slices.Contains([]string{VALIDATOR, ISOLATEDVALIDATOR}, config.Role) { for _, a := range config.GenesisKeyper { if !common.IsHexAddress(a) { return errors.Errorf("--genesis-keyper argument '%s' is not an address", a) @@ -133,13 +139,17 @@ func initFiles(_ *cobra.Command, config *Config, _ []string) error { // set up according to the network role: https://docs.tendermint.com/v0.34/tendermint-core/validators.html switch config.Role { - case VALIDATOR: + case VALIDATOR: // standard validator mode, network exposed + tendermintCfg.P2P.PexReactor = true + tendermintCfg.Mode = cfg.ModeValidator + tendermintCfg.P2P.AddrBookStrict = true + case ISOLATEDVALIDATOR: // validator mode behind a sentry node tendermintCfg.P2P.PexReactor = false tendermintCfg.P2P.AddrBookStrict = false - case "sentry": + case SENTRY: // even though "sentry" nodes are documented, there is no special mode tendermintCfg.P2P.PexReactor = true tendermintCfg.P2P.AddrBookStrict = false - case "seed": + case SEED: tendermintCfg.P2P.PexReactor = true tendermintCfg.P2P.AddrBookStrict = false default: @@ -170,24 +180,25 @@ func adjustPort(address string, keyperIndex int) (string, error) { func initFilesWithConfig(tendermintConfig *cfg.Config, config *Config, appState app.GenesisAppState) error { var err error - // private validator - privValKeyFile := tendermintConfig.PrivValidatorKeyFile() - privValStateFile := tendermintConfig.PrivValidatorStateFile() - var pv *privval.FilePV - if tmos.FileExists(privValKeyFile) { - pv = privval.LoadFilePV(privValKeyFile, privValStateFile) - log.Info(). - Str("privValKeyFile", privValKeyFile). - Str("stateFile", privValStateFile). - Msg("Found private validator") - } else { - pv = privval.GenFilePV(privValKeyFile, privValStateFile) - pv.Save() - log.Info(). - Str("privValKeyFile", privValKeyFile). - Str("stateFile", privValStateFile). - Msg("Generated private validator") - } + if slices.Contains([]string{VALIDATOR, ISOLATEDVALIDATOR}, config.Role) { + // private validator + privValKeyFile := tendermintConfig.PrivValidatorKeyFile() + privValStateFile := tendermintConfig.PrivValidatorStateFile() + var pv *privval.FilePV + if tmos.FileExists(privValKeyFile) { + pv = privval.LoadFilePV(privValKeyFile, privValStateFile) + log.Info(). + Str("privValKeyFile", privValKeyFile). + Str("stateFile", privValStateFile). + Msg("Found private validator") + } else { + pv = privval.GenFilePV(privValKeyFile, privValStateFile) + pv.Save() + log.Info(). + Str("privValKeyFile", privValKeyFile). + Str("stateFile", privValStateFile). + Msg("Generated private validator") + } validatorPubKeyPath := filepath.Join(tendermintConfig.RootDir, "config", "priv_validator_pubkey.hex") validatorPublicKeyHex := hex.EncodeToString(pv.Key.PubKey.Bytes()) @@ -227,7 +238,7 @@ func initFilesWithConfig(tendermintConfig *cfg.Config, config *Config, appState } log.Info().Str("path", genFile).Msg("Generated genesis file") } - + nodeKeyFile := tendermintConfig.NodeKeyFile() if tmos.FileExists(nodeKeyFile) { log.Info().Str("path", nodeKeyFile).Msg("Found node key") diff --git a/rolling-shutter/docs/rolling-shutter_chain_init.md b/rolling-shutter/docs/rolling-shutter_chain_init.md index a6b11a42..ab881350 100644 --- a/rolling-shutter/docs/rolling-shutter_chain_init.md +++ b/rolling-shutter/docs/rolling-shutter_chain_init.md @@ -16,7 +16,7 @@ rolling-shutter chain init [flags] --index int keyper index --initial-eon uint initial eon --listen-address string tendermint RPC listen address (default "tcp://127.0.0.1:26657") - --role string tendermint node role (validator, sentry, seed) (default "validator") + --role string tendermint node role (validator, isolated-validator, sentry, seed) (default "validator") --root string root directory ``` diff --git a/rolling-shutter/go.mod b/rolling-shutter/go.mod index 3b1ef4fc..378a8c93 100644 --- a/rolling-shutter/go.mod +++ b/rolling-shutter/go.mod @@ -42,6 +42,7 @@ require ( go.opentelemetry.io/otel/trace v1.14.0 go.opentelemetry.io/proto/otlp v0.19.0 golang.org/x/crypto v0.12.0 + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 google.golang.org/protobuf v1.30.0 gotest.tools v2.2.0+incompatible @@ -215,7 +216,6 @@ require ( go.uber.org/fx v1.20.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect - golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/sys v0.11.0 // indirect From 2e36e745254e048f273a4b3482cc385a12288bb1 Mon Sep 17 00:00:00 2001 From: Konrad Feldmeier Date: Mon, 25 Sep 2023 15:30:14 +0200 Subject: [PATCH 3/3] Simplify init-chain script --- docker/02-init-chain.sh | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/docker/02-init-chain.sh b/docker/02-init-chain.sh index 9b3a3a6b..e1130baf 100755 --- a/docker/02-init-chain.sh +++ b/docker/02-init-chain.sh @@ -16,13 +16,19 @@ ${BB} rm -rf data/deployments # has geth as dependency $DC up deploy-contracts +# setup chain-seed $DC run --rm --no-deps chain-seed init \ --root /chain \ --blocktime 1 \ --listen-address tcp://0.0.0.0:${TM_RPC_PORT} \ --role seed -for num in 0 1 2 3; do +seed_node=$(cat data/chain-seed/config/node_key.json.id)@chain-seed:${TM_P2P_PORT} + +${BB} sed -i "/^moniker/c\moniker = \"chain-seed\"" data/chain-seed/config/config.toml + +# configure validators and keypers 0-3 +for num in {0..3}; do validator_cmd=chain-$num-validator $DC run --rm --no-deps ${validator_cmd} init \ @@ -32,31 +38,26 @@ for num in 0 1 2 3; do --listen-address tcp://0.0.0.0:${TM_RPC_PORT} \ --role validator - ${BB} sed -i "/ValidatorPublicKey/c\ValidatorPublicKey = \"$(cat data/${validator_cmd}/config/priv_validator_pubkey.hex)\"" /config/keyper-${num}.toml + validator_id=$(cat data/${validator_cmd}/config/node_key.json.id) + validator_node=${validator_id}@${validator_cmd}:${TM_P2P_PORT} + validator_config_path=data/${validator_cmd}/config/config.toml + # share genesis if [ $num -eq 0 ]; then - for destination in data/chain-seed/config/ data/chain-{1..3}-validator/config/ ; do + for destination in data/chain-seed/config/ data/chain-{1..3}-validator/config/; do ${BB} cp -v data/chain-0-validator/config/genesis.json "${destination}" done fi -done -seed_node=$(cat data/chain-seed/config/node_key.json.id)@chain-seed:${TM_P2P_PORT} -validator_0_node=$(cat data/chain-0-validator/config/node_key.json.id)@chain-0-validator:${TM_P2P_PORT} - -${BB} sed -i "/^moniker/c\moniker = \"chain-seed\"" data/chain-seed/config/config.toml - -for num in {0..3}; do - validator_cmd=chain-$num-validator - - validator_id=$(cat data/${validator_cmd}/config/node_key.json.id) - validator_node=${validator_id}@${validator_cmd}:${TM_P2P_PORT} + # set validator publickey for keyper + ${BB} sed -i "/ValidatorPublicKey/c\ValidatorPublicKey = \"$(cat data/${validator_cmd}/config/priv_validator_pubkey.hex)\"" /config/keyper-${num}.toml - # set seed node for validator - ${BB} sed -i "/^bootstrap_peers =/c\bootstrap_peers = \"${validator_0_node}\"" data/${validator_cmd}/config/config.toml - ${BB} sed -i "/^external_address =/c\external_address = \"${validator_cmd}:${TM_P2P_PORT}\"" data/${validator_cmd}/config/config.toml - ${BB} sed -i "/^pex/c\pex = true" data/${validator_cmd}/config/config.toml - ${BB} sed -i "/^moniker/c\moniker = \"${validator_cmd}\"" data/${validator_cmd}/config/config.toml + # set seed node for chain bootstrap + ${BB} sed -i "/^bootstrap_peers =/c\bootstrap_peers = \"${seed_node}\"" "${validator_config_path}" + # fix external address for docker internal communication + ${BB} sed -i "/^external_address =/c\external_address = \"${validator_cmd}:${TM_P2P_PORT}\"" "${validator_config_path}" + # give a nice name + ${BB} sed -i "/^moniker/c\moniker = \"${validator_cmd}\"" "${validator_config_path}" done