From 4c97d097e9cfc1843ed0e770513e69b35bbf8221 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 15 Feb 2023 14:38:01 -0500 Subject: [PATCH 01/14] updates --- server/grpc/server.go | 53 +++++++++++++++++++++++++++++++------------ server/start.go | 41 ++++++++++++++++++++------------- 2 files changed, 63 insertions(+), 31 deletions(-) diff --git a/server/grpc/server.go b/server/grpc/server.go index 79a9be3dca24..1de5777510a8 100644 --- a/server/grpc/server.go +++ b/server/grpc/server.go @@ -1,10 +1,11 @@ package grpc import ( + "context" "fmt" "net" - "time" + "cosmossdk.io/log" "google.golang.org/grpc" "github.com/cosmos/cosmos-sdk/client" @@ -17,8 +18,9 @@ import ( _ "github.com/cosmos/cosmos-sdk/types/tx/amino" // Import amino.proto file for reflection ) -// StartGRPCServer starts a gRPC server on the given address. -func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config.GRPCConfig) (*grpc.Server, error) { +// NewGRPCServer returns a correctly configured and initialized gRPC server. +// Note, the caller is responsible for starting the server. See StartGRPCServer. +func NewGRPCServer(clientCtx client.Context, app types.Application, cfg config.GRPCConfig) (*grpc.Server, error) { maxSendMsgSize := cfg.MaxSendMsgSize if maxSendMsgSize == 0 { maxSendMsgSize = config.DefaultGRPCMaxSendMsgSize @@ -46,6 +48,7 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config for _, m := range clientCtx.TxConfig.SignModeHandler().Modes() { modes[m.String()] = (int32)(m) } + return modes }(), ChainID: clientCtx.ChainID, @@ -60,25 +63,45 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config // the gRPC server exposes. gogoreflection.Register(grpcSrv) + return grpcSrv, nil +} + +// StartGRPCServer starts the provided gRPC server on the address specified in cfg. +// Note, this creates a blocking process if the server is started successfully. +// Otherwise, an error is returned. The caller is expected to provide a Context +// that is properly canceled or closed to indicate the server should be stopped. +func StartGRPCServer(ctx context.Context, logger log.Logger, cfg config.GRPCConfig, grpcSrv *grpc.Server) error { listener, err := net.Listen("tcp", cfg.Address) if err != nil { - return nil, err + return fmt.Errorf("failed to listen on address %s: %w", cfg.Address, err) } - errCh := make(chan error) + errCh := make(chan error, 1) + + // Start the gRPC in an external goroutine as Serve is blocking and will return + // an error upon failure, which we'll send on the error channel that will be + // consumed by the for block below. go func() { - err = grpcSrv.Serve(listener) - if err != nil { - errCh <- fmt.Errorf("failed to serve: %w", err) - } + logger.Info("starting gRPC server...", "address", cfg.Address) + errCh <- grpcSrv.Serve(listener) }() - select { - case err := <-errCh: - return nil, err + // Start a block loop to wait for an indication to stop the server or that + // the server failed to start properly. + for { + select { + case <-ctx.Done(): + // The calling process cancelled or closed the provided context, so we must + // gracefully stop the gRPC server. + + logger.Info("stopping gRPC server...", "address", cfg.Address) + grpcSrv.GracefulStop() - case <-time.After(types.ServerStartTime): - // assume server started successfully - return grpcSrv, nil + return nil + + case err := <-errCh: + logger.Error("failed to start gRPC server", "err", err) + return err + } } } diff --git a/server/start.go b/server/start.go index 7ad4ca73adee..c232af0558db 100644 --- a/server/start.go +++ b/server/start.go @@ -1,6 +1,7 @@ package server import ( + "context" "fmt" "net" "os" @@ -14,6 +15,7 @@ import ( pvm "github.com/cometbft/cometbft/privval" "github.com/cometbft/cometbft/proxy" "github.com/cometbft/cometbft/rpc/client/local" + "github.com/neilotoole/errgroup" "github.com/spf13/cobra" "github.com/spf13/pflag" "google.golang.org/grpc" @@ -260,33 +262,34 @@ func startStandAlone(ctx *Context, appCreator types.AppCreator) error { return WaitForQuitSignals() } -func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.AppCreator) error { - cfg := ctx.Config +func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types.AppCreator) error { + cfg := svrCtx.Config home := cfg.RootDir - db, err := openDB(home, GetAppDBBackend(ctx.Viper)) + db, err := openDB(home, GetAppDBBackend(svrCtx.Viper)) if err != nil { return err } - traceWriterFile := ctx.Viper.GetString(flagTraceStore) + traceWriterFile := svrCtx.Viper.GetString(flagTraceStore) traceWriter, err := openTraceWriter(traceWriterFile) if err != nil { return err } - // Clean up the traceWriter when the server is shutting down. + // clean up the traceWriter when the server is shutting down var traceWriterCleanup func() + // if flagTraceStore is not used then traceWriter is nil if traceWriter != nil { traceWriterCleanup = func() { if err = traceWriter.Close(); err != nil { - ctx.Logger.Error("failed to close trace writer", "err", err) + svrCtx.Logger.Error("failed to close trace writer", "err", err) } } } - config, err := serverconfig.GetConfig(ctx.Viper) + config, err := serverconfig.GetConfig(svrCtx.Viper) if err != nil { return err } @@ -295,24 +298,25 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } - app := appCreator(ctx.Logger, db, traceWriter, ctx.Viper) + app := appCreator(svrCtx.Logger, db, traceWriter, svrCtx.Viper) nodeKey, err := p2p.LoadOrGenNodeKey(cfg.NodeKeyFile()) if err != nil { return err } + genDocProvider := node.DefaultGenesisDocProviderFunc(cfg) var ( tmNode *node.Node - gRPCOnly = ctx.Viper.GetBool(flagGRPCOnly) + gRPCOnly = svrCtx.Viper.GetBool(flagGRPCOnly) ) if gRPCOnly { - ctx.Logger.Info("starting node in gRPC only mode; CometBFT is disabled") + svrCtx.Logger.Info("starting node in gRPC only mode; CometBFT is disabled") config.GRPC.Enable = true } else { - ctx.Logger.Info("starting node with ABCI CometBFT in-process") + svrCtx.Logger.Info("starting node with ABCI CometBFT in-process") tmNode, err = node.NewNode( cfg, @@ -322,7 +326,7 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App genDocProvider, node.DefaultDBProvider, node.DefaultMetricsProvider(cfg.Instrumentation), - ctx.Logger, + svrCtx.Logger, ) if err != nil { return err @@ -356,6 +360,9 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App grpcSrv *grpc.Server ) + ctx, cancel := context.WithCancel(context.Background()) + g, ctx := errgroup.WithContext(ctx) + if config.API.Enable { genDoc, err := genDocProvider() if err != nil { @@ -382,7 +389,7 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App grpcAddress := fmt.Sprintf("127.0.0.1:%s", port) - // If grpc is enabled, configure grpc client for grpc gateway. + // if gRPC is enabled, configure gRPC client for gRPC gateway grpcClient, err := grpc.Dial( grpcAddress, grpc.WithTransportCredentials(insecure.NewCredentials()), @@ -397,18 +404,19 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } clientCtx = clientCtx.WithGRPCClient(grpcClient) - ctx.Logger.Debug("grpc client assigned to client context", "target", grpcAddress) + svrCtx.Logger.Debug("grpc client assigned to client context", "target", grpcAddress) // start grpc server grpcSrv, err = servergrpc.StartGRPCServer(clientCtx, app, config.GRPC) if err != nil { return err } + defer grpcSrv.Stop() } // configure api server - apiSrv = api.New(clientCtx, ctx.Logger.With("module", "api-server"), grpcSrv) + apiSrv = api.New(clientCtx, svrCtx.Logger.With("module", "api-server"), grpcSrv) app.RegisterAPIRoutes(apiSrv, config.API) if config.Telemetry.Enabled { apiSrv.SetTelemetry(metrics) @@ -460,7 +468,7 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App _ = apiSrv.Close() } - ctx.Logger.Info("exiting...") + svrCtx.Logger.Info("exiting...") }() // wait for signal capture and gracefully return @@ -471,6 +479,7 @@ func startTelemetry(cfg serverconfig.Config) (*telemetry.Metrics, error) { if !cfg.Telemetry.Enabled { return nil, nil } + return telemetry.New(cfg.Telemetry) } From 8454b243cba873a82d89f0d58757566c0e7c3df0 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 15 Feb 2023 19:37:26 -0500 Subject: [PATCH 02/14] updates --- go.mod | 1 + go.sum | 2 + server/api/server.go | 54 ++++++--- server/grpc/server.go | 6 +- server/start.go | 211 +++++++++++++++++++----------------- server/types/app.go | 11 +- server/util.go | 36 +++--- testutil/network/network.go | 37 ++++--- testutil/network/util.go | 36 +++--- 9 files changed, 223 insertions(+), 171 deletions(-) diff --git a/go.mod b/go.mod index 6d49eb1e9239..dc5212fcc05e 100644 --- a/go.mod +++ b/go.mod @@ -124,6 +124,7 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 83371f21cedb..a2c289c8264c 100644 --- a/go.sum +++ b/go.sum @@ -624,6 +624,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/server/api/server.go b/server/api/server.go index ff5a0e823bbe..12ec0a443b30 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -1,6 +1,7 @@ package api import ( + "context" "fmt" "net" "net/http" @@ -29,11 +30,10 @@ type Server struct { Router *mux.Router GRPCGatewayRouter *runtime.ServeMux ClientCtx client.Context + GRPCSrv *grpc.Server + logger log.Logger + metrics *telemetry.Metrics - GRPCSrv *grpc.Server - - logger log.Logger - metrics *telemetry.Metrics // Start() is blocking and generally called from a separate goroutine. // Close() can be called asynchronously and access shared memory // via the listener. Therefore, we sync access to Start and Close with @@ -51,6 +51,7 @@ func CustomGRPCHeaderMatcher(key string) (string, bool) { switch strings.ToLower(key) { case grpctypes.GRPCBlockHeightHeader: return grpctypes.GRPCBlockHeightHeader, true + default: return runtime.DefaultHeaderMatcher(key) } @@ -88,9 +89,12 @@ func New(clientCtx client.Context, logger log.Logger, grpcSrv *grpc.Server) *Ser // Start starts the API server. Internally, the API server leverages CometBFT's // JSON RPC server. Configuration options are provided via config.APIConfig -// and are delegated to the CometBFT JSON RPC server. The process is -// non-blocking, so an external signal handler must be used. -func (s *Server) Start(cfg config.Config) error { +// and are delegated to the CometBFT JSON RPC server. +// +// Note, this creates a blocking process if the server is started successfully. +// Otherwise, an error is returned. The caller is expected to provide a Context +// that is properly canceled or closed to indicate the server should be stopped. +func (s *Server) Start(ctx context.Context, cfg config.Config) error { s.mtx.Lock() cmtCfg := tmrpcserver.DefaultConfig() @@ -134,13 +138,37 @@ func (s *Server) Start(cfg config.Config) error { // register grpc-gateway routes (after grpc-web server as the first match is used) s.Router.PathPrefix("/").Handler(s.GRPCGatewayRouter) - s.logger.Info("starting API server...") - if cfg.API.EnableUnsafeCORS { - allowAllCORS := handlers.CORS(handlers.AllowedHeaders([]string{"Content-Type"})) - return tmrpcserver.Serve(s.listener, allowAllCORS(s.Router), s.logger, cmtCfg) - } + errCh := make(chan error, 1) + + // Start the API in an external goroutine as Serve is blocking and will return + // an error upon failure, which we'll send on the error channel that will be + // consumed by the for block below. + go func(enableUnsafeCORS bool) { + s.logger.Info("starting API server...", "address", cfg.API.Address) - return tmrpcserver.Serve(s.listener, s.Router, s.logger, cmtCfg) + if enableUnsafeCORS { + allowAllCORS := handlers.CORS(handlers.AllowedHeaders([]string{"Content-Type"})) + errCh <- tmrpcserver.Serve(s.listener, allowAllCORS(s.Router), s.logger, cmtCfg) + } else { + errCh <- tmrpcserver.Serve(s.listener, s.Router, s.logger, cmtCfg) + } + }(cfg.API.EnableUnsafeCORS) + + // Start a blocking loop to wait for an indication to stop the server or that + // the server failed to start properly. + for { + select { + case <-ctx.Done(): + // The calling process cancelled or closed the provided context, so we must + // gracefully stop the API server. + s.logger.Info("stopping API server...", "address", cfg.API.Address) + return s.Close() + + case err := <-errCh: + s.logger.Error("failed to start API server", "err", err) + return err + } + } } // Close closes the API server. diff --git a/server/grpc/server.go b/server/grpc/server.go index 1de5777510a8..de8d8fcf7643 100644 --- a/server/grpc/server.go +++ b/server/grpc/server.go @@ -56,7 +56,7 @@ func NewGRPCServer(clientCtx client.Context, app types.Application, cfg config.G InterfaceRegistry: clientCtx.InterfaceRegistry, }) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to register reflection service: %w", err) } // Reflection allows external clients to see what services and methods @@ -67,6 +67,7 @@ func NewGRPCServer(clientCtx client.Context, app types.Application, cfg config.G } // StartGRPCServer starts the provided gRPC server on the address specified in cfg. +// // Note, this creates a blocking process if the server is started successfully. // Otherwise, an error is returned. The caller is expected to provide a Context // that is properly canceled or closed to indicate the server should be stopped. @@ -86,14 +87,13 @@ func StartGRPCServer(ctx context.Context, logger log.Logger, cfg config.GRPCConf errCh <- grpcSrv.Serve(listener) }() - // Start a block loop to wait for an indication to stop the server or that + // Start a blocking loop to wait for an indication to stop the server or that // the server failed to start properly. for { select { case <-ctx.Done(): // The calling process cancelled or closed the provided context, so we must // gracefully stop the gRPC server. - logger.Info("stopping gRPC server...", "address", cfg.Address) grpcSrv.GracefulStop() diff --git a/server/start.go b/server/start.go index c232af0558db..8016237a7b96 100644 --- a/server/start.go +++ b/server/start.go @@ -4,9 +4,6 @@ import ( "context" "fmt" "net" - "os" - "runtime/pprof" - "time" "github.com/cometbft/cometbft/abci/server" tcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" @@ -137,22 +134,20 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. withCMT, _ := cmd.Flags().GetBool(flagWithComet) if !withCMT { serverCtx.Logger.Info("starting ABCI without CometBFT") - return wrapCPUProfile(serverCtx, func() error { - return startStandAlone(serverCtx, appCreator) - }) - } + return startStandAlone(serverCtx, appCreator) - // amino is needed here for backwards compatibility of REST routes - err = wrapCPUProfile(serverCtx, func() error { - return startInProcess(serverCtx, clientCtx, appCreator) - }) - errCode, ok := err.(ErrorCode) - if !ok { - return err + // TODO: ... + // return wrapCPUProfile(serverCtx, func() error { + // return startStandAlone(serverCtx, appCreator) + // }) } - serverCtx.Logger.Debug(fmt.Sprintf("received quit signal: %d", errCode.Code)) - return nil + // TODO: ... + // err = wrapCPUProfile(serverCtx, func() error { + // return startInProcess(serverCtx, clientCtx, appCreator) + // }) + + return startInProcess(serverCtx, clientCtx, appCreator) }, } @@ -210,31 +205,30 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. return cmd } -func startStandAlone(ctx *Context, appCreator types.AppCreator) error { - addr := ctx.Viper.GetString(flagAddress) - transport := ctx.Viper.GetString(flagTransport) - home := ctx.Viper.GetString(flags.FlagHome) +func startStandAlone(svrCtx *Context, appCreator types.AppCreator) error { + addr := svrCtx.Viper.GetString(flagAddress) + transport := svrCtx.Viper.GetString(flagTransport) + home := svrCtx.Viper.GetString(flags.FlagHome) - db, err := openDB(home, GetAppDBBackend(ctx.Viper)) + db, err := openDB(home, GetAppDBBackend(svrCtx.Viper)) if err != nil { return err } - traceWriterFile := ctx.Viper.GetString(flagTraceStore) + traceWriterFile := svrCtx.Viper.GetString(flagTraceStore) traceWriter, err := openTraceWriter(traceWriterFile) if err != nil { return err } - app := appCreator(ctx.Logger, db, traceWriter, ctx.Viper) + app := appCreator(svrCtx.Logger, db, traceWriter, svrCtx.Viper) - config, err := serverconfig.GetConfig(ctx.Viper) + config, err := serverconfig.GetConfig(svrCtx.Viper) if err != nil { return err } - _, err = startTelemetry(config) - if err != nil { + if _, err := startTelemetry(config); err != nil { return err } @@ -243,23 +237,33 @@ func startStandAlone(ctx *Context, appCreator types.AppCreator) error { return fmt.Errorf("error creating listener: %v", err) } - svr.SetLogger(ctx.Logger.With("module", "abci-server")) + svr.SetLogger(svrCtx.Logger.With("module", "abci-server")) - err = svr.Start() - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } + ctx, cancelFn := context.WithCancel(context.Background()) + g, ctx := errgroup.WithContext(ctx) - defer func() { - if err = svr.Stop(); err != nil { - fmt.Println(err.Error()) - os.Exit(1) + // listen for quit signals so the calling parent process can gracefully exit + ListenForQuitSignals(cancelFn, svrCtx.Logger) + + g.Go(func() error { + if err := svr.Start(); err != nil { + svrCtx.Logger.Error("failed to start out-of-process ABCI server", "err", err) + return err } - }() - // Wait for SIGINT or SIGTERM signal - return WaitForQuitSignals() + // start a blocking loop to wait for an indication to stop the server + for { + select { + case <-ctx.Done(): + // The calling process cancelled or closed the provided context, so we + // must gracefully stop the ABCI server. + svrCtx.Logger.Info("stopping the ABCI server...") + return svr.Stop() + } + } + }) + + return g.Wait() } func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types.AppCreator) error { @@ -360,9 +364,12 @@ func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types. grpcSrv *grpc.Server ) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancelFn := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) + // listen for quit signals so the calling parent process can gracefully exit + ListenForQuitSignals(cancelFn, svrCtx.Logger) + if config.API.Enable { genDoc, err := genDocProvider() if err != nil { @@ -404,57 +411,56 @@ func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types. } clientCtx = clientCtx.WithGRPCClient(grpcClient) - svrCtx.Logger.Debug("grpc client assigned to client context", "target", grpcAddress) + svrCtx.Logger.Debug("gRPC client assigned to client context", "target", grpcAddress) - // start grpc server - grpcSrv, err = servergrpc.StartGRPCServer(clientCtx, app, config.GRPC) + grpcSrv, err := servergrpc.NewGRPCServer(clientCtx, app, config.GRPC) if err != nil { return err } - defer grpcSrv.Stop() + // Start the gRPC server in a goroutine. Note, the provided ctx will ensure + // that the server is gracefully shut down. + g.Go(func() error { + return servergrpc.StartGRPCServer(ctx, svrCtx.Logger.With("module", "grpc-server"), config.GRPC, grpcSrv) + }) } - // configure api server apiSrv = api.New(clientCtx, svrCtx.Logger.With("module", "api-server"), grpcSrv) app.RegisterAPIRoutes(apiSrv, config.API) + if config.Telemetry.Enabled { apiSrv.SetTelemetry(metrics) } - errCh := make(chan error) - - go func() { - if err := apiSrv.Start(config); err != nil { - errCh <- err - } - }() - - select { - case err := <-errCh: - return err - case <-time.After(types.ServerStartTime): // assume server started successfully - } + g.Go(func() error { + return apiSrv.Start(ctx, config) + }) } - // If gRPC is enabled but API is not, we need to start the gRPC server + // If gRPC is enabled but the API is not, we need to start the gRPC server // without the API server. If the API server is enabled, we've already // started the grpc server. if config.GRPC.Enable && !config.API.Enable { - grpcSrv, err = servergrpc.StartGRPCServer(clientCtx, app, config.GRPC) + grpcSrv, err := servergrpc.NewGRPCServer(clientCtx, app, config.GRPC) if err != nil { return err } - defer grpcSrv.Stop() + + // Start the gRPC server in a goroutine. Note, the provided ctx will ensure + // that the server is gracefully shut down. + g.Go(func() error { + return servergrpc.StartGRPCServer(ctx, svrCtx.Logger.With("module", "grpc-server"), config.GRPC, grpcSrv) + }) } // At this point it is safe to block the process if we're in gRPC only mode as // we do not need to handle any CometBFT related processes. if gRPCOnly { // wait for signal capture and gracefully return - return WaitForQuitSignals() + return g.Wait() } + // deferred cleanup function defer func() { if tmNode != nil && tmNode.IsRunning() { _ = tmNode.Stop() @@ -463,16 +469,10 @@ func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types. if traceWriterCleanup != nil { traceWriterCleanup() } - - if apiSrv != nil { - _ = apiSrv.Close() - } - - svrCtx.Logger.Info("exiting...") }() // wait for signal capture and gracefully return - return WaitForQuitSignals() + return g.Wait() } func startTelemetry(cfg serverconfig.Config) (*telemetry.Metrics, error) { @@ -483,39 +483,52 @@ func startTelemetry(cfg serverconfig.Config) (*telemetry.Metrics, error) { return telemetry.New(cfg.Telemetry) } -// wrapCPUProfile runs callback in a goroutine, then wait for quit signals. -func wrapCPUProfile(ctx *Context, callback func() error) error { - if cpuProfile := ctx.Viper.GetString(flagCPUProfile); cpuProfile != "" { - f, err := os.Create(cpuProfile) - if err != nil { - return err - } +// // wrapCPUProfile runs callback in a goroutine, then wait for quit signals. +// func wrapCPUProfile(svrCtx *Context, callback func() error) error { +// if cpuProfile := svrCtx.Viper.GetString(flagCPUProfile); cpuProfile != "" { +// f, err := os.Create(cpuProfile) +// if err != nil { +// return err +// } - ctx.Logger.Info("starting CPU profiler", "profile", cpuProfile) - if err := pprof.StartCPUProfile(f); err != nil { - return err - } +// svrCtx.Logger.Info("starting CPU profiler", "profile", cpuProfile) - defer func() { - ctx.Logger.Info("stopping CPU profiler", "profile", cpuProfile) - pprof.StopCPUProfile() - if err := f.Close(); err != nil { - ctx.Logger.Info("failed to close cpu-profile file", "profile", cpuProfile, "err", err.Error()) - } - }() - } +// if err := pprof.StartCPUProfile(f); err != nil { +// return err +// } - errCh := make(chan error) - go func() { - errCh <- callback() - }() +// defer func() { +// svrCtx.Logger.Info("stopping CPU profiler", "profile", cpuProfile) +// pprof.StopCPUProfile() - select { - case err := <-errCh: - return err +// if err := f.Close(); err != nil { +// svrCtx.Logger.Info("failed to close cpu-profile file", "profile", cpuProfile, "err", err.Error()) +// } +// }() +// } - case <-time.After(types.ServerStartTime): - } +// ctx, cancelFn := context.WithCancel(context.Background()) +// g, ctx := errgroup.WithContext(ctx) - return WaitForQuitSignals() -} +// // Wait for the parent process to capture any termination signals in a +// // non-block manner, where the error code will be sent on a channel. +// errCh := WatchForQuitSignals(cancelFn) + +// g.Go(func() error { +// return callback() +// }) + +// // errCh := make(chan error) +// // go func() { +// // errCh <- callback() +// // }() + +// // select { +// // case err := <-errCh: +// // return err + +// // case <-time.After(types.ServerStartTime): +// // } + +// // return WaitForQuitSignals() +// } diff --git a/server/types/app.go b/server/types/app.go index d5c8fb1f10c2..659ad93e20f3 100644 --- a/server/types/app.go +++ b/server/types/app.go @@ -3,28 +3,21 @@ package types import ( "encoding/json" "io" - "time" - - dbm "github.com/cosmos/cosmos-db" "cosmossdk.io/log" + storetypes "cosmossdk.io/store/types" abci "github.com/cometbft/cometbft/abci/types" cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" cmttypes "github.com/cometbft/cometbft/types" + dbm "github.com/cosmos/cosmos-db" "github.com/cosmos/gogoproto/grpc" "github.com/spf13/cobra" - storetypes "cosmossdk.io/store/types" - "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/server/api" "github.com/cosmos/cosmos-sdk/server/config" ) -// ServerStartTime defines the time duration that the server need to stay running after startup -// for the startup be considered successful -const ServerStartTime = 5 * time.Second - type ( // AppOptions defines an interface that is passed into an application // constructor, typically used to set BaseApp options that are either supplied diff --git a/server/util.go b/server/util.go index 712eec749276..117cc0469a37 100644 --- a/server/util.go +++ b/server/util.go @@ -1,6 +1,7 @@ package server import ( + "context" "errors" "fmt" "io" @@ -14,23 +15,22 @@ import ( "syscall" "time" - dbm "github.com/cosmos/cosmos-db" - + "cosmossdk.io/log" + "cosmossdk.io/store" + "cosmossdk.io/store/snapshots" + snapshottypes "cosmossdk.io/store/snapshots/types" + storetypes "cosmossdk.io/store/types" cmtcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" cmtcfg "github.com/cometbft/cometbft/config" cmtcli "github.com/cometbft/cometbft/libs/cli" cmtflags "github.com/cometbft/cometbft/libs/cli/flags" cmtlog "github.com/cometbft/cometbft/libs/log" + dbm "github.com/cosmos/cosmos-db" "github.com/spf13/cast" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" - "cosmossdk.io/store" - "cosmossdk.io/store/snapshots" - snapshottypes "cosmossdk.io/store/snapshots/types" - storetypes "cosmossdk.io/store/types" - "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/server/config" @@ -363,12 +363,22 @@ func TrapSignal(cleanupFunc func()) { }() } -// WaitForQuitSignals waits for SIGINT and SIGTERM and returns. -func WaitForQuitSignals() ErrorCode { - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - sig := <-sigs - return ErrorCode{Code: int(sig.(syscall.Signal)) + 128} +// ListenForQuitSignals listens for SIGINT and SIGTERM. When a signal is received, +// the cleanup function is called, indicating the caller can gracefully exit or +// return. +// +// Note, this performs a non-blocking process so the caller must ensure the +// corresponding context derived from the cancelFn is used correctly. +func ListenForQuitSignals(cancelFn context.CancelFunc, logger log.Logger) { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + + go func() { + sig := <-sigCh + cancelFn() + + logger.Info("caught signal", "signal", sig.String()) + }() } // GetAppDBBackend gets the backend type to use for the application DBs. diff --git a/testutil/network/network.go b/testutil/network/network.go index 8b7f9703aa07..da6d1c84ba4b 100644 --- a/testutil/network/network.go +++ b/testutil/network/network.go @@ -18,16 +18,16 @@ import ( "cosmossdk.io/depinject" "cosmossdk.io/log" sdkmath "cosmossdk.io/math" + pruningtypes "cosmossdk.io/store/pruning/types" cmtlog "github.com/cometbft/cometbft/libs/log" cmtrand "github.com/cometbft/cometbft/libs/rand" "github.com/cometbft/cometbft/node" cmtclient "github.com/cometbft/cometbft/rpc/client" dbm "github.com/cosmos/cosmos-db" + "github.com/neilotoole/errgroup" "github.com/spf13/cobra" "google.golang.org/grpc" - pruningtypes "cosmossdk.io/store/pruning/types" - "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/grpc/cmtservice" @@ -261,10 +261,12 @@ type ( ValAddress sdk.ValAddress RPCClient cmtclient.Client - tmNode *node.Node - api *api.Server - grpc *grpc.Server - grpcWeb *http.Server + tmNode *node.Node + api *api.Server + grpc *grpc.Server + grpcWeb *http.Server + errGroup *errgroup.Group + cancelFn context.CancelFunc } // ValidatorI expose a validator's context and configuration @@ -734,24 +736,25 @@ func (n *Network) Cleanup() { n.Logger.Log("cleaning up test network...") for _, v := range n.Validators { - if v.tmNode != nil && v.tmNode.IsRunning() { - _ = v.tmNode.Stop() - } + // cancel the validator's context which will signal to the gRPC and API + // goroutines that they should gracefully exit. + v.cancelFn() - if v.api != nil { - _ = v.api.Close() + if err := v.errGroup.Wait(); err != nil { + n.Logger.Log("unexpected error waiting for validator gRPC and API processes to exit", "err", err) } - if v.grpc != nil { - v.grpc.Stop() - if v.grpcWeb != nil { - _ = v.grpcWeb.Close() + if v.tmNode != nil && v.tmNode.IsRunning() { + if err := v.tmNode.Stop(); err != nil { + n.Logger.Log("failed to stop validator Tendermint node", "err", err) } } + + if v.grpcWeb != nil { + _ = v.grpcWeb.Close() + } } - // Give a brief pause for things to finish closing in other processes. Hopefully this helps with the address-in-use errors. - // 100ms chosen randomly. time.Sleep(100 * time.Millisecond) if n.Config.CleanupDir { diff --git a/testutil/network/util.go b/testutil/network/util.go index 7608e770da3b..110cb241e921 100644 --- a/testutil/network/util.go +++ b/testutil/network/util.go @@ -1,12 +1,12 @@ package network import ( + "context" "encoding/json" "fmt" "net" "os" "path/filepath" - "time" "github.com/cometbft/cometbft/node" "github.com/cometbft/cometbft/p2p" @@ -15,10 +15,10 @@ import ( "github.com/cometbft/cometbft/rpc/client/local" "github.com/cometbft/cometbft/types" cmttime "github.com/cometbft/cometbft/types/time" + "github.com/neilotoole/errgroup" "github.com/cosmos/cosmos-sdk/server/api" servergrpc "github.com/cosmos/cosmos-sdk/server/grpc" - srvtypes "github.com/cosmos/cosmos-sdk/server/types" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" "github.com/cosmos/cosmos-sdk/x/genutil" @@ -75,12 +75,24 @@ func startInProcess(cfg Config, val *Validator) error { app.RegisterNodeService(val.ClientCtx) } - if val.AppConfig.GRPC.Enable { - grpcSrv, err := servergrpc.StartGRPCServer(val.ClientCtx, app, val.AppConfig.GRPC) + ctx := context.Background() + ctx, val.cancelFn = context.WithCancel(ctx) + val.errGroup, ctx = errgroup.WithContext(ctx) + + grpcCfg := val.AppConfig.GRPC + + if grpcCfg.Enable { + grpcSrv, err := servergrpc.NewGRPCServer(val.ClientCtx, app, grpcCfg) if err != nil { return err } + // Start the gRPC server in a goroutine. Note, the provided ctx will ensure + // that the server is gracefully shut down. + val.errGroup.Go(func() error { + return servergrpc.StartGRPCServer(ctx, logger.With("module", "grpc-server"), grpcCfg, grpcSrv) + }) + val.grpc = grpcSrv } @@ -88,19 +100,9 @@ func startInProcess(cfg Config, val *Validator) error { apiSrv := api.New(val.ClientCtx, logger.With("module", "api-server"), val.grpc) app.RegisterAPIRoutes(apiSrv, val.AppConfig.API) - errCh := make(chan error) - - go func() { - if err := apiSrv.Start(*val.AppConfig); err != nil { - errCh <- err - } - }() - - select { - case err := <-errCh: - return err - case <-time.After(srvtypes.ServerStartTime): // assume server started successfully - } + val.errGroup.Go(func() error { + return apiSrv.Start(ctx, *val.AppConfig) + }) val.api = apiSrv } From f2d26aeeb63d4090ec99ffdfe857b34e483e24d5 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 15 Feb 2023 19:45:01 -0500 Subject: [PATCH 03/14] updates --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index dc5212fcc05e..ec3a986a5720 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 github.com/mattn/go-isatty v0.0.17 + github.com/neilotoole/errgroup v0.1.6 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/common v0.39.0 @@ -124,7 +125,6 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pmezard/go-difflib v1.0.0 // indirect From ab04e115b4f2fb423201fd4ff7a06338b4d43098 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 15 Feb 2023 19:53:21 -0500 Subject: [PATCH 04/14] cl++ --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe8b696d5fc5..5d3c81e0d73a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Improvements +* (server) [#15041](https://github.com/cosmos/cosmos-sdk/pull/15041) Remove unnecessary sleeps from gRPC and API server initiation. The servers will start and accept requests as soon as they're ready. * (cli) [#14659](https://github.com/cosmos/cosmos-sdk/pull/14659) Added ability to query blocks by either height/hash `simd q block --type=height|hash `. * (store) [#14410](https://github.com/cosmos/cosmos-sdk/pull/14410) `rootmulti.Store.loadVersion` has validation to check if all the module stores' height is correct, it will error if any module store has incorrect height. * (x/evidence) [#14757](https://github.com/cosmos/cosmos-sdk/pull/14757) Evidence messages do not need to implement a `.Type()` anymore. @@ -176,6 +177,11 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### API Breaking Changes +* (server) [#15041](https://github.com/cosmos/cosmos-sdk/pull/15041) Refactor how gRPC and API servers are started to remove unnecessary sleeps: + * Remove `ServerStartTime` constant. + * Rename `WaitForQuitSignals` to `ListenForQuitSignals`. Note, this function is no longer blocking. Thus the caller is expected to provide a `context.CancelFunc` which indicates that when a signal is caught, that any spawned processes can gracefully exit. + * `api.Server#Start` now accepts a `context.Context`. The caller is responsible for ensuring that the context is canceled such that the API server can gracefully exit. The caller does not need to stop the server. + * To start the gRPC server you must first create the server via `NewGRPCServer`, after which you can start the gRPC server via `StartGRPCServer` which accepts a `context.Context`. The caller is responsible for ensuring that the context is canceled such that the gRPC server can gracefully exit. The caller does not need to stop the server. * (testutil) [#14991](https://github.com/cosmos/cosmos-sdk/pull/14991) The `testutil/testdata_pulsar` package has moved to `testutil/testdata/testpb`. * (simapp) [#14977](https://github.com/cosmos/cosmos-sdk/pull/14977) Move simulation helpers functions (`AppStateFn` and `AppStateRandomizedFn`) to `testutil/sims`. These takes an extra genesisState argument which is the default state of the app. * (x/gov) [#14720](https://github.com/cosmos/cosmos-sdk/pull/14720) Add an expedited field in the gov v1 proposal and `MsgNewMsgProposal`. From 6c573813916f08aedd6d492e5276211f42eff5de Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 15 Feb 2023 20:06:55 -0500 Subject: [PATCH 05/14] updates --- simapp/go.mod | 1 + simapp/go.sum | 2 ++ tests/go.mod | 1 + tests/go.sum | 2 ++ tools/confix/go.mod | 1 + tools/confix/go.sum | 2 ++ tools/cosmovisor/go.sum | 1 + tools/rosetta/go.sum | 1 + x/evidence/go.mod | 1 + x/evidence/go.sum | 2 ++ x/feegrant/go.mod | 1 + x/feegrant/go.sum | 2 ++ x/nft/go.mod | 1 + x/nft/go.sum | 2 ++ x/upgrade/go.mod | 1 + x/upgrade/go.sum | 2 ++ 16 files changed, 23 insertions(+) diff --git a/simapp/go.mod b/simapp/go.mod index 73acd3f18c16..af871aeb56dd 100644 --- a/simapp/go.mod +++ b/simapp/go.mod @@ -140,6 +140,7 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/simapp/go.sum b/simapp/go.sum index b46cb77224b7..532c279f21cb 100644 --- a/simapp/go.sum +++ b/simapp/go.sum @@ -672,6 +672,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/tests/go.mod b/tests/go.mod index e721a47b3e3c..a932f032b462 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -134,6 +134,7 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/tests/go.sum b/tests/go.sum index 650a4bb9853a..54567127324e 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -667,6 +667,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/tools/confix/go.mod b/tools/confix/go.mod index 9e125550ba36..e7806900d504 100644 --- a/tools/confix/go.mod +++ b/tools/confix/go.mod @@ -107,6 +107,7 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/tools/confix/go.sum b/tools/confix/go.sum index 1c5200c34198..8788fae90358 100644 --- a/tools/confix/go.sum +++ b/tools/confix/go.sum @@ -619,6 +619,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/tools/cosmovisor/go.sum b/tools/cosmovisor/go.sum index c86bea76978a..4c5d602dc968 100644 --- a/tools/cosmovisor/go.sum +++ b/tools/cosmovisor/go.sum @@ -631,6 +631,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/tools/rosetta/go.sum b/tools/rosetta/go.sum index 903be1ac2ffe..5b7629e138c8 100644 --- a/tools/rosetta/go.sum +++ b/tools/rosetta/go.sum @@ -588,6 +588,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/x/evidence/go.mod b/x/evidence/go.mod index 239980c42e76..654f58e52e73 100644 --- a/x/evidence/go.mod +++ b/x/evidence/go.mod @@ -106,6 +106,7 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/x/evidence/go.sum b/x/evidence/go.sum index 18b3b1937eaf..786dade4bc9c 100644 --- a/x/evidence/go.sum +++ b/x/evidence/go.sum @@ -615,6 +615,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/x/feegrant/go.mod b/x/feegrant/go.mod index 7982f2207c90..336cece61a1e 100644 --- a/x/feegrant/go.mod +++ b/x/feegrant/go.mod @@ -108,6 +108,7 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/x/feegrant/go.sum b/x/feegrant/go.sum index d6febe0c9333..ee9690c7a2ec 100644 --- a/x/feegrant/go.sum +++ b/x/feegrant/go.sum @@ -618,6 +618,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/x/nft/go.mod b/x/nft/go.mod index c3cdd46f5fbb..48cdc01b340c 100644 --- a/x/nft/go.mod +++ b/x/nft/go.mod @@ -105,6 +105,7 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/x/nft/go.sum b/x/nft/go.sum index 18b3b1937eaf..786dade4bc9c 100644 --- a/x/nft/go.sum +++ b/x/nft/go.sum @@ -615,6 +615,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/x/upgrade/go.mod b/x/upgrade/go.mod index 9678a28b58b7..1457fb304e1a 100644 --- a/x/upgrade/go.mod +++ b/x/upgrade/go.mod @@ -128,6 +128,7 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/x/upgrade/go.sum b/x/upgrade/go.sum index 709ee0b11b4b..8ca8c13fd403 100644 --- a/x/upgrade/go.sum +++ b/x/upgrade/go.sum @@ -664,6 +664,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= From 4e9e264612c5bc16e61a07b6c145df13aeb93273 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 15 Feb 2023 20:30:03 -0500 Subject: [PATCH 06/14] updates --- server/start.go | 94 ++++++++++++++++++++----------------------------- 1 file changed, 38 insertions(+), 56 deletions(-) diff --git a/server/start.go b/server/start.go index 8016237a7b96..ab464baaf7fb 100644 --- a/server/start.go +++ b/server/start.go @@ -4,7 +4,10 @@ import ( "context" "fmt" "net" + "os" + "runtime/pprof" + pruningtypes "cosmossdk.io/store/pruning/types" "github.com/cometbft/cometbft/abci/server" tcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" "github.com/cometbft/cometbft/node" @@ -18,8 +21,6 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - pruningtypes "cosmossdk.io/store/pruning/types" - "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" @@ -134,20 +135,15 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. withCMT, _ := cmd.Flags().GetBool(flagWithComet) if !withCMT { serverCtx.Logger.Info("starting ABCI without CometBFT") - return startStandAlone(serverCtx, appCreator) - // TODO: ... - // return wrapCPUProfile(serverCtx, func() error { - // return startStandAlone(serverCtx, appCreator) - // }) + return wrapCPUProfile(serverCtx, func() error { + return startStandAlone(serverCtx, appCreator) + }) } - // TODO: ... - // err = wrapCPUProfile(serverCtx, func() error { - // return startInProcess(serverCtx, clientCtx, appCreator) - // }) - - return startInProcess(serverCtx, clientCtx, appCreator) + return wrapCPUProfile(serverCtx, func() error { + return startInProcess(serverCtx, clientCtx, appCreator) + }) }, } @@ -483,52 +479,38 @@ func startTelemetry(cfg serverconfig.Config) (*telemetry.Metrics, error) { return telemetry.New(cfg.Telemetry) } -// // wrapCPUProfile runs callback in a goroutine, then wait for quit signals. -// func wrapCPUProfile(svrCtx *Context, callback func() error) error { -// if cpuProfile := svrCtx.Viper.GetString(flagCPUProfile); cpuProfile != "" { -// f, err := os.Create(cpuProfile) -// if err != nil { -// return err -// } - -// svrCtx.Logger.Info("starting CPU profiler", "profile", cpuProfile) - -// if err := pprof.StartCPUProfile(f); err != nil { -// return err -// } - -// defer func() { -// svrCtx.Logger.Info("stopping CPU profiler", "profile", cpuProfile) -// pprof.StopCPUProfile() - -// if err := f.Close(); err != nil { -// svrCtx.Logger.Info("failed to close cpu-profile file", "profile", cpuProfile, "err", err.Error()) -// } -// }() -// } - -// ctx, cancelFn := context.WithCancel(context.Background()) -// g, ctx := errgroup.WithContext(ctx) +// wrapCPUProfile starts CPU profiling, if enabled, and executes the provided +// callbackFn in a separate goroutine, then will wait for that callback to +// return. +// +// NOTE: We expect the caller to handle graceful shutdown and signal handling. +func wrapCPUProfile(svrCtx *Context, callbackFn func() error) error { + if cpuProfile := svrCtx.Viper.GetString(flagCPUProfile); cpuProfile != "" { + f, err := os.Create(cpuProfile) + if err != nil { + return err + } -// // Wait for the parent process to capture any termination signals in a -// // non-block manner, where the error code will be sent on a channel. -// errCh := WatchForQuitSignals(cancelFn) + svrCtx.Logger.Info("starting CPU profiler", "profile", cpuProfile) -// g.Go(func() error { -// return callback() -// }) + if err := pprof.StartCPUProfile(f); err != nil { + return err + } -// // errCh := make(chan error) -// // go func() { -// // errCh <- callback() -// // }() + defer func() { + svrCtx.Logger.Info("stopping CPU profiler", "profile", cpuProfile) + pprof.StopCPUProfile() -// // select { -// // case err := <-errCh: -// // return err + if err := f.Close(); err != nil { + svrCtx.Logger.Info("failed to close cpu-profile file", "profile", cpuProfile, "err", err.Error()) + } + }() + } -// // case <-time.After(types.ServerStartTime): -// // } + errCh := make(chan error) + go func() { + errCh <- callbackFn() + }() -// // return WaitForQuitSignals() -// } + return <-errCh +} From 6f64b675685487ca1864c2a9365c25d4d0bcabb1 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Thu, 16 Feb 2023 10:05:25 -0500 Subject: [PATCH 07/14] updates --- go.mod | 2 +- go.sum | 3 +-- server/api/server.go | 26 ++++++++++++-------------- server/grpc/server.go | 30 ++++++++++++++---------------- server/start.go | 8 +------- testutil/network/network.go | 2 +- testutil/network/util.go | 2 +- 7 files changed, 31 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index ec3a986a5720..c6b75c15de1d 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,6 @@ require ( github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 github.com/mattn/go-isatty v0.0.17 - github.com/neilotoole/errgroup v0.1.6 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/common v0.39.0 @@ -56,6 +55,7 @@ require ( github.com/tendermint/go-amino v0.16.0 golang.org/x/crypto v0.6.0 golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 + golang.org/x/sync v0.1.0 google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.28.2-0.20230208135220-49eaa78c6c9c diff --git a/go.sum b/go.sum index a2c289c8264c..39f0adecbf95 100644 --- a/go.sum +++ b/go.sum @@ -624,8 +624,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= -github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -991,6 +989,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/server/api/server.go b/server/api/server.go index 12ec0a443b30..4ea289fce99a 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -138,7 +138,7 @@ func (s *Server) Start(ctx context.Context, cfg config.Config) error { // register grpc-gateway routes (after grpc-web server as the first match is used) s.Router.PathPrefix("/").Handler(s.GRPCGatewayRouter) - errCh := make(chan error, 1) + errCh := make(chan error) // Start the API in an external goroutine as Serve is blocking and will return // an error upon failure, which we'll send on the error channel that will be @@ -154,20 +154,18 @@ func (s *Server) Start(ctx context.Context, cfg config.Config) error { } }(cfg.API.EnableUnsafeCORS) - // Start a blocking loop to wait for an indication to stop the server or that + // Start a blocking select to wait for an indication to stop the server or that // the server failed to start properly. - for { - select { - case <-ctx.Done(): - // The calling process cancelled or closed the provided context, so we must - // gracefully stop the API server. - s.logger.Info("stopping API server...", "address", cfg.API.Address) - return s.Close() - - case err := <-errCh: - s.logger.Error("failed to start API server", "err", err) - return err - } + select { + case <-ctx.Done(): + // The calling process cancelled or closed the provided context, so we must + // gracefully stop the API server. + s.logger.Info("stopping API server...", "address", cfg.API.Address) + return s.Close() + + case err := <-errCh: + s.logger.Error("failed to start API server", "err", err) + return err } } diff --git a/server/grpc/server.go b/server/grpc/server.go index de8d8fcf7643..4c809e8655ed 100644 --- a/server/grpc/server.go +++ b/server/grpc/server.go @@ -77,7 +77,7 @@ func StartGRPCServer(ctx context.Context, logger log.Logger, cfg config.GRPCConf return fmt.Errorf("failed to listen on address %s: %w", cfg.Address, err) } - errCh := make(chan error, 1) + errCh := make(chan error) // Start the gRPC in an external goroutine as Serve is blocking and will return // an error upon failure, which we'll send on the error channel that will be @@ -87,21 +87,19 @@ func StartGRPCServer(ctx context.Context, logger log.Logger, cfg config.GRPCConf errCh <- grpcSrv.Serve(listener) }() - // Start a blocking loop to wait for an indication to stop the server or that + // Start a blocking select to wait for an indication to stop the server or that // the server failed to start properly. - for { - select { - case <-ctx.Done(): - // The calling process cancelled or closed the provided context, so we must - // gracefully stop the gRPC server. - logger.Info("stopping gRPC server...", "address", cfg.Address) - grpcSrv.GracefulStop() - - return nil - - case err := <-errCh: - logger.Error("failed to start gRPC server", "err", err) - return err - } + select { + case <-ctx.Done(): + // The calling process cancelled or closed the provided context, so we must + // gracefully stop the gRPC server. + logger.Info("stopping gRPC server...", "address", cfg.Address) + grpcSrv.GracefulStop() + + return nil + + case err := <-errCh: + logger.Error("failed to start gRPC server", "err", err) + return err } } diff --git a/server/start.go b/server/start.go index ab464baaf7fb..067c00cfa092 100644 --- a/server/start.go +++ b/server/start.go @@ -15,9 +15,9 @@ import ( pvm "github.com/cometbft/cometbft/privval" "github.com/cometbft/cometbft/proxy" "github.com/cometbft/cometbft/rpc/client/local" - "github.com/neilotoole/errgroup" "github.com/spf13/cobra" "github.com/spf13/pflag" + "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -164,7 +164,6 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')") cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks") cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune CometBFT blocks") - cmd.Flags().Bool(FlagAPIEnable, false, "Define if the API server should be enabled") cmd.Flags().Bool(FlagAPISwagger, false, "Define if swagger documentation should automatically be registered (Note: the API must also be enabled)") cmd.Flags().String(FlagAPIAddress, serverconfig.DefaultAPIAddress, "the API server address to listen on") @@ -173,18 +172,13 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. cmd.Flags().Uint(FlagRPCWriteTimeout, 0, "Define the CometBFT RPC write timeout (in seconds)") cmd.Flags().Uint(FlagRPCMaxBodyBytes, 1000000, "Define the CometBFT maximum request body (in bytes)") cmd.Flags().Bool(FlagAPIEnableUnsafeCORS, false, "Define if CORS should be enabled (unsafe - use it at your own risk)") - cmd.Flags().Bool(flagGRPCOnly, false, "Start the node in gRPC query only mode (no CometBFT process is started)") cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") cmd.Flags().String(flagGRPCAddress, serverconfig.DefaultGRPCAddress, "the gRPC server address to listen on") - cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled)") - cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") - cmd.Flags().Bool(FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree") - cmd.Flags().Int(FlagMempoolMaxTxs, mempool.DefaultMaxTx, "Sets MaxTx value for the app-side mempool") // support old flags name for backwards compatibility diff --git a/testutil/network/network.go b/testutil/network/network.go index da6d1c84ba4b..5388c0bf47b4 100644 --- a/testutil/network/network.go +++ b/testutil/network/network.go @@ -24,8 +24,8 @@ import ( "github.com/cometbft/cometbft/node" cmtclient "github.com/cometbft/cometbft/rpc/client" dbm "github.com/cosmos/cosmos-db" - "github.com/neilotoole/errgroup" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" "google.golang.org/grpc" "github.com/cosmos/cosmos-sdk/baseapp" diff --git a/testutil/network/util.go b/testutil/network/util.go index 110cb241e921..0555076a0b35 100644 --- a/testutil/network/util.go +++ b/testutil/network/util.go @@ -15,7 +15,7 @@ import ( "github.com/cometbft/cometbft/rpc/client/local" "github.com/cometbft/cometbft/types" cmttime "github.com/cometbft/cometbft/types/time" - "github.com/neilotoole/errgroup" + "golang.org/x/sync/errgroup" "github.com/cosmos/cosmos-sdk/server/api" servergrpc "github.com/cosmos/cosmos-sdk/server/grpc" From 08a31ae3066c4536a47c8afa15c6dab6368f63d7 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Thu, 16 Feb 2023 10:40:01 -0500 Subject: [PATCH 08/14] updates --- server/start.go | 108 +++++++++++++++++++++--------------------------- 1 file changed, 46 insertions(+), 62 deletions(-) diff --git a/server/start.go b/server/start.go index 067c00cfa092..1f929ab19627 100644 --- a/server/start.go +++ b/server/start.go @@ -335,8 +335,8 @@ func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types. // service if API or gRPC is enabled, and avoid doing so in the general // case, because it spawns a new local CometBFT RPC client. if (config.API.Enable || config.GRPC.Enable) && tmNode != nil { - // re-assign for making the client available below - // do not use := to avoid shadowing clientCtx + // Re-assign for making the client available below do not use := to avoid + // shadowing the clientCtx variable. clientCtx = clientCtx.WithClient(local.New(tmNode)) app.RegisterTxService(clientCtx) @@ -360,61 +360,61 @@ func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types. // listen for quit signals so the calling parent process can gracefully exit ListenForQuitSignals(cancelFn, svrCtx.Logger) - if config.API.Enable { - genDoc, err := genDocProvider() + if config.GRPC.Enable { + _, port, err := net.SplitHostPort(config.GRPC.Address) if err != nil { return err } - clientCtx := clientCtx.WithHomeDir(home).WithChainID(genDoc.ChainID) - - if config.GRPC.Enable { - _, port, err := net.SplitHostPort(config.GRPC.Address) - if err != nil { - return err - } + maxSendMsgSize := config.GRPC.MaxSendMsgSize + if maxSendMsgSize == 0 { + maxSendMsgSize = serverconfig.DefaultGRPCMaxSendMsgSize + } - maxSendMsgSize := config.GRPC.MaxSendMsgSize - if maxSendMsgSize == 0 { - maxSendMsgSize = serverconfig.DefaultGRPCMaxSendMsgSize - } + maxRecvMsgSize := config.GRPC.MaxRecvMsgSize + if maxRecvMsgSize == 0 { + maxRecvMsgSize = serverconfig.DefaultGRPCMaxRecvMsgSize + } - maxRecvMsgSize := config.GRPC.MaxRecvMsgSize - if maxRecvMsgSize == 0 { - maxRecvMsgSize = serverconfig.DefaultGRPCMaxRecvMsgSize - } + grpcAddress := fmt.Sprintf("127.0.0.1:%s", port) + + // if gRPC is enabled, configure gRPC client for gRPC gateway + grpcClient, err := grpc.Dial( + grpcAddress, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions( + grpc.ForceCodec(codec.NewProtoCodec(clientCtx.InterfaceRegistry).GRPCCodec()), + grpc.MaxCallRecvMsgSize(maxRecvMsgSize), + grpc.MaxCallSendMsgSize(maxSendMsgSize), + ), + ) + if err != nil { + return err + } - grpcAddress := fmt.Sprintf("127.0.0.1:%s", port) - - // if gRPC is enabled, configure gRPC client for gRPC gateway - grpcClient, err := grpc.Dial( - grpcAddress, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultCallOptions( - grpc.ForceCodec(codec.NewProtoCodec(clientCtx.InterfaceRegistry).GRPCCodec()), - grpc.MaxCallRecvMsgSize(maxRecvMsgSize), - grpc.MaxCallSendMsgSize(maxSendMsgSize), - ), - ) - if err != nil { - return err - } + clientCtx = clientCtx.WithGRPCClient(grpcClient) + svrCtx.Logger.Debug("gRPC client assigned to client context", "target", grpcAddress) - clientCtx = clientCtx.WithGRPCClient(grpcClient) - svrCtx.Logger.Debug("gRPC client assigned to client context", "target", grpcAddress) + grpcSrv, err = servergrpc.NewGRPCServer(clientCtx, app, config.GRPC) + if err != nil { + return err + } - grpcSrv, err := servergrpc.NewGRPCServer(clientCtx, app, config.GRPC) - if err != nil { - return err - } + // Start the gRPC server in a goroutine. Note, the provided ctx will ensure + // that the server is gracefully shut down. + g.Go(func() error { + return servergrpc.StartGRPCServer(ctx, svrCtx.Logger.With("module", "grpc-server"), config.GRPC, grpcSrv) + }) + } - // Start the gRPC server in a goroutine. Note, the provided ctx will ensure - // that the server is gracefully shut down. - g.Go(func() error { - return servergrpc.StartGRPCServer(ctx, svrCtx.Logger.With("module", "grpc-server"), config.GRPC, grpcSrv) - }) + if config.API.Enable { + genDoc, err := genDocProvider() + if err != nil { + return err } + clientCtx := clientCtx.WithHomeDir(home).WithChainID(genDoc.ChainID) + apiSrv = api.New(clientCtx, svrCtx.Logger.With("module", "api-server"), grpcSrv) app.RegisterAPIRoutes(apiSrv, config.API) @@ -427,23 +427,7 @@ func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types. }) } - // If gRPC is enabled but the API is not, we need to start the gRPC server - // without the API server. If the API server is enabled, we've already - // started the grpc server. - if config.GRPC.Enable && !config.API.Enable { - grpcSrv, err := servergrpc.NewGRPCServer(clientCtx, app, config.GRPC) - if err != nil { - return err - } - - // Start the gRPC server in a goroutine. Note, the provided ctx will ensure - // that the server is gracefully shut down. - g.Go(func() error { - return servergrpc.StartGRPCServer(ctx, svrCtx.Logger.With("module", "grpc-server"), config.GRPC, grpcSrv) - }) - } - - // At this point it is safe to block the process if we're in gRPC only mode as + // At this point it is safe to block the process if we're in gRPC-only mode as // we do not need to handle any CometBFT related processes. if gRPCOnly { // wait for signal capture and gracefully return From e7c165d618341e36a38ebb0830c2bab7767b6192 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Thu, 16 Feb 2023 10:43:53 -0500 Subject: [PATCH 09/14] updates --- server/start.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/start.go b/server/start.go index 1f929ab19627..39479fd67080 100644 --- a/server/start.go +++ b/server/start.go @@ -434,6 +434,14 @@ func startInProcess(svrCtx *Context, clientCtx client.Context, appCreator types. return g.Wait() } + // In case the operator has both gRPC and API servers disabled, there is + // nothing blocking this root process, so we need to block manually, so we'll + // create an empty blocking loop. + g.Go(func() error { + <-ctx.Done() + return nil + }) + // deferred cleanup function defer func() { if tmNode != nil && tmNode.IsRunning() { From 22fd7382d1282b49fd4766dcdbe0a99b69862402 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Thu, 16 Feb 2023 10:50:32 -0500 Subject: [PATCH 10/14] updates --- server/start.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/server/start.go b/server/start.go index 39479fd67080..18b2d06a7dbf 100644 --- a/server/start.go +++ b/server/start.go @@ -241,15 +241,13 @@ func startStandAlone(svrCtx *Context, appCreator types.AppCreator) error { return err } - // start a blocking loop to wait for an indication to stop the server - for { - select { - case <-ctx.Done(): - // The calling process cancelled or closed the provided context, so we - // must gracefully stop the ABCI server. - svrCtx.Logger.Info("stopping the ABCI server...") - return svr.Stop() - } + // start a blocking select to wait for an indication to stop the server + select { + case <-ctx.Done(): + // The calling process cancelled or closed the provided context, so we + // must gracefully stop the ABCI server. + svrCtx.Logger.Info("stopping the ABCI server...") + return svr.Stop() } }) From d5a8c2110a2dc267cd12c816be97b36a962b8867 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Thu, 16 Feb 2023 10:56:24 -0500 Subject: [PATCH 11/14] updates --- simapp/go.mod | 2 +- simapp/go.sum | 3 +-- tests/go.mod | 2 +- tests/go.sum | 3 +-- tools/confix/go.mod | 2 +- tools/confix/go.sum | 3 +-- tools/cosmovisor/go.sum | 1 - tools/rosetta/go.sum | 1 - x/evidence/go.mod | 2 +- x/evidence/go.sum | 3 +-- x/feegrant/go.mod | 2 +- x/feegrant/go.sum | 3 +-- x/nft/go.mod | 2 +- x/nft/go.sum | 3 +-- x/upgrade/go.mod | 2 +- x/upgrade/go.sum | 3 +-- 16 files changed, 14 insertions(+), 23 deletions(-) diff --git a/simapp/go.mod b/simapp/go.mod index af871aeb56dd..a4a9499d13d8 100644 --- a/simapp/go.mod +++ b/simapp/go.mod @@ -140,7 +140,6 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect @@ -171,6 +170,7 @@ require ( golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 // indirect golang.org/x/net v0.6.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/simapp/go.sum b/simapp/go.sum index 532c279f21cb..501fa297f892 100644 --- a/simapp/go.sum +++ b/simapp/go.sum @@ -672,8 +672,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= -github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -1047,6 +1045,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/tests/go.mod b/tests/go.mod index a932f032b462..81262e6511b6 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -134,7 +134,6 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect @@ -168,6 +167,7 @@ require ( golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 // indirect golang.org/x/net v0.6.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index 54567127324e..1b742d34328f 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -667,8 +667,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= -github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -1049,6 +1047,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/tools/confix/go.mod b/tools/confix/go.mod index e7806900d504..e322a07a964a 100644 --- a/tools/confix/go.mod +++ b/tools/confix/go.mod @@ -107,7 +107,6 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect @@ -137,6 +136,7 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/net v0.6.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/tools/confix/go.sum b/tools/confix/go.sum index 8788fae90358..d60fe2b367e1 100644 --- a/tools/confix/go.sum +++ b/tools/confix/go.sum @@ -619,8 +619,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= -github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -993,6 +991,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/tools/cosmovisor/go.sum b/tools/cosmovisor/go.sum index 4c5d602dc968..c86bea76978a 100644 --- a/tools/cosmovisor/go.sum +++ b/tools/cosmovisor/go.sum @@ -631,7 +631,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/tools/rosetta/go.sum b/tools/rosetta/go.sum index 5b7629e138c8..903be1ac2ffe 100644 --- a/tools/rosetta/go.sum +++ b/tools/rosetta/go.sum @@ -588,7 +588,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/x/evidence/go.mod b/x/evidence/go.mod index 654f58e52e73..ba9b45e60c4d 100644 --- a/x/evidence/go.mod +++ b/x/evidence/go.mod @@ -106,7 +106,6 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect @@ -137,6 +136,7 @@ require ( golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 // indirect golang.org/x/net v0.6.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/x/evidence/go.sum b/x/evidence/go.sum index 786dade4bc9c..0b9360cf6a41 100644 --- a/x/evidence/go.sum +++ b/x/evidence/go.sum @@ -615,8 +615,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= -github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -989,6 +987,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/x/feegrant/go.mod b/x/feegrant/go.mod index 336cece61a1e..c2d3b2145af2 100644 --- a/x/feegrant/go.mod +++ b/x/feegrant/go.mod @@ -108,7 +108,6 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect @@ -139,6 +138,7 @@ require ( golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 // indirect golang.org/x/net v0.6.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/x/feegrant/go.sum b/x/feegrant/go.sum index ee9690c7a2ec..89f7424ae671 100644 --- a/x/feegrant/go.sum +++ b/x/feegrant/go.sum @@ -618,8 +618,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= -github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -992,6 +990,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/x/nft/go.mod b/x/nft/go.mod index 48cdc01b340c..4102f2e06a60 100644 --- a/x/nft/go.mod +++ b/x/nft/go.mod @@ -105,7 +105,6 @@ require ( github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect @@ -136,6 +135,7 @@ require ( golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 // indirect golang.org/x/net v0.6.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/x/nft/go.sum b/x/nft/go.sum index 786dade4bc9c..0b9360cf6a41 100644 --- a/x/nft/go.sum +++ b/x/nft/go.sum @@ -615,8 +615,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= -github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -989,6 +987,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/x/upgrade/go.mod b/x/upgrade/go.mod index 1457fb304e1a..2b95e1bc80f1 100644 --- a/x/upgrade/go.mod +++ b/x/upgrade/go.mod @@ -128,7 +128,6 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/neilotoole/errgroup v0.1.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect github.com/pkg/errors v0.9.1 // indirect @@ -160,6 +159,7 @@ require ( golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 // indirect golang.org/x/net v0.6.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/x/upgrade/go.sum b/x/upgrade/go.sum index 8ca8c13fd403..4a3d334e5ae1 100644 --- a/x/upgrade/go.sum +++ b/x/upgrade/go.sum @@ -664,8 +664,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= -github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -1046,6 +1044,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 7d980917f175d71f3598e25cf762b410ee53dbb8 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 22 Feb 2023 09:57:17 -0500 Subject: [PATCH 12/14] updates --- testutil/network/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testutil/network/network.go b/testutil/network/network.go index d65b2179163b..347d0d317b20 100644 --- a/testutil/network/network.go +++ b/testutil/network/network.go @@ -746,7 +746,7 @@ func (n *Network) Cleanup() { if v.tmNode != nil && v.tmNode.IsRunning() { if err := v.tmNode.Stop(); err != nil { - n.Logger.Log("failed to stop validator Tendermint node", "err", err) + n.Logger.Log("failed to stop validator CometBFT node", "err", err) } } From 9742f1690f603645c8fa9f8d31879102b728945c Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 22 Feb 2023 09:58:41 -0500 Subject: [PATCH 13/14] updates --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 43f0ce597461..55d9e5ae279d 100644 --- a/go.mod +++ b/go.mod @@ -54,8 +54,8 @@ require ( github.com/tendermint/go-amino v0.16.0 golang.org/x/crypto v0.6.0 golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb - google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 golang.org/x/sync v0.1.0 + google.golang.org/genproto v0.0.0-20230202175211-008b39050e57 google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.28.2-0.20230208135220-49eaa78c6c9c gotest.tools/v3 v3.4.0 From 425db712205b0657488fd5678d57c052d49679f5 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Wed, 22 Feb 2023 10:03:14 -0500 Subject: [PATCH 14/14] updates --- server/start.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/server/start.go b/server/start.go index ff4a4e465539..787467ae94f8 100644 --- a/server/start.go +++ b/server/start.go @@ -243,14 +243,11 @@ func startStandAlone(svrCtx *Context, appCreator types.AppCreator) error { return err } - // start a blocking select to wait for an indication to stop the server - select { - case <-ctx.Done(): - // The calling process cancelled or closed the provided context, so we - // must gracefully stop the ABCI server. - svrCtx.Logger.Info("stopping the ABCI server...") - return svr.Stop() - } + // Wait for the calling process to be cancelled or close the provided context, + // so we can gracefully stop the ABCI server. + <-ctx.Done() + svrCtx.Logger.Info("stopping the ABCI server...") + return svr.Stop() }) return g.Wait()