From bb8c0ac22c2981c246aa96be0ac391f23e8782b7 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Mon, 17 Dec 2018 18:24:09 +0530 Subject: [PATCH 01/11] cmd/swarm/swarm-snapshot: add binary to create network snapshots --- cmd/swarm/swarm-snapshot/create.go | 165 ++++++++++++++++++ cmd/swarm/swarm-snapshot/create_test.go | 75 ++++++++ cmd/swarm/swarm-snapshot/helper.go | 40 +++++ cmd/swarm/swarm-snapshot/main.go | 110 ++++++++++++ cmd/swarm/swarm-snapshot/run_test.go | 49 ++++++ .../swarm-snapshot/testdata/snapshot.json | 1 + cmd/swarm/swarm-snapshot/verify.go | 85 +++++++++ cmd/swarm/swarm-snapshot/verify_test.go | 33 ++++ p2p/simulations/network.go | 37 ++++ p2p/simulations/simulation.go | 8 +- swarm/network/kademlia.go | 2 + .../simulations/discovery/discovery_test.go | 39 +---- 12 files changed, 607 insertions(+), 37 deletions(-) create mode 100644 cmd/swarm/swarm-snapshot/create.go create mode 100644 cmd/swarm/swarm-snapshot/create_test.go create mode 100644 cmd/swarm/swarm-snapshot/helper.go create mode 100644 cmd/swarm/swarm-snapshot/main.go create mode 100644 cmd/swarm/swarm-snapshot/run_test.go create mode 100644 cmd/swarm/swarm-snapshot/testdata/snapshot.json create mode 100644 cmd/swarm/swarm-snapshot/verify.go create mode 100644 cmd/swarm/swarm-snapshot/verify_test.go diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go new file mode 100644 index 000000000000..7526c7d8c3bf --- /dev/null +++ b/cmd/swarm/swarm-snapshot/create.go @@ -0,0 +1,165 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/network/simulation" + cli "gopkg.in/urfave/cli.v1" +) + +const testMinProxBinSize = 2 +const NoConnectionTimeout = 2 * time.Second + +func create(ctx *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + + if len(ctx.Args()) < 1 { + return errors.New("argument should be the filename to verify or write-to") + } + filename, err := touchPath(ctx.Args()[0]) + if err != nil { + return err + } + err = discoverySnapshot(filename, 10) + if err != nil { + utils.Fatalf("Simulation failed: %s", err) + } + + return err +} + +func discoverySnapshot(filename string, nodes int) error { + //disable discovery if topology is specified + discovery = topology == "" + log.Debug("discoverySnapshot", "filename", filename, "nodes", nodes, "discovery", discovery) + i := 0 + var lock sync.Mutex + var pivotNodeID enode.ID + sim := simulation.New(map[string]simulation.ServiceFunc{ + "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + lock.Lock() + i++ + if i == pivot { + pivotNodeID = ctx.Config.ID + } + lock.Unlock() + + addr := network.NewAddr(ctx.Config.Node()) + kp := network.NewKadParams() + kp.MinProxBinSize = testMinProxBinSize + + kad := network.NewKademlia(addr.Over(), kp) + hp := network.NewHiveParams() + hp.KeepAliveInterval = time.Duration(200) * time.Millisecond + hp.Discovery = discovery + + config := &network.BzzConfig{ + OverlayAddr: addr.Over(), + UnderlayAddr: addr.Under(), + HiveParams: hp, + } + return network.NewBzz(config, kad, nil, nil, nil), nil, nil + }, + }) + defer sim.Close() + + _, err := sim.AddNodes(10) + if err != nil { + utils.Fatalf("%v", err) + } + + events := make(chan *simulations.Event) + sub := sim.Net.Events().Subscribe(events) + select { + case ev := <-events: + //only catch node up events + if ev.Type == simulations.EventTypeConn { + utils.Fatalf("this shouldn't happen as connections weren't initiated yet") + } + case <-time.After(NoConnectionTimeout): + } + + sub.Unsubscribe() + + if len(sim.Net.Conns) > 0 { + utils.Fatalf("no connections should exist after just adding nodes") + } + + err := sim.Net.ConnectNodesRing(nil) + if err != nil { + utils.Fatalf("had an error connecting the nodes in a %v topology: %v", topology, err) + } + + if discovery { + ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelSimRun() + + if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + utils.Fatalf("%v", err) + } + } + + var snap *simulations.Snapshot + if len(services) > 0 { + var addServices []string + var removeServices []string + for _, osvc := range strings.Split(services, ",") { + if strings.Index(osvc, "+") == 0 { + addServices = append(addServices, osvc[1:]) + } else if strings.Index(osvc, "-") == 0 { + removeServices = append(removeServices, osvc[1:]) + } else { + panic("stick to the rules, you know what they are") + } + } + snap, err = sim.Net.SnapshotWithServices(addServices, removeServices) + } else { + snap, err = sim.Net.Snapshot() + } + + if err != nil { + return errors.New("no shapshot dude") + } + jsonsnapshot, err := json.Marshal(snap) + if err != nil { + return fmt.Errorf("corrupt json snapshot: %v", err) + } + err = ioutil.WriteFile(filename, jsonsnapshot, 0666) + if err != nil { + return err + } + + return nil +} diff --git a/cmd/swarm/swarm-snapshot/create_test.go b/cmd/swarm/swarm-snapshot/create_test.go new file mode 100644 index 000000000000..d6f593913b3e --- /dev/null +++ b/cmd/swarm/swarm-snapshot/create_test.go @@ -0,0 +1,75 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/ethereum/go-ethereum/log" +) + +func init() { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) +} + +//TestSnapshotCreate is a high level e2e test that tests for snapshot generation +func TestSnapshotCreate(t *testing.T) { + + for _, v := range []struct { + name string + args []string + }{ + { + name: "no topology - discovery enabled", + args: []string{ + "c", + }, + }, + { + name: "yes topology - discovery disabled", + args: []string{ + "--topology", + "ring", + "c", + }, + }, + } { + t.Run(v.name, func(t *testing.T) { + file, err := ioutil.TempFile("", "swarm-snapshot") + defer os.Remove(file.Name()) + if err != nil { + t.Fatal(err) + } + + file.Close() + snap := runSnapshot(t, append(v.args, file.Name())...) + + snap.ExpectExit() + if snap.ExitStatus() != 0 { + t.Fatal("expected exit code 0") + } + + _, err = os.Stat(file.Name()) + if err != nil { + t.Fatal("could not stat snapshot json") + } + }) + } +} diff --git a/cmd/swarm/swarm-snapshot/helper.go b/cmd/swarm/swarm-snapshot/helper.go new file mode 100644 index 000000000000..8c4fab928dcd --- /dev/null +++ b/cmd/swarm/swarm-snapshot/helper.go @@ -0,0 +1,40 @@ +package main + +import ( + "os" + "path" + "path/filepath" +) + +func touchPath(filename string) (string, error) { + if path.IsAbs(filename) { + if _, err := os.Stat(filename); err == nil { + // path exists, we will override the file + return filename, nil + } + } + + d, f := path.Split(filename) + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return "", err + } + + _, err = os.Stat(path.Join(dir, filename)) + if err == nil { + // path exists, we will override + return filename, nil + } + + dirPath := path.Join(dir, d) + filePath := path.Join(dirPath, f) + if d != "" { + err = os.MkdirAll(dirPath, os.ModeDir) + if err != nil { + return "", err + } + } + + filename = filePath + return filename, nil +} diff --git a/cmd/swarm/swarm-snapshot/main.go b/cmd/swarm/swarm-snapshot/main.go new file mode 100644 index 000000000000..6338b2886b0c --- /dev/null +++ b/cmd/swarm/swarm-snapshot/main.go @@ -0,0 +1,110 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "os" + "sort" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + cli "gopkg.in/urfave/cli.v1" +) + +var ( + gitCommit string // Git SHA1 commit hash of the release (set via linker flags) +) + +var ( + topology string + services string + pivot int + nodes int + verbosity int +) + +var app = utils.NewApp("", "Swarm Snapshot Util") +var discovery = true + +func init() { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + + app.Name = "swarm-snapshot" + app.Usage = "" + + app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: "topology", + Value: "chain", + Usage: "the desired topology to connect the nodes in (star, ring, chain, full)", + Destination: &topology, + }, + cli.IntFlag{ + Name: "pivot", + Value: 0, + Usage: "pivot node zero-index", + Destination: &pivot, + }, + cli.IntFlag{ + Name: "nodes", + Value: 10, + Usage: "swarm nodes", + Destination: &nodes, + }, + cli.IntFlag{ + Name: "verbosity", + Value: 1, + Usage: "verbosity", + Destination: &verbosity, + }, + cli.StringFlag{ + Name: "services", + Value: "", + Usage: "comma separated list of services to boot the nodes with", + Destination: &services, + }, + } + + app.Commands = []cli.Command{ + { + Name: "create", + Aliases: []string{"c"}, + Usage: "create a swarm snapshot", + Action: create, + }, + { + Name: "verify", + Aliases: []string{"v"}, + Usage: "verify a swarm snapshot", + Action: verify, + }, + } + + sort.Sort(cli.FlagsByName(app.Flags)) + sort.Sort(cli.CommandsByName(app.Commands)) +} + +func main() { + err := app.Run(os.Args) + if err != nil { + log.Error(err.Error()) + + os.Exit(1) + } + os.Exit(0) +} diff --git a/cmd/swarm/swarm-snapshot/run_test.go b/cmd/swarm/swarm-snapshot/run_test.go new file mode 100644 index 000000000000..3a66ac987fca --- /dev/null +++ b/cmd/swarm/swarm-snapshot/run_test.go @@ -0,0 +1,49 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func init() { + reexec.Register("swarm-snapshot", func() { + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) +} + +func runSnapshot(t *testing.T, args ...string) *cmdtest.TestCmd { + tt := cmdtest.NewTestCmd(t, nil) + tt.Run("swarm-snapshot", args...) + return tt +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/cmd/swarm/swarm-snapshot/testdata/snapshot.json b/cmd/swarm/swarm-snapshot/testdata/snapshot.json new file mode 100644 index 000000000000..dd6e49dbe182 --- /dev/null +++ b/cmd/swarm/swarm-snapshot/testdata/snapshot.json @@ -0,0 +1 @@ +{"nodes":[{"node":{"config":{"id":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","private_key":"3ff3a0c7dc8b63cac9255233d84cccad53d61f8c9e47539329c07e4f248fc6dd","name":"node_279f0a191892115b8db4e2689cc5e1bc19019e8f5228f4f093f35c48fa2f606fd00a2ec5cf0fa4657ad3606e14643c3be8d49350ad2673f2ad12e17c0174b0d8","services":["bzz"],"enable_msg_events":true,"port":62851},"up":true}},{"node":{"config":{"id":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","private_key":"1ab5fd3f1885661af1829b42683ca1379ca90b1b2c5a0132027346c74001a154","name":"node_2c3d17a3b019d9d32922f06aef03ac03f67777e382f0b86c6e4dac07e590f2dbf4cbd1cc7ed136b52bba897e7163dee926caa627111db6d948581ad1b2edbc3f","services":["bzz"],"enable_msg_events":true,"port":62852},"up":true}},{"node":{"config":{"id":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","private_key":"a90fea5197ac84a162db7a06ec739e075ee334026e792d83a3269526888c8b5d","name":"node_6e327c2d9d3ffdae707ed067d15b83ad0ea81b96bb2a07331436f01c1e194ed90f630ef0eb6511f4849ede30c403e3c4ff3c2cb8021317de97f3ed5b1d990f49","services":["bzz"],"enable_msg_events":true,"port":62853},"up":true}},{"node":{"config":{"id":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","private_key":"76b6f09c28f5568e7724e9850116bdbd6356152fefedc4b0a9747361d712964b","name":"node_9aa1df8ddddd035e81ffc950f063714c3b6be24013108f8b99c32c25fc3a8582904f68c73fe63ebfb96143a12ab398f36c4070a3055c039491ca2f20be6774c2","services":["bzz"],"enable_msg_events":true,"port":62854},"up":true}},{"node":{"config":{"id":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","private_key":"f7e7d60abb7d43d973131804e684ec99e77e81bf38165dda29f3ae026eb169f1","name":"node_a1226991909b92dd4f22150b798e2b6312ffc62bad7a47d250e811c7e60cf5ceb6a87998f295e0bd392c0f5c15213115000781e498dd7eae79b67c7e7e19cf81","services":["bzz"],"enable_msg_events":true,"port":62855},"up":true}},{"node":{"config":{"id":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","private_key":"f88cd88c2c219d13b699ba7c7cf9adf16fc10be5d8e1f35b2ea260e7987ab76e","name":"node_1dcbfc79418f9a665e3154e07663b296ce086e54c23458b7113a011dca62e635245cab144d392716e5e90b60336b60927270b72bb4a1c5c85fbf959060fb3ee4","services":["bzz"],"enable_msg_events":true,"port":62856},"up":true}},{"node":{"config":{"id":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","private_key":"8fa1673a72b2ab000a4a7c8ad4540484d5286b69c93a5c608ef0977f783b7c4c","name":"node_571afb6ce94d28cb7d5cd49b3af35d2b1e44c917a9d3d207ab6c7ae960eb625377334e62317f6c69b3004e2eef92682649df828f4a645a75309ee7014ee4a940","services":["bzz"],"enable_msg_events":true,"port":62857},"up":true}},{"node":{"config":{"id":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","private_key":"daa9c74fbb0b897acdf20f1b955a608b086ce5b45dc8e2b76a44f277d25e3377","name":"node_f1e669bb6e5b89c04e07e3dcdc445d498ac54c8ab2e9a62e622a492b6f9781bfa27f75ef7448d06caa28d9afa40c1c089066358cbcca40018bc0f903eab9508e","services":["bzz"],"enable_msg_events":true,"port":62858},"up":true}},{"node":{"config":{"id":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","private_key":"eb94b600be5949e3ca545e81b4d0a5fff3ae9f51f63c722d10f5098f9e1a883a","name":"node_526d25e0aa47e64d8f681255348a3d3c94fc68bedcb7111532f4ed50d47ef064b1e97e3a6bcd13ac8eaf44ce676b22d2b343a94dfa38b48cf964bfc3f338d4bd","services":["bzz"],"enable_msg_events":true,"port":62859},"up":true}},{"node":{"config":{"id":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","private_key":"4e5d074bc46efbd0d18310e6a74e72e98b92080329d9c4e2b5ae99eb2fcfaf89","name":"node_fa81c440829adcbb3eba9196dc8255e319a976e53db0aceedcda25c2605f21b8778bf8c89e08744f7455c909568855f2a5be7bc88646796d21598ddda810390a","services":["bzz"],"enable_msg_events":true,"port":62860},"up":true}},{"node":{"config":{"id":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","private_key":"73c752d26f3438acdc54e9eaae6b1194d3706c65932939a593f1c8c5d451daf6","name":"node_5702eeccd7cb8c18dd25e5700919a544fb5e15fd53c5d4b56226021a9031a2206a694eb3b7334e4bd24fc510f7c1ea4f7a98e17e3394b4bd8d356a4d048b47dc","services":["bzz"],"enable_msg_events":true,"port":62861},"up":true}},{"node":{"config":{"id":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","private_key":"2e6321d9dda81fc0f863f09868b280dea3300105cf9d224294aa138caf10b4a6","name":"node_6311f5a830849b47094d84129fd987fcd3d511d000bf0a8b888c9a457545950fdae6ae10a2a85b47721a1d1ece8c1c62d866a26577380048d30f205c4e7cd7e8","services":["bzz"],"enable_msg_events":true,"port":62862},"up":true}},{"node":{"config":{"id":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","private_key":"200b7caaf8a33b8ea47947eb842fb8d8aac90951c70bb2f555f902380eb7f1ee","name":"node_dbef367277df652c8d3380efcf5f9d445d77fa6e7957b50b2467f90b7190e70d4eb4febbbffa6b3fc2b04bbf74dc7191e2b98923c3a62ecb80ba5336195994c7","services":["bzz"],"enable_msg_events":true,"port":62863},"up":true}},{"node":{"config":{"id":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","private_key":"b58a24805bb0c417e5666f2bf3f3342b8359d3c0d30a631c0104fcdfb6d0809b","name":"node_a4cdf723b3d7ead36bcb1bba582c8758271d5f08651694e1def0ae2ceca079d4fca81730a84a14749a81bc3ea41a66f1320788310823de5d1ceecd608ed5b88e","services":["bzz"],"enable_msg_events":true,"port":62864},"up":true}},{"node":{"config":{"id":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","private_key":"422d3b2c1b37b851ad524c8f26700fb5eec0d9a2fff1cb157ae931baa52d9521","name":"node_67cc8be0fd092c82e53efc53c418db257eff1135ad0058fd10e3eb77b205cb3e6fa87dfed4ec008c371f77546d4c117d8c34734bfe594da4c96fc08e0d2ede32","services":["bzz"],"enable_msg_events":true,"port":62865},"up":true}},{"node":{"config":{"id":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","private_key":"7871010c8b88b9594c71b9f43aecfefdca39da348e31f47ef2645c20c100d072","name":"node_4da7d1c9577afb52a5fe1de9f99824ceff314828c581caed8a3a33168df699d7b3befb4e888419264a9eeb3c9e5e9dac468e01edd71eb6f0ef41d15f7651042d","services":["bzz"],"enable_msg_events":true,"port":62866},"up":true}}],"conns":[{"one":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","other":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","up":true},{"one":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","other":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","up":true},{"one":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","other":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","up":true},{"one":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","other":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","up":true},{"one":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","other":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","up":true},{"one":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","other":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","up":true},{"one":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","other":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","up":true},{"one":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","other":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","up":true},{"one":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","other":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","up":true},{"one":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","other":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","up":true},{"one":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","other":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","up":true},{"one":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","other":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","up":true},{"one":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","other":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","up":true},{"one":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","other":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","up":true},{"one":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","other":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","up":true},{"one":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","other":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","up":true},{"one":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","other":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","up":true},{"one":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","up":true},{"one":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","other":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","up":true},{"one":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","other":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","up":true},{"one":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","up":true},{"one":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","other":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","up":true},{"one":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","other":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","up":true},{"one":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","other":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","up":true},{"one":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","other":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","up":true},{"one":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","other":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","up":true},{"one":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","other":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","up":true},{"one":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","other":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","up":true},{"one":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","other":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","up":true},{"one":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","up":true},{"one":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","other":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","up":true},{"one":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","other":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","up":true}]} diff --git a/cmd/swarm/swarm-snapshot/verify.go b/cmd/swarm/swarm-snapshot/verify.go new file mode 100644 index 000000000000..3312d7681fec --- /dev/null +++ b/cmd/swarm/swarm-snapshot/verify.go @@ -0,0 +1,85 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/network/simulation" + cli "gopkg.in/urfave/cli.v1" +) + +func verify(ctx *cli.Context) error { + if len(ctx.Args()) < 1 { + return errors.New("argument should be the filename to verify or write-to") + } + filename, err := touchPath(ctx.Args()[0]) + if err != nil { + return err + } + err = verifySnapshot(filename) + if err != nil { + utils.Fatalf("Simulation failed: %s", err) + } + + return err +} + +func verifySnapshot(filename string) error { + sim := simulation.New(map[string]simulation.ServiceFunc{ + "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + addr := network.NewAddr(ctx.Config.Node()) + + kp := network.NewKadParams() + kp.MinProxBinSize = testMinProxBinSize + + kad := network.NewKademlia(addr.Over(), kp) + hp := network.NewHiveParams() + hp.KeepAliveInterval = time.Duration(200) * time.Millisecond + hp.Discovery = true //discovery + + config := &network.BzzConfig{ + OverlayAddr: addr.Over(), + UnderlayAddr: addr.Under(), + HiveParams: hp, + } + return network.NewBzz(config, kad, nil, nil, nil), nil, nil + + }, + }) + defer sim.Close() + err := sim.UploadSnapshot(filename) + if err != nil { + utils.Fatalf("%v", err) + } + + ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelSimRun() + + if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + utils.Fatalf("%v", err) + } + + return nil +} diff --git a/cmd/swarm/swarm-snapshot/verify_test.go b/cmd/swarm/swarm-snapshot/verify_test.go new file mode 100644 index 000000000000..45646be078a7 --- /dev/null +++ b/cmd/swarm/swarm-snapshot/verify_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "testing" +) + +func TestSnapshotVerify(t *testing.T) { + snap := runSnapshot(t, + "v", + "testdata/snapshot.json", + ) + + snap.ExpectExit() + if snap.ExitStatus() != 0 { + t.Fatal("expected exit code 0") + } +} diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 86f7dc9bef44..ffbc523f6d2f 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -80,6 +80,43 @@ func (net *Network) Events() *event.Feed { return &net.events } +func triggerChecks(trigger chan enode.ID, net *Network, id enode.ID) error { + node := net.GetNode(id) + if node == nil { + return fmt.Errorf("unknown node: %s", id) + } + client, err := node.Client() + if err != nil { + return err + } + events := make(chan *p2p.PeerEvent) + sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents") + if err != nil { + return fmt.Errorf("error getting peer events for node %v: %s", id, err) + } + go func() { + defer sub.Unsubscribe() + + tick := time.NewTicker(time.Second) + defer tick.Stop() + + for { + select { + case <-events: + trigger <- id + case <-tick.C: + trigger <- id + case err := <-sub.Err(): + if err != nil { + log.Error(fmt.Sprintf("error getting peer events for node %v", id), "err", err) + } + return + } + } + }() + return nil +} + // NewNodeWithConfig adds a new node to the network with the given config, // returning an error if a node with the same ID or name already exists func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { diff --git a/p2p/simulations/simulation.go b/p2p/simulations/simulation.go index ae62c42b9c2d..b7e6204e5d95 100644 --- a/p2p/simulations/simulation.go +++ b/p2p/simulations/simulation.go @@ -49,9 +49,11 @@ func (s *Simulation) Run(ctx context.Context, step *Step) (result *StepResult) { defer stop() // perform the action - if err := step.Action(ctx); err != nil { - result.Error = err - return + if step.Action != nil { + if err := step.Action(ctx); err != nil { + result.Error = err + return + } } // wait for all node expectations to either pass, error or timeout diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index ec53f70a358e..1c0f6fec56f3 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -640,6 +640,8 @@ func (k *Kademlia) saturation() int { }) // TODO evaluate whether this check cannot just as well be done within the eachbin depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) + + // if in the iterator above we iterated deeper than the neighbourhood depth - return depth if depth < prev { return depth } diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index e5121c477581..75f34930df77 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -23,11 +23,9 @@ import ( "flag" "fmt" "io/ioutil" - "math/rand" "os" "path" "strings" - "sync" "testing" "time" @@ -247,25 +245,14 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul action := func(ctx context.Context) error { return nil } - wg := sync.WaitGroup{} for i := range ids { // collect the overlay addresses, to addrs = append(addrs, ids[i].Bytes()) - for j := 0; j < conns; j++ { - var k int - if j == 0 { - k = (i + 1) % len(ids) - } else { - k = rand.Intn(len(ids)) - } - wg.Add(1) - go func(i, k int) { - defer wg.Done() - net.Connect(ids[i], ids[k]) - }(i, k) - } } - wg.Wait() + err := net.ConnectNodesChain(nil) + if err != nil { + return nil, err + } log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) // construct the peer pot, so that kademlia health can be checked ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) @@ -457,23 +444,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt return nil } - //connects in a chain - wg := sync.WaitGroup{} - //connects in a ring - for i := range ids { - for j := 1; j <= conns; j++ { - k := (i + j) % len(ids) - if k == i { - k = (k + 1) % len(ids) - } - wg.Add(1) - go func(i, k int) { - defer wg.Done() - net.Connect(ids[i], ids[k]) - }(i, k) - } - } - wg.Wait() + net.ConnectNodesChain(nil) log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) // construct the peer pot, so that kademlia health can be checked check := func(ctx context.Context, id enode.ID) (bool, error) { From 8cc3fe6eda49cc12441a8ac1f961664a374bd4c1 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 9 Jan 2019 17:37:39 +0100 Subject: [PATCH 02/11] cmd/swarm/swarm-snapshot: refactor and extend tests --- cmd/swarm/swarm-snapshot/create.go | 124 ++++++++++++------------ cmd/swarm/swarm-snapshot/create_test.go | 115 +++++++++++++++++----- cmd/swarm/swarm-snapshot/helper.go | 40 -------- cmd/swarm/swarm-snapshot/main.go | 85 ++++++---------- cmd/swarm/swarm-snapshot/run_test.go | 2 +- cmd/swarm/swarm-snapshot/verify.go | 43 ++++---- cmd/swarm/swarm-snapshot/verify_test.go | 2 +- 7 files changed, 203 insertions(+), 208 deletions(-) delete mode 100644 cmd/swarm/swarm-snapshot/helper.go diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go index 7526c7d8c3bf..72a1446f1152 100644 --- a/cmd/swarm/swarm-snapshot/create.go +++ b/cmd/swarm/swarm-snapshot/create.go @@ -23,14 +23,14 @@ import ( "fmt" "io/ioutil" "os" + "path" + "path/filepath" "strings" "sync" "time" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" @@ -38,12 +38,11 @@ import ( cli "gopkg.in/urfave/cli.v1" ) -const testMinProxBinSize = 2 -const NoConnectionTimeout = 2 * time.Second +const noConnectionTimeout = 2 * time.Second func create(ctx *cli.Context) error { log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) if len(ctx.Args()) < 1 { return errors.New("argument should be the filename to verify or write-to") @@ -52,38 +51,19 @@ func create(ctx *cli.Context) error { if err != nil { return err } - err = discoverySnapshot(filename, 10) - if err != nil { - utils.Fatalf("Simulation failed: %s", err) - } - - return err + return createSnapshot(filename, ctx.Int("nodes"), ctx.String("services")) } -func discoverySnapshot(filename string, nodes int) error { - //disable discovery if topology is specified - discovery = topology == "" - log.Debug("discoverySnapshot", "filename", filename, "nodes", nodes, "discovery", discovery) - i := 0 - var lock sync.Mutex - var pivotNodeID enode.ID - sim := simulation.New(map[string]simulation.ServiceFunc{ - "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { - lock.Lock() - i++ - if i == pivot { - pivotNodeID = ctx.Config.ID - } - lock.Unlock() +func createSnapshot(filename string, nodes int, services string) (err error) { + log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services) + sim := simulation.New(map[string]simulation.ServiceFunc{ + bzzServiceName: func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) - kp := network.NewKadParams() - kp.MinProxBinSize = testMinProxBinSize - - kad := network.NewKademlia(addr.Over(), kp) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) hp := network.NewHiveParams() hp.KeepAliveInterval = time.Duration(200) * time.Millisecond - hp.Discovery = discovery + hp.Discovery = true // discovery must be enabled when creating a snapshot config := &network.BzzConfig{ OverlayAddr: addr.Over(), @@ -95,9 +75,9 @@ func discoverySnapshot(filename string, nodes int) error { }) defer sim.Close() - _, err := sim.AddNodes(10) + _, err = sim.AddNodes(nodes) if err != nil { - utils.Fatalf("%v", err) + return fmt.Errorf("add nodes: %v", err) } events := make(chan *simulations.Event) @@ -106,60 +86,84 @@ func discoverySnapshot(filename string, nodes int) error { case ev := <-events: //only catch node up events if ev.Type == simulations.EventTypeConn { - utils.Fatalf("this shouldn't happen as connections weren't initiated yet") + return errors.New("unexpected connection events") } - case <-time.After(NoConnectionTimeout): + case <-time.After(noConnectionTimeout): } - sub.Unsubscribe() if len(sim.Net.Conns) > 0 { - utils.Fatalf("no connections should exist after just adding nodes") + return errors.New("no connections should exist after just adding nodes") } - err := sim.Net.ConnectNodesRing(nil) + err = sim.Net.ConnectNodesRing(nil) if err != nil { - utils.Fatalf("had an error connecting the nodes in a %v topology: %v", topology, err) + return fmt.Errorf("connect nodes: %v", err) } - if discovery { - ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancelSimRun() - - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { - utils.Fatalf("%v", err) - } + ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelSimRun() + if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + return fmt.Errorf("wait for healthy kademlia: %v", err) } var snap *simulations.Snapshot if len(services) > 0 { - var addServices []string + addServices := strings.Split(services, ",") var removeServices []string - for _, osvc := range strings.Split(services, ",") { - if strings.Index(osvc, "+") == 0 { - addServices = append(addServices, osvc[1:]) - } else if strings.Index(osvc, "-") == 0 { - removeServices = append(removeServices, osvc[1:]) - } else { - panic("stick to the rules, you know what they are") + var hasBzz bool + for _, s := range addServices { + if s == bzzServiceName { + hasBzz = true + break } } + if !hasBzz { + removeServices = append(removeServices, bzzServiceName) + } snap, err = sim.Net.SnapshotWithServices(addServices, removeServices) } else { snap, err = sim.Net.Snapshot() } - if err != nil { - return errors.New("no shapshot dude") + return fmt.Errorf("create snapshot: %v", err) } jsonsnapshot, err := json.Marshal(snap) if err != nil { - return fmt.Errorf("corrupt json snapshot: %v", err) + return fmt.Errorf("json encode snapshot: %v", err) } - err = ioutil.WriteFile(filename, jsonsnapshot, 0666) + return ioutil.WriteFile(filename, jsonsnapshot, 0666) +} + +func touchPath(filename string) (string, error) { + if path.IsAbs(filename) { + if _, err := os.Stat(filename); err == nil { + // path exists, we will override the file + return filename, nil + } + } + + d, f := path.Split(filename) + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) if err != nil { - return err + return "", err + } + + _, err = os.Stat(path.Join(dir, filename)) + if err == nil { + // path exists, we will override + return filename, nil + } + + dirPath := path.Join(dir, d) + filePath := path.Join(dirPath, f) + if d != "" { + err = os.MkdirAll(dirPath, os.ModeDir) + if err != nil { + return "", err + } } - return nil + filename = filePath + return filename, nil } diff --git a/cmd/swarm/swarm-snapshot/create_test.go b/cmd/swarm/swarm-snapshot/create_test.go index d6f593913b3e..e8f419376a37 100644 --- a/cmd/swarm/swarm-snapshot/create_test.go +++ b/cmd/swarm/swarm-snapshot/create_test.go @@ -17,58 +17,121 @@ package main import ( + "encoding/json" + "fmt" "io/ioutil" "os" + "sort" + "strconv" + "strings" "testing" - "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/simulations" ) -func init() { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) -} - //TestSnapshotCreate is a high level e2e test that tests for snapshot generation func TestSnapshotCreate(t *testing.T) { - for _, v := range []struct { - name string - args []string + name string + nodes int + services string }{ { - name: "no topology - discovery enabled", - args: []string{ - "c", - }, + name: "defaults", + }, + { + name: "more nodes", + nodes: defaultNodes + 5, }, { - name: "yes topology - discovery disabled", - args: []string{ - "--topology", - "ring", - "c", - }, + name: "services", + services: "stream,pss,zorglub", + }, + { + name: "services with " + bzzServiceName, + services: bzzServiceName + ",pss", }, } { t.Run(v.name, func(t *testing.T) { + t.Parallel() + file, err := ioutil.TempFile("", "swarm-snapshot") - defer os.Remove(file.Name()) if err != nil { t.Fatal(err) } + defer os.Remove(file.Name()) + + if err = file.Close(); err != nil { + t.Error(err) + } - file.Close() - snap := runSnapshot(t, append(v.args, file.Name())...) + args := []string{"create"} + if v.nodes > 0 { + args = append(args, "--nodes", strconv.Itoa(v.nodes)) + } + if v.services != "" { + args = append(args, "--services", v.services) + } + testCmd := runSnapshot(t, append(args, file.Name())...) - snap.ExpectExit() - if snap.ExitStatus() != 0 { + testCmd.ExpectExit() + if testCmd.ExitStatus() != 0 { t.Fatal("expected exit code 0") } - _, err = os.Stat(file.Name()) + f, err := os.Open(file.Name()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := f.Close() + if err != nil { + t.Error("closing snapshot file", "err", err) + } + }() + + b, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + var snap simulations.Snapshot + err = json.Unmarshal(b, &snap) + if err != nil { + t.Fatal(err) + } + + wantNodes := v.nodes + if wantNodes == 0 { + wantNodes = defaultNodes + } + gotNodes := len(snap.Nodes) + if gotNodes != wantNodes { + t.Errorf("got %v nodes, want %v", gotNodes, wantNodes) + } + + if len(snap.Conns) == 0 { + t.Error("no connections in a snapshot") + } + + var wantServices []string + if v.services != "" { + wantServices = strings.Split(v.services, ",") + } else { + wantServices = []string{bzzServiceName} + } + sort.Strings(wantServices) + + for i, n := range snap.Nodes { + gotServices := n.Node.Config.Services + sort.Strings(gotServices) + if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) { + t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices) + } + } + + err = verifySnapshot(file.Name()) if err != nil { - t.Fatal("could not stat snapshot json") + t.Error(err) } }) } diff --git a/cmd/swarm/swarm-snapshot/helper.go b/cmd/swarm/swarm-snapshot/helper.go deleted file mode 100644 index 8c4fab928dcd..000000000000 --- a/cmd/swarm/swarm-snapshot/helper.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "os" - "path" - "path/filepath" -) - -func touchPath(filename string) (string, error) { - if path.IsAbs(filename) { - if _, err := os.Stat(filename); err == nil { - // path exists, we will override the file - return filename, nil - } - } - - d, f := path.Split(filename) - dir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return "", err - } - - _, err = os.Stat(path.Join(dir, filename)) - if err == nil { - // path exists, we will override - return filename, nil - } - - dirPath := path.Join(dir, d) - filePath := path.Join(dirPath, f) - if d != "" { - err = os.MkdirAll(dirPath, os.ModeDir) - if err != nil { - return "", err - } - } - - filename = filePath - return filename, nil -} diff --git a/cmd/swarm/swarm-snapshot/main.go b/cmd/swarm/swarm-snapshot/main.go index 6338b2886b0c..238e1d991d80 100644 --- a/cmd/swarm/swarm-snapshot/main.go +++ b/cmd/swarm/swarm-snapshot/main.go @@ -18,65 +18,38 @@ package main import ( "os" - "sort" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/log" cli "gopkg.in/urfave/cli.v1" ) -var ( - gitCommit string // Git SHA1 commit hash of the release (set via linker flags) -) +var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) -var ( - topology string - services string - pivot int - nodes int - verbosity int +const ( + defaultNodes = 10 + bzzServiceName = "bzz" ) -var app = utils.NewApp("", "Swarm Snapshot Util") -var discovery = true +func main() { + err := newApp().Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} -func init() { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) +func newApp() (app *cli.App) { + app = utils.NewApp(gitCommit, "Swarm Snapshot Utility") app.Name = "swarm-snapshot" app.Usage = "" app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "topology", - Value: "chain", - Usage: "the desired topology to connect the nodes in (star, ring, chain, full)", - Destination: &topology, - }, cli.IntFlag{ - Name: "pivot", - Value: 0, - Usage: "pivot node zero-index", - Destination: &pivot, - }, - cli.IntFlag{ - Name: "nodes", - Value: 10, - Usage: "swarm nodes", - Destination: &nodes, - }, - cli.IntFlag{ - Name: "verbosity", - Value: 1, - Usage: "verbosity", - Destination: &verbosity, - }, - cli.StringFlag{ - Name: "services", - Value: "", - Usage: "comma separated list of services to boot the nodes with", - Destination: &services, + Name: "verbosity", + Value: 1, + Usage: "verbosity level", }, } @@ -86,25 +59,27 @@ func init() { Aliases: []string{"c"}, Usage: "create a swarm snapshot", Action: create, + Flags: append(app.Flags, + cli.IntFlag{ + Name: "nodes", + Value: defaultNodes, + Usage: "number of nodes", + }, + cli.StringFlag{ + Name: "services", + Value: bzzServiceName, + Usage: "comma separated list of services to boot the nodes with", + }, + ), }, { Name: "verify", Aliases: []string{"v"}, Usage: "verify a swarm snapshot", Action: verify, + Flags: app.Flags, }, } - sort.Sort(cli.FlagsByName(app.Flags)) - sort.Sort(cli.CommandsByName(app.Commands)) -} - -func main() { - err := app.Run(os.Args) - if err != nil { - log.Error(err.Error()) - - os.Exit(1) - } - os.Exit(0) + return app } diff --git a/cmd/swarm/swarm-snapshot/run_test.go b/cmd/swarm/swarm-snapshot/run_test.go index 3a66ac987fca..d9a041597e70 100644 --- a/cmd/swarm/swarm-snapshot/run_test.go +++ b/cmd/swarm/swarm-snapshot/run_test.go @@ -27,7 +27,7 @@ import ( func init() { reexec.Register("swarm-snapshot", func() { - if err := app.Run(os.Args); err != nil { + if err := newApp().Run(os.Args); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/cmd/swarm/swarm-snapshot/verify.go b/cmd/swarm/swarm-snapshot/verify.go index 3312d7681fec..398335b4235b 100644 --- a/cmd/swarm/swarm-snapshot/verify.go +++ b/cmd/swarm/swarm-snapshot/verify.go @@ -19,10 +19,12 @@ package main import ( "context" "errors" + "fmt" + "os" "sync" "time" - "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" @@ -31,33 +33,23 @@ import ( ) func verify(ctx *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + if len(ctx.Args()) < 1 { - return errors.New("argument should be the filename to verify or write-to") - } - filename, err := touchPath(ctx.Args()[0]) - if err != nil { - return err + return errors.New("argument should be the filename to verify") } - err = verifySnapshot(filename) - if err != nil { - utils.Fatalf("Simulation failed: %s", err) - } - - return err + return verifySnapshot(ctx.Args()[0]) } -func verifySnapshot(filename string) error { +func verifySnapshot(filename string) (err error) { sim := simulation.New(map[string]simulation.ServiceFunc{ - "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + bzzServiceName: func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) - - kp := network.NewKadParams() - kp.MinProxBinSize = testMinProxBinSize - - kad := network.NewKademlia(addr.Over(), kp) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) hp := network.NewHiveParams() hp.KeepAliveInterval = time.Duration(200) * time.Millisecond - hp.Discovery = true //discovery + hp.Discovery = false // discovery must be false when uploading a snapshot config := &network.BzzConfig{ OverlayAddr: addr.Over(), @@ -65,20 +57,21 @@ func verifySnapshot(filename string) error { HiveParams: hp, } return network.NewBzz(config, kad, nil, nil, nil), nil, nil - }, }) defer sim.Close() - err := sim.UploadSnapshot(filename) + + err = sim.UploadSnapshot(filename) if err != nil { - utils.Fatalf("%v", err) + return fmt.Errorf("upload snapshot: %v", err) } ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) defer cancelSimRun() - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { - utils.Fatalf("%v", err) + _, err = sim.WaitTillHealthy(ctx, 2) + if err != nil { + return fmt.Errorf("wait for healthy kademlia: %v", err) } return nil diff --git a/cmd/swarm/swarm-snapshot/verify_test.go b/cmd/swarm/swarm-snapshot/verify_test.go index 45646be078a7..2cdedee01be5 100644 --- a/cmd/swarm/swarm-snapshot/verify_test.go +++ b/cmd/swarm/swarm-snapshot/verify_test.go @@ -22,7 +22,7 @@ import ( func TestSnapshotVerify(t *testing.T) { snap := runSnapshot(t, - "v", + "verify", "testdata/snapshot.json", ) From fe1ddb0b145f0e4ef5a93e73e09f22ad8a3be180 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 10 Jan 2019 10:29:25 +0100 Subject: [PATCH 03/11] p2p/simulations: remove unused triggerChecks func and fix linter --- p2p/simulations/network.go | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index ffbc523f6d2f..86f7dc9bef44 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -80,43 +80,6 @@ func (net *Network) Events() *event.Feed { return &net.events } -func triggerChecks(trigger chan enode.ID, net *Network, id enode.ID) error { - node := net.GetNode(id) - if node == nil { - return fmt.Errorf("unknown node: %s", id) - } - client, err := node.Client() - if err != nil { - return err - } - events := make(chan *p2p.PeerEvent) - sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents") - if err != nil { - return fmt.Errorf("error getting peer events for node %v: %s", id, err) - } - go func() { - defer sub.Unsubscribe() - - tick := time.NewTicker(time.Second) - defer tick.Stop() - - for { - select { - case <-events: - trigger <- id - case <-tick.C: - trigger <- id - case err := <-sub.Err(): - if err != nil { - log.Error(fmt.Sprintf("error getting peer events for node %v", id), "err", err) - } - return - } - } - }() - return nil -} - // NewNodeWithConfig adds a new node to the network with the given config, // returning an error if a node with the same ID or name already exists func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) { From 186dd02c3d8becbd9e652ed849364a8184bfc443 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 10 Jan 2019 11:15:10 +0100 Subject: [PATCH 04/11] internal/cmdtest: raise the timeout for killing TestCmd --- internal/cmdtest/test_cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index f8b1b43c0119..59c276f30c38 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -227,7 +227,7 @@ func (tt *TestCmd) Kill() { } func (tt *TestCmd) withKillTimeout(fn func()) { - timeout := time.AfterFunc(5*time.Second, func() { + timeout := time.AfterFunc(15*time.Second, func() { tt.Log("killing the child process (timeout)") tt.Kill() }) From b22605257614fa743168ef734052ffa07dc4e3a2 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 10 Jan 2019 12:04:23 +0100 Subject: [PATCH 05/11] cmd/swarm/swarm-snapshot: add more comments and other minor adjustments --- cmd/swarm/swarm-snapshot/create.go | 36 +++++++++++++++---------- cmd/swarm/swarm-snapshot/create_test.go | 16 ++++++----- cmd/swarm/swarm-snapshot/main.go | 18 ++++++++----- cmd/swarm/swarm-snapshot/verify.go | 5 +++- cmd/swarm/swarm-snapshot/verify_test.go | 6 +++-- 5 files changed, 52 insertions(+), 29 deletions(-) diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go index 72a1446f1152..e5236b865d02 100644 --- a/cmd/swarm/swarm-snapshot/create.go +++ b/cmd/swarm/swarm-snapshot/create.go @@ -38,8 +38,9 @@ import ( cli "gopkg.in/urfave/cli.v1" ) -const noConnectionTimeout = 2 * time.Second +const noConnectionTimeout = 1 * time.Second +// create is used as the entry function for "create" app command. func create(ctx *cli.Context) error { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) @@ -51,14 +52,16 @@ func create(ctx *cli.Context) error { if err != nil { return err } - return createSnapshot(filename, ctx.Int("nodes"), ctx.String("services")) + return createSnapshot(filename, ctx.Int("nodes"), strings.Split(ctx.String("services"), ",")) } -func createSnapshot(filename string, nodes int, services string) (err error) { +// createSnapshot creates a new snapshot on filesystem with provided filename, +// number of nodes and service names. +func createSnapshot(filename string, nodes int, services []string) (err error) { log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services) sim := simulation.New(map[string]simulation.ServiceFunc{ - bzzServiceName: func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) kad := network.NewKademlia(addr.Over(), network.NewKadParams()) hp := network.NewHiveParams() @@ -80,6 +83,8 @@ func createSnapshot(filename string, nodes int, services string) (err error) { return fmt.Errorf("add nodes: %v", err) } + // wait for two some time to ensure no connections + // are established events := make(chan *simulations.Event) sub := sim.Net.Events().Subscribe(events) select { @@ -109,19 +114,21 @@ func createSnapshot(filename string, nodes int, services string) (err error) { var snap *simulations.Snapshot if len(services) > 0 { - addServices := strings.Split(services, ",") + // If service names are provided, include them in the snapshot. + // But, check if "bzz" service is not among them to remove it + // form the snapshot as it exists on snapshot creation. var removeServices []string - var hasBzz bool - for _, s := range addServices { - if s == bzzServiceName { - hasBzz = true + var wantBzz bool + for _, s := range services { + if s == "bzz" { + wantBzz = true break } } - if !hasBzz { - removeServices = append(removeServices, bzzServiceName) + if !wantBzz { + removeServices = []string{"bzz"} } - snap, err = sim.Net.SnapshotWithServices(addServices, removeServices) + snap, err = sim.Net.SnapshotWithServices(services, removeServices) } else { snap, err = sim.Net.Snapshot() } @@ -135,6 +142,8 @@ func createSnapshot(filename string, nodes int, services string) (err error) { return ioutil.WriteFile(filename, jsonsnapshot, 0666) } +// touchPath creates an empty file and all subdirectories +// that are missing. func touchPath(filename string) (string, error) { if path.IsAbs(filename) { if _, err := os.Stat(filename); err == nil { @@ -164,6 +173,5 @@ func touchPath(filename string) (string, error) { } } - filename = filePath - return filename, nil + return filePath, nil } diff --git a/cmd/swarm/swarm-snapshot/create_test.go b/cmd/swarm/swarm-snapshot/create_test.go index e8f419376a37..603a5f41002b 100644 --- a/cmd/swarm/swarm-snapshot/create_test.go +++ b/cmd/swarm/swarm-snapshot/create_test.go @@ -29,7 +29,9 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations" ) -//TestSnapshotCreate is a high level e2e test that tests for snapshot generation +// TestSnapshotCreate is a high level e2e test that tests for snapshot generation. +// It runs a few "create" commands with different flag values and loads generated +// snapshot files to validate their content. func TestSnapshotCreate(t *testing.T) { for _, v := range []struct { name string @@ -48,8 +50,8 @@ func TestSnapshotCreate(t *testing.T) { services: "stream,pss,zorglub", }, { - name: "services with " + bzzServiceName, - services: bzzServiceName + ",pss", + name: "services with bzz", + services: "bzz,pss", }, } { t.Run(v.name, func(t *testing.T) { @@ -75,8 +77,8 @@ func TestSnapshotCreate(t *testing.T) { testCmd := runSnapshot(t, append(args, file.Name())...) testCmd.ExpectExit() - if testCmd.ExitStatus() != 0 { - t.Fatal("expected exit code 0") + if code := testCmd.ExitStatus(); code != 0 { + t.Fatalf("command exit code %v, expected 0", code) } f, err := os.Open(file.Name()) @@ -117,8 +119,10 @@ func TestSnapshotCreate(t *testing.T) { if v.services != "" { wantServices = strings.Split(v.services, ",") } else { - wantServices = []string{bzzServiceName} + wantServices = []string{"bzz"} } + // sort service names so they can be comparable + // as strings to every node sorted services sort.Strings(wantServices) for i, n := range snap.Nodes { diff --git a/cmd/swarm/swarm-snapshot/main.go b/cmd/swarm/swarm-snapshot/main.go index 238e1d991d80..6bf9e76b9e63 100644 --- a/cmd/swarm/swarm-snapshot/main.go +++ b/cmd/swarm/swarm-snapshot/main.go @@ -26,10 +26,8 @@ import ( var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) -const ( - defaultNodes = 10 - bzzServiceName = "bzz" -) +// default value for "create" command --nodes flag +const defaultNodes = 10 func main() { err := newApp().Run(os.Args) @@ -39,12 +37,15 @@ func main() { } } +// newApp construct a new instance of Swarm Snapshot Utility. +// Method Run is called on it in the main function and in tests. func newApp() (app *cli.App) { app = utils.NewApp(gitCommit, "Swarm Snapshot Utility") app.Name = "swarm-snapshot" app.Usage = "" + // app flags (for all commands) app.Flags = []cli.Flag{ cli.IntFlag{ Name: "verbosity", @@ -59,6 +60,9 @@ func newApp() (app *cli.App) { Aliases: []string{"c"}, Usage: "create a swarm snapshot", Action: create, + // Flags only for "create" command. + // Allow app flags to be specified after the + // command argument. Flags: append(app.Flags, cli.IntFlag{ Name: "nodes", @@ -67,7 +71,7 @@ func newApp() (app *cli.App) { }, cli.StringFlag{ Name: "services", - Value: bzzServiceName, + Value: "bzz", Usage: "comma separated list of services to boot the nodes with", }, ), @@ -77,7 +81,9 @@ func newApp() (app *cli.App) { Aliases: []string{"v"}, Usage: "verify a swarm snapshot", Action: verify, - Flags: app.Flags, + // Allow app flags to be specified after the + // command argument. + Flags: app.Flags, }, } diff --git a/cmd/swarm/swarm-snapshot/verify.go b/cmd/swarm/swarm-snapshot/verify.go index 398335b4235b..43bde1fdc1f4 100644 --- a/cmd/swarm/swarm-snapshot/verify.go +++ b/cmd/swarm/swarm-snapshot/verify.go @@ -32,6 +32,7 @@ import ( cli "gopkg.in/urfave/cli.v1" ) +// verify is used as the entry function for "verify" app command. func verify(ctx *cli.Context) error { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) @@ -42,9 +43,11 @@ func verify(ctx *cli.Context) error { return verifySnapshot(ctx.Args()[0]) } +// verifySnapshot constructs a simulation, uploads a snapshot from +// a file with provided filename and validates that kademlia is healthy. func verifySnapshot(filename string) (err error) { sim := simulation.New(map[string]simulation.ServiceFunc{ - bzzServiceName: func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) kad := network.NewKademlia(addr.Over(), network.NewKadParams()) hp := network.NewHiveParams() diff --git a/cmd/swarm/swarm-snapshot/verify_test.go b/cmd/swarm/swarm-snapshot/verify_test.go index 2cdedee01be5..f8c615bf84eb 100644 --- a/cmd/swarm/swarm-snapshot/verify_test.go +++ b/cmd/swarm/swarm-snapshot/verify_test.go @@ -20,6 +20,8 @@ import ( "testing" ) +// TestSnapshotVerify executes "verify" command with a +// pregenerated snapshot file that is valid. func TestSnapshotVerify(t *testing.T) { snap := runSnapshot(t, "verify", @@ -27,7 +29,7 @@ func TestSnapshotVerify(t *testing.T) { ) snap.ExpectExit() - if snap.ExitStatus() != 0 { - t.Fatal("expected exit code 0") + if code := snap.ExitStatus(); code != 0 { + t.Fatalf("command exit code %v, expected 0", code) } } From a208208425654c924a2b3ec7c77e03a0c6f05a96 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 11 Jan 2019 13:57:19 +0100 Subject: [PATCH 06/11] cmd/swarm/swarm-snapshot: remove redundant check in createSnapshot --- cmd/swarm/swarm-snapshot/create.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go index e5236b865d02..155b612590e1 100644 --- a/cmd/swarm/swarm-snapshot/create.go +++ b/cmd/swarm/swarm-snapshot/create.go @@ -97,10 +97,6 @@ func createSnapshot(filename string, nodes int, services []string) (err error) { } sub.Unsubscribe() - if len(sim.Net.Conns) > 0 { - return errors.New("no connections should exist after just adding nodes") - } - err = sim.Net.ConnectNodesRing(nil) if err != nil { return fmt.Errorf("connect nodes: %v", err) From a40801deadefe9e3012a6913f9f07b5594027aa8 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 11 Jan 2019 14:01:35 +0100 Subject: [PATCH 07/11] cmd/swarm/swarm-snapshot: change comment wording --- cmd/swarm/swarm-snapshot/create.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go index 155b612590e1..fa5a6629d9d3 100644 --- a/cmd/swarm/swarm-snapshot/create.go +++ b/cmd/swarm/swarm-snapshot/create.go @@ -143,7 +143,7 @@ func createSnapshot(filename string, nodes int, services []string) (err error) { func touchPath(filename string) (string, error) { if path.IsAbs(filename) { if _, err := os.Stat(filename); err == nil { - // path exists, we will override the file + // path exists, we will overwrite the file return filename, nil } } @@ -156,7 +156,7 @@ func touchPath(filename string) (string, error) { _, err = os.Stat(path.Join(dir, filename)) if err == nil { - // path exists, we will override + // path exists, we will overwrite return filename, nil } From 621e2b64ee1b3b6a6b46c9f3b4d10aaef40386bf Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 11 Jan 2019 14:08:29 +0100 Subject: [PATCH 08/11] p2p/simulations: revert Simulation.Run from master https://github.com/ethersphere/go-ethereum/pull/1077/files#r247078904 --- p2p/simulations/simulation.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/p2p/simulations/simulation.go b/p2p/simulations/simulation.go index b7e6204e5d95..ae62c42b9c2d 100644 --- a/p2p/simulations/simulation.go +++ b/p2p/simulations/simulation.go @@ -49,11 +49,9 @@ func (s *Simulation) Run(ctx context.Context, step *Step) (result *StepResult) { defer stop() // perform the action - if step.Action != nil { - if err := step.Action(ctx); err != nil { - result.Error = err - return - } + if err := step.Action(ctx); err != nil { + result.Error = err + return } // wait for all node expectations to either pass, error or timeout From 85208256d3cdd14abb21a561aba0afbce8abb158 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Tue, 15 Jan 2019 12:04:12 +0530 Subject: [PATCH 09/11] cmd/swarm/swarm-snapshot: address pr comments --- cmd/swarm/swarm-snapshot/create.go | 6 +- cmd/swarm/swarm-snapshot/create_test.go | 4 - cmd/swarm/swarm-snapshot/main.go | 9 --- .../swarm-snapshot/testdata/snapshot.json | 1 - cmd/swarm/swarm-snapshot/verify.go | 81 ------------------- cmd/swarm/swarm-snapshot/verify_test.go | 35 -------- internal/cmdtest/test_cmd.go | 2 +- 7 files changed, 4 insertions(+), 134 deletions(-) delete mode 100644 cmd/swarm/swarm-snapshot/testdata/snapshot.json delete mode 100644 cmd/swarm/swarm-snapshot/verify.go delete mode 100644 cmd/swarm/swarm-snapshot/verify_test.go diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go index fa5a6629d9d3..eab46bf377a3 100644 --- a/cmd/swarm/swarm-snapshot/create.go +++ b/cmd/swarm/swarm-snapshot/create.go @@ -104,7 +104,7 @@ func createSnapshot(filename string, nodes int, services []string) (err error) { ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) defer cancelSimRun() - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return fmt.Errorf("wait for healthy kademlia: %v", err) } @@ -143,7 +143,7 @@ func createSnapshot(filename string, nodes int, services []string) (err error) { func touchPath(filename string) (string, error) { if path.IsAbs(filename) { if _, err := os.Stat(filename); err == nil { - // path exists, we will overwrite the file + // path exists, overwrite return filename, nil } } @@ -156,7 +156,7 @@ func touchPath(filename string) (string, error) { _, err = os.Stat(path.Join(dir, filename)) if err == nil { - // path exists, we will overwrite + // path exists, overwrite return filename, nil } diff --git a/cmd/swarm/swarm-snapshot/create_test.go b/cmd/swarm/swarm-snapshot/create_test.go index 603a5f41002b..dbd5b12cd2a8 100644 --- a/cmd/swarm/swarm-snapshot/create_test.go +++ b/cmd/swarm/swarm-snapshot/create_test.go @@ -133,10 +133,6 @@ func TestSnapshotCreate(t *testing.T) { } } - err = verifySnapshot(file.Name()) - if err != nil { - t.Error(err) - } }) } } diff --git a/cmd/swarm/swarm-snapshot/main.go b/cmd/swarm/swarm-snapshot/main.go index 6bf9e76b9e63..184727e4d724 100644 --- a/cmd/swarm/swarm-snapshot/main.go +++ b/cmd/swarm/swarm-snapshot/main.go @@ -76,15 +76,6 @@ func newApp() (app *cli.App) { }, ), }, - { - Name: "verify", - Aliases: []string{"v"}, - Usage: "verify a swarm snapshot", - Action: verify, - // Allow app flags to be specified after the - // command argument. - Flags: app.Flags, - }, } return app diff --git a/cmd/swarm/swarm-snapshot/testdata/snapshot.json b/cmd/swarm/swarm-snapshot/testdata/snapshot.json deleted file mode 100644 index dd6e49dbe182..000000000000 --- a/cmd/swarm/swarm-snapshot/testdata/snapshot.json +++ /dev/null @@ -1 +0,0 @@ -{"nodes":[{"node":{"config":{"id":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","private_key":"3ff3a0c7dc8b63cac9255233d84cccad53d61f8c9e47539329c07e4f248fc6dd","name":"node_279f0a191892115b8db4e2689cc5e1bc19019e8f5228f4f093f35c48fa2f606fd00a2ec5cf0fa4657ad3606e14643c3be8d49350ad2673f2ad12e17c0174b0d8","services":["bzz"],"enable_msg_events":true,"port":62851},"up":true}},{"node":{"config":{"id":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","private_key":"1ab5fd3f1885661af1829b42683ca1379ca90b1b2c5a0132027346c74001a154","name":"node_2c3d17a3b019d9d32922f06aef03ac03f67777e382f0b86c6e4dac07e590f2dbf4cbd1cc7ed136b52bba897e7163dee926caa627111db6d948581ad1b2edbc3f","services":["bzz"],"enable_msg_events":true,"port":62852},"up":true}},{"node":{"config":{"id":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","private_key":"a90fea5197ac84a162db7a06ec739e075ee334026e792d83a3269526888c8b5d","name":"node_6e327c2d9d3ffdae707ed067d15b83ad0ea81b96bb2a07331436f01c1e194ed90f630ef0eb6511f4849ede30c403e3c4ff3c2cb8021317de97f3ed5b1d990f49","services":["bzz"],"enable_msg_events":true,"port":62853},"up":true}},{"node":{"config":{"id":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","private_key":"76b6f09c28f5568e7724e9850116bdbd6356152fefedc4b0a9747361d712964b","name":"node_9aa1df8ddddd035e81ffc950f063714c3b6be24013108f8b99c32c25fc3a8582904f68c73fe63ebfb96143a12ab398f36c4070a3055c039491ca2f20be6774c2","services":["bzz"],"enable_msg_events":true,"port":62854},"up":true}},{"node":{"config":{"id":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","private_key":"f7e7d60abb7d43d973131804e684ec99e77e81bf38165dda29f3ae026eb169f1","name":"node_a1226991909b92dd4f22150b798e2b6312ffc62bad7a47d250e811c7e60cf5ceb6a87998f295e0bd392c0f5c15213115000781e498dd7eae79b67c7e7e19cf81","services":["bzz"],"enable_msg_events":true,"port":62855},"up":true}},{"node":{"config":{"id":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","private_key":"f88cd88c2c219d13b699ba7c7cf9adf16fc10be5d8e1f35b2ea260e7987ab76e","name":"node_1dcbfc79418f9a665e3154e07663b296ce086e54c23458b7113a011dca62e635245cab144d392716e5e90b60336b60927270b72bb4a1c5c85fbf959060fb3ee4","services":["bzz"],"enable_msg_events":true,"port":62856},"up":true}},{"node":{"config":{"id":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","private_key":"8fa1673a72b2ab000a4a7c8ad4540484d5286b69c93a5c608ef0977f783b7c4c","name":"node_571afb6ce94d28cb7d5cd49b3af35d2b1e44c917a9d3d207ab6c7ae960eb625377334e62317f6c69b3004e2eef92682649df828f4a645a75309ee7014ee4a940","services":["bzz"],"enable_msg_events":true,"port":62857},"up":true}},{"node":{"config":{"id":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","private_key":"daa9c74fbb0b897acdf20f1b955a608b086ce5b45dc8e2b76a44f277d25e3377","name":"node_f1e669bb6e5b89c04e07e3dcdc445d498ac54c8ab2e9a62e622a492b6f9781bfa27f75ef7448d06caa28d9afa40c1c089066358cbcca40018bc0f903eab9508e","services":["bzz"],"enable_msg_events":true,"port":62858},"up":true}},{"node":{"config":{"id":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","private_key":"eb94b600be5949e3ca545e81b4d0a5fff3ae9f51f63c722d10f5098f9e1a883a","name":"node_526d25e0aa47e64d8f681255348a3d3c94fc68bedcb7111532f4ed50d47ef064b1e97e3a6bcd13ac8eaf44ce676b22d2b343a94dfa38b48cf964bfc3f338d4bd","services":["bzz"],"enable_msg_events":true,"port":62859},"up":true}},{"node":{"config":{"id":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","private_key":"4e5d074bc46efbd0d18310e6a74e72e98b92080329d9c4e2b5ae99eb2fcfaf89","name":"node_fa81c440829adcbb3eba9196dc8255e319a976e53db0aceedcda25c2605f21b8778bf8c89e08744f7455c909568855f2a5be7bc88646796d21598ddda810390a","services":["bzz"],"enable_msg_events":true,"port":62860},"up":true}},{"node":{"config":{"id":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","private_key":"73c752d26f3438acdc54e9eaae6b1194d3706c65932939a593f1c8c5d451daf6","name":"node_5702eeccd7cb8c18dd25e5700919a544fb5e15fd53c5d4b56226021a9031a2206a694eb3b7334e4bd24fc510f7c1ea4f7a98e17e3394b4bd8d356a4d048b47dc","services":["bzz"],"enable_msg_events":true,"port":62861},"up":true}},{"node":{"config":{"id":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","private_key":"2e6321d9dda81fc0f863f09868b280dea3300105cf9d224294aa138caf10b4a6","name":"node_6311f5a830849b47094d84129fd987fcd3d511d000bf0a8b888c9a457545950fdae6ae10a2a85b47721a1d1ece8c1c62d866a26577380048d30f205c4e7cd7e8","services":["bzz"],"enable_msg_events":true,"port":62862},"up":true}},{"node":{"config":{"id":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","private_key":"200b7caaf8a33b8ea47947eb842fb8d8aac90951c70bb2f555f902380eb7f1ee","name":"node_dbef367277df652c8d3380efcf5f9d445d77fa6e7957b50b2467f90b7190e70d4eb4febbbffa6b3fc2b04bbf74dc7191e2b98923c3a62ecb80ba5336195994c7","services":["bzz"],"enable_msg_events":true,"port":62863},"up":true}},{"node":{"config":{"id":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","private_key":"b58a24805bb0c417e5666f2bf3f3342b8359d3c0d30a631c0104fcdfb6d0809b","name":"node_a4cdf723b3d7ead36bcb1bba582c8758271d5f08651694e1def0ae2ceca079d4fca81730a84a14749a81bc3ea41a66f1320788310823de5d1ceecd608ed5b88e","services":["bzz"],"enable_msg_events":true,"port":62864},"up":true}},{"node":{"config":{"id":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","private_key":"422d3b2c1b37b851ad524c8f26700fb5eec0d9a2fff1cb157ae931baa52d9521","name":"node_67cc8be0fd092c82e53efc53c418db257eff1135ad0058fd10e3eb77b205cb3e6fa87dfed4ec008c371f77546d4c117d8c34734bfe594da4c96fc08e0d2ede32","services":["bzz"],"enable_msg_events":true,"port":62865},"up":true}},{"node":{"config":{"id":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","private_key":"7871010c8b88b9594c71b9f43aecfefdca39da348e31f47ef2645c20c100d072","name":"node_4da7d1c9577afb52a5fe1de9f99824ceff314828c581caed8a3a33168df699d7b3befb4e888419264a9eeb3c9e5e9dac468e01edd71eb6f0ef41d15f7651042d","services":["bzz"],"enable_msg_events":true,"port":62866},"up":true}}],"conns":[{"one":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","other":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","up":true},{"one":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","other":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","up":true},{"one":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","other":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","up":true},{"one":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","other":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","up":true},{"one":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","other":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","up":true},{"one":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","other":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","up":true},{"one":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","other":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","up":true},{"one":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","other":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","up":true},{"one":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","other":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","up":true},{"one":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","other":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","up":true},{"one":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","other":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","up":true},{"one":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","other":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","up":true},{"one":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","other":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","up":true},{"one":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","other":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","up":true},{"one":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","other":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","up":true},{"one":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","other":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","up":true},{"one":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","other":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","up":true},{"one":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","up":true},{"one":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","other":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","up":true},{"one":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","other":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","up":true},{"one":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","up":true},{"one":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","other":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","up":true},{"one":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"a873e971042dfd5aaeb56d7d9089e35bc8dd410451a66740ebe7ea0752870120","other":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","up":true},{"one":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","other":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","up":true},{"one":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","other":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","up":true},{"one":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","other":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","up":true},{"one":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","other":"af99d46c187735ff037e1264f2dbca2985694d906de0e2c9e52d5513a6feb331","up":true},{"one":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","other":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","up":true},{"one":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","other":"55cc9bd08390eec7bc050924e3011b251c6c4e088ac2adda27493064b75f0ab8","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"9121c3fea7ccc99775fb5ade460f7a0cb76003231d13d44ed3e4f3cca1947227","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"f64d3106d2bb99e69e26c07de168f7fceaf85672859f7c1c038fd4dd6b6d7588","up":true},{"one":"6d8b6881d36bc5f4a4a82f5d707807af2eef2eaf4527eb6b16f62cd75a92bb8c","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"fc373cf4e892501073ed87b98231f0d001e6db70a1a08148035fbdfa2dffb3d3","other":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","up":true},{"one":"3c4287b7722f71eb03a3d54fb6b18aeb0bf8495cffbe8c103fe90229f08392ca","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"615c7fcf7a80635e42719c0ae1fb924ba703ee39e021898be56b81ef0f575f62","other":"4ce65548fc8490c02e4c6c0549be7329a8a05c090d62870ad1075e52e204a3b5","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"9fcdef2b818e8a4fcc07fedb706d9f88cc59703230af4f52043c84f9bd37cc57","up":true},{"one":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","other":"a3790562c8209b7e96e7abef9ad28de2e470a01927234e4848966e0cd15652e9","up":true},{"one":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","other":"91129c7135bff2b7cb1e1162bdd5399192790bdb4fc2eaa5b4cea53fe7a69d8c","up":true},{"one":"8bf97ec29b6b3ca638ed82fa30d03d0d8309c9e8f1db2071f4a37de805327108","other":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","up":true},{"one":"84842bc52628c0bed88539ee40f211de307b3e728ab5c0ca78032b86028b49db","other":"fb555885b96d5742a10e688bd7c1bb842f0434c319bf076d07f545e4c0550601","up":true}]} diff --git a/cmd/swarm/swarm-snapshot/verify.go b/cmd/swarm/swarm-snapshot/verify.go deleted file mode 100644 index 43bde1fdc1f4..000000000000 --- a/cmd/swarm/swarm-snapshot/verify.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "context" - "errors" - "fmt" - "os" - "sync" - "time" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" - "github.com/ethereum/go-ethereum/swarm/network" - "github.com/ethereum/go-ethereum/swarm/network/simulation" - cli "gopkg.in/urfave/cli.v1" -) - -// verify is used as the entry function for "verify" app command. -func verify(ctx *cli.Context) error { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - - if len(ctx.Args()) < 1 { - return errors.New("argument should be the filename to verify") - } - return verifySnapshot(ctx.Args()[0]) -} - -// verifySnapshot constructs a simulation, uploads a snapshot from -// a file with provided filename and validates that kademlia is healthy. -func verifySnapshot(filename string) (err error) { - sim := simulation.New(map[string]simulation.ServiceFunc{ - "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { - addr := network.NewAddr(ctx.Config.Node()) - kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - hp := network.NewHiveParams() - hp.KeepAliveInterval = time.Duration(200) * time.Millisecond - hp.Discovery = false // discovery must be false when uploading a snapshot - - config := &network.BzzConfig{ - OverlayAddr: addr.Over(), - UnderlayAddr: addr.Under(), - HiveParams: hp, - } - return network.NewBzz(config, kad, nil, nil, nil), nil, nil - }, - }) - defer sim.Close() - - err = sim.UploadSnapshot(filename) - if err != nil { - return fmt.Errorf("upload snapshot: %v", err) - } - - ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancelSimRun() - - _, err = sim.WaitTillHealthy(ctx, 2) - if err != nil { - return fmt.Errorf("wait for healthy kademlia: %v", err) - } - - return nil -} diff --git a/cmd/swarm/swarm-snapshot/verify_test.go b/cmd/swarm/swarm-snapshot/verify_test.go deleted file mode 100644 index f8c615bf84eb..000000000000 --- a/cmd/swarm/swarm-snapshot/verify_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "testing" -) - -// TestSnapshotVerify executes "verify" command with a -// pregenerated snapshot file that is valid. -func TestSnapshotVerify(t *testing.T) { - snap := runSnapshot(t, - "verify", - "testdata/snapshot.json", - ) - - snap.ExpectExit() - if code := snap.ExitStatus(); code != 0 { - t.Fatalf("command exit code %v, expected 0", code) - } -} diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index 59c276f30c38..f8b1b43c0119 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -227,7 +227,7 @@ func (tt *TestCmd) Kill() { } func (tt *TestCmd) withKillTimeout(fn func()) { - timeout := time.AfterFunc(15*time.Second, func() { + timeout := time.AfterFunc(5*time.Second, func() { tt.Log("killing the child process (timeout)") tt.Kill() }) From b866ba97b86e59efad42cfdc63d92ca449075396 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Tue, 15 Jan 2019 15:21:57 +0530 Subject: [PATCH 10/11] swarm/network/simulations/discovery: removed snapshot write to file --- .../simulations/discovery/discovery_test.go | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 75f34930df77..352ee20ede23 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -18,8 +18,6 @@ package discovery import ( "context" - "encoding/json" - "errors" "flag" "fmt" "io/ioutil" @@ -86,7 +84,6 @@ func getDbStore(nodeID string) (*state.DBStore, error) { var ( nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") - snapshotFile = flag.String("snapshot", "", "path to create snapshot file in") loglevel = flag.Int("loglevel", 3, "verbosity of logs") rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery") @@ -296,40 +293,6 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul if result.Error != nil { return result, nil } - - if *snapshotFile != "" { - var err error - var snap *simulations.Snapshot - if len(*serviceOverride) > 0 { - var addServices []string - var removeServices []string - for _, osvc := range strings.Split(*serviceOverride, ",") { - if strings.Index(osvc, "+") == 0 { - addServices = append(addServices, osvc[1:]) - } else if strings.Index(osvc, "-") == 0 { - removeServices = append(removeServices, osvc[1:]) - } else { - panic("stick to the rules, you know what they are") - } - } - snap, err = net.SnapshotWithServices(addServices, removeServices) - } else { - snap, err = net.Snapshot() - } - - if err != nil { - return nil, errors.New("no shapshot dude") - } - jsonsnapshot, err := json.Marshal(snap) - if err != nil { - return nil, fmt.Errorf("corrupt json snapshot: %v", err) - } - log.Info("writing snapshot", "file", *snapshotFile) - err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, 0755) - if err != nil { - return nil, err - } - } return result, nil } From e37ade067191f05408d58c8ab3285e7d15a272f9 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Wed, 16 Jan 2019 13:38:02 +0530 Subject: [PATCH 11/11] cmd/swarm/swarm-snapshot, swarm/network/simulations: removed redundant connection event check, fixed lint error --- cmd/swarm/swarm-snapshot/create.go | 16 ---------------- .../simulations/discovery/discovery_test.go | 9 ++++----- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go index eab46bf377a3..127fde8ae245 100644 --- a/cmd/swarm/swarm-snapshot/create.go +++ b/cmd/swarm/swarm-snapshot/create.go @@ -38,8 +38,6 @@ import ( cli "gopkg.in/urfave/cli.v1" ) -const noConnectionTimeout = 1 * time.Second - // create is used as the entry function for "create" app command. func create(ctx *cli.Context) error { log.PrintOrigins(true) @@ -83,20 +81,6 @@ func createSnapshot(filename string, nodes int, services []string) (err error) { return fmt.Errorf("add nodes: %v", err) } - // wait for two some time to ensure no connections - // are established - events := make(chan *simulations.Event) - sub := sim.Net.Events().Subscribe(events) - select { - case ev := <-events: - //only catch node up events - if ev.Type == simulations.EventTypeConn { - return errors.New("unexpected connection events") - } - case <-time.After(noConnectionTimeout): - } - sub.Unsubscribe() - err = sim.Net.ConnectNodesRing(nil) if err != nil { return fmt.Errorf("connect nodes: %v", err) diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 352ee20ede23..7d03789870c5 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -82,11 +82,10 @@ func getDbStore(nodeID string) (*state.DBStore, error) { } var ( - nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") - initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") - loglevel = flag.Int("loglevel", 3, "verbosity of logs") - rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") - serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery") + nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") + initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") + loglevel = flag.Int("loglevel", 3, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") ) func init() {