From 443177148da88cffdb4faed6aaf73a1c02d9b89d Mon Sep 17 00:00:00 2001 From: Miao Luo Date: Tue, 31 Oct 2017 00:20:00 -0700 Subject: [PATCH] vFile: handle swarm node promotion and demotion (#1868) When a node is promoted from worker to manager, the helper thread will join ETCD cluster according to swarm information; On the other hand, when the node is demoted from manager to worker, the helper thread should stop the watcher, delete itself from ETCD member list, and clean up the ETCD data directory. --- .../drivers/vfile/kvstore/etcdops/etcdops.go | 212 +++++++++++++++--- misc/scripts/build.sh | 2 +- tests/constants/dockercli/cmd.go | 9 + tests/e2e/vfile_demote_promote_test.go | 184 +++++++++++++++ tests/utils/dockercli/swarm.go | 57 +++++ tests/utils/verification/volumeproperties.go | 7 +- 6 files changed, 437 insertions(+), 34 deletions(-) create mode 100644 tests/e2e/vfile_demote_promote_test.go diff --git a/client_plugin/drivers/vfile/kvstore/etcdops/etcdops.go b/client_plugin/drivers/vfile/kvstore/etcdops/etcdops.go index 07694e696..04f934294 100644 --- a/client_plugin/drivers/vfile/kvstore/etcdops/etcdops.go +++ b/client_plugin/drivers/vfile/kvstore/etcdops/etcdops.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "os/exec" "strconv" "strings" @@ -38,6 +39,7 @@ import ( etcdClusterToken: ID of the cluster to create/join etcdListenURL: etcd listening interface etcdScheme: Protocol used for communication + etcdDataDir: Data directory for ETCD etcdClusterStateNew: Used to indicate the formation of a new cluster etcdClusterStateExisting: Used to indicate that this node is joining @@ -58,12 +60,13 @@ const ( etcdClusterToken = "vfile-etcd-cluster" etcdListenURL = "0.0.0.0" etcdScheme = "http://" + etcdDataDir = "/etcd-data" etcdClusterStateNew = "new" etcdClusterStateExisting = "existing" etcdRequestTimeout = 2 * time.Second etcdUpdateTimeout = 10 * time.Second checkSleepDuration = time.Second - gcTicker = 30 * time.Second + gcTicker = 15 * time.Second etcdClientCreateError = "Failed to create etcd client" swarmUnhealthyErrorMsg = "Swarm cluster maybe unhealthy" etcdSingleRef = "1" @@ -74,6 +77,13 @@ type EtcdKVS struct { dockerOps *dockerops.DockerOps nodeID string nodeAddr string + // isManager indicates if current node is a manager or not + isManager bool + // etcdCMD records the cmd struct for ETCD process + // it's used for stopping ETCD process when node is demoted + etcdCMD *exec.Cmd + // watcher is used for killing the watch request when node is demoted + watcher *etcdClient.Client } // VFileVolConnectivityData - Contains metadata of vFile volumes @@ -101,12 +111,15 @@ func NewKvStore(dockerOps *dockerops.DockerOps) *EtcdKVS { dockerOps: dockerOps, nodeID: nodeID, nodeAddr: addr, + isManager: isManager, } if !isManager { log.WithFields( log.Fields{"nodeID": nodeID}, ).Info("Swarm node role: worker. Return from NewKvStore ") + // start helper before return + go e.etcdHelper() return e } @@ -135,21 +148,13 @@ func NewKvStore(dockerOps *dockerops.DockerOps) *EtcdKVS { ).Error("Failed to start ETCD Cluster ") return nil } + // start helper before return + go e.etcdHelper() return e } - // if manager, first find out who's leader, then proceed to join ETCD cluster - leaderAddr, err := dockerOps.GetSwarmLeader() - if err != nil { - log.WithFields( - log.Fields{ - "nodeID": nodeID, - "error": err}, - ).Error("Failed to get swarm leader address ") - return nil - } - - err = e.joinEtcdCluster(leaderAddr) + // if manager, join ETCD cluster + err = e.joinEtcdCluster() if err != nil { log.WithFields(log.Fields{ "nodeID": nodeID, @@ -157,6 +162,8 @@ func NewKvStore(dockerOps *dockerops.DockerOps) *EtcdKVS { ).Error("Failed to join ETCD Cluster") return nil } + // start helper before return + go e.etcdHelper() return e } @@ -167,6 +174,7 @@ func (e *EtcdKVS) startEtcdCluster() error { log.Infof("startEtcdCluster on node with nodeID %s and nodeAddr %s", nodeID, nodeAddr) lines := []string{ "--name", nodeID, + "--data-dir", etcdDataDir, "--advertise-client-urls", etcdScheme + nodeAddr + etcdClientPort, "--initial-advertise-peer-urls", etcdScheme + nodeAddr + etcdPeerPort, "--listen-client-urls", etcdScheme + etcdListenURL + etcdClientPort, @@ -177,17 +185,27 @@ func (e *EtcdKVS) startEtcdCluster() error { } // start the routine to create an etcd cluster - go etcdService(lines) + e.etcdStartService(lines) // check if etcd cluster is successfully started, then start the watcher return e.checkLocalEtcd() } // joinEtcdCluster function is called by a non-leader swarm manager to join a ETCD cluster -func (e *EtcdKVS) joinEtcdCluster(leaderAddr string) error { +func (e *EtcdKVS) joinEtcdCluster() error { nodeAddr := e.nodeAddr nodeID := e.nodeID - log.Infof("joinEtcdCluster on node with nodeID %s and nodeAddr %s leaderAddr %s", nodeID, nodeAddr, leaderAddr) + log.Infof("joinEtcdCluster on node with nodeID %s and nodeAddr %s", nodeID, nodeAddr) + + leaderAddr, err := e.dockerOps.GetSwarmLeader() + if err != nil { + log.WithFields( + log.Fields{ + "nodeID": nodeID, + "error": err}, + ).Error("Failed to get swarm leader address ") + return err + } etcd, err := addrToEtcdClient(leaderAddr) if err != nil { @@ -196,6 +214,7 @@ func (e *EtcdKVS) joinEtcdCluster(leaderAddr string) error { "leaderAddr": leaderAddr, "nodeID": nodeID}, ).Error("Failed to join ETCD cluster on manager ") + return err } defer etcd.Close() @@ -280,6 +299,7 @@ func (e *EtcdKVS) joinEtcdCluster(leaderAddr string) error { lines := []string{ "--name", nodeID, + "--data-dir", etcdDataDir, "--advertise-client-urls", etcdScheme + nodeAddr + etcdClientPort, "--initial-advertise-peer-urls", etcdScheme + nodeAddr + etcdPeerPort, "--listen-client-urls", etcdScheme + etcdListenURL + etcdClientPort, @@ -290,20 +310,104 @@ func (e *EtcdKVS) joinEtcdCluster(leaderAddr string) error { } // start the routine for joining an etcd cluster - go etcdService(lines) + e.etcdStartService(lines) // check if successfully joined the etcd cluster, then start the watcher return e.checkLocalEtcd() } -// etcdService function starts a routine of etcd -func etcdService(cmd []string) { - _, err := exec.Command("/bin/etcd", cmd...).Output() +// leaveEtcdCluster function is called when a manager is demoted +func (e *EtcdKVS) leaveEtcdCluster() error { + etcd, err := addrToEtcdClient(e.nodeAddr) + if err != nil { + log.WithFields( + log.Fields{"nodeAddr": e.nodeAddr, + "nodeID": e.nodeID}, + ).Error("Failed to create ETCD client from own address") + return err + } + defer etcd.Close() + + // list all current ETCD members + ctx, cancel := context.WithTimeout(context.Background(), etcdRequestTimeout) + lresp, err := etcd.MemberList(ctx) + cancel() + if err != nil { + log.WithFields( + log.Fields{"nodeAddr": e.nodeAddr, + "error": err}, + ).Error("Failed to list member for ETCD") + return err + } + + // create the peer URL for filtering ETCD member information + // each ETCD member has a unique peer URL + peerAddr := etcdScheme + e.nodeAddr + etcdPeerPort + for _, member := range lresp.Members { + // loop all current etcd members to find if there is already a member with the same peerAddr + if member.PeerURLs[0] == peerAddr { + log.WithFields( + log.Fields{"nodeID": e.nodeID, + "peerAddr": peerAddr}, + ).Info("Remove self from ETCD member due to demotion. ") + + ctx, cancel = context.WithTimeout(context.Background(), etcdRequestTimeout) + _, err = etcd.MemberRemove(ctx, member.ID) + cancel() + if err != nil { + log.WithFields( + log.Fields{"peerAddr": peerAddr, + "member.ID": member.ID}, + ).Error("Failed to remove this node from ETCD ") + return err + } + + // the same peerAddr can only join at once. no need to continue. + log.WithFields( + log.Fields{"peerAddr": peerAddr, + "member.ID": member.ID}, + ).Info("Successfully removed self from ETCD ") + break + } + } + + e.etcdStopService() + return nil +} + +// etcdStartService function starts an ETCD process +func (e *EtcdKVS) etcdStartService(lines []string) { + cmd := exec.Command("/bin/etcd", lines...) + err := cmd.Start() if err != nil { log.WithFields( log.Fields{"error": err, "cmd": cmd}, ).Error("Failed to start ETCD command ") + return + } + + e.etcdCMD = cmd +} + +// etcdStopService function stops the ETCD process +func (e *EtcdKVS) etcdStopService() { + // stop watcher + e.watcher.Close() + + // stop ETCD process + if err := e.etcdCMD.Process.Kill(); err != nil { + log.Errorf("Failed to stop ETCD process. Error: %v", err) + return + } + + // clean up ETCD data + if err := os.RemoveAll(etcdDataDir); err != nil { + log.Errorf("Failed to remove ETCD data directory. Error: %v", err) + return } + + log.Infof("Stopped ETCD service due to demotion") + e.etcdCMD = nil } // checkLocalEtcd function check if local ETCD endpoint is successfully started or not @@ -325,8 +429,9 @@ func (e *EtcdKVS) checkLocalEtcd() error { "error": err}, ).Warningf("Failed to get ETCD client, retry before timeout ") } else { + log.Infof("Local ETCD client is up successfully, start watcher") + e.watcher = cli go e.etcdWatcher(cli) - go e.serviceAndVolumeGC(cli) return nil } case <-timer.C: @@ -337,7 +442,6 @@ func (e *EtcdKVS) checkLocalEtcd() error { // etcdWatcher function sets up a watcher to monitor all the changes to global refcounts in the KV store func (e *EtcdKVS) etcdWatcher(cli *etcdClient.Client) { - // TODO: when the manager is demoted to worker, the watcher should be cancelled watchCh := cli.Watch(context.Background(), kvstore.VolPrefixGRef, etcdClient.WithPrefix(), etcdClient.WithPrevKV()) for wresp := range watchCh { @@ -347,20 +451,30 @@ func (e *EtcdKVS) etcdWatcher(cli *etcdClient.Client) { } } -// serviceAndVolumeGC: garbage collector for orphan services or volumes -func (e *EtcdKVS) serviceAndVolumeGC(cli *etcdClient.Client) { +// etcdHelper: a helper thread which does the following tasks with time interval +// 1. clean up orphan services or orphan internal volumes +// 2. monitor the role of the node, start/shutdown etcd service accordingly +func (e *EtcdKVS) etcdHelper() { ticker := time.NewTicker(gcTicker) quit := make(chan struct{}) for { select { case <-ticker.C: - // find all the vFile volume services - volumesToVerify, err := e.dockerOps.ListVolumesFromServices() + // check the role of this node + err := e.etcdRoleCheck() if err != nil { - log.Warningf("Failed to get vFile volumes according to docker services") - } else { - e.cleanOrphanService(volumesToVerify) + log.Warningf("Failed to do role check") + } + + if e.isManager { + // find all the vFile volume services + volumesToVerify, err := e.dockerOps.ListVolumesFromServices() + if err != nil { + log.Warningf("Failed to get vFile volumes according to docker services") + } else { + e.cleanOrphanService(volumesToVerify) + } } case <-quit: ticker.Stop() @@ -369,6 +483,46 @@ func (e *EtcdKVS) serviceAndVolumeGC(cli *etcdClient.Client) { } } +func (e *EtcdKVS) etcdRoleCheck() error { + nodeID, _, isManager, err := e.dockerOps.GetSwarmInfo() + if err != nil { + log.WithFields( + log.Fields{"error": err}, + ).Error("Failed to get swarm Info from docker client ") + return err + } + + if isManager { + if !e.isManager { + log.Infof("Node is promoted to manager, prepare to join ETCD cluster") + err = e.joinEtcdCluster() + if err != nil { + log.WithFields(log.Fields{ + "nodeID": nodeID, + "error": err}, + ).Error("Failed to join ETCD Cluster in etcdRoleCheck") + return err + } + e.isManager = true + } + } else { + if e.isManager { + log.Infof("Node is demoted from manager to worker, prepare to leave ETCD cluster") + err = e.leaveEtcdCluster() + if err != nil { + log.WithFields(log.Fields{ + "nodeID": nodeID, + "error": err}, + ).Error("Failed to leave ETCD Cluster in etcdRoleCheck") + return err + } + e.isManager = false + } + } + + return nil +} + // cleanOrphanService: stop orphan services func (e *EtcdKVS) cleanOrphanService(volumesToVerify []string) { volStates, err := e.KvMapFromPrefix(string(kvstore.VolPrefixState)) diff --git a/misc/scripts/build.sh b/misc/scripts/build.sh index 3ff8a4803..0d0b1b91b 100755 --- a/misc/scripts/build.sh +++ b/misc/scripts/build.sh @@ -123,7 +123,7 @@ then $DOCKER run --rm -v $PWD/..:$dir -w $dir $pylint_container $MAKE_ESX pylint else docker_socket=/var/run/docker.sock - if [ -z $SSH_KEY_OPT ] + if [ -z '$SSH_KEY_OPT' ] then SSH_KEY_OPT="-i /root/.ssh/id_rsa" fi diff --git a/tests/constants/dockercli/cmd.go b/tests/constants/dockercli/cmd.go index fe81e2b83..05536c94a 100644 --- a/tests/constants/dockercli/cmd.go +++ b/tests/constants/dockercli/cmd.go @@ -88,6 +88,15 @@ const ( // ListNodes list all docker swarm nodes ListNodes = dockerNode + "ls " + // InspectNode inspects a swarm node + InspectNode = dockerNode + "inspect " + + // PromoteNode promotes a swarm worker to manager + PromoteNode = dockerNode + "promote " + + // DemoteNode demotes a swarm manager to worker + DemoteNode = dockerNode + "demote " + // CreateService create a docker service CreateService = dockerService + "create " diff --git a/tests/e2e/vfile_demote_promote_test.go b/tests/e2e/vfile_demote_promote_test.go new file mode 100644 index 000000000..29873e26e --- /dev/null +++ b/tests/e2e/vfile_demote_promote_test.go @@ -0,0 +1,184 @@ +// Copyright 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This test suite includes test cases to verify basic functionality +// before and after swarm node demote/promote operation + +// +build runoncevfile + +package e2e + +import ( + "log" + "strconv" + "time" + + "github.com/vmware/docker-volume-vsphere/tests/utils/dockercli" + "github.com/vmware/docker-volume-vsphere/tests/utils/inputparams" + "github.com/vmware/docker-volume-vsphere/tests/utils/misc" + "github.com/vmware/docker-volume-vsphere/tests/utils/verification" + . "gopkg.in/check.v1" +) + +// vFile Demote/Promote test: +// This test is used to check vFile volume functionality after swarm node role change. +// In this test, we first create a vFile volume before the role change. +// Then a worker node is promoted to manager, and the original manager is demoted to worker. +// After the role change, we create another vFile volume. +// At last, we attach to both of the two vFile volumes to verify their functionality. + +type VFileDemotePromoteTestSuite struct { + config *inputparams.TestConfig + esx string + master string + worker1 string + volName1 string + volName2 string + container1Name string + container2Name string +} + +func (s *VFileDemotePromoteTestSuite) SetUpSuite(c *C) { + s.config = inputparams.GetTestConfig() + if s.config == nil { + c.Skip("Unable to retrieve test config, skipping basic vfile tests") + } + + s.esx = s.config.EsxHost + s.master = inputparams.GetSwarmManager1() + s.worker1 = inputparams.GetSwarmWorker1() +} + +func (s *VFileDemotePromoteTestSuite) SetUpTest(c *C) { + s.volName1 = inputparams.GetVFileVolumeName() + s.volName2 = inputparams.GetVFileVolumeName() + s.container1Name = inputparams.GetUniqueContainerName(c.TestName()) + s.container2Name = inputparams.GetUniqueContainerName(c.TestName()) +} + +var _ = Suite(&VFileDemotePromoteTestSuite{}) + +// All VMs are created in a shared datastore +// Test steps: +// 1. Create the 1st volume on worker1 +// 2. Verify the 1st volume is available +// 3. Attach the 1st volume on worker1 +// 4. Promote worker1 to manager +// 5. Sleep 20 seconds +// 6. Demote original manager to worker +// 7. Sleep 20 seconds +// 8. Create the 2nd volume on new worker (s.master) +// 9. Verify the 2nd volume is available +// 10. Attach the 2nd volume on new worker (s.master) +// 11. Verify the global refcounts of the two volumes are 1 +// 12. Remove the both containers +// 14. Verify the global refcounts of the two volumes are back to 0 +// 15. Verify status of both volumes are detached +// 16. Remove two volumes +// 17. Reset swarm roles +func (s *VFileDemotePromoteTestSuite) TestSwarmRoleChange(c *C) { + misc.LogTestStart(c.TestName()) + + out, err := dockercli.CreateVFileVolume(s.worker1, s.volName1) + c.Assert(err, IsNil, Commentf(out)) + + accessible := verification.CheckVolumeAvailability(s.master, s.volName1) + c.Assert(accessible, Equals, true, Commentf("Volume %s is not available", s.volName2)) + + out, err = dockercli.AttachVFileVolume(s.worker1, s.volName1, s.container1Name) + c.Assert(err, IsNil, Commentf(out)) + + err = dockercli.PromoteNode(s.master, s.worker1) + c.Assert(err, IsNil, Commentf("Failed to promote worker1 %s to manager", s.worker1)) + + log.Printf("Wait 20 seconds for new manager to be updated") + time.Sleep(20 * time.Second) + + err = dockercli.DemoteNode(s.worker1, s.master) + c.Assert(err, IsNil, Commentf("Failed to demote manager %s", s.master)) + + log.Printf("Wait 20 seconds for new worker to be updated") + time.Sleep(20 * time.Second) + + out, err = dockercli.CreateVFileVolume(s.master, s.volName2) + c.Assert(err, IsNil, Commentf(out)) + + accessible = verification.CheckVolumeAvailability(s.worker1, s.volName2) + c.Assert(accessible, Equals, true, Commentf("Volume %s is not available", s.volName1)) + + out, err = dockercli.AttachVFileVolume(s.master, s.volName2, s.container2Name) + c.Assert(err, IsNil, Commentf(out)) + + out = verification.GetVFileVolumeGlobalRefcount(s.volName1, s.master) + grefc, _ := strconv.Atoi(out) + c.Assert(grefc, Equals, 1, Commentf("Expected volume %s global refcount to be 1, found %s", s.volName1, out)) + + out = verification.GetVFileVolumeGlobalRefcount(s.volName2, s.worker1) + grefc, _ = strconv.Atoi(out) + c.Assert(grefc, Equals, 1, Commentf("Expected volume %s global refcount to be 1, found %s", s.volName2, out)) + + out, err = dockercli.RemoveContainer(s.worker1, s.container1Name) + c.Assert(err, IsNil, Commentf(out)) + + out = verification.GetVFileVolumeGlobalRefcount(s.volName1, s.master) + grefc, _ = strconv.Atoi(out) + c.Assert(grefc, Equals, 0, Commentf("Expected volume %s global refcount to be 0, found %s", s.volName1, out)) + + out, err = dockercli.RemoveContainer(s.master, s.container2Name) + c.Assert(err, IsNil, Commentf(out)) + + out = verification.GetVFileVolumeGlobalRefcount(s.volName2, s.worker1) + grefc, _ = strconv.Atoi(out) + c.Assert(grefc, Equals, 0, Commentf("Expected volume %s global refcount to be 0, found %s", s.volName2, out)) + + log.Printf("Wait 20 seconds for volume status back to Ready") + time.Sleep(20 * time.Second) + + out = verification.GetVFileVolumeStatusHost(s.volName1, s.master) + log.Println("GetVFileVolumeStatusHost return out[%s] for volume %s", out, s.volName1) + c.Assert(out, Equals, "Ready", Commentf("Volume %s status is expected to be [Ready], actual status is [%s]", + s.volName1, out)) + + out = verification.GetVFileVolumeStatusHost(s.volName2, s.worker1) + log.Println("GetVFileVolumeStatusHost return out[%s] for volume %s", out, s.volName2) + c.Assert(out, Equals, "Ready", Commentf("Volume %s status is expected to be [Ready], actual status is [%s]", + s.volName2, out)) + + accessible = verification.CheckVolumeAvailability(s.master, s.volName1) + c.Assert(accessible, Equals, true, Commentf("Volume %s is not available", s.volName1)) + + accessible = verification.CheckVolumeAvailability(s.worker1, s.volName2) + c.Assert(accessible, Equals, true, Commentf("Volume %s is not available", s.volName2)) + + out, err = dockercli.DeleteVolume(s.master, s.volName1) + c.Assert(err, IsNil, Commentf(out)) + + out, err = dockercli.DeleteVolume(s.worker1, s.volName2) + c.Assert(err, IsNil, Commentf(out)) + + log.Println("Finished swarm promote/demote test for vFile, start to reset the testbed swarm roles...") + err = dockercli.PromoteNode(s.worker1, s.master) + c.Assert(err, IsNil, Commentf("Failed to reset manager role for %s ", s.master)) + + log.Printf("Wait 20 seconds for original manager to be updated") + time.Sleep(20 * time.Second) + + err = dockercli.DemoteNode(s.master, s.worker1) + c.Assert(err, IsNil, Commentf("Failed to reset worker role for %s", s.worker1)) + + log.Printf("Wait 20 seconds for original worker to be updated") + time.Sleep(20 * time.Second) + + misc.LogTestEnd(c.TestName()) +} diff --git a/tests/utils/dockercli/swarm.go b/tests/utils/dockercli/swarm.go index 28159e56a..021b5b937 100644 --- a/tests/utils/dockercli/swarm.go +++ b/tests/utils/dockercli/swarm.go @@ -17,8 +17,11 @@ package dockercli import ( + "fmt" "log" + "net" "strconv" + "strings" "github.com/vmware/docker-volume-vsphere/tests/constants/dockercli" "github.com/vmware/docker-volume-vsphere/tests/utils/ssh" @@ -83,3 +86,57 @@ func GetAllContainers(hostName, filter string) (string, error) { cmd := dockercli.ListContainers + "--filter name='" + filter + "' --format '{{.Names}}'" return ssh.InvokeCommand(hostName, cmd) } + +// PromoteNode promotes a worker node to manager node +func PromoteNode(hostIP, targetIP string) error { + log.Printf("Promoting worker node IP %s to manager", targetIP) + out, err := ssh.InvokeCommand(hostIP, dockercli.ListNodes+"-q") + if err != nil { + return err + } + + IDs := strings.Split(out, "\n") + for _, nodeID := range IDs { + nodeIP, err := ssh.InvokeCommand(hostIP, dockercli.InspectNode+nodeID+" --format \"{{.Status.Addr}}\"") + if err != nil { + return err + } + if nodeIP == targetIP { + log.Printf("Found the ID of target IP is %s", nodeID) + _, err = ssh.InvokeCommand(hostIP, dockercli.PromoteNode+nodeID) + return err + } + } + + err = fmt.Errorf("No node from 'docker node ls' matches with target IP") + return err +} + +// DemoteNode demotes a manager node to worker node +func DemoteNode(hostIP, targetIP string) error { + log.Printf("Demoting manager node ID %s to worker", targetIP) + out, err := ssh.InvokeCommand(hostIP, dockercli.ListNodes+"-q") + if err != nil { + return err + } + + IDs := strings.Split(out, "\n") + for _, nodeID := range IDs { + nodeIP, err := ssh.InvokeCommand(hostIP, dockercli.InspectNode+nodeID+" --format \"{{.ManagerStatus.Addr}}\"") + if err != nil { + continue + } + nodeIP, _, err = net.SplitHostPort(nodeIP) + if err != nil { + continue + } + if nodeIP == targetIP { + log.Printf("Found the ID of target IP is %s", nodeID) + _, err = ssh.InvokeCommand(hostIP, dockercli.DemoteNode+nodeID) + return err + } + } + + err = fmt.Errorf("No node from 'docker node ls' matches with target IP") + return err +} diff --git a/tests/utils/verification/volumeproperties.go b/tests/utils/verification/volumeproperties.go index 1004dda8d..eff53136d 100644 --- a/tests/utils/verification/volumeproperties.go +++ b/tests/utils/verification/volumeproperties.go @@ -146,7 +146,7 @@ func getVolumeStatusHost(name, hostName string) string { // vfile driver func GetVFileVolumeStatusHost(name, hostName string) string { cmd := dockercli.InspectVolume + " --format \"{{index .Status \\\"Volume Status\\\"}}\" " + name - log.Printf("GetVFileVolumeStatusHost: cmd[]", cmd) + log.Printf("Check the volume status of vFile volume %s", name) out, _ := ssh.InvokeCommand(hostName, cmd) return out } @@ -160,9 +160,8 @@ func GetVFileVolumeStatusHost(name, hostName string) string { // string: String containing global refcount func GetVFileVolumeGlobalRefcount(name, hostName string) string { cmd := dockercli.InspectVolume + - " --format \"{{index .Status \\\"Global Refcount\\\"}}\" " + - name - log.Printf("GetVFileVolumeGlobalRefcount: cmd[]", cmd) + " --format \"{{index .Status \\\"Global Refcount\\\"}}\" " + name + log.Printf("Check the global refcount of vFile volume %s", name) out, _ := ssh.InvokeCommand(hostName, cmd) return out }