From 5454bb9982a91461a0f42526d036ca1746af7838 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Tue, 13 Nov 2018 21:23:07 -0800 Subject: [PATCH 1/3] etcdserver/*: add "etcd_cluster_version" metric Signed-off-by: Gyuho Lee --- etcdserver/membership/cluster.go | 4 +++- etcdserver/membership/metrics.go | 31 +++++++++++++++++++++++++++++++ etcdserver/server.go | 1 + 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 etcdserver/membership/metrics.go diff --git a/etcdserver/membership/cluster.go b/etcdserver/membership/cluster.go index 2330219f18a..8485b0081a9 100644 --- a/etcdserver/membership/cluster.go +++ b/etcdserver/membership/cluster.go @@ -36,6 +36,7 @@ import ( "github.com/coreos/etcd/store" "github.com/coreos/etcd/version" "github.com/coreos/go-semver/semver" + "github.com/prometheus/client_golang/prometheus" ) // RaftCluster is a list of Members that belong to the same raft cluster @@ -376,7 +377,8 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version if c.be != nil { mustSaveClusterVersionToBackend(c.be, ver) } - onSet(ver) + ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": ver.String()}).Set(1) + onSet(ver) } func (c *RaftCluster) IsReadyToAddNewMember() bool { diff --git a/etcdserver/membership/metrics.go b/etcdserver/membership/metrics.go new file mode 100644 index 00000000000..b3212bc80cd --- /dev/null +++ b/etcdserver/membership/metrics.go @@ -0,0 +1,31 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package membership + +import "github.com/prometheus/client_golang/prometheus" + +var ( + ClusterVersionMetrics = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "etcd", + Subsystem: "cluster", + Name: "version", + Help: "Which version is running. 1 for 'cluster_version' label with current cluster version", + }, + []string{"cluster_version"}) +) + +func init() { + prometheus.MustRegister(ClusterVersionMetrics) +} diff --git a/etcdserver/server.go b/etcdserver/server.go index 73b9fa1077e..48e7baf7c36 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -591,6 +591,7 @@ func (s *EtcdServer) start() { s.readNotifier = newNotifier() if s.ClusterVersion() != nil { plog.Infof("starting server... [version: %v, cluster version: %v]", version.Version, version.Cluster(s.ClusterVersion().String())) + membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": s.ClusterVersion().String()}).Set(1) } else { plog.Infof("starting server... [version: %v, cluster version: to_be_decided]", version.Version) } From a038dc6464f2e4f79bcd5ae0af22907408b8d69f Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Tue, 13 Nov 2018 21:48:38 -0800 Subject: [PATCH 2/3] tests/e2e: test cluster version Signed-off-by: Gyuho Lee --- e2e/etcd_release_upgrade_test.go | 9 +++++ tests/e2e/metrics_test.go | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/e2e/metrics_test.go diff --git a/e2e/etcd_release_upgrade_test.go b/e2e/etcd_release_upgrade_test.go index af958e89213..066feac6b2a 100644 --- a/e2e/etcd_release_upgrade_test.go +++ b/e2e/etcd_release_upgrade_test.go @@ -101,6 +101,15 @@ func TestReleaseUpgrade(t *testing.T) { } } } + + // expect upgraded cluster version + ver := version.Version + if strings.HasSuffix(ver, "+git") { + ver = strings.Replace(ver, "+git", "", 1) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, ver), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } } func TestReleaseUpgradeWithRestart(t *testing.T) { diff --git a/tests/e2e/metrics_test.go b/tests/e2e/metrics_test.go new file mode 100644 index 00000000000..7a32f003147 --- /dev/null +++ b/tests/e2e/metrics_test.go @@ -0,0 +1,59 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "strings" + "testing" + + "go.etcd.io/etcd/version" +) + +func TestV3MetricsSecure(t *testing.T) { + cfg := configTLS + cfg.clusterSize = 1 + cfg.metricsURLScheme = "https" + testCtl(t, metricsTest) +} + +func TestV3MetricsInsecure(t *testing.T) { + cfg := configTLS + cfg.clusterSize = 1 + cfg.metricsURLScheme = "http" + testCtl(t, metricsTest) +} + +func metricsTest(cx ctlCtx) { + if err := ctlV3Put(cx, "k", "v", ""); err != nil { + cx.t.Fatal(err) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: `etcd_debugging_mvcc_keys_total 1`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } + ver := version.Version + if strings.HasSuffix(ver, "+git") { + ver = strings.Replace(ver, "+git", "", 1) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, ver), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health":"true"}`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } +} From fc5f94144b4416e9d5dea222efd22f21d5d74e58 Mon Sep 17 00:00:00 2001 From: Wenjia Zhang Date: Wed, 16 Oct 2019 13:27:07 -0700 Subject: [PATCH 3/3] Add cluster version fix #11233, #11254, #11265 --- e2e/etcd_release_upgrade_test.go | 6 +----- etcdserver/membership/cluster.go | 8 ++++++-- etcdserver/server.go | 2 +- tests/e2e/metrics_test.go | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/e2e/etcd_release_upgrade_test.go b/e2e/etcd_release_upgrade_test.go index 066feac6b2a..3ece2f8c726 100644 --- a/e2e/etcd_release_upgrade_test.go +++ b/e2e/etcd_release_upgrade_test.go @@ -103,11 +103,7 @@ func TestReleaseUpgrade(t *testing.T) { } // expect upgraded cluster version - ver := version.Version - if strings.HasSuffix(ver, "+git") { - ver = strings.Replace(ver, "+git", "", 1) - } - if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, ver), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { cx.t.Fatalf("failed get with curl (%v)", err) } } diff --git a/etcdserver/membership/cluster.go b/etcdserver/membership/cluster.go index 8485b0081a9..8e175153085 100644 --- a/etcdserver/membership/cluster.go +++ b/etcdserver/membership/cluster.go @@ -369,6 +369,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version } else { plog.Noticef("set the initial cluster version to %v", version.Cluster(ver.String())) } + oldVer := c.version c.version = ver mustDetectDowngrade(c.version) if c.store != nil { @@ -377,8 +378,11 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version if c.be != nil { mustSaveClusterVersionToBackend(c.be, ver) } - ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": ver.String()}).Set(1) - onSet(ver) + if oldVer != nil { + ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(oldVer.String())}).Set(0) + } + ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(ver.String())}).Set(1) + onSet(ver) } func (c *RaftCluster) IsReadyToAddNewMember() bool { diff --git a/etcdserver/server.go b/etcdserver/server.go index 48e7baf7c36..9d28bd61d50 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -591,7 +591,7 @@ func (s *EtcdServer) start() { s.readNotifier = newNotifier() if s.ClusterVersion() != nil { plog.Infof("starting server... [version: %v, cluster version: %v]", version.Version, version.Cluster(s.ClusterVersion().String())) - membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": s.ClusterVersion().String()}).Set(1) + membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(s.ClusterVersion().String())}).Set(1) } else { plog.Infof("starting server... [version: %v, cluster version: to_be_decided]", version.Version) } diff --git a/tests/e2e/metrics_test.go b/tests/e2e/metrics_test.go index 7a32f003147..aade2718f15 100644 --- a/tests/e2e/metrics_test.go +++ b/tests/e2e/metrics_test.go @@ -19,7 +19,7 @@ import ( "strings" "testing" - "go.etcd.io/etcd/version" + "github.com/coreos/etcd/version" ) func TestV3MetricsSecure(t *testing.T) { @@ -50,7 +50,7 @@ func metricsTest(cx ctlCtx) { if strings.HasSuffix(ver, "+git") { ver = strings.Replace(ver, "+git", "", 1) } - if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, ver), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { cx.t.Fatalf("failed get with curl (%v)", err) } if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health":"true"}`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {