From 2165cd0702879f9e5bcaa4f8df857240a8faa968 Mon Sep 17 00:00:00 2001 From: Basit Hasan Date: Sun, 6 Aug 2023 23:22:07 +0530 Subject: [PATCH 1/4] add leader election mechanism Signed-off-by: Basit Hasan --- cmd/controller/main.go | 78 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index f9f3aa0e1..4ee2775c3 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -4,18 +4,72 @@ package main import ( + "context" "flag" - "os" - "time" - + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/klog/v2" + "os" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "time" ) // Version of kapp-controller is set via ldflags at build-time from the most recent git tag; see hack/build.sh var Version = "develop" +var ( + client *clientset.Clientset +) + +func getNewLock(lockname, podname, namespace string) *resourcelock.LeaseLock { + return &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: lockname, + Namespace: namespace, + }, + Client: client.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: podname, + }, + } +} + +func runLeaderElection(ctx context.Context, lock *resourcelock.LeaseLock, podname string, ctrlOpts Options, log logr.Logger) { + // Start the leader election for running kapp-controller + log.Info("Waiting for leader election") + leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ + Lock: lock, + ReleaseOnCancel: true, + LeaseDuration: 15 * time.Second, + RenewDeadline: 10 * time.Second, + RetryPeriod: 2 * time.Second, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(c context.Context) { + err := Run(ctrlOpts, log.WithName("controller")) + if err != nil { + klog.Errorf("Error while running as leader: %v", err) + } + }, + OnStoppedLeading: func() { + klog.Fatalf("no longer the leader, staying inactive.") + os.Exit(0) + }, + OnNewLeader: func(identity string) { + //Notify when a new leader is elected + if identity == podname { + return + } + klog.InfoS("new leader elected", "id", identity) + }, + }, + }) +} + func main() { ctrlOpts := Options{} var sidecarexec bool @@ -36,19 +90,27 @@ func main() { sidecarexecMain() return } - log := zap.New(zap.UseDevMode(false)).WithName("kc") logf.SetLogger(log) klog.SetLogger(log) - mainLog := log.WithName("main") mainLog.Info("kapp-controller", "version", Version) + var ( + leaseLockName string + leaseLockNamespace string + podName = os.Getenv("POD_NAME") + ) + config, err := rest.InClusterConfig() + client = clientset.NewForConfigOrDie(config) - err := Run(ctrlOpts, log.WithName("controller")) if err != nil { - mainLog.Error(err, "Exited run with error") - os.Exit(1) + klog.Fatalf("failed to get kubeconfig") } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + lock := getNewLock(leaseLockName, podName, leaseLockNamespace) + runLeaderElection(lock, ctx, podName, ctrlOpts, log) os.Exit(0) } From fb19b0d09f57c1a80f2b4c54e5ccab25b39bd2fb Mon Sep 17 00:00:00 2001 From: Basit Hasan Date: Sun, 27 Aug 2023 11:28:51 +0530 Subject: [PATCH 2/4] minor fix Signed-off-by: Basit Hasan --- cmd/controller/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 4ee2775c3..07aa82d2b 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -110,7 +110,7 @@ func main() { defer cancel() lock := getNewLock(leaseLockName, podName, leaseLockNamespace) - runLeaderElection(lock, ctx, podName, ctrlOpts, log) + runLeaderElection(ctx, lock, podName, ctrlOpts, log) os.Exit(0) } From 4f6755710729db65df2ba4dd8801a7ce5a04dedd Mon Sep 17 00:00:00 2001 From: Basit Hasan Date: Sun, 27 Aug 2023 11:42:10 +0530 Subject: [PATCH 3/4] move leader election to different file Signed-off-by: Basit Hasan --- cmd/controller/leader_election.go | 61 +++++++++++++++++++++++++++++++ cmd/controller/main.go | 52 -------------------------- 2 files changed, 61 insertions(+), 52 deletions(-) create mode 100644 cmd/controller/leader_election.go diff --git a/cmd/controller/leader_election.go b/cmd/controller/leader_election.go new file mode 100644 index 000000000..c43d2df3e --- /dev/null +++ b/cmd/controller/leader_election.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/klog/v2" + "os" + "time" +) + +var ( + client *clientset.Clientset +) + +func getNewLock(lockname, podname, namespace string) *resourcelock.LeaseLock { + return &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: lockname, + Namespace: namespace, + }, + Client: client.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: podname, + }, + } +} + +func runLeaderElection(ctx context.Context, lock *resourcelock.LeaseLock, podname string, ctrlOpts Options, log logr.Logger) { + // Start the leader election for running kapp-controller + log.Info("Waiting for leader election") + leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ + Lock: lock, + ReleaseOnCancel: true, + LeaseDuration: 15 * time.Second, + RenewDeadline: 10 * time.Second, + RetryPeriod: 2 * time.Second, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(c context.Context) { + err := Run(ctrlOpts, log.WithName("controller")) + if err != nil { + klog.Errorf("Error while running as leader: %v", err) + } + }, + OnStoppedLeading: func() { + klog.Fatalf("no longer the leader, staying inactive.") + os.Exit(0) + }, + OnNewLeader: func(identity string) { + //Notify when a new leader is elected + if identity == podname { + return + } + klog.InfoS("new leader elected", "id", identity) + }, + }, + }) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 07aa82d2b..a11fb5357 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -6,12 +6,8 @@ package main import ( "context" "flag" - "github.com/go-logr/logr" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/leaderelection" - "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/klog/v2" "os" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -22,54 +18,6 @@ import ( // Version of kapp-controller is set via ldflags at build-time from the most recent git tag; see hack/build.sh var Version = "develop" -var ( - client *clientset.Clientset -) - -func getNewLock(lockname, podname, namespace string) *resourcelock.LeaseLock { - return &resourcelock.LeaseLock{ - LeaseMeta: metav1.ObjectMeta{ - Name: lockname, - Namespace: namespace, - }, - Client: client.CoordinationV1(), - LockConfig: resourcelock.ResourceLockConfig{ - Identity: podname, - }, - } -} - -func runLeaderElection(ctx context.Context, lock *resourcelock.LeaseLock, podname string, ctrlOpts Options, log logr.Logger) { - // Start the leader election for running kapp-controller - log.Info("Waiting for leader election") - leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ - Lock: lock, - ReleaseOnCancel: true, - LeaseDuration: 15 * time.Second, - RenewDeadline: 10 * time.Second, - RetryPeriod: 2 * time.Second, - Callbacks: leaderelection.LeaderCallbacks{ - OnStartedLeading: func(c context.Context) { - err := Run(ctrlOpts, log.WithName("controller")) - if err != nil { - klog.Errorf("Error while running as leader: %v", err) - } - }, - OnStoppedLeading: func() { - klog.Fatalf("no longer the leader, staying inactive.") - os.Exit(0) - }, - OnNewLeader: func(identity string) { - //Notify when a new leader is elected - if identity == podname { - return - } - klog.InfoS("new leader elected", "id", identity) - }, - }, - }) -} - func main() { ctrlOpts := Options{} var sidecarexec bool From 9be70db4580d5889aea4405b10f184c7ca47cb23 Mon Sep 17 00:00:00 2001 From: Basit Hasan Date: Sun, 27 Aug 2023 12:10:10 +0530 Subject: [PATCH 4/4] add header Signed-off-by: Basit Hasan --- cmd/controller/leader_election.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/controller/leader_election.go b/cmd/controller/leader_election.go index c43d2df3e..3af170ca0 100644 --- a/cmd/controller/leader_election.go +++ b/cmd/controller/leader_election.go @@ -1,3 +1,6 @@ +// Copyright 2020 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + package main import (