Add --tear-down-event flag to delay tear down

cmd/cluster-bootstrap/start.go

@@ -21,10 +21,11 @@ var (
 	}
 
 	startOpts struct {
-		assetDir        string
-		podManifestPath string
-		strict          bool
-		requiredPods    []string
+		assetDir             string
+		podManifestPath      string
+		strict               bool
+		requiredPods         []string
+		waitForTearDownEvent string
 	}
 )
 
@@ -41,14 +42,16 @@ func init() {
 	cmdStart.Flags().StringVar(&startOpts.podManifestPath, "pod-manifest-path", "/etc/kubernetes/manifests", "The location where the kubelet is configured to look for static pod manifests.")
 	cmdStart.Flags().BoolVar(&startOpts.strict, "strict", false, "Strict mode will cause start command to exit early if any manifests in the asset directory cannot be created.")
 	cmdStart.Flags().StringSliceVar(&startOpts.requiredPods, "required-pods", defaultRequiredPods, "List of pods with their namespace (written as <namespace>/<pod-name>) that are required to be running and ready before the start command does the pivot.")
+	cmdStart.Flags().StringVar(&startOpts.waitForTearDownEvent, "tear-down-event", "", "if this optional event name of the form <ns>/<event-name> is given, the event is waited for before tearing down the bootstrap control plane")
 }
 
 func runCmdStart(cmd *cobra.Command, args []string) error {
 	bk, err := start.NewStartCommand(start.Config{
-		AssetDir:        startOpts.assetDir,
-		PodManifestPath: startOpts.podManifestPath,
-		Strict:          startOpts.strict,
-		RequiredPods:    startOpts.requiredPods,
+		AssetDir:             startOpts.assetDir,
+		PodManifestPath:      startOpts.podManifestPath,
+		Strict:               startOpts.strict,
+		RequiredPods:         startOpts.requiredPods,
+		WaitForTearDownEvent: startOpts.waitForTearDownEvent,
 	})
 	if err != nil {
 		return err

pkg/start/start.go

@@ -1,38 +1,48 @@
 package start
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
-const assetTimeout = 20 * time.Minute
+const (
+	// how long we wait until the bootstrap pods to be running
+	bootstrapPodsRunningTimeout = 20 * time.Minute
+)
 
 type Config struct {
-	AssetDir        string
-	PodManifestPath string
-	Strict          bool
-	RequiredPods    []string
+	AssetDir             string
+	PodManifestPath      string
+	Strict               bool
+	RequiredPods         []string
+	WaitForTearDownEvent string
 }
 
 type startCommand struct {
-	podManifestPath string
-	assetDir        string
-	strict          bool
-	requiredPods    []string
+	podManifestPath      string
+	assetDir             string
+	strict               bool
+	requiredPods         []string
+	waitForTearDownEvent string
 }
 
 func NewStartCommand(config Config) (*startCommand, error) {
 	return &startCommand{
-		assetDir:        config.AssetDir,
-		podManifestPath: config.PodManifestPath,
-		strict:          config.Strict,
-		requiredPods:    config.RequiredPods,
+		assetDir:             config.AssetDir,
+		podManifestPath:      config.PodManifestPath,
+		strict:               config.Strict,
+		requiredPods:         config.RequiredPods,
+		waitForTearDownEvent: config.WaitForTearDownEvent,
 	}, nil
 }
 
@@ -66,11 +76,11 @@ func (b *startCommand) Run() error {
 		return err
 	}
 
-	if err = CreateAssets(restConfig, filepath.Join(b.assetDir, AssetPathManifests), assetTimeout, b.strict); err != nil {
+	if err = CreateAssets(restConfig, filepath.Join(b.assetDir, AssetPathManifests), bootstrapPodsRunningTimeout, b.strict); err != nil {
 		return err
 	}
 
-	if err = WaitUntilPodsRunning(client, b.requiredPods, assetTimeout); err != nil {
+	if err = WaitUntilPodsRunning(client, b.requiredPods, bootstrapPodsRunningTimeout); err != nil {
 		return err
 	}
 
@@ -80,6 +90,20 @@ func (b *startCommand) Run() error {
 		return err
 	}
 
+	// optionally wait for tear down event coming from the installer. This is necessary to
+	// remove the bootstrap node from the AWS load balancer.
+	if len(b.waitForTearDownEvent) != 0 {
+		ss := strings.Split(b.waitForTearDownEvent, "/")
+		if len(ss) != 2 {
+			return fmt.Errorf("tear down event name of format <namespace>/<event-name> expected, got: %q", b.waitForTearDownEvent)
+		}
+		ns, name := ss[0], ss[1]
+		if err := waitForEvent(context.TODO(), client, ns, name); err != nil {
+			return err
+		}
+		UserOutput("Got %s event.", b.waitForTearDownEvent)
+	}
+
 	return nil
 }
 
@@ -91,6 +115,18 @@ func UserOutput(format string, a ...interface{}) {
 	fmt.Printf(format, a...)
 }
 
+func waitForEvent(ctx context.Context, client kubernetes.Interface, ns, name string) error {
+	return wait.PollImmediateUntil(time.Second, func() (done bool, err error) {
+		if _, err := client.CoreV1().Events(ns).Get(name, metav1.GetOptions{}); err != nil && apierrors.IsNotFound(err) {
+			return false, nil
+		} else if err != nil {
+			UserOutput("Error waiting for %s/%s event: %v", ns, name, err)
+			return false, nil
+		}
+		return true, nil
+	}, ctx.Done())
+}
+
 func makeBootstrapSuccessEvent(ns, name string) *corev1.Event {
 	currentTime := metav1.Time{Time: time.Now()}
 	event := &corev1.Event{