Run reset after k0s install has been run but host doesn't come up

If "k0s install" has been run but the host fails to become ready, run
"k0s reset" in the phase's cleanup.

Signed-off-by: Kimmo Lehto <klehto@mirantis.com>

action/apply.go

@@ -84,6 +84,7 @@ func (a Apply) Run() error {
 
 	if result = a.Manager.Run(); result != nil {
 		analytics.Client.Publish("apply-failure", map[string]interface{}{"clusterID": a.Manager.Config.Spec.K0s.Metadata.ClusterID})
+		log.Info(phase.Colorize.Red("==> Apply failed").String())
 		return result
 	}
 

phase/initialize_k0s.go

@@ -8,6 +8,7 @@ import (
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	"github.com/k0sproject/k0sctl/pkg/node"
 	"github.com/k0sproject/k0sctl/pkg/retry"
+	"github.com/k0sproject/rig/exec"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -40,11 +41,18 @@ func (p *InitializeK0s) ShouldRun() bool {
 // CleanUp cleans up the environment override file
 func (p *InitializeK0s) CleanUp() {
 	h := p.leader
+
+	log.Infof("%s: cleaning up", h)
 	if len(h.Environment) > 0 {
 		if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
 			log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
 		}
 	}
+	if h.Metadata.K0sInstalled {
+		if err := h.Exec(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h)); err != nil {
+			log.Warnf("%s: k0s reset failed", h)
+		}
+	}
 }
 
 // Run the phase
@@ -72,6 +80,8 @@ func (p *InitializeK0s) Run() error {
 		return err
 	}
 
+	h.Metadata.K0sInstalled = true
+
 	if len(h.Environment) > 0 {
 		log.Infof("%s: updating service environment", h)
 		if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil {

phase/install_controllers.go

@@ -28,9 +28,8 @@ func (p *InstallControllers) Title() string {
 // Prepare the phase
 func (p *InstallControllers) Prepare(config *v1beta1.Cluster) error {
 	p.Config = config
-	var controllers cluster.Hosts = p.Config.Spec.Hosts.Controllers()
 	p.leader = p.Config.Spec.K0sLeader()
-	p.hosts = controllers.Filter(func(h *cluster.Host) bool {
+	p.hosts = p.Config.Spec.Hosts.Controllers().Filter(func(h *cluster.Host) bool {
 		return !h.Reset && !h.Metadata.NeedsUpgrade && (h != p.leader && h.Metadata.K0sRunningVersion == nil)
 	})
 
@@ -44,13 +43,22 @@ func (p *InstallControllers) ShouldRun() bool {
 
 // CleanUp cleans up the environment override files on hosts
 func (p *InstallControllers) CleanUp() {
-	for _, h := range p.hosts {
+	_ = p.hosts.Filter(func(h *cluster.Host) bool {
+		return !h.Metadata.Ready
+	}).ParallelEach(func(h *cluster.Host) error {
+		log.Infof("%s: cleaning up", h)
 		if len(h.Environment) > 0 {
 			if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
-				log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+				log.Warnf("%s: failed to clean up service environment: %v", h, err)
 			}
 		}
-	}
+		if h.Metadata.K0sInstalled {
+			if err := h.Exec(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h)); err != nil {
+				log.Warnf("%s: k0s reset failed", h)
+			}
+		}
+		return nil
+	})
 }
 
 // Run the phase
@@ -119,6 +127,7 @@ func (p *InstallControllers) Run() error {
 		if err = h.Exec(cmd); err != nil {
 			return err
 		}
+		h.Metadata.K0sInstalled = true
 
 		if len(h.Environment) > 0 {
 			log.Infof("%s: updating service environment", h)
@@ -140,6 +149,7 @@ func (p *InstallControllers) Run() error {
 		if err := p.waitJoined(h); err != nil {
 			return err
 		}
+		h.Metadata.Ready = true
 	}
 
 	return nil

phase/install_workers.go

@@ -28,8 +28,7 @@ func (p *InstallWorkers) Title() string {
 // Prepare the phase
 func (p *InstallWorkers) Prepare(config *v1beta1.Cluster) error {
 	p.Config = config
-	var workers cluster.Hosts = p.Config.Spec.Hosts.Workers()
-	p.hosts = workers.Filter(func(h *cluster.Host) bool {
+	p.hosts = p.Config.Spec.Hosts.Workers().Filter(func(h *cluster.Host) bool {
 		return !h.Reset && !h.Metadata.NeedsUpgrade && (h.Metadata.K0sRunningVersion == nil || !h.Metadata.Ready)
 	})
 	p.leader = p.Config.Spec.K0sLeader()
@@ -42,15 +41,24 @@ func (p *InstallWorkers) ShouldRun() bool {
 	return len(p.hosts) > 0
 }
 
-// CleanUp cleans up the environment override files on hosts
+// CleanUp attempts to clean up any changes after a failed install
 func (p *InstallWorkers) CleanUp() {
-	for _, h := range p.hosts {
+	_ = p.hosts.Filter(func(h *cluster.Host) bool {
+		return !h.Metadata.Ready
+	}).ParallelEach(func(h *cluster.Host) error {
+		log.Infof("%s: cleaning up", h)
 		if len(h.Environment) > 0 {
 			if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
-				log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+				log.Warnf("%s: failed to clean up service environment: %v", h, err)
 			}
 		}
-	}
+		if h.Metadata.K0sInstalled {
+			if err := h.Exec(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h)); err != nil {
+				log.Warnf("%s: k0s reset failed", h)
+			}
+		}
+		return nil
+	})
 }
 
 // Run the phase
@@ -137,6 +145,8 @@ func (p *InstallWorkers) Run() error {
 			return err
 		}
 
+		h.Metadata.K0sInstalled = true
+
 		if len(h.Environment) > 0 {
 			log.Infof("%s: updating service environment", h)
 			if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil {

pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go

@@ -175,6 +175,7 @@ type HostMetadata struct {
 	K0sBinaryVersion  *version.Version
 	K0sBinaryTempFile string
 	K0sRunningVersion *version.Version
+	K0sInstalled      bool
 	Arch              string
 	IsK0sLeader       bool
 	Hostname          string