From 97e3e10891b140470853a14503164fd80c6308d0 Mon Sep 17 00:00:00 2001 From: Kimmo Lehto Date: Wed, 18 Oct 2023 10:19:02 +0300 Subject: [PATCH] Run reset after k0s install has been run but host doesn't come up If "k0s install" has been run but the host fails to become ready, run "k0s reset" in the phase's cleanup. Signed-off-by: Kimmo Lehto --- action/apply.go | 1 + phase/initialize_k0s.go | 10 +++++++++ phase/install_controllers.go | 20 ++++++++++++----- phase/install_workers.go | 22 ++++++++++++++----- .../v1beta1/cluster/host.go | 1 + 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/action/apply.go b/action/apply.go index 11b2a26a..e6091823 100644 --- a/action/apply.go +++ b/action/apply.go @@ -84,6 +84,7 @@ func (a Apply) Run() error { if result = a.Manager.Run(); result != nil { analytics.Client.Publish("apply-failure", map[string]interface{}{"clusterID": a.Manager.Config.Spec.K0s.Metadata.ClusterID}) + log.Info(phase.Colorize.Red("==> Apply failed").String()) return result } diff --git a/phase/initialize_k0s.go b/phase/initialize_k0s.go index a2bf1726..0c9125f5 100644 --- a/phase/initialize_k0s.go +++ b/phase/initialize_k0s.go @@ -8,6 +8,7 @@ import ( "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster" "github.com/k0sproject/k0sctl/pkg/node" "github.com/k0sproject/k0sctl/pkg/retry" + "github.com/k0sproject/rig/exec" log "github.com/sirupsen/logrus" ) @@ -40,11 +41,18 @@ func (p *InitializeK0s) ShouldRun() bool { // CleanUp cleans up the environment override file func (p *InitializeK0s) CleanUp() { h := p.leader + + log.Infof("%s: cleaning up", h) if len(h.Environment) > 0 { if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil { log.Warnf("%s: failed to clean up service environment: %s", h, err.Error()) } } + if h.Metadata.K0sInstalled { + if err := h.Exec(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h)); err != nil { + log.Warnf("%s: k0s reset failed", h) + } + } } // Run the phase @@ -72,6 +80,8 @@ func (p *InitializeK0s) Run() error { return err } + h.Metadata.K0sInstalled = true + if len(h.Environment) > 0 { log.Infof("%s: updating service environment", h) if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil { diff --git a/phase/install_controllers.go b/phase/install_controllers.go index 19caa577..3d37708a 100644 --- a/phase/install_controllers.go +++ b/phase/install_controllers.go @@ -28,9 +28,8 @@ func (p *InstallControllers) Title() string { // Prepare the phase func (p *InstallControllers) Prepare(config *v1beta1.Cluster) error { p.Config = config - var controllers cluster.Hosts = p.Config.Spec.Hosts.Controllers() p.leader = p.Config.Spec.K0sLeader() - p.hosts = controllers.Filter(func(h *cluster.Host) bool { + p.hosts = p.Config.Spec.Hosts.Controllers().Filter(func(h *cluster.Host) bool { return !h.Reset && !h.Metadata.NeedsUpgrade && (h != p.leader && h.Metadata.K0sRunningVersion == nil) }) @@ -44,13 +43,22 @@ func (p *InstallControllers) ShouldRun() bool { // CleanUp cleans up the environment override files on hosts func (p *InstallControllers) CleanUp() { - for _, h := range p.hosts { + _ = p.hosts.Filter(func(h *cluster.Host) bool { + return !h.Metadata.Ready + }).ParallelEach(func(h *cluster.Host) error { + log.Infof("%s: cleaning up", h) if len(h.Environment) > 0 { if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil { - log.Warnf("%s: failed to clean up service environment: %s", h, err.Error()) + log.Warnf("%s: failed to clean up service environment: %v", h, err) } } - } + if h.Metadata.K0sInstalled { + if err := h.Exec(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h)); err != nil { + log.Warnf("%s: k0s reset failed", h) + } + } + return nil + }) } // Run the phase @@ -119,6 +127,7 @@ func (p *InstallControllers) Run() error { if err = h.Exec(cmd); err != nil { return err } + h.Metadata.K0sInstalled = true if len(h.Environment) > 0 { log.Infof("%s: updating service environment", h) @@ -140,6 +149,7 @@ func (p *InstallControllers) Run() error { if err := p.waitJoined(h); err != nil { return err } + h.Metadata.Ready = true } return nil diff --git a/phase/install_workers.go b/phase/install_workers.go index 2c1edaf1..5909ba85 100644 --- a/phase/install_workers.go +++ b/phase/install_workers.go @@ -28,8 +28,7 @@ func (p *InstallWorkers) Title() string { // Prepare the phase func (p *InstallWorkers) Prepare(config *v1beta1.Cluster) error { p.Config = config - var workers cluster.Hosts = p.Config.Spec.Hosts.Workers() - p.hosts = workers.Filter(func(h *cluster.Host) bool { + p.hosts = p.Config.Spec.Hosts.Workers().Filter(func(h *cluster.Host) bool { return !h.Reset && !h.Metadata.NeedsUpgrade && (h.Metadata.K0sRunningVersion == nil || !h.Metadata.Ready) }) p.leader = p.Config.Spec.K0sLeader() @@ -42,15 +41,24 @@ func (p *InstallWorkers) ShouldRun() bool { return len(p.hosts) > 0 } -// CleanUp cleans up the environment override files on hosts +// CleanUp attempts to clean up any changes after a failed install func (p *InstallWorkers) CleanUp() { - for _, h := range p.hosts { + _ = p.hosts.Filter(func(h *cluster.Host) bool { + return !h.Metadata.Ready + }).ParallelEach(func(h *cluster.Host) error { + log.Infof("%s: cleaning up", h) if len(h.Environment) > 0 { if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil { - log.Warnf("%s: failed to clean up service environment: %s", h, err.Error()) + log.Warnf("%s: failed to clean up service environment: %v", h, err) } } - } + if h.Metadata.K0sInstalled { + if err := h.Exec(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h)); err != nil { + log.Warnf("%s: k0s reset failed", h) + } + } + return nil + }) } // Run the phase @@ -137,6 +145,8 @@ func (p *InstallWorkers) Run() error { return err } + h.Metadata.K0sInstalled = true + if len(h.Environment) > 0 { log.Infof("%s: updating service environment", h) if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil { diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go index fbece209..28522b64 100644 --- a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go +++ b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go @@ -175,6 +175,7 @@ type HostMetadata struct { K0sBinaryVersion *version.Version K0sBinaryTempFile string K0sRunningVersion *version.Version + K0sInstalled bool Arch string IsK0sLeader bool Hostname string