Reinstall already installed k0s to reconfigure installFlags

.github/actions/smoke-test-cache/action.yaml

@@ -1,4 +1,5 @@
 name: Smoke test cache steps
+description: Cache smoke test binaries
 runs:
   using: composite
   steps:

.github/workflows/smoke.yml

@@ -262,6 +262,25 @@ jobs:
         env:
           LINUX_IMAGE: ${{ matrix.image }}
         run: make smoke-controller-swap
+
+  smoke-reinstall:
+    strategy:
+      matrix:
+        image:
+          - quay.io/k0sproject/bootloose-alpine3.18
+          - quay.io/k0sproject/bootloose-ubuntu20.04
+
+    name: Reinstall (modify install flags)
+    needs: build
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/smoke-test-cache
+      - name: Run smoke tests
+        env:
+          LINUX_IMAGE: ${{ matrix.image }}
+        run: make smoke-reinstall
 
   smoke-init:
     name: Init sub-command smoke test

Makefile

@@ -51,7 +51,7 @@ build-all: $(addprefix bin/,$(bins)) bin/checksums.md
 clean:
 	rm -rf bin/ k0sctl
 
-smoketests := smoke-basic smoke-basic-rootless smoke-files smoke-upgrade smoke-reset smoke-os-override smoke-init smoke-backup-restore smoke-dynamic smoke-basic-openssh smoke-dryrun smoke-downloadurl smoke-controller-swap
+smoketests := smoke-basic smoke-basic-rootless smoke-files smoke-upgrade smoke-reset smoke-os-override smoke-init smoke-backup-restore smoke-dynamic smoke-basic-openssh smoke-dryrun smoke-downloadurl smoke-controller-swap smoke-reinstall
 .PHONY: $(smoketests)
 $(smoketests): k0sctl
 	$(MAKE) -C smoke-test $@

action/apply.go

@@ -75,6 +75,7 @@ func (a Apply) Run() error {
 		&phase.InstallWorkers{},
 		&phase.UpgradeControllers{},
 		&phase.UpgradeWorkers{NoDrain: a.NoDrain},
+		&phase.Reinstall{},
 		&phase.ResetWorkers{NoDrain: a.NoDrain},
 		&phase.ResetControllers{NoDrain: a.NoDrain},
 		&phase.RunHooks{Stage: "after", Action: "apply"},
@@ -108,7 +109,7 @@ func (a Apply) Run() error {
 
 	duration := time.Since(start).Truncate(time.Second)
 	text := fmt.Sprintf("==> Finished in %s", duration)
-	log.Infof(phase.Colorize.Green(text).String())
+	log.Info(phase.Colorize.Green(text).String())
 
 	for _, host := range a.Manager.Config.Spec.Hosts {
 		if host.Reset {
@@ -141,8 +142,8 @@ func (a Apply) Run() error {
 			cmd.WriteString(a.ConfigPath)
 		}
 
-		log.Infof("Tip: To access the cluster you can now fetch the admin kubeconfig using:")
-		log.Infof("     " + phase.Colorize.Cyan(cmd.String()).String())
+		log.Info("Tip: To access the cluster you can now fetch the admin kubeconfig using:")
+		log.Info("     " + phase.Colorize.Cyan(cmd.String()).String())
 	}
 
 	return nil

action/backup.go

@@ -44,6 +44,6 @@ func (b Backup) Run() error {
 
 	duration := time.Since(start).Truncate(time.Second)
 	text := fmt.Sprintf("==> Finished in %s", duration)
-	log.Infof(phase.Colorize.Green(text).String())
+	log.Info(phase.Colorize.Green(text).String())
 	return nil
 }

action/reset.go

@@ -78,7 +78,7 @@ func (r Reset) Run() error {
 
 	duration := time.Since(start).Truncate(time.Second)
 	text := fmt.Sprintf("==> Finished in %s", duration)
-	log.Infof(phase.Colorize.Green(text).String())
+	log.Info(phase.Colorize.Green(text).String())
 
 	return nil
 }

phase/gather_k0s_facts.go

@@ -74,6 +74,7 @@ func (p *GatherK0sFacts) Run() error {
 		return err
 	}
 	p.leader = p.Config.Spec.K0sLeader()
+	p.leader.Metadata.IsK0sLeader = true
 
 	if id, err := p.Config.Spec.K0s.GetClusterID(p.leader); err == nil {
 		p.Config.Spec.K0s.Metadata.ClusterID = id
@@ -271,9 +272,14 @@ func (p *GatherK0sFacts) investigateK0s(h *cluster.Host) error {
 
 	h.Metadata.NeedsUpgrade = p.needsUpgrade(h)
 
+	if len(status.Args) > 2 {
+		// status.Args contains the binary path and the role as the first two elements, which we can ignore here.
+		h.Metadata.K0sStatusArgs = status.Args[2:]
+	}
+
 	log.Infof("%s: is running k0s %s version %s", h, h.Role, h.Metadata.K0sRunningVersion)
 	if h.IsController() {
-		for _, a := range status.Args {
+		for _, a := range h.Metadata.K0sStatusArgs {
 			if strings.HasPrefix(a, "--enable-dynamic-config") && !strings.HasSuffix(a, "false") {
 				if !p.Config.Spec.K0s.DynamicConfig {
 					log.Warnf("%s: controller has dynamic config enabled, but spec.k0s.dynamicConfig was not set in configuration, proceeding in dynamic config mode", h)

phase/install_controllers.go

@@ -77,7 +77,6 @@ func (p *InstallControllers) After() error {
 			log.Warnf("%s: failed to invalidate worker join token: %v", p.leader, err)
 		}
 		_ = p.Wet(h, "overwrite k0s join token file", func() error {
-
 			if err := h.Configurer.WriteFile(h, h.K0sJoinTokenPath(), "# overwritten by k0sctl after join\n", "0600"); err != nil {
 				log.Warnf("%s: failed to overwrite the join token file at %s", h, h.K0sJoinTokenPath())
 			}
@@ -105,7 +104,6 @@ func (p *InstallControllers) Run() error {
 		}
 		return nil
 	})
-
 	if err != nil {
 		return err
 	}
@@ -162,6 +160,7 @@ func (p *InstallControllers) Run() error {
 			return err
 		}
 		h.Metadata.K0sInstalled = true
+		h.Metadata.K0sRunningVersion = p.Config.Spec.K0s.Version
 
 		if p.IsWet() {
 			if len(h.Environment) > 0 {

phase/reinstall.go

@@ -0,0 +1,137 @@
+package phase
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
+	"github.com/k0sproject/k0sctl/pkg/node"
+	"github.com/k0sproject/k0sctl/pkg/retry"
+	"github.com/k0sproject/rig/exec"
+	log "github.com/sirupsen/logrus"
+)
+
+type Reinstall struct {
+	GenericPhase
+	hosts cluster.Hosts
+}
+
+// Title for the phase
+func (p *Reinstall) Title() string {
+	return "Reinstall"
+}
+
+// Prepare the phase
+func (p *Reinstall) Prepare(config *v1beta1.Cluster) error {
+	p.Config = config
+	p.hosts = p.Config.Spec.Hosts.Filter(func(h *cluster.Host) bool {
+		return !h.Metadata.K0sInstalled && h.Metadata.K0sRunningVersion != nil && !h.Reset && h.FlagsChanged()
+	})
+
+	return nil
+}
+
+// ShouldRun is true when there are hosts that needs to be reinstalled
+func (p *Reinstall) ShouldRun() bool {
+	return cluster.K0sForceFlagSince.Check(p.Config.Spec.K0s.Version) && len(p.hosts) > 0
+}
+
+// Run the phase
+func (p *Reinstall) Run() error {
+	if !cluster.K0sForceFlagSince.Check(p.Config.Spec.K0s.Version) {
+		log.Warnf("k0s version %s does not support install --force flag, installFlags won't be reconfigured", p.Config.Spec.K0s.Version)
+		return nil
+	}
+	controllers := p.hosts.Controllers()
+	if len(controllers) > 0 {
+		log.Infof("Reinstalling %d controllers sequentially", len(controllers))
+		err := controllers.Each(func(h *cluster.Host) error {
+			return p.reinstall(h)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	workers := p.hosts.Workers()
+	if len(workers) == 0 {
+		return nil
+	}
+
+	concurrentReinstalls := int(math.Floor(float64(len(p.hosts)) * 0.10))
+	if concurrentReinstalls == 0 {
+		concurrentReinstalls = 1
+	}
+
+	log.Infof("Reinstalling max %d workers in parallel", concurrentReinstalls)
+
+	return p.hosts.BatchedParallelEach(concurrentReinstalls, p.reinstall)
+}
+
+func (p *Reinstall) reinstall(h *cluster.Host) error {
+	if p.Config.Spec.K0s.DynamicConfig && h.Role != "worker" {
+		h.InstallFlags.AddOrReplace("--enable-dynamic-config")
+	}
+
+	h.InstallFlags.AddOrReplace("--force")
+
+	cmd, err := h.K0sInstallCommand()
+	if err != nil {
+		return err
+	}
+	log.Infof("%s: reinstalling k0s", h)
+	err = p.Wet(h, fmt.Sprintf("reinstall k0s using `%s", strings.ReplaceAll(cmd, h.Configurer.K0sBinaryPath(), "k0s")), func() error {
+		if err := h.Exec(cmd, exec.Sudo(h)); err != nil {
+			return fmt.Errorf("failed to reinstall k0s: %w", err)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = p.Wet(h, "restart k0s service", func() error {
+		if err := h.Configurer.RestartService(h, h.K0sServiceName()); err != nil {
+			return fmt.Errorf("failed to restart k0s: %w", err)
+		}
+		log.Infof("%s: waiting for the k0s service to start", h)
+		if err := retry.Timeout(context.TODO(), retry.DefaultTimeout, node.ServiceRunningFunc(h, h.K0sServiceName())); err != nil {
+			return fmt.Errorf("k0s did not restart: %w", err)
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("restart after reinstall: %w", err)
+	}
+
+	if h != p.Config.Spec.K0sLeader() {
+		return nil
+	}
+
+	if NoWait || !p.IsWet() {
+		log.Warnf("%s: skipping scheduler and system pod checks because --no-wait given", h)
+		return nil
+	}
+
+	log.Infof("%s: waiting for the scheduler to become ready", h)
+	if err := retry.Timeout(context.TODO(), retry.DefaultTimeout, node.ScheduledEventsAfterFunc(h, time.Now())); err != nil {
+		if !Force {
+			return fmt.Errorf("failed to observe scheduling events after api start-up, you can ignore this check by using --force: %w", err)
+		}
+		log.Warnf("%s: failed to observe scheduling events after api start-up: %s", h, err)
+	}
+
+	log.Infof("%s: waiting for system pods to become ready", h)
+	if err := retry.Timeout(context.TODO(), retry.DefaultTimeout, node.SystemPodsRunningFunc(h)); err != nil {
+		if !Force {
+			return fmt.Errorf("all system pods not running after api start-up, you can ignore this check by using --force: %w", err)
+		}
+		log.Warnf("%s: failed to observe system pods running after api start-up: %s", h, err)
+	}
+
+	return nil
+}

phase/upgrade_controllers.go

@@ -9,6 +9,7 @@ import (
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	"github.com/k0sproject/k0sctl/pkg/node"
 	"github.com/k0sproject/k0sctl/pkg/retry"
+	"github.com/k0sproject/rig/exec"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -95,6 +96,27 @@ func (p *UpgradeControllers) Run() error {
 			}
 		}
 
+		err = p.Wet(h, "reinstall k0s service", func() error {
+			if p.Config.Spec.K0s.DynamicConfig {
+				h.InstallFlags.AddOrReplace("--enable-dynamic-config")
+			}
+
+			h.InstallFlags.AddOrReplace("--force")
+
+			cmd, err := h.K0sInstallCommand()
+			if err != nil {
+				return err
+			}
+			if err := h.Exec(cmd, exec.Sudo(h)); err != nil {
+				return fmt.Errorf("failed to reinstall k0s: %w", err)
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+		h.Metadata.K0sInstalled = true
+
 		log.Debugf("%s: restart service", h)
 		err = p.Wet(h, "start k0s service with the new binary", func() error {
 			if err := h.Configurer.StartService(h, h.K0sServiceName()); err != nil {
@@ -119,6 +141,8 @@ func (p *UpgradeControllers) Run() error {
 				return fmt.Errorf("kube api did not become ready: %w", err)
 			}
 		}
+
+		h.Metadata.K0sRunningVersion = p.Config.Spec.K0s.Version
 	}
 
 	leader := p.Config.Spec.K0sLeader()

phase/upgrade_workers.go

@@ -9,6 +9,7 @@ import (
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	"github.com/k0sproject/k0sctl/pkg/node"
 	"github.com/k0sproject/k0sctl/pkg/retry"
+	"github.com/k0sproject/rig/exec"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -182,6 +183,23 @@ func (p *UpgradeWorkers) upgradeWorker(h *cluster.Host) error {
 		}
 	}
 
+	err = p.Wet(h, "reinstall k0s service", func() error {
+		h.InstallFlags.AddOrReplace("--force")
+
+		cmd, err := h.K0sInstallCommand()
+		if err != nil {
+			return err
+		}
+		if err := h.Exec(cmd, exec.Sudo(h)); err != nil {
+			return fmt.Errorf("failed to reinstall k0s: %w", err)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	h.Metadata.K0sInstalled = true
+
 	log.Debugf("%s: restart service", h)
 	err = p.Wet(h, "restart k0s service", func() error {
 		if err := h.Configurer.StartService(h, h.K0sServiceName()); err != nil {