[Release-1.30] Testing And Secrets-Encryption Backports for 2024-09

.drone.yml

@@ -644,6 +644,10 @@ steps:
         UPGRADE_CHANNEL="latest"
       else
         UPGRADE_CHANNEL=$(echo $DRONE_BRANCH | sed 's/release-/v/')
+        # Check if the UPGRADE_CHANNEL exists, in the case of new minor releases it won't
+        if ! curl --head --silent --fail https://update.k3s.io/v1-release/channels/$UPGRADE_CHANNEL; then
+          UPGRADE_CHANNEL="latest"
+        fi
       fi
       E2E_RELEASE_CHANNEL=$UPGRADE_CHANNEL go test -v -timeout=45m ./upgradecluster_test.go  -ci -local
       cp ./coverage.out /tmp/artifacts/upgrade-coverage.out

.github/workflows/e2e.yaml

@@ -52,7 +52,7 @@ jobs:
         with:
           path: |
              ~/.vagrant.d/boxes
-          key: vagrant-box-ubuntu-2204
+          key: vagrant-box-ubuntu-2404
       - name: "Vagrant Plugin(s)"
         run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
 

.github/workflows/install.yaml

@@ -30,7 +30,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vm: [centos-7, rocky-8, rocky-9, fedora, opensuse-leap, ubuntu-2204]
+        vm: [centos-9, rocky-8, rocky-9, fedora, opensuse-leap, ubuntu-2404]
       max-parallel: 3
     defaults:
       run:
@@ -65,7 +65,7 @@ jobs:
           vagrant ssh -c "sudo mv /tmp/k3s /usr/local/bin/k3s"
           vagrant provision --provision-with=k3s-upload
       - name: Add binary to PATH
-        if: matrix.vm == 'centos-7' || matrix.vm == 'rocky-8' || matrix.vm == 'rocky-9' || matrix.vm == 'opensuse-leap'
+        if: matrix.vm == 'centos-9' || matrix.vm == 'rocky-8' || matrix.vm == 'rocky-9' || matrix.vm == 'opensuse-leap'
         run: vagrant provision --provision-with=add-bin-path
       - name: "⏩ Install K3s"
         run: |

.github/workflows/nightly-install.yaml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         channel: [stable, latest]
-        vm: [rocky-8, fedora, opensuse-leap, ubuntu-2204]
+        vm: [rocky-9, fedora, opensuse-leap, ubuntu-2404]
       max-parallel: 4
     defaults:
       run:

Dockerfile.test

@@ -40,7 +40,7 @@ FROM vagrantlibvirt/vagrant-libvirt:0.12.1 AS test-e2e
 RUN apt-get update && apt-get install -y docker.io
 ENV VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1
 RUN vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
-RUN vagrant box add generic/ubuntu2204 --provider libvirt --force
+RUN vagrant box add bento/ubuntu-24.04 --provider libvirt --force
 RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \
     chmod +x ./kubectl; \
     mv ./kubectl /usr/local/bin/kubectl

pkg/daemons/config/types.go

@@ -227,8 +227,6 @@ type Control struct {
 	ClusterInit              bool
 	ClusterReset             bool
 	ClusterResetRestorePath  string
-	EncryptForce             bool
-	EncryptSkip              bool
 	MinTLSVersion            string
 	CipherSuites             []string
 	TLSMinVersion            uint16   `json:"-"`

pkg/daemons/control/server.go

@@ -14,7 +14,6 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
-	"github.com/k3s-io/k3s/pkg/secretsencrypt"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
@@ -61,18 +60,6 @@ func Server(ctx context.Context, cfg *config.Control) error {
 		if err := apiServer(ctx, cfg); err != nil {
 			return err
 		}
-		if cfg.EncryptSecrets {
-			controllerName := "reencrypt-secrets"
-			cfg.Runtime.ClusterControllerStarts[controllerName] = func(ctx context.Context) {
-				// cfg.Runtime.Core is populated before this callback is triggered
-				if err := secretsencrypt.Register(ctx,
-					controllerName,
-					cfg,
-					cfg.Runtime.Core.Core().V1().Node()); err != nil {
-					logrus.Errorf("Failed to register %s controller: %v", controllerName, err)
-				}
-			}
-		}
 	}
 
 	// Wait for an apiserver to become available before starting additional controllers,

pkg/secretsencrypt/config.go

@@ -27,13 +27,19 @@ import (
 )
 
 const (
-	EncryptionStart             string = "start"
-	EncryptionPrepare           string = "prepare"
-	EncryptionRotate            string = "rotate"
-	EncryptionRotateKeys        string = "rotate_keys"
-	EncryptionReencryptRequest  string = "reencrypt_request"
-	EncryptionReencryptActive   string = "reencrypt_active"
-	EncryptionReencryptFinished string = "reencrypt_finished"
+	EncryptionStart             string  = "start"
+	EncryptionPrepare           string  = "prepare"
+	EncryptionRotate            string  = "rotate"
+	EncryptionRotateKeys        string  = "rotate_keys"
+	EncryptionReencryptRequest  string  = "reencrypt_request"
+	EncryptionReencryptActive   string  = "reencrypt_active"
+	EncryptionReencryptFinished string  = "reencrypt_finished"
+	SecretListPageSize          int64   = 20
+	SecretQPS                   float32 = 200
+	SecretBurst                 int     = 200
+	SecretsUpdateErrorEvent     string  = "SecretsUpdateError"
+	SecretsProgressEvent        string  = "SecretsProgress"
+	SecretsUpdateCompleteEvent  string  = "SecretsUpdateComplete"
 )
 
 var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash"
@@ -178,7 +184,9 @@ func BootstrapEncryptionHashAnnotation(node *corev1.Node, runtime *config.Contro
 	return nil
 }
 
-func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, stage string) error {
+// WriteEncryptionHashAnnotation writes the encryption hash to the node annotation and optionally to a file.
+// The file is used to track the last stage of the reencryption process.
+func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, skipFile bool, stage string) error {
 	encryptionConfigHash, err := GenEncryptionConfigHash(runtime)
 	if err != nil {
 		return err
@@ -192,6 +200,9 @@ func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.
 		return err
 	}
 	logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
+	if skipFile {
+		return nil
+	}
 	return os.WriteFile(runtime.EncryptionHash, []byte(ann), 0600)
 }
 
@@ -253,10 +264,11 @@ func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics b
 			return true, err
 		}
 		tsMetric := mf["apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds"]
-		successMetric := mf["apiserver_encryption_config_controller_automatic_reload_success_total"]
+		// Potentially multiple metrics with different success/failure labels
+		totalMetrics := mf["apiserver_encryption_config_controller_automatic_reloads_total"]
 
 		// First time, no metrics exist, so return zeros
-		if tsMetric == nil && successMetric == nil && initialMetrics {
+		if tsMetric == nil && totalMetrics == nil && initialMetrics {
 			return true, nil
 		}
 
@@ -265,8 +277,8 @@ func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics b
 			return false, nil
 		}
 
-		if successMetric == nil {
-			lastFailure = "encryption config success metric not found"
+		if totalMetrics == nil {
+			lastFailure = "encryption config total metric not found"
 			return false, nil
 		}
 
@@ -275,8 +287,14 @@ func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics b
 			return true, fmt.Errorf("encryption reload time is incorrectly ahead of current time")
 		}
 
-		reloadSuccessCounter = int64(successMetric.GetMetric()[0].GetCounter().GetValue())
-
+		for _, totalMetric := range totalMetrics.GetMetric() {
+			logrus.Debugf("totalMetric: %+v", totalMetric)
+			for _, label := range totalMetric.GetLabel() {
+				if label.GetName() == "status" && label.GetValue() == "success" {
+					reloadSuccessCounter = int64(totalMetric.GetCounter().GetValue())
+				}
+			}
+		}
 		return true, nil
 	})