feat: add ability to auto label clusters from k8s clusterinfo (argopr…

…oj#17289) * feat: add ability to auto label clusters This gives the ability to automatically label cluster secrets on a cluster-by-cluster basis. If `enableClusterInfoLabels` is set on a cluster secret, the controller will (eventually) label the cluster secret with the current k8s version detected by the cluster info. This needs documentation, e2e tests, as well as CLI/UI additions. Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com> * refactor: use labels instead of secret data This is easier to work with, especially in the context where we need this feature. Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com> * docs: add description on how to use dynamic labeling Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com> --------- Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com> Signed-off-by: penglongli <pelenli@tencent.com>
penglongli · Mar 6, 2024 · 1519091 · 1519091
1 parent 5d64db0
commit 1519091
Show file tree

Hide file tree

Showing 5 changed files with 141 additions and 5 deletions.
diff --git a/common/common.go b/common/common.go
@@ -149,10 +149,14 @@ const (
 	LabelKeyAppInstance = "app.kubernetes.io/instance"
 	// LabelKeyAppName is the label key to use to uniquely identify the name of the Kubernetes application
 	LabelKeyAppName = "app.kubernetes.io/name"
+	// LabelKeyAutoLabelClusterInfo if set to true will automatically add extra labels from the cluster info (currently it only adds a k8s version label)
+	LabelKeyAutoLabelClusterInfo = "argocd.argoproj.io/auto-label-cluster-info"
 	// LabelKeyLegacyApplicationName is the legacy label (v0.10 and below) and is superseded by 'app.kubernetes.io/instance'
 	LabelKeyLegacyApplicationName = "applications.argoproj.io/app-name"
 	// LabelKeySecretType contains the type of argocd secret (currently: 'cluster', 'repository', 'repo-config' or 'repo-creds')
 	LabelKeySecretType = "argocd.argoproj.io/secret-type"
+	// LabelKeyClusterKubernetesVersion contains the kubernetes version of the cluster secret if it has been enabled
+	LabelKeyClusterKubernetesVersion = "argocd.argoproj.io/kubernetes-version"
 	// LabelValueSecretTypeCluster indicates a secret type of cluster
 	LabelValueSecretTypeCluster = "cluster"
 	// LabelValueSecretTypeRepository indicates a secret type of repository

diff --git a/controller/clusterinfoupdater.go b/controller/clusterinfoupdater.go
@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"github.com/argoproj/argo-cd/v2/common"
 	"time"
 
 	"github.com/argoproj/argo-cd/v2/util/env"
@@ -101,8 +102,11 @@ func (c *clusterInfoUpdater) updateClusters() {
 	}
 	_ = kube.RunAllAsync(len(clustersFiltered), func(i int) error {
 		cluster := clustersFiltered[i]
-		if err := c.updateClusterInfo(ctx, cluster, infoByServer[cluster.Server]); err != nil {
-			log.Warnf("Failed to save clusters info: %v", err)
+		clusterInfo := infoByServer[cluster.Server]
+		if err := c.updateClusterInfo(ctx, cluster, clusterInfo); err != nil {
+			log.Warnf("Failed to save cluster info: %v", err)
+		} else if err := updateClusterLabels(ctx, clusterInfo, cluster, c.db.UpdateCluster); err != nil {
+			log.Warnf("Failed to update cluster labels: %v", err)
 		}
 		return nil
 	})
@@ -114,6 +118,12 @@ func (c *clusterInfoUpdater) updateClusterInfo(ctx context.Context, cluster appv
 	if err != nil {
 		return fmt.Errorf("error while fetching the apps list: %w", err)
 	}
+
+	updated := c.getUpdatedClusterInfo(ctx, apps, cluster, info, metav1.Now())
+	return c.cache.SetClusterInfo(cluster.Server, &updated)
+}
+
+func (c *clusterInfoUpdater) getUpdatedClusterInfo(ctx context.Context, apps []*appv1.Application, cluster appv1.Cluster, info *cache.ClusterInfo, now metav1.Time) appv1.ClusterInfo {
 	var appCount int64
 	for _, a := range apps {
 		if c.projGetter != nil {
@@ -129,7 +139,6 @@ func (c *clusterInfoUpdater) updateClusterInfo(ctx context.Context, cluster appv
 			appCount += 1
 		}
 	}
-	now := metav1.Now()
 	clusterInfo := appv1.ClusterInfo{
 		ConnectionState:   appv1.ConnectionState{ModifiedAt: &now},
 		ApplicationsCount: appCount,
@@ -156,5 +165,15 @@ func (c *clusterInfoUpdater) updateClusterInfo(ctx context.Context, cluster appv
 		}
 	}
 
-	return c.cache.SetClusterInfo(cluster.Server, &clusterInfo)
+	return clusterInfo
+}
+
+func updateClusterLabels(ctx context.Context, clusterInfo *cache.ClusterInfo, cluster appv1.Cluster, updateCluster func(context.Context, *appv1.Cluster) (*appv1.Cluster, error)) error {
+	if clusterInfo != nil && cluster.Labels[common.LabelKeyAutoLabelClusterInfo] == "true" && cluster.Labels[common.LabelKeyClusterKubernetesVersion] != clusterInfo.K8SVersion {
+		cluster.Labels[common.LabelKeyClusterKubernetesVersion] = clusterInfo.K8SVersion
+		_, err := updateCluster(ctx, &cluster)
+		return err
+	}
+
+	return nil
 }
diff --git a/controller/clusterinfoupdater_test.go b/controller/clusterinfoupdater_test.go
@@ -2,6 +2,7 @@ package controller
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"testing"
 	"time"
@@ -98,3 +99,92 @@ func TestClusterSecretUpdater(t *testing.T) {
 		assert.Equal(t, test.ExpectedStatus, clusterInfo.ConnectionState.Status)
 	}
 }
+
+func TestUpdateClusterLabels(t *testing.T) {
+	shouldNotBeInvoked := func(ctx context.Context, cluster *v1alpha1.Cluster) (*v1alpha1.Cluster, error) {
+		shouldNotHappen := errors.New("if an error happens here, something's wrong")
+		assert.NoError(t, shouldNotHappen)
+		return nil, shouldNotHappen
+	}
+	tests := []struct {
+		name          string
+		clusterInfo   *clustercache.ClusterInfo
+		cluster       v1alpha1.Cluster
+		updateCluster func(context.Context, *v1alpha1.Cluster) (*v1alpha1.Cluster, error)
+		wantErr       assert.ErrorAssertionFunc
+	}{
+		{
+			"enableClusterInfoLabels = false",
+			&clustercache.ClusterInfo{
+				Server:     "kubernetes.svc.local",
+				K8SVersion: "1.28",
+			},
+			v1alpha1.Cluster{
+				Server: "kubernetes.svc.local",
+				Labels: nil,
+			},
+			shouldNotBeInvoked,
+			assert.NoError,
+		},
+		{
+			"clusterInfo = nil",
+			nil,
+			v1alpha1.Cluster{
+				Server: "kubernetes.svc.local",
+				Labels: map[string]string{"argocd.argoproj.io/auto-label-cluster-info": "true"},
+			},
+			shouldNotBeInvoked,
+			assert.NoError,
+		},
+		{
+			"clusterInfo.k8sversion == cluster k8s label",
+			&clustercache.ClusterInfo{
+				Server:     "kubernetes.svc.local",
+				K8SVersion: "1.28",
+			},
+			v1alpha1.Cluster{
+				Server: "kubernetes.svc.local",
+				Labels: map[string]string{"argocd.argoproj.io/kubernetes-version": "1.28", "argocd.argoproj.io/auto-label-cluster-info": "true"},
+			},
+			shouldNotBeInvoked,
+			assert.NoError,
+		},
+		{
+			"clusterInfo.k8sversion != cluster k8s label, no error",
+			&clustercache.ClusterInfo{
+				Server:     "kubernetes.svc.local",
+				K8SVersion: "1.28",
+			},
+			v1alpha1.Cluster{
+				Server: "kubernetes.svc.local",
+				Labels: map[string]string{"argocd.argoproj.io/kubernetes-version": "1.27", "argocd.argoproj.io/auto-label-cluster-info": "true"},
+			},
+			func(ctx context.Context, cluster *v1alpha1.Cluster) (*v1alpha1.Cluster, error) {
+				assert.Equal(t, cluster.Labels["argocd.argoproj.io/kubernetes-version"], "1.28")
+				return nil, nil
+			},
+			assert.NoError,
+		},
+		{
+			"clusterInfo.k8sversion != cluster k8s label, some error",
+			&clustercache.ClusterInfo{
+				Server:     "kubernetes.svc.local",
+				K8SVersion: "1.28",
+			},
+			v1alpha1.Cluster{
+				Server: "kubernetes.svc.local",
+				Labels: map[string]string{"argocd.argoproj.io/kubernetes-version": "1.27", "argocd.argoproj.io/auto-label-cluster-info": "true"},
+			},
+			func(ctx context.Context, cluster *v1alpha1.Cluster) (*v1alpha1.Cluster, error) {
+				assert.Equal(t, cluster.Labels["argocd.argoproj.io/kubernetes-version"], "1.28")
+				return nil, errors.New("some error happened while saving")
+			},
+			assert.Error,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.wantErr(t, updateClusterLabels(context.Background(), tt.clusterInfo, tt.cluster, tt.updateCluster), fmt.Sprintf("updateClusterLabels(%v, %v, %v)", context.Background(), tt.clusterInfo, tt.cluster))
+		})
+	}
+}
diff --git a/docs/operator-manual/applicationset/Generators-Cluster.md b/docs/operator-manual/applicationset/Generators-Cluster.md
@@ -136,6 +136,29 @@ However, if you do wish to target both local and non-local clusters, while also
 
 These steps might seem counterintuitive, but the act of changing one of the default values for the local cluster causes the Argo CD Web UI to create a new secret for this cluster. In the Argo CD namespace, you should now see a Secret resource named `cluster-(cluster suffix)` with label `argocd.argoproj.io/secret-type": "cluster"`. You may also create a local [cluster secret declaratively](../../declarative-setup/#clusters), or with the CLI using `argocd cluster add "(context name)" --in-cluster`, rather than through the Web UI.
 
+### Fetch clusters based on their K8s version
+
+There is also the possibility to fetch clusters based upon their Kubernetes version. To do this, the label `argocd.argoproj.io/auto-label-cluster-info` needs to be set to `true` on the cluster secret. 
+Once that has been set, the controller will dynamically label the cluster secret with the Kubernetes version it is running on. To retrieve that value, you need to use the
+`argocd.argoproj.io/kubernetes-version`, as the example below demonstrates:
+
+```yaml
+spec:
+  goTemplate: true
+  generators:
+  - clusters:
+      selector:
+        matchLabels:
+          argocd.argoproj.io/kubernetes-version: 1.28
+        # matchExpressions are also supported.
+        #matchExpressions:
+        #  - key: argocd.argoproj.io/kubernetes-version
+        #    operator: In
+        #    values:
+        #      - "1.27"
+        #      - "1.28"
+```
+
 ### Pass additional key-value pairs via `values` field
 
 You may pass additional, arbitrary string key-value pairs via the `values` field of the cluster generator. Values added via the `values` field are added as `values.(field)`

diff --git a/util/db/cluster.go b/util/db/cluster.go
@@ -316,7 +316,7 @@ func (db *db) DeleteCluster(ctx context.Context, server string) error {
 	return db.settingsMgr.ResyncInformers()
 }
 
-// clusterToData converts a cluster object to string data for serialization to a secret
+// clusterToSecret converts a cluster object to string data for serialization to a secret
 func clusterToSecret(c *appv1.Cluster, secret *apiv1.Secret) error {
 	data := make(map[string][]byte)
 	data["server"] = []byte(strings.TrimRight(c.Server, "/"))