Merge branch 'master' of github.com:argoproj/argo-cd into fix/trackin…

…g-annotation-reference-pruning

USERS.md

@@ -27,6 +27,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [BigPanda](https://bigpanda.io)
 1. [BioBox Analytics](https://biobox.io)
 1. [BMW Group](https://www.bmwgroup.com/)
+1. [Bulder Bank](https://bulderbank.no)
 1. [Boozt](https://www.booztgroup.com/)
 1. [Boticario](https://www.boticario.com.br/)
 1. [Camptocamp](https://camptocamp.com)
@@ -37,6 +38,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [Chargetrip](https://chargetrip.com)
 1. [Chime](https://www.chime.com)
 1. [Cisco ET&I](https://eti.cisco.com/)
+1. [Cobalt](https://www.cobalt.io/)
 1. [Codefresh](https://www.codefresh.io/)
 1. [Codility](https://www.codility.com/)
 1. [Commonbond](https://commonbond.co/)
@@ -211,3 +213,4 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [Trendyol](https://www.trendyol.com/)
 1. [RapidAPI](https://www.rapidapi.com/)
 1. [Pismo](https://pismo.io/)
+1. [South China Morning Post (SCMP)](https://www.scmp.com/)

applicationset/generators/cluster.go

@@ -3,6 +3,7 @@ package generators
 import (
 	"context"
 	"fmt"
+	"github.com/valyala/fasttemplate"
 	"regexp"
 	"strings"
 	"time"
@@ -37,6 +38,8 @@ type ClusterGenerator struct {
 	settingsManager *settings.SettingsManager
 }
 
+var render = &utils.Render{}
+
 func NewClusterGenerator(c client.Client, ctx context.Context, clientset kubernetes.Interface, namespace string) Generator {
 
 	settingsManager := settings.NewSettingsManager(ctx, clientset, namespace)
@@ -104,21 +107,24 @@ func (g *ClusterGenerator) GenerateParams(
 			// If there is no secret for the cluster, it's the local cluster, so handle it here.
 			params := map[string]string{}
 			params["name"] = cluster.Name
+			params["nameNormalized"] = cluster.Name
 			params["server"] = cluster.Server
 
-			for key, value := range appSetGenerator.Clusters.Values {
-				params[fmt.Sprintf("values.%s", key)] = value
+			err = appendTemplatedValues(appSetGenerator.Clusters.Values, params)
+			if err != nil {
+				return nil, err
 			}
 
-			log.WithField("cluster", "local cluster").Info("matched local cluster")
-
 			res = append(res, params)
+
+			log.WithField("cluster", "local cluster").Info("matched local cluster")
 		}
 	}
 
 	// For each matching cluster secret (non-local clusters only)
 	for _, cluster := range secretsFound {
 		params := map[string]string{}
+
 		params["name"] = string(cluster.Data["name"])
 		params["nameNormalized"] = sanitizeName(string(cluster.Data["name"]))
 		params["server"] = string(cluster.Data["server"])
@@ -128,17 +134,52 @@ func (g *ClusterGenerator) GenerateParams(
 		for key, value := range cluster.ObjectMeta.Labels {
 			params[fmt.Sprintf("metadata.labels.%s", key)] = value
 		}
-		for key, value := range appSetGenerator.Clusters.Values {
-			params[fmt.Sprintf("values.%s", key)] = value
+
+		err = appendTemplatedValues(appSetGenerator.Clusters.Values, params)
+		if err != nil {
+			return nil, err
 		}
-		log.WithField("cluster", cluster.Name).Info("matched cluster secret")
 
 		res = append(res, params)
+
+		log.WithField("cluster", cluster.Name).Info("matched cluster secret")
 	}
 
 	return res, nil
 }
 
+func appendTemplatedValues(clusterValues map[string]string, params map[string]string) error {
+	// We create a local map to ensure that we do not fall victim to a billion-laughs attack. We iterate through the
+	// cluster values map and only replace values in said map if it has already been whitelisted in the params map.
+	// Once we iterate through all the cluster values we can then safely merge the `tmp` map into the main params map.
+	tmp := map[string]string{}
+
+	for key, value := range clusterValues {
+		result, err := replaceTemplatedString(value, params)
+
+		if err != nil {
+			return err
+		}
+
+		tmp[fmt.Sprintf("values.%s", key)] = result
+	}
+
+	for key, value := range tmp {
+		params[key] = value
+	}
+
+	return nil
+}
+
+func replaceTemplatedString(value string, params map[string]string) (string, error) {
+	fstTmpl := fasttemplate.New(value, "{{", "}}")
+	replacedTmplStr, err := render.Replace(fstTmpl, params, true)
+	if err != nil {
+		return "", err
+	}
+	return replacedTmplStr, nil
+}
+
 func (g *ClusterGenerator) getSecretsByClusterName(appSetGenerator *argoappsetv1alpha1.ApplicationSetGenerator) (map[string]corev1.Secret, error) {
 	// List all Clusters:
 	clusterSecretList := &corev1.SecretList{}

applicationset/generators/cluster_test.go

@@ -94,15 +94,23 @@ func TestGenerateParams(t *testing.T) {
 		{
 			name:     "no label selector",
 			selector: metav1.LabelSelector{},
-			values:   nil,
-			expected: []map[string]string{
-				{"name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar",
+			values: map[string]string{
+				"lol1":  "lol",
+				"lol2":  "{{values.lol1}}{{values.lol1}}",
+				"lol3":  "{{values.lol2}}{{values.lol2}}{{values.lol2}}",
+				"foo":   "bar",
+				"bar":   "{{ metadata.annotations.foo.argoproj.io }}",
+				"bat":   "{{ metadata.labels.environment }}",
+				"aaa":   "{{ server }}",
+				"no-op": "{{ this-does-not-exist }}",
+			}, expected: []map[string]string{
+				{"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "production", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "production", "values.aaa": "https://production-01.example.com", "name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar",
 					"metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"},
 
-				{"name": "staging-01", "nameNormalized": "staging-01", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo",
+				{"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "staging", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "staging", "values.aaa": "https://staging-01.example.com", "name": "staging-01", "nameNormalized": "staging-01", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo",
 					"metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"},
 
-				{"name": "in-cluster", "server": "https://kubernetes.default.svc"},
+				{"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "{{ metadata.annotations.foo.argoproj.io }}", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "{{ metadata.labels.environment }}", "values.aaa": "https://kubernetes.default.svc", "nameNormalized": "in-cluster", "name": "in-cluster", "server": "https://kubernetes.default.svc"},
 			},
 			clientError:   false,
 			expectedError: nil,

applicationset/generators/git.go

@@ -162,8 +162,10 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
 		}
 		params["path"] = path.Dir(filePath)
 		params["path.basename"] = path.Base(params["path"])
+		params["path.filename"] = path.Base(filePath)
 		params["path.basenameNormalized"] = sanitizeName(path.Base(params["path"]))
-		for k, v := range strings.Split(strings.TrimSuffix(params["path"], params["path.basename"]), "/") {
+		params["path.filenameNormalized"] = sanitizeName(path.Base(params["path.filename"]))
+		for k, v := range strings.Split(params["path"], "/") {
 			if len(v) > 0 {
 				params["path["+strconv.Itoa(k)+"]"] = v
 			}
@@ -213,7 +215,7 @@ func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argopro
 		params["path"] = a
 		params["path.basename"] = path.Base(a)
 		params["path.basenameNormalized"] = sanitizeName(path.Base(a))
-		for k, v := range strings.Split(strings.TrimSuffix(params["path"], params["path.basename"]), "/") {
+		for k, v := range strings.Split(params["path"], "/") {
 			if len(v) > 0 {
 				params["path["+strconv.Itoa(k)+"]"] = v
 			}

applicationset/generators/git_test.go

@@ -47,6 +47,28 @@ func (a argoCDServiceMock) GetDirectories(ctx context.Context, repoURL string, r
 	return args.Get(0).([]string), args.Error(1)
 }
 
+func Test_generateParamsFromGitFile(t *testing.T) {
+	params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
+foo:
+  bar: baz
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, []map[string]string{
+		{
+			"foo.bar":                 "baz",
+			"path":                    "path/dir",
+			"path.basename":           "dir",
+			"path.filename":           "file_name.yaml",
+			"path.basenameNormalized": "dir",
+			"path.filenameNormalized": "file-name.yaml",
+			"path[0]":                 "path",
+			"path[1]":                 "dir",
+		},
+	}, params)
+}
+
 func TestGitGenerateParamsFromDirectories(t *testing.T) {
 
 	cases := []struct {
@@ -68,9 +90,9 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 			},
 			repoError: nil,
 			expected: []map[string]string{
-				{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"},
-				{"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"},
-				{"path": "app_3", "path.basename": "app_3", "path.basenameNormalized": "app-3"},
+				{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1", "path[0]": "app1"},
+				{"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2", "path[0]": "app2"},
+				{"path": "app_3", "path.basename": "app_3", "path.basenameNormalized": "app-3", "path[0]": "app_3"},
 			},
 			expectedError: nil,
 		},
@@ -85,8 +107,8 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 			},
 			repoError: nil,
 			expected: []map[string]string{
-				{"path": "p1/app2", "path.basename": "app2", "path[0]": "p1", "path.basenameNormalized": "app2"},
-				{"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3"},
+				{"path": "p1/app2", "path.basename": "app2", "path[0]": "p1", "path[1]": "app2", "path.basenameNormalized": "app2"},
+				{"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path[2]": "app3", "path.basenameNormalized": "app3"},
 			},
 			expectedError: nil,
 		},
@@ -102,9 +124,9 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 			},
 			repoError: nil,
 			expected: []map[string]string{
-				{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"},
-				{"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"},
-				{"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path.basenameNormalized": "app3"},
+				{"path": "app1", "path.basename": "app1", "path[0]": "app1", "path.basenameNormalized": "app1"},
+				{"path": "app2", "path.basename": "app2", "path[0]": "app2", "path.basenameNormalized": "app2"},
+				{"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path[1]": "app3", "path.basenameNormalized": "app3"},
 			},
 			expectedError: nil,
 		},
@@ -120,9 +142,9 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 			},
 			repoError: nil,
 			expected: []map[string]string{
-				{"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"},
-				{"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"},
-				{"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path.basenameNormalized": "app3"},
+				{"path": "app1", "path.basename": "app1", "path[0]": "app1", "path.basenameNormalized": "app1"},
+				{"path": "app2", "path.basename": "app2", "path[0]": "app2", "path.basenameNormalized": "app2"},
+				{"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path[1]": "app3", "path.basenameNormalized": "app3"},
 			},
 			expectedError: nil,
 		},
@@ -238,7 +260,10 @@ func TestGitGenerateParamsFromFiles(t *testing.T) {
 					"path":                    "cluster-config/production",
 					"path.basename":           "production",
 					"path[0]":                 "cluster-config",
+					"path[1]":                 "production",
 					"path.basenameNormalized": "production",
+					"path.filename":           "config.json",
+					"path.filenameNormalized": "config.json",
 				},
 				{
 					"cluster.owner":           "foo.bar@example.com",
@@ -247,7 +272,10 @@ func TestGitGenerateParamsFromFiles(t *testing.T) {
 					"path":                    "cluster-config/staging",
 					"path.basename":           "staging",
 					"path[0]":                 "cluster-config",
+					"path[1]":                 "staging",
 					"path.basenameNormalized": "staging",
+					"path.filename":           "config.json",
+					"path.filenameNormalized": "config.json",
 				},
 			},
 			expectedError: nil,
@@ -305,7 +333,10 @@ func TestGitGenerateParamsFromFiles(t *testing.T) {
 					"path":                    "cluster-config/production",
 					"path.basename":           "production",
 					"path[0]":                 "cluster-config",
+					"path[1]":                 "production",
 					"path.basenameNormalized": "production",
+					"path.filename":           "config.json",
+					"path.filenameNormalized": "config.json",
 				},
 				{
 					"cluster.owner":           "john.doe@example.com",
@@ -314,7 +345,10 @@ func TestGitGenerateParamsFromFiles(t *testing.T) {
 					"path":                    "cluster-config/production",
 					"path.basename":           "production",
 					"path[0]":                 "cluster-config",
+					"path[1]":                 "production",
 					"path.basenameNormalized": "production",
+					"path.filename":           "config.json",
+					"path.filenameNormalized": "config.json",
 				},
 			},
 			expectedError: nil,
@@ -353,7 +387,10 @@ cluster:
 					"path":                    "cluster-config/production",
 					"path.basename":           "production",
 					"path[0]":                 "cluster-config",
+					"path[1]":                 "production",
 					"path.basenameNormalized": "production",
+					"path.filename":           "config.yaml",
+					"path.filenameNormalized": "config.yaml",
 				},
 				{
 					"cluster.owner":           "foo.bar@example.com",
@@ -362,7 +399,10 @@ cluster:
 					"path":                    "cluster-config/staging",
 					"path.basename":           "staging",
 					"path[0]":                 "cluster-config",
+					"path[1]":                 "staging",
 					"path.basenameNormalized": "staging",
+					"path.filename":           "config.yaml",
+					"path.filenameNormalized": "config.yaml",
 				},
 			},
 			expectedError: nil,
@@ -393,7 +433,10 @@ cluster:
 					"path":                    "cluster-config/production",
 					"path.basename":           "production",
 					"path[0]":                 "cluster-config",
+					"path[1]":                 "production",
 					"path.basenameNormalized": "production",
+					"path.filename":           "config.yaml",
+					"path.filenameNormalized": "config.yaml",
 				},
 				{
 					"cluster.owner":           "john.doe@example.com",
@@ -402,7 +445,10 @@ cluster:
 					"path":                    "cluster-config/production",
 					"path.basename":           "production",
 					"path[0]":                 "cluster-config",
+					"path[1]":                 "production",
 					"path.basenameNormalized": "production",
+					"path.filename":           "config.yaml",
+					"path.filenameNormalized": "config.yaml",
 				},
 			},
 			expectedError: nil,

applicationset/generators/pull_request.go

@@ -80,12 +80,19 @@ func (g *PullRequestGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha
 		"_": "-",
 	}
 
+	var shortSHALength int
 	for _, pull := range pulls {
+		shortSHALength = 8
+		if len(pull.HeadSHA) < 8 {
+			shortSHALength = len(pull.HeadSHA)
+		}
+
 		params = append(params, map[string]string{
-			"number":      strconv.Itoa(pull.Number),
-			"branch":      pull.Branch,
-			"branch_slug": slug.Make(pull.Branch),
-			"head_sha":    pull.HeadSHA,
+			"number":         strconv.Itoa(pull.Number),
+			"branch":         pull.Branch,
+			"branch_slug":    slug.Make(pull.Branch),
+			"head_sha":       pull.HeadSHA,
+			"head_short_sha": pull.HeadSHA[:shortSHALength],
 		})
 	}
 	return params, nil