From 340406bf6c2b96dfde033883391dd35466b46e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Legrand?= Date: Sat, 14 Oct 2023 00:01:52 +0200 Subject: [PATCH] add plugin tar inclusion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Clément Legrand --- USERS.md | 1 + .../commands/argocd_repo_server.go | 3 ++ cmpserver/plugin/plugin_test.go | 6 +-- .../config-management-plugins.md | 12 ++++- go.mod | 1 + go.sum | 2 + .../argocd-repo-server-deployment.yaml | 6 +++ manifests/core-install.yaml | 6 +++ manifests/ha/install.yaml | 6 +++ manifests/ha/namespace-install.yaml | 6 +++ manifests/install.yaml | 6 +++ manifests/namespace-install.yaml | 6 +++ reposerver/repository/repository.go | 42 ++++++++++------- util/app/discovery/discovery.go | 22 ++++----- util/app/discovery/discovery_test.go | 14 +++--- util/cmp/stream.go | 8 ++-- util/cmp/stream_test.go | 6 +-- util/io/files/tar.go | 7 +-- util/io/files/tar_test.go | 47 ++++++++++++++++--- util/io/files/testdata/app/git/index | 0 .../src/domain/service/deploy/helmfile.yaml | 0 .../src/domain/service/deploy/template.tpl | 0 22 files changed, 152 insertions(+), 55 deletions(-) create mode 100644 util/io/files/testdata/app/git/index create mode 100644 util/io/files/testdata/app/src/domain/service/deploy/helmfile.yaml create mode 100644 util/io/files/testdata/app/src/domain/service/deploy/template.tpl diff --git a/USERS.md b/USERS.md index 6c4bc8e5e3eac..5056512f6b4c6 100644 --- a/USERS.md +++ b/USERS.md @@ -117,6 +117,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Grupo MasMovil](https://grupomasmovil.com/en/) 1. [Handelsbanken](https://www.handelsbanken.se) 1. [Healy](https://www.healyworld.net) +1. [Heetch](https://www.heetch.com) 1. [Helio](https://helio.exchange) 1. [Hetki](https://hetki.ai) 1. [hipages](https://hipages.com.au/) diff --git a/cmd/argocd-repo-server/commands/argocd_repo_server.go b/cmd/argocd-repo-server/commands/argocd_repo_server.go index 69358d2a91efd..c72baf2645a08 100644 --- a/cmd/argocd-repo-server/commands/argocd_repo_server.go +++ b/cmd/argocd-repo-server/commands/argocd_repo_server.go @@ -62,6 +62,7 @@ func NewCommand() *cobra.Command { disableTLS bool maxCombinedDirectoryManifestsSize string cmpTarExcludedGlobs []string + cmpTarIncludedGlobs []string allowOutOfBoundsSymlinks bool streamedManifestMaxTarSize string streamedManifestMaxExtractedSize string @@ -119,6 +120,7 @@ func NewCommand() *cobra.Command { SubmoduleEnabled: gitSubmoduleEnabled, MaxCombinedDirectoryManifestsSize: maxCombinedDirectoryManifestsQuantity, CMPTarExcludedGlobs: cmpTarExcludedGlobs, + CMPTarIncludedGlobs: cmpTarIncludedGlobs, AllowOutOfBoundsSymlinks: allowOutOfBoundsSymlinks, StreamedManifestMaxExtractedSize: streamedManifestMaxExtractedSizeQuantity.ToDec().Value(), StreamedManifestMaxTarSize: streamedManifestMaxTarSizeQuantity.ToDec().Value(), @@ -200,6 +202,7 @@ func NewCommand() *cobra.Command { command.Flags().BoolVar(&disableTLS, "disable-tls", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_TLS", false), "Disable TLS on the gRPC endpoint") command.Flags().StringVar(&maxCombinedDirectoryManifestsSize, "max-combined-directory-manifests-size", env.StringFromEnv("ARGOCD_REPO_SERVER_MAX_COMBINED_DIRECTORY_MANIFESTS_SIZE", "10M"), "Max combined size of manifest files in a directory-type Application") command.Flags().StringArrayVar(&cmpTarExcludedGlobs, "plugin-tar-exclude", env.StringsFromEnv("ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS", []string{}, ";"), "Globs to filter when sending tarballs to plugins.") + command.Flags().StringArrayVar(&cmpTarIncludedGlobs, "plugin-tar-include", env.StringsFromEnv("ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS", []string{}, ";"), "Globs to include files when sending tarballs to plugins.") command.Flags().BoolVar(&allowOutOfBoundsSymlinks, "allow-oob-symlinks", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS", false), "Allow out-of-bounds symlinks in repositories (not recommended)") command.Flags().StringVar(&streamedManifestMaxTarSize, "streamed-manifest-max-tar-size", env.StringFromEnv("ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE", "100M"), "Maximum size of streamed manifest archives") command.Flags().StringVar(&streamedManifestMaxExtractedSize, "streamed-manifest-max-extracted-size", env.StringFromEnv("ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE", "1G"), "Maximum size of streamed manifest archives when extracted") diff --git a/cmpserver/plugin/plugin_test.go b/cmpserver/plugin/plugin_test.go index b253dc414cbdc..1e55a06793119 100644 --- a/cmpserver/plugin/plugin_test.go +++ b/cmpserver/plugin/plugin_test.go @@ -541,7 +541,7 @@ type MockGenerateManifestStream struct { } func NewMockGenerateManifestStream(repoPath, appPath string, env []string) (*MockGenerateManifestStream, error) { - tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil) + tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil, nil) if err != nil { return nil, err } @@ -615,7 +615,7 @@ type MockMatchRepositoryStream struct { } func NewMockMatchRepositoryStream(repoPath, appPath string, env []string) (*MockMatchRepositoryStream, error) { - tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil) + tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil, nil) if err != nil { return nil, err } @@ -688,7 +688,7 @@ type MockParametersAnnouncementStream struct { } func NewMockParametersAnnouncementStream(repoPath, appPath string, env []string) (*MockParametersAnnouncementStream, error) { - tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil) + tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil, nil) if err != nil { return nil, err } diff --git a/docs/operator-manual/config-management-plugins.md b/docs/operator-manual/config-management-plugins.md index b77a4bdd7ba21..2b2c52287fad6 100644 --- a/docs/operator-manual/config-management-plugins.md +++ b/docs/operator-manual/config-management-plugins.md @@ -342,7 +342,7 @@ If you are actively developing a sidecar-installed CMP, keep a few things in min | -- | -- | | `no matches for kind "ConfigManagementPlugin" in version "argoproj.io/v1alpha1"` | The `ConfigManagementPlugin` CRD was deprecated in Argo CD 2.4 and removed in 2.8. This error means you've tried to put the configuration for your plugin directly into Kubernetes as a CRD. Refer to this [section of documentation](#write-the-plugin-configuration-file) for how to write the plugin configuration file and place it properly in the sidecar. | -## Plugin tar stream exclusions +## Plugin tar stream exclusions or inclusions In order to increase the speed of manifest generation, certain files and folders can be excluded from being sent to your plugin. We recommend excluding your `.git` folder if it isn't necessary. Use Go's @@ -354,6 +354,16 @@ You can set it one of three ways: 2. The `reposerver.plugin.tar.exclusions` key if you are using `argocd-cmd-params-cm` 3. Directly setting `ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS` environment variable on the repo server. +In addition, if you're using a monorepo mixed with code and GitOps resources, you probably want to only include files +involved in manifests generation in the tar archive. Use Go's [filepatch.Match](https://pkg.go.dev/path/filepath#Match) syntax to include +desired files. For example, `*.y*ml` to include `.yaml` or `.yml` files. + +You can set it one of three ways: + +1. The `--plugin-tar-include` argument on the repo server. +2. The `reposerver.plugin.tar.inclusions` key if you are using `argocd-cmd-params-cm` +3. Directly setting `ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS` environment variable on the repo server. + For option 1, the flag can be repeated multiple times. For option 2 and 3, you can specify multiple globs by separating them with semicolons. diff --git a/go.mod b/go.mod index 2dc71e929b0e9..c62c7253171b4 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/argoproj/notifications-engine v0.4.1-0.20230905144632-9dcecdc3eebf github.com/argoproj/pkg v0.13.7-0.20230626144333-d56162821bd1 github.com/aws/aws-sdk-go v1.44.317 + github.com/bmatcuk/doublestar v1.3.4 github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/bombsimon/logrusr/v2 v2.0.1 github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 diff --git a/go.sum b/go.sum index fcf4a561a7511..240ee81c65122 100644 --- a/go.sum +++ b/go.sum @@ -752,6 +752,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= +github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc= github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM= diff --git a/manifests/base/repo-server/argocd-repo-server-deployment.yaml b/manifests/base/repo-server/argocd-repo-server-deployment.yaml index 728c80c14bc2a..12b5eada5c385 100644 --- a/manifests/base/repo-server/argocd-repo-server-deployment.yaml +++ b/manifests/base/repo-server/argocd-repo-server-deployment.yaml @@ -132,6 +132,12 @@ spec: name: argocd-cmd-params-cm key: reposerver.plugin.tar.exclusions optional: true + - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS + valueFrom: + configMapKeyRef: + key: reposerver.plugin.tar.inclusions + name: argocd-cmd-params-cm + optional: true - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS valueFrom: configMapKeyRef: diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml index 86e636e12ef59..4452c11bb2e15 100644 --- a/manifests/core-install.yaml +++ b/manifests/core-install.yaml @@ -21006,6 +21006,12 @@ spec: key: reposerver.plugin.tar.exclusions name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS + valueFrom: + configMapKeyRef: + key: reposerver.plugin.tar.inclusions + name: argocd-cmd-params-cm + optional: true - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS valueFrom: configMapKeyRef: diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml index 043083de6be84..2b996d7471d17 100644 --- a/manifests/ha/install.yaml +++ b/manifests/ha/install.yaml @@ -22492,6 +22492,12 @@ spec: key: reposerver.plugin.tar.exclusions name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS + valueFrom: + configMapKeyRef: + key: reposerver.plugin.tar.inclusions + name: argocd-cmd-params-cm + optional: true - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS valueFrom: configMapKeyRef: diff --git a/manifests/ha/namespace-install.yaml b/manifests/ha/namespace-install.yaml index 524ec8ace3e6c..d7d9543fd7f81 100644 --- a/manifests/ha/namespace-install.yaml +++ b/manifests/ha/namespace-install.yaml @@ -2148,6 +2148,12 @@ spec: key: reposerver.plugin.tar.exclusions name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS + valueFrom: + configMapKeyRef: + key: reposerver.plugin.tar.inclusions + name: argocd-cmd-params-cm + optional: true - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS valueFrom: configMapKeyRef: diff --git a/manifests/install.yaml b/manifests/install.yaml index a08c3a5cd1302..4778d1db5b8df 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -21538,6 +21538,12 @@ spec: key: reposerver.plugin.tar.exclusions name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS + valueFrom: + configMapKeyRef: + key: reposerver.plugin.tar.inclusions + name: argocd-cmd-params-cm + optional: true - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS valueFrom: configMapKeyRef: diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml index cfdf7756ff134..9ab350873ea35 100644 --- a/manifests/namespace-install.yaml +++ b/manifests/namespace-install.yaml @@ -1194,6 +1194,12 @@ spec: key: reposerver.plugin.tar.exclusions name: argocd-cmd-params-cm optional: true + - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS + valueFrom: + configMapKeyRef: + key: reposerver.plugin.tar.inclusions + name: argocd-cmd-params-cm + optional: true - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS valueFrom: configMapKeyRef: diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index f66bc71ac4621..d40009d211b24 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -103,6 +103,7 @@ type RepoServerInitConstants struct { SubmoduleEnabled bool MaxCombinedDirectoryManifestsSize resource.Quantity CMPTarExcludedGlobs []string + CMPTarIncludedGlobs []string AllowOutOfBoundsSymlinks bool StreamedManifestMaxExtractedSize int64 StreamedManifestMaxTarSize int64 @@ -220,7 +221,7 @@ func (s *Service) ListApps(ctx context.Context, q *apiclient.ListAppsRequest) (* } defer io.Close(closer) - apps, err := discovery.Discover(ctx, gitClient.Root(), gitClient.Root(), q.EnabledSourceTypes, s.initConstants.CMPTarExcludedGlobs) + apps, err := discovery.Discover(ctx, gitClient.Root(), gitClient.Root(), q.EnabledSourceTypes, s.initConstants.CMPTarExcludedGlobs, s.initConstants.CMPTarIncludedGlobs) if err != nil { return nil, fmt.Errorf("error discovering applications: %w", err) } @@ -782,7 +783,7 @@ func (s *Service) runManifestGenAsync(ctx context.Context, repoRoot, commitSHA, } } - manifestGenResult, err = GenerateManifests(ctx, opContext.appPath, repoRoot, commitSHA, q, false, s.gitCredsStore, s.initConstants.MaxCombinedDirectoryManifestsSize, s.gitRepoPaths, WithCMPTarDoneChannel(ch.tarDoneCh), WithCMPTarExcludedGlobs(s.initConstants.CMPTarExcludedGlobs)) + manifestGenResult, err = GenerateManifests(ctx, opContext.appPath, repoRoot, commitSHA, q, false, s.gitCredsStore, s.initConstants.MaxCombinedDirectoryManifestsSize, s.gitRepoPaths, WithCMPTarDoneChannel(ch.tarDoneCh), WithCMPTarExcludedGlobs(s.initConstants.CMPTarExcludedGlobs), WithCMPTarIncludedGlobs(s.initConstants.CMPTarIncludedGlobs)) } refSourceCommitSHAs := make(map[string]string) if len(repoRefs) > 0 { @@ -1321,6 +1322,7 @@ type GenerateManifestOpt func(*generateManifestOpt) type generateManifestOpt struct { cmpTarDoneCh chan<- bool cmpTarExcludedGlobs []string + cmpTarIncludedGlobs []string } func newGenerateManifestOpt(opts ...GenerateManifestOpt) *generateManifestOpt { @@ -1348,6 +1350,14 @@ func WithCMPTarExcludedGlobs(excludedGlobs []string) GenerateManifestOpt { } } +// WithCMPTarIncludedGlobs defines globs for files to include when streaming the tarball +// to a CMP sidecar. +func WithCMPTarIncludedGlobs(includedGlobs []string) GenerateManifestOpt { + return func(o *generateManifestOpt) { + o.cmpTarIncludedGlobs = includedGlobs + } +} + // GenerateManifests generates manifests from a path. Overrides are applied as a side effect on the given ApplicationSource. func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string, q *apiclient.ManifestRequest, isLocal bool, gitCredsStore git.CredsStore, maxCombinedManifestQuantity resource.Quantity, gitRepoPaths io.TempPaths, opts ...GenerateManifestOpt) (*apiclient.ManifestResponse, error) { opt := newGenerateManifestOpt(opts...) @@ -1355,7 +1365,7 @@ func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string, resourceTracking := argo.NewResourceTracking() - appSourceType, err := GetAppSourceType(ctx, q.ApplicationSource, appPath, repoRoot, q.AppName, q.EnabledSourceTypes, opt.cmpTarExcludedGlobs) + appSourceType, err := GetAppSourceType(ctx, q.ApplicationSource, appPath, repoRoot, q.AppName, q.EnabledSourceTypes, opt.cmpTarExcludedGlobs, opt.cmpTarIncludedGlobs) if err != nil { return nil, fmt.Errorf("error getting app source type: %w", err) } @@ -1381,7 +1391,7 @@ func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string, pluginName = q.ApplicationSource.Plugin.Name } // if pluginName is provided it has to be `-` or just `` if plugin version is empty - targetObjs, err = runConfigManagementPluginSidecars(ctx, appPath, repoRoot, pluginName, env, q, opt.cmpTarDoneCh, opt.cmpTarExcludedGlobs) + targetObjs, err = runConfigManagementPluginSidecars(ctx, appPath, repoRoot, pluginName, env, q, opt.cmpTarDoneCh, opt.cmpTarExcludedGlobs, opt.cmpTarIncludedGlobs) if err != nil { err = fmt.Errorf("plugin sidecar failed. %s", err.Error()) } @@ -1518,7 +1528,7 @@ func mergeSourceParameters(source *v1alpha1.ApplicationSource, path, appName str } // GetAppSourceType returns explicit application source type or examines a directory and determines its application source type -func GetAppSourceType(ctx context.Context, source *v1alpha1.ApplicationSource, appPath, repoPath, appName string, enableGenerateManifests map[string]bool, tarExcludedGlobs []string) (v1alpha1.ApplicationSourceType, error) { +func GetAppSourceType(ctx context.Context, source *v1alpha1.ApplicationSource, appPath, repoPath, appName string, enableGenerateManifests map[string]bool, tarExcludedGlobs, tarIncludedGlobs []string) (v1alpha1.ApplicationSourceType, error) { err := mergeSourceParameters(source, appPath, appName) if err != nil { return "", fmt.Errorf("error while parsing source parameters: %v", err) @@ -1535,7 +1545,7 @@ func GetAppSourceType(ctx context.Context, source *v1alpha1.ApplicationSource, a } return *appSourceType, nil } - appType, err := discovery.AppType(ctx, appPath, repoPath, enableGenerateManifests, tarExcludedGlobs) + appType, err := discovery.AppType(ctx, appPath, repoPath, enableGenerateManifests, tarExcludedGlobs, tarIncludedGlobs) if err != nil { return "", fmt.Errorf("error getting app source type: %v", err) } @@ -1882,7 +1892,7 @@ func getPluginParamEnvs(envVars []string, plugin *v1alpha1.ApplicationSourcePlug return env, nil } -func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath, pluginName string, envVars *v1alpha1.Env, q *apiclient.ManifestRequest, tarDoneCh chan<- bool, tarExcludedGlobs []string) ([]*unstructured.Unstructured, error) { +func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath, pluginName string, envVars *v1alpha1.Env, q *apiclient.ManifestRequest, tarDoneCh chan<- bool, tarExcludedGlobs, tarIncludedGlobs []string) ([]*unstructured.Unstructured, error) { // compute variables. env, err := getPluginEnvs(envVars, q) if err != nil { @@ -1890,14 +1900,14 @@ func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath, p } // detect config management plugin server - conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, appPath, repoPath, pluginName, env, tarExcludedGlobs) + conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, appPath, repoPath, pluginName, env, tarExcludedGlobs, tarIncludedGlobs) if err != nil { return nil, err } defer io.Close(conn) // generate manifests using commands provided in plugin config file in detected cmp-server sidecar - cmpManifests, err := generateManifestsCMP(ctx, appPath, repoPath, env, cmpClient, tarDoneCh, tarExcludedGlobs) + cmpManifests, err := generateManifestsCMP(ctx, appPath, repoPath, env, cmpClient, tarDoneCh, tarExcludedGlobs, tarIncludedGlobs) if err != nil { return nil, fmt.Errorf("error generating manifests in cmp: %s", err) } @@ -1920,7 +1930,7 @@ func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath, p // generateManifestsCMP will send the appPath files to the cmp-server over a gRPC stream. // The cmp-server will generate the manifests. Returns a response object with the generated // manifests. -func generateManifestsCMP(ctx context.Context, appPath, repoPath string, env []string, cmpClient pluginclient.ConfigManagementPluginServiceClient, tarDoneCh chan<- bool, tarExcludedGlobs []string) (*pluginclient.ManifestResponse, error) { +func generateManifestsCMP(ctx context.Context, appPath, repoPath string, env []string, cmpClient pluginclient.ConfigManagementPluginServiceClient, tarDoneCh chan<- bool, tarExcludedGlobs, tarIncludedGlobs []string) (*pluginclient.ManifestResponse, error) { generateManifestStream, err := cmpClient.GenerateManifest(ctx, grpc_retry.Disable()) if err != nil { return nil, fmt.Errorf("error getting generateManifestStream: %w", err) @@ -1929,7 +1939,7 @@ func generateManifestsCMP(ctx context.Context, appPath, repoPath string, env []s cmp.WithTarDoneChan(tarDoneCh), } - err = cmp.SendRepoStream(generateManifestStream.Context(), appPath, repoPath, generateManifestStream, env, tarExcludedGlobs, opts...) + err = cmp.SendRepoStream(generateManifestStream.Context(), appPath, repoPath, generateManifestStream, env, tarExcludedGlobs, tarIncludedGlobs, opts...) if err != nil { return nil, fmt.Errorf("error sending file to cmp-server: %s", err) } @@ -1947,7 +1957,7 @@ func (s *Service) GetAppDetails(ctx context.Context, q *apiclient.RepoServerAppD return err } - appSourceType, err := GetAppSourceType(ctx, q.Source, opContext.appPath, repoRoot, q.AppName, q.EnabledSourceTypes, s.initConstants.CMPTarExcludedGlobs) + appSourceType, err := GetAppSourceType(ctx, q.Source, opContext.appPath, repoRoot, q.AppName, q.EnabledSourceTypes, s.initConstants.CMPTarExcludedGlobs, s.initConstants.CMPTarIncludedGlobs) if err != nil { return err } @@ -1964,7 +1974,7 @@ func (s *Service) GetAppDetails(ctx context.Context, q *apiclient.RepoServerAppD return err } case v1alpha1.ApplicationSourceTypePlugin: - if err := populatePluginAppDetails(ctx, res, opContext.appPath, repoRoot, q, s.gitCredsStore, s.initConstants.CMPTarExcludedGlobs); err != nil { + if err := populatePluginAppDetails(ctx, res, opContext.appPath, repoRoot, q, s.gitCredsStore, s.initConstants.CMPTarExcludedGlobs, s.initConstants.CMPTarIncludedGlobs); err != nil { return fmt.Errorf("failed to populate plugin app details: %w", err) } } @@ -2123,7 +2133,7 @@ func populateKustomizeAppDetails(res *apiclient.RepoAppDetailsResponse, q *apicl return nil } -func populatePluginAppDetails(ctx context.Context, res *apiclient.RepoAppDetailsResponse, appPath string, repoPath string, q *apiclient.RepoServerAppDetailsQuery, store git.CredsStore, tarExcludedGlobs []string) error { +func populatePluginAppDetails(ctx context.Context, res *apiclient.RepoAppDetailsResponse, appPath string, repoPath string, q *apiclient.RepoServerAppDetailsQuery, store git.CredsStore, tarExcludedGlobs, tarIncludedGlobs []string) error { res.Plugin = &apiclient.PluginAppSpec{} envVars := []string{ @@ -2143,7 +2153,7 @@ func populatePluginAppDetails(ctx context.Context, res *apiclient.RepoAppDetails pluginName = q.Source.Plugin.Name } // detect config management plugin server (sidecar) - conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, appPath, repoPath, pluginName, env, tarExcludedGlobs) + conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, appPath, repoPath, pluginName, env, tarExcludedGlobs, tarIncludedGlobs) if err != nil { return fmt.Errorf("failed to detect CMP for app: %w", err) } @@ -2154,7 +2164,7 @@ func populatePluginAppDetails(ctx context.Context, res *apiclient.RepoAppDetails return fmt.Errorf("error getting parametersAnnouncementStream: %w", err) } - err = cmp.SendRepoStream(parametersAnnouncementStream.Context(), appPath, repoPath, parametersAnnouncementStream, env, tarExcludedGlobs) + err = cmp.SendRepoStream(parametersAnnouncementStream.Context(), appPath, repoPath, parametersAnnouncementStream, env, tarExcludedGlobs, tarIncludedGlobs) if err != nil { return fmt.Errorf("error sending file to cmp-server: %s", err) } diff --git a/util/app/discovery/discovery.go b/util/app/discovery/discovery.go index 21fbe5fd4bf36..60552c42e1c48 100644 --- a/util/app/discovery/discovery.go +++ b/util/app/discovery/discovery.go @@ -31,11 +31,11 @@ func IsManifestGenerationEnabled(sourceType v1alpha1.ApplicationSourceType, enab return enabled } -func Discover(ctx context.Context, appPath, repoPath string, enableGenerateManifests map[string]bool, tarExcludedGlobs []string) (map[string]string, error) { +func Discover(ctx context.Context, appPath, repoPath string, enableGenerateManifests map[string]bool, tarExcludedGlobs, tarIncludedGlobs []string) (map[string]string, error) { apps := make(map[string]string) // Check if it is CMP - conn, _, err := DetectConfigManagementPlugin(ctx, appPath, repoPath, "", []string{}, tarExcludedGlobs) + conn, _, err := DetectConfigManagementPlugin(ctx, appPath, repoPath, "", []string{}, tarExcludedGlobs, tarIncludedGlobs) if err == nil { // Found CMP io.Close(conn) @@ -67,8 +67,8 @@ func Discover(ctx context.Context, appPath, repoPath string, enableGenerateManif return apps, err } -func AppType(ctx context.Context, appPath, repoPath string, enableGenerateManifests map[string]bool, tarExcludedGlobs []string) (string, error) { - apps, err := Discover(ctx, appPath, repoPath, enableGenerateManifests, tarExcludedGlobs) +func AppType(ctx context.Context, appPath, repoPath string, enableGenerateManifests map[string]bool, tarExcludedGlobs, tarIncludedGlobs []string) (string, error) { + apps, err := Discover(ctx, appPath, repoPath, enableGenerateManifests, tarExcludedGlobs, tarIncludedGlobs) if err != nil { return "", err } @@ -85,7 +85,7 @@ func AppType(ctx context.Context, appPath, repoPath string, enableGenerateManife // check cmpSupports() // if supported return conn for the cmp-server -func DetectConfigManagementPlugin(ctx context.Context, appPath, repoPath, pluginName string, env []string, tarExcludedGlobs []string) (io.Closer, pluginclient.ConfigManagementPluginServiceClient, error) { +func DetectConfigManagementPlugin(ctx context.Context, appPath, repoPath, pluginName string, env []string, tarExcludedGlobs, tarIncludedGlobs []string) (io.Closer, pluginclient.ConfigManagementPluginServiceClient, error) { var conn io.Closer var cmpClient pluginclient.ConfigManagementPluginServiceClient var connFound bool @@ -98,7 +98,7 @@ func DetectConfigManagementPlugin(ctx context.Context, appPath, repoPath, plugin if pluginName != "" { // check if the given plugin supports the repo - conn, cmpClient, connFound = cmpSupports(ctx, pluginSockFilePath, appPath, repoPath, fmt.Sprintf("%v.sock", pluginName), env, tarExcludedGlobs, true) + conn, cmpClient, connFound = cmpSupports(ctx, pluginSockFilePath, appPath, repoPath, fmt.Sprintf("%v.sock", pluginName), env, tarExcludedGlobs, tarIncludedGlobs, true) if !connFound { return nil, nil, fmt.Errorf("couldn't find cmp-server plugin with name %q supporting the given repository", pluginName) } @@ -109,7 +109,7 @@ func DetectConfigManagementPlugin(ctx context.Context, appPath, repoPath, plugin } for _, file := range fileList { if file.Type() == os.ModeSocket { - conn, cmpClient, connFound = cmpSupports(ctx, pluginSockFilePath, appPath, repoPath, file.Name(), env, tarExcludedGlobs, false) + conn, cmpClient, connFound = cmpSupports(ctx, pluginSockFilePath, appPath, repoPath, file.Name(), env, tarExcludedGlobs, tarIncludedGlobs, false) if connFound { break } @@ -125,13 +125,13 @@ func DetectConfigManagementPlugin(ctx context.Context, appPath, repoPath, plugin // matchRepositoryCMP will send the repoPath to the cmp-server. The cmp-server will // inspect the files and return true if the repo is supported for manifest generation. // Will return false otherwise. -func matchRepositoryCMP(ctx context.Context, appPath, repoPath string, client pluginclient.ConfigManagementPluginServiceClient, env []string, tarExcludedGlobs []string) (bool, bool, error) { +func matchRepositoryCMP(ctx context.Context, appPath, repoPath string, client pluginclient.ConfigManagementPluginServiceClient, env []string, tarExcludedGlobs, tarIncludedGlobs []string) (bool, bool, error) { matchRepoStream, err := client.MatchRepository(ctx, grpc_retry.Disable()) if err != nil { return false, false, fmt.Errorf("error getting stream client: %s", err) } - err = cmp.SendRepoStream(ctx, appPath, repoPath, matchRepoStream, env, tarExcludedGlobs) + err = cmp.SendRepoStream(ctx, appPath, repoPath, matchRepoStream, env, tarExcludedGlobs, tarIncludedGlobs) if err != nil { return false, false, fmt.Errorf("error sending stream: %s", err) } @@ -142,7 +142,7 @@ func matchRepositoryCMP(ctx context.Context, appPath, repoPath string, client pl return resp.GetIsSupported(), resp.GetIsDiscoveryEnabled(), nil } -func cmpSupports(ctx context.Context, pluginSockFilePath, appPath, repoPath, fileName string, env []string, tarExcludedGlobs []string, namedPlugin bool) (io.Closer, pluginclient.ConfigManagementPluginServiceClient, bool) { +func cmpSupports(ctx context.Context, pluginSockFilePath, appPath, repoPath, fileName string, env []string, tarExcludedGlobs, tarIncludedGlobs []string, namedPlugin bool) (io.Closer, pluginclient.ConfigManagementPluginServiceClient, bool) { absPluginSockFilePath, err := filepath.Abs(pluginSockFilePath) if err != nil { log.Errorf("error getting absolute path for plugin socket dir %v, %v", pluginSockFilePath, err) @@ -165,7 +165,7 @@ func cmpSupports(ctx context.Context, pluginSockFilePath, appPath, repoPath, fil return nil, nil, false } - isSupported, isDiscoveryEnabled, err := matchRepositoryCMP(ctx, appPath, repoPath, cmpClient, env, tarExcludedGlobs) + isSupported, isDiscoveryEnabled, err := matchRepositoryCMP(ctx, appPath, repoPath, cmpClient, env, tarExcludedGlobs, tarIncludedGlobs) if err != nil { log.WithFields(log.Fields{ common.SecurityField: common.SecurityMedium, diff --git a/util/app/discovery/discovery_test.go b/util/app/discovery/discovery_test.go index 54eb30aff4fd1..771a1942eb467 100644 --- a/util/app/discovery/discovery_test.go +++ b/util/app/discovery/discovery_test.go @@ -10,7 +10,7 @@ import ( ) func TestDiscover(t *testing.T) { - apps, err := Discover(context.Background(), "./testdata", "./testdata", map[string]bool{}, []string{}) + apps, err := Discover(context.Background(), "./testdata", "./testdata", map[string]bool{}, []string{}, []string{}) assert.NoError(t, err) assert.Equal(t, map[string]string{ "foo": "Kustomize", @@ -19,15 +19,15 @@ func TestDiscover(t *testing.T) { } func TestAppType(t *testing.T) { - appType, err := AppType(context.Background(), "./testdata/foo", "./testdata", map[string]bool{}, []string{}) + appType, err := AppType(context.Background(), "./testdata/foo", "./testdata", map[string]bool{}, []string{}, []string{}) assert.NoError(t, err) assert.Equal(t, "Kustomize", appType) - appType, err = AppType(context.Background(), "./testdata/baz", "./testdata", map[string]bool{}, []string{}) + appType, err = AppType(context.Background(), "./testdata/baz", "./testdata", map[string]bool{}, []string{}, []string{}) assert.NoError(t, err) assert.Equal(t, "Helm", appType) - appType, err = AppType(context.Background(), "./testdata", "./testdata", map[string]bool{}, []string{}) + appType, err = AppType(context.Background(), "./testdata", "./testdata", map[string]bool{}, []string{}, []string{}) assert.NoError(t, err) assert.Equal(t, "Directory", appType) } @@ -37,15 +37,15 @@ func TestAppType_Disabled(t *testing.T) { string(v1alpha1.ApplicationSourceTypeKustomize): false, string(v1alpha1.ApplicationSourceTypeHelm): false, } - appType, err := AppType(context.Background(), "./testdata/foo", "./testdata", enableManifestGeneration, []string{}) + appType, err := AppType(context.Background(), "./testdata/foo", "./testdata", enableManifestGeneration, []string{}, []string{}) assert.NoError(t, err) assert.Equal(t, "Directory", appType) - appType, err = AppType(context.Background(), "./testdata/baz", "./testdata", enableManifestGeneration, []string{}) + appType, err = AppType(context.Background(), "./testdata/baz", "./testdata", enableManifestGeneration, []string{}, []string{}) assert.NoError(t, err) assert.Equal(t, "Directory", appType) - appType, err = AppType(context.Background(), "./testdata", "./testdata", enableManifestGeneration, []string{}) + appType, err = AppType(context.Background(), "./testdata", "./testdata", enableManifestGeneration, []string{}, []string{}) assert.NoError(t, err) assert.Equal(t, "Directory", appType) } diff --git a/util/cmp/stream.go b/util/cmp/stream.go index 8a0d1a97283d0..cf5f1a3060621 100644 --- a/util/cmp/stream.go +++ b/util/cmp/stream.go @@ -85,10 +85,10 @@ func WithTarDoneChan(ch chan<- bool) SenderOption { // SendRepoStream will compress the files under the given repoPath and send // them using the plugin stream sender. -func SendRepoStream(ctx context.Context, appPath, repoPath string, sender StreamSender, env []string, excludedGlobs []string, opts ...SenderOption) error { +func SendRepoStream(ctx context.Context, appPath, repoPath string, sender StreamSender, env []string, excludedGlobs, includedGlobs []string, opts ...SenderOption) error { opt := newSenderOption(opts...) - tgz, mr, err := GetCompressedRepoAndMetadata(repoPath, appPath, env, excludedGlobs, opt) + tgz, mr, err := GetCompressedRepoAndMetadata(repoPath, appPath, env, excludedGlobs, includedGlobs, opt) if err != nil { return err } @@ -106,9 +106,9 @@ func SendRepoStream(ctx context.Context, appPath, repoPath string, sender Stream return nil } -func GetCompressedRepoAndMetadata(repoPath string, appPath string, env []string, excludedGlobs []string, opt *senderOption) (*os.File, *pluginclient.AppStreamRequest, error) { +func GetCompressedRepoAndMetadata(repoPath string, appPath string, env []string, excludedGlobs, includedGlobs []string, opt *senderOption) (*os.File, *pluginclient.AppStreamRequest, error) { // compress all files in repoPath in tgz - tgz, filesWritten, checksum, err := tgzstream.CompressFiles(repoPath, nil, excludedGlobs) + tgz, filesWritten, checksum, err := tgzstream.CompressFiles(repoPath, includedGlobs, excludedGlobs) if err != nil { return nil, nil, fmt.Errorf("error compressing repo files: %w", err) } diff --git a/util/cmp/stream_test.go b/util/cmp/stream_test.go index 93a76cf10e673..ff350a97bd108 100644 --- a/util/cmp/stream_test.go +++ b/util/cmp/stream_test.go @@ -59,7 +59,7 @@ func TestReceiveApplicationStream(t *testing.T) { close(streamMock.messages) os.RemoveAll(workdir) }() - go streamMock.sendFile(context.Background(), t, appDir, streamMock, []string{"env1", "env2"}, []string{"DUMMY.md", "dum*"}) + go streamMock.sendFile(context.Background(), t, appDir, streamMock, []string{"env1", "env2"}, []string{"DUMMY.md", "dum*"}, []string{}) // when env, err := cmp.ReceiveRepoStream(context.Background(), streamMock, workdir, false) @@ -82,12 +82,12 @@ func TestReceiveApplicationStream(t *testing.T) { }) } -func (m *streamMock) sendFile(ctx context.Context, t *testing.T, basedir string, sender cmp.StreamSender, env []string, excludedGlobs []string) { +func (m *streamMock) sendFile(ctx context.Context, t *testing.T, basedir string, sender cmp.StreamSender, env []string, excludedGlobs, includedBlobs []string) { t.Helper() defer func() { m.done <- true }() - err := cmp.SendRepoStream(ctx, basedir, basedir, sender, env, excludedGlobs) + err := cmp.SendRepoStream(ctx, basedir, basedir, sender, env, excludedGlobs, includedBlobs) require.NoError(t, err) } diff --git a/util/io/files/tar.go b/util/io/files/tar.go index 13973f732fe72..2b95dbd596e6b 100644 --- a/util/io/files/tar.go +++ b/util/io/files/tar.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" + "github.com/bmatcuk/doublestar" log "github.com/sirupsen/logrus" ) @@ -158,10 +159,10 @@ func (t *tgz) tgzFile(path string, fi os.FileInfo, err error) error { return fmt.Errorf("relative path error: %s", err) } - if t.inclusions != nil && base != "." && !fi.IsDir() { + if t.inclusions != nil && len(t.inclusions) > 0 && base != "." && !fi.IsDir() { included := false for _, inclusionPattern := range t.inclusions { - found, err := filepath.Match(inclusionPattern, base) + found, err := doublestar.Match(inclusionPattern, relativePath) if err != nil { return fmt.Errorf("error verifying inclusion pattern %q: %w", inclusionPattern, err) } @@ -176,7 +177,7 @@ func (t *tgz) tgzFile(path string, fi os.FileInfo, err error) error { } if t.exclusions != nil { for _, exclusionPattern := range t.exclusions { - found, err := filepath.Match(exclusionPattern, relativePath) + found, err := doublestar.Match(exclusionPattern, relativePath) if err != nil { return fmt.Errorf("error verifying exclusion pattern %q: %w", exclusionPattern, err) } diff --git a/util/io/files/tar_test.go b/util/io/files/tar_test.go index 1817fa5ce9353..6278cbfe088b9 100644 --- a/util/io/files/tar_test.go +++ b/util/io/files/tar_test.go @@ -51,12 +51,12 @@ func TestTgz(t *testing.T) { filesWritten, err := files.Tgz(getTestAppDir(t), nil, exclusions, f.file) // then - assert.Equal(t, 3, filesWritten) + assert.Equal(t, 6, filesWritten) assert.NoError(t, err) prepareRead(f) files, err := read(f.file) require.NoError(t, err) - assert.Equal(t, 8, len(files)) + assert.Equal(t, 16, len(files)) assert.Contains(t, files, "README.md") assert.Contains(t, files, "applicationset/latest/kustomization.yaml") assert.Contains(t, files, "applicationset/stable/kustomization.yaml") @@ -74,12 +74,12 @@ func TestTgz(t *testing.T) { filesWritten, err := files.Tgz(getTestAppDir(t), nil, exclusions, f.file) // then - assert.Equal(t, 2, filesWritten) + assert.Equal(t, 5, filesWritten) assert.NoError(t, err) prepareRead(f) files, err := read(f.file) require.NoError(t, err) - assert.Equal(t, 7, len(files)) + assert.Equal(t, 15, len(files)) assert.Contains(t, files, "applicationset/latest/kustomization.yaml") assert.Contains(t, files, "applicationset/stable/kustomization.yaml") }) @@ -94,14 +94,47 @@ func TestTgz(t *testing.T) { filesWritten, err := files.Tgz(getTestAppDir(t), nil, exclusions, f.file) // then - assert.Equal(t, 1, filesWritten) + assert.Equal(t, 4, filesWritten) assert.NoError(t, err) prepareRead(f) files, err := read(f.file) require.NoError(t, err) - assert.Equal(t, 5, len(files)) + assert.Equal(t, 13, len(files)) assert.Contains(t, files, "applicationset/stable/kustomization.yaml") }) + + t.Run("will include and exclude files from both inclusion and exclusion lists", func(t *testing.T) { + // given + t.Parallel() + inclusions := []string{ + "**/*.y*ml", + "**/deploy/**", + } + exclusions := []string{ + "git/**", + } + f := setup(t) + defer teardown(f) + + // when + filesWritten, err := files.Tgz(getTestAppDir(t), inclusions, exclusions, f.file) + + // then + assert.Equal(t, 4, filesWritten) + assert.NoError(t, err) + prepareRead(f) + + files, err := read(f.file) + require.NoError(t, err) + + assert.Equal(t, 13, len(files)) + assert.Contains(t, files, "applicationset/stable/kustomization.yaml") + assert.Contains(t, files, "applicationset/stable/kustomization.yaml") + assert.NotContains(t, files, "git/index") + assert.Contains(t, files, "src/domain/service/deploy/template.tpl") + assert.Contains(t, files, "src/domain/service/deploy/helmfile.yaml") + assert.NotContains(t, files, "README.md") + }) } func TestUntgz(t *testing.T) { @@ -174,7 +207,7 @@ func TestUntgz(t *testing.T) { // then require.NoError(t, err) names := readFiles(t, destDir) - assert.Equal(t, 8, len(names)) + assert.Equal(t, 16, len(names)) assert.Contains(t, names, "README.md") assert.Contains(t, names, "applicationset/latest/kustomization.yaml") assert.Contains(t, names, "applicationset/stable/kustomization.yaml") diff --git a/util/io/files/testdata/app/git/index b/util/io/files/testdata/app/git/index new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/util/io/files/testdata/app/src/domain/service/deploy/helmfile.yaml b/util/io/files/testdata/app/src/domain/service/deploy/helmfile.yaml new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/util/io/files/testdata/app/src/domain/service/deploy/template.tpl b/util/io/files/testdata/app/src/domain/service/deploy/template.tpl new file mode 100644 index 0000000000000..e69de29bb2d1d