From fc125b0fd5147bf2c2934ed0ad11f6e955d067ab Mon Sep 17 00:00:00 2001 From: Jont828 Date: Thu, 11 May 2023 12:30:25 -0400 Subject: [PATCH] Support add-on providers in clusterctl --- cmd/clusterctl/api/v1alpha3/labels.go | 2 ++ cmd/clusterctl/api/v1alpha3/provider_type.go | 9 +++++++- .../client/config/providers_client.go | 21 +++++++++++++++--- .../client/config/providers_client_test.go | 8 +++++++ cmd/clusterctl/client/config_test.go | 2 ++ cmd/clusterctl/client/delete.go | 8 +++++++ cmd/clusterctl/client/init.go | 7 ++++++ cmd/clusterctl/client/upgrade.go | 10 ++++++++- .../cmd/config_repositories_test.go | 12 ++++++++-- cmd/clusterctl/cmd/delete.go | 11 +++++++--- cmd/clusterctl/cmd/generate_provider.go | 22 ++++++++++++++----- cmd/clusterctl/cmd/init.go | 6 ++++- cmd/clusterctl/cmd/init_list_images.go | 1 + cmd/clusterctl/cmd/upgrade_apply.go | 11 +++++++--- .../hack/create-local-repository.py | 8 ++++++- docs/book/src/clusterctl/provider-contract.md | 1 + hack/tools/tilt-prepare/main.go | 4 ++++ test/e2e/clusterctl_upgrade.go | 12 +++++++++- test/e2e/e2e_suite_test.go | 1 + test/e2e/self_hosted.go | 1 + test/framework/clusterctl/client.go | 15 ++++++++++--- .../clusterctl/clusterctl_helpers.go | 9 ++++++-- test/framework/clusterctl/e2e_config.go | 8 ++++++- test/framework/clusterctl/repository.go | 2 +- 24 files changed, 162 insertions(+), 29 deletions(-) diff --git a/cmd/clusterctl/api/v1alpha3/labels.go b/cmd/clusterctl/api/v1alpha3/labels.go index 5b848c29c65c..a6dacf143ccd 100644 --- a/cmd/clusterctl/api/v1alpha3/labels.go +++ b/cmd/clusterctl/api/v1alpha3/labels.go @@ -54,6 +54,8 @@ func ManifestLabel(name string, providerType ProviderType) string { return fmt.Sprintf("ipam-%s", name) case RuntimeExtensionProviderType: return fmt.Sprintf("runtime-extension-%s", name) + case AddonProviderType: + return fmt.Sprintf("addon-%s", name) default: return name } diff --git a/cmd/clusterctl/api/v1alpha3/provider_type.go b/cmd/clusterctl/api/v1alpha3/provider_type.go index ba67861bfd46..824861c15455 100644 --- a/cmd/clusterctl/api/v1alpha3/provider_type.go +++ b/cmd/clusterctl/api/v1alpha3/provider_type.go @@ -95,7 +95,8 @@ func (p *Provider) GetProviderType() ProviderType { InfrastructureProviderType, ControlPlaneProviderType, IPAMProviderType, - RuntimeExtensionProviderType: + RuntimeExtensionProviderType, + AddonProviderType: return t default: return ProviderTypeUnknown @@ -129,6 +130,10 @@ const ( // runtime extensions. RuntimeExtensionProviderType = ProviderType("RuntimeExtensionProvider") + // AddonProviderType is the type associated with codebases that provide + // add-on capabilities. + AddonProviderType = ProviderType("AddonProvider") + // ProviderTypeUnknown is used when the type is unknown. ProviderTypeUnknown = ProviderType("") ) @@ -148,6 +153,8 @@ func (p ProviderType) Order() int { return 4 case RuntimeExtensionProviderType: return 5 + case AddonProviderType: + return 6 default: return 99 } diff --git a/cmd/clusterctl/client/config/providers_client.go b/cmd/clusterctl/client/config/providers_client.go index fa75ad540f32..9ccc1812fa2f 100644 --- a/cmd/clusterctl/client/config/providers_client.go +++ b/cmd/clusterctl/client/config/providers_client.go @@ -81,6 +81,11 @@ const ( KubeKeyK3sControlPlaneProviderName = "kubekey-k3s" ) +// Add-on providers. +const ( + HelmAddonProviderName = "helm" +) + // Other. const ( // ProvidersConfigKey is a constant for finding provider configurations with the ProvidersClient. @@ -276,6 +281,7 @@ func (p *providersClient) defaults() []Provider { url: "https://github.com/canonical/cluster-api-bootstrap-provider-microk8s/releases/latest/bootstrap-components.yaml", providerType: clusterctlv1.BootstrapProviderType, }, + // ControlPlane providers &provider{ name: KubeadmControlPlaneProviderName, @@ -302,6 +308,13 @@ func (p *providersClient) defaults() []Provider { url: "https://github.com/kubernetes-sigs/cluster-api-provider-nested/releases/latest/control-plane-components.yaml", providerType: clusterctlv1.ControlPlaneProviderType, }, + + // Add-on providers + &provider{ + name: HelmAddonProviderName, + url: "https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/latest/addon-components.yaml", + providerType: clusterctlv1.AddonProviderType, + }, } return defaults @@ -401,16 +414,18 @@ func validateProvider(r Provider) error { clusterctlv1.InfrastructureProviderType, clusterctlv1.ControlPlaneProviderType, clusterctlv1.IPAMProviderType, - clusterctlv1.RuntimeExtensionProviderType: + clusterctlv1.RuntimeExtensionProviderType, + clusterctlv1.AddonProviderType: break default: - return errors.Errorf("invalid provider type. Allowed values are [%s, %s, %s, %s, %s, %s]", + return errors.Errorf("invalid provider type. Allowed values are [%s, %s, %s, %s, %s, %s, %s]", clusterctlv1.CoreProviderType, clusterctlv1.BootstrapProviderType, clusterctlv1.InfrastructureProviderType, clusterctlv1.ControlPlaneProviderType, clusterctlv1.IPAMProviderType, - clusterctlv1.RuntimeExtensionProviderType) + clusterctlv1.RuntimeExtensionProviderType, + clusterctlv1.AddonProviderType) } return nil } diff --git a/cmd/clusterctl/client/config/providers_client_test.go b/cmd/clusterctl/client/config/providers_client_test.go index a54f3de3f061..d71418aa1688 100644 --- a/cmd/clusterctl/client/config/providers_client_test.go +++ b/cmd/clusterctl/client/config/providers_client_test.go @@ -23,6 +23,7 @@ import ( "testing" . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/test" @@ -41,6 +42,10 @@ func Test_providers_List(t *testing.T) { }) defaultsAndZZZ := append(defaults, NewProvider("zzz", "https://zzz/infrastructure-components.yaml", "InfrastructureProvider")) + // AddonProviders are at the end of the list so we want to make sure this InfrastructureProvider is before the AddonProviders. + sort.Slice(defaultsAndZZZ, func(i, j int) bool { + return defaultsAndZZZ[i].Less(defaultsAndZZZ[j]) + }) defaultsWithOverride := append([]Provider{}, defaults...) defaultsWithOverride[0] = NewProvider(defaults[0].Name(), "https://zzz/infrastructure-components.yaml", defaults[0].Type()) @@ -135,6 +140,9 @@ func Test_providers_List(t *testing.T) { wantErr: true, }, } + + format.MaxLength = 15000 // This way it doesn't truncate the output on test failure + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) diff --git a/cmd/clusterctl/client/config_test.go b/cmd/clusterctl/client/config_test.go index 690ab55af094..fff810040b56 100644 --- a/cmd/clusterctl/client/config_test.go +++ b/cmd/clusterctl/client/config_test.go @@ -90,6 +90,7 @@ func Test_clusterctlClient_GetProvidersConfig(t *testing.T) { config.VclusterProviderName, config.VirtinkProviderName, config.VSphereProviderName, + config.HelmAddonProviderName, }, wantErr: false, }, @@ -136,6 +137,7 @@ func Test_clusterctlClient_GetProvidersConfig(t *testing.T) { config.VclusterProviderName, config.VirtinkProviderName, config.VSphereProviderName, + config.HelmAddonProviderName, }, wantErr: false, }, diff --git a/cmd/clusterctl/client/delete.go b/cmd/clusterctl/client/delete.go index 77ee268a3b15..3797abb3e9e2 100644 --- a/cmd/clusterctl/client/delete.go +++ b/cmd/clusterctl/client/delete.go @@ -48,6 +48,9 @@ type DeleteOptions struct { // RuntimeExtensionProviders and versions (e.g. test:v0.0.1) to delete from the management cluster. RuntimeExtensionProviders []string + // AddonProviders and versions (e.g. helm:v0.1.0) to delete from the management cluster. + AddonProviders []string + // DeleteAll set for deletion of all the providers. DeleteAll bool @@ -122,6 +125,11 @@ func (c *clusterctlClient) Delete(options DeleteOptions) error { return err } + providers, err = appendProviders(providers, clusterctlv1.AddonProviderType, options.AddonProviders...) + if err != nil { + return err + } + for _, provider := range providers { // Try to detect the namespace where the provider lives provider.Namespace, err = clusterClient.ProviderInventory().GetProviderNamespace(provider.ProviderName, provider.GetProviderType()) diff --git a/cmd/clusterctl/client/init.go b/cmd/clusterctl/client/init.go index 3c4540a6401d..9057b90d789c 100644 --- a/cmd/clusterctl/client/init.go +++ b/cmd/clusterctl/client/init.go @@ -59,6 +59,9 @@ type InitOptions struct { // RuntimeExtensionProviders and versions (e.g. test:v0.0.1) to add to the management cluster. RuntimeExtensionProviders []string + // AddonProviders and versions (e.g. helm:v0.1.0) to add to the management cluster. + AddonProviders []string + // TargetNamespace defines the namespace where the providers should be deployed. If unspecified, each provider // will be installed in a provider's default namespace. TargetNamespace string @@ -255,6 +258,10 @@ func (c *clusterctlClient) setupInstaller(cluster cluster.Client, options InitOp return nil, err } + if err := c.addToInstaller(addOptions, clusterctlv1.AddonProviderType, options.AddonProviders...); err != nil { + return nil, err + } + return installer, nil } diff --git a/cmd/clusterctl/client/upgrade.go b/cmd/clusterctl/client/upgrade.go index 849ed23cc889..c077aeaba34e 100644 --- a/cmd/clusterctl/client/upgrade.go +++ b/cmd/clusterctl/client/upgrade.go @@ -121,6 +121,9 @@ type ApplyUpgradeOptions struct { // RuntimeExtensionProviders instance and versions (e.g. runtime-extension-system/test:v0.0.1) to upgrade to. This field can be used as alternative to Contract. RuntimeExtensionProviders []string + // AddonProviders instance and versions (e.g. caaph-system/helm:v0.1.0) to upgrade to. This field can be used as alternative to Contract. + AddonProviders []string + // WaitProviders instructs the upgrade apply command to wait till the providers are successfully upgraded. WaitProviders bool @@ -169,7 +172,8 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error { len(options.ControlPlaneProviders) > 0 || len(options.InfrastructureProviders) > 0 || len(options.IPAMProviders) > 0 || - len(options.RuntimeExtensionProviders) > 0 + len(options.RuntimeExtensionProviders) > 0 || + len(options.AddonProviders) > 0 opts := cluster.UpgradeOptions{ WaitProviders: options.WaitProviders, @@ -207,6 +211,10 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error { if err != nil { return err } + upgradeItems, err = addUpgradeItems(clusterClient, upgradeItems, clusterctlv1.AddonProviderType, options.AddonProviders...) + if err != nil { + return err + } // Execute the upgrade using the custom upgrade items return clusterClient.ProviderUpgrader().ApplyCustomPlan(opts, upgradeItems...) diff --git a/cmd/clusterctl/cmd/config_repositories_test.go b/cmd/clusterctl/cmd/config_repositories_test.go index 079737bad555..430dd70df2fd 100644 --- a/cmd/clusterctl/cmd/config_repositories_test.go +++ b/cmd/clusterctl/cmd/config_repositories_test.go @@ -23,6 +23,7 @@ import ( "path/filepath" "testing" + "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" ) @@ -45,11 +46,13 @@ func Test_runGetRepositories(t *testing.T) { out, err := io.ReadAll(buf) g.Expect(err).ToNot(HaveOccurred()) + var diff string if val == RepositoriesOutputText { - g.Expect(string(out)).To(Equal(expectedOutputText)) + diff = cmp.Diff(expectedOutputText, string(out)) } else if val == RepositoriesOutputYaml { - g.Expect(string(out)).To(Equal(expectedOutputYaml)) + diff = cmp.Diff(expectedOutputYaml, string(out)) } + g.Expect(diff).To(BeEmpty()) // Use diff to compare as Gomega output does not actually print the string values on failure } }) @@ -137,6 +140,7 @@ vcd InfrastructureProvider https://github.com/vmware/cluster-a vcluster InfrastructureProvider https://github.com/loft-sh/cluster-api-provider-vcluster/releases/latest/ infrastructure-components.yaml virtink InfrastructureProvider https://github.com/smartxworks/cluster-api-provider-virtink/releases/latest/ infrastructure-components.yaml vsphere InfrastructureProvider https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/latest/ infrastructure-components.yaml +helm AddonProvider https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/latest/ addon-components.yaml ` var expectedOutputYaml = `- File: core_components.yaml @@ -287,4 +291,8 @@ var expectedOutputYaml = `- File: core_components.yaml Name: vsphere ProviderType: InfrastructureProvider URL: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/latest/ +- File: addon-components.yaml + Name: helm + ProviderType: AddonProvider + URL: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/latest/ ` diff --git a/cmd/clusterctl/cmd/delete.go b/cmd/clusterctl/cmd/delete.go index 662a505b93b3..9d512fc8a999 100644 --- a/cmd/clusterctl/cmd/delete.go +++ b/cmd/clusterctl/cmd/delete.go @@ -32,6 +32,7 @@ type deleteOptions struct { infrastructureProviders []string ipamProviders []string runtimeExtensionProviders []string + addonProviders []string includeNamespace bool includeCRDs bool deleteAll bool @@ -109,6 +110,8 @@ func init() { "IPAM providers and versions (e.g. infoblox:v0.0.1) to delete from the management cluster") deleteCmd.Flags().StringSliceVar(&dd.runtimeExtensionProviders, "runtime-extension", nil, "Runtime extension providers and versions (e.g. test:v0.0.1) to delete from the management cluster") + deleteCmd.Flags().StringSliceVar(&dd.addonProviders, "addon", nil, + "Add-on providers and versions (e.g. helm:v0.1.0) to delete from the management cluster") deleteCmd.Flags().BoolVar(&dd.deleteAll, "all", false, "Force deletion of all the providers") @@ -127,14 +130,15 @@ func runDelete() error { (len(dd.controlPlaneProviders) > 0) || (len(dd.infrastructureProviders) > 0) || (len(dd.ipamProviders) > 0) || - (len(dd.runtimeExtensionProviders) > 0) + (len(dd.runtimeExtensionProviders) > 0) || + (len(dd.addonProviders) > 0) if dd.deleteAll && hasProviderNames { - return errors.New("The --all flag can't be used in combination with --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension") + return errors.New("The --all flag can't be used in combination with --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon") } if !dd.deleteAll && !hasProviderNames { - return errors.New("At least one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension should be specified or the --all flag should be set") + return errors.New("At least one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon should be specified or the --all flag should be set") } return c.Delete(client.DeleteOptions{ @@ -147,6 +151,7 @@ func runDelete() error { ControlPlaneProviders: dd.controlPlaneProviders, IPAMProviders: dd.ipamProviders, RuntimeExtensionProviders: dd.runtimeExtensionProviders, + AddonProviders: dd.addonProviders, DeleteAll: dd.deleteAll, }) } diff --git a/cmd/clusterctl/cmd/generate_provider.go b/cmd/clusterctl/cmd/generate_provider.go index c9a335f0162f..89fba0590bad 100644 --- a/cmd/clusterctl/cmd/generate_provider.go +++ b/cmd/clusterctl/cmd/generate_provider.go @@ -31,6 +31,7 @@ type generateProvidersOptions struct { infrastructureProvider string ipamProvider string runtimeExtensionProvider string + addonProvider string targetNamespace string textOutput bool raw bool @@ -89,6 +90,8 @@ func init() { "IPAM provider and version (e.g. infoblox:v0.0.1)") generateProviderCmd.Flags().StringVar(&gpo.runtimeExtensionProvider, "runtime-extension", "", "Runtime extension provider and version (e.g. test:v0.0.1)") + generateProviderCmd.Flags().StringVar(&gpo.addonProvider, "addon", "", + "Add-on provider and version (e.g. helm:v0.1.0)") generateProviderCmd.Flags().StringVarP(&gpo.targetNamespace, "target-namespace", "n", "", "The target namespace where the provider should be deployed. If unspecified, the components default namespace is used.") generateProviderCmd.Flags().BoolVar(&gpo.textOutput, "describe", false, @@ -134,41 +137,48 @@ func parseProvider() (string, clusterctlv1.ProviderType, error) { providerType := clusterctlv1.CoreProviderType if gpo.bootstrapProvider != "" { if providerName != "" { - return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension should be set") + return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon should be set") } providerName = gpo.bootstrapProvider providerType = clusterctlv1.BootstrapProviderType } if gpo.controlPlaneProvider != "" { if providerName != "" { - return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension should be set") + return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon should be set") } providerName = gpo.controlPlaneProvider providerType = clusterctlv1.ControlPlaneProviderType } if gpo.infrastructureProvider != "" { if providerName != "" { - return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension should be set") + return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon should be set") } providerName = gpo.infrastructureProvider providerType = clusterctlv1.InfrastructureProviderType } if gpo.ipamProvider != "" { if providerName != "" { - return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension should be set") + return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon should be set") } providerName = gpo.ipamProvider providerType = clusterctlv1.IPAMProviderType } if gpo.runtimeExtensionProvider != "" { if providerName != "" { - return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension should be set") + return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon should be set") } providerName = gpo.runtimeExtensionProvider providerType = clusterctlv1.RuntimeExtensionProviderType } + if gpo.addonProvider != "" { + if providerName != "" { + return "", "", errors.New("only one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon should be set") + } + providerName = gpo.addonProvider + providerType = clusterctlv1.AddonProviderType + } if providerName == "" { - return "", "", errors.New("at least one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension should be set") + return "", "", errors.New("at least one of --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon should be set") } return providerName, providerType, nil diff --git a/cmd/clusterctl/cmd/init.go b/cmd/clusterctl/cmd/init.go index 9d6eafbdbf9d..f53342cd81ea 100644 --- a/cmd/clusterctl/cmd/init.go +++ b/cmd/clusterctl/cmd/init.go @@ -33,6 +33,7 @@ type initOptions struct { infrastructureProviders []string ipamProviders []string runtimeExtensionProviders []string + addonProviders []string targetNamespace string validate bool waitProviders bool @@ -73,7 +74,7 @@ var initCmd = &cobra.Command{ clusterctl init --infrastructure=aws:v0.4.1 # Initialize a management cluster with a custom kubeconfig path and the given infrastructure provider. - clusterctl init --kubeconfig=foo.yaml --infrastructure=aws + clusterctl init --kubeconfig=foo.yaml --infrastructure=aws # Initialize a management cluster with multiple infrastructure providers. clusterctl init --infrastructure=aws,vsphere @@ -103,6 +104,8 @@ func init() { "IPAM providers and versions (e.g. infoblox:v0.0.1) to add to the management cluster.") initCmd.PersistentFlags().StringSliceVar(&initOpts.runtimeExtensionProviders, "runtime-extension", nil, "Runtime extension providers and versions (e.g. test:v0.0.1) to add to the management cluster.") + initCmd.PersistentFlags().StringSliceVar(&initOpts.addonProviders, "addon", nil, + "Add-on providers and versions (e.g. helm:v0.1.0) to add to the management cluster.") initCmd.Flags().StringVarP(&initOpts.targetNamespace, "target-namespace", "n", "", "The target namespace where the providers should be deployed. If unspecified, the provider components' default namespace is used.") initCmd.Flags().BoolVar(&initOpts.waitProviders, "wait-providers", false, @@ -130,6 +133,7 @@ func runInit() error { InfrastructureProviders: initOpts.infrastructureProviders, IPAMProviders: initOpts.ipamProviders, RuntimeExtensionProviders: initOpts.runtimeExtensionProviders, + AddonProviders: initOpts.addonProviders, TargetNamespace: initOpts.targetNamespace, LogUsageInstructions: true, WaitProviders: initOpts.waitProviders, diff --git a/cmd/clusterctl/cmd/init_list_images.go b/cmd/clusterctl/cmd/init_list_images.go index f087fa804e84..12cf787ebf8e 100644 --- a/cmd/clusterctl/cmd/init_list_images.go +++ b/cmd/clusterctl/cmd/init_list_images.go @@ -61,6 +61,7 @@ func runInitListImages() error { InfrastructureProviders: initOpts.infrastructureProviders, IPAMProviders: initOpts.ipamProviders, RuntimeExtensionProviders: initOpts.runtimeExtensionProviders, + AddonProviders: initOpts.addonProviders, LogUsageInstructions: false, } diff --git a/cmd/clusterctl/cmd/upgrade_apply.go b/cmd/clusterctl/cmd/upgrade_apply.go index 41175669f4f7..fa0fdafa3d65 100644 --- a/cmd/clusterctl/cmd/upgrade_apply.go +++ b/cmd/clusterctl/cmd/upgrade_apply.go @@ -35,6 +35,7 @@ type upgradeApplyOptions struct { infrastructureProviders []string ipamProviders []string runtimeExtensionProviders []string + addonProviders []string waitProviders bool waitProviderTimeout int } @@ -85,6 +86,8 @@ func init() { "IPAM providers and versions (e.g. infoblox:v0.0.1) to upgrade to. This flag can be used as alternative to --contract.") upgradeApplyCmd.Flags().StringSliceVar(&ua.runtimeExtensionProviders, "runtime-extension", nil, "Runtime extension providers and versions (e.g. test:v0.0.1) to upgrade to. This flag can be used as alternative to --contract.") + upgradeApplyCmd.Flags().StringSliceVar(&ua.addonProviders, "addon", nil, + "Add-on providers and versions (e.g. helm:v0.1.0) to upgrade to. This flag can be used as alternative to --contract.") upgradeApplyCmd.Flags().BoolVar(&ua.waitProviders, "wait-providers", false, "Wait for providers to be upgraded.") upgradeApplyCmd.Flags().IntVar(&ua.waitProviderTimeout, "wait-provider-timeout", 5*60, @@ -102,13 +105,14 @@ func runUpgradeApply() error { (len(ua.controlPlaneProviders) > 0) || (len(ua.infrastructureProviders) > 0) || (len(ua.ipamProviders) > 0) || - (len(ua.runtimeExtensionProviders) > 0) + (len(ua.runtimeExtensionProviders) > 0) || + (len(ua.addonProviders) > 0) if ua.contract == "" && !hasProviderNames { - return errors.New("Either the --contract flag or at least one of the following flags has to be set: --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension") + return errors.New("Either the --contract flag or at least one of the following flags has to be set: --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon") } if ua.contract != "" && hasProviderNames { - return errors.New("The --contract flag can't be used in combination with --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension") + return errors.New("The --contract flag can't be used in combination with --core, --bootstrap, --control-plane, --infrastructure, --ipam, --extension, --addon") } return c.ApplyUpgrade(client.ApplyUpgradeOptions{ @@ -120,6 +124,7 @@ func runUpgradeApply() error { InfrastructureProviders: ua.infrastructureProviders, IPAMProviders: ua.ipamProviders, RuntimeExtensionProviders: ua.runtimeExtensionProviders, + AddonProviders: ua.addonProviders, WaitProviders: ua.waitProviders, WaitProviderTimeout: time.Duration(ua.waitProviderTimeout) * time.Second, }) diff --git a/cmd/clusterctl/hack/create-local-repository.py b/cmd/clusterctl/hack/create-local-repository.py index fb9746e7b7b5..0fecab5b168f 100755 --- a/cmd/clusterctl/hack/create-local-repository.py +++ b/cmd/clusterctl/hack/create-local-repository.py @@ -209,6 +209,8 @@ def splitNameAndType(provider): return provider[len('ipam-'):], 'IPAMProvider' if provider.startswith('runtime-extension-'): return provider[len('runtime-extension-'):], 'RuntimeExtensionProvider' + if provider.startswith('addon-'): + return provider[len('addon-'):], 'AddonProvider' return None, None def CoreProviderFlag(): @@ -229,6 +231,9 @@ def IPAMProviderFlag(): def RuntimeExtensionProviderFlag(): return '--runtime-extension' +def AddonProviderFlag(): + return '--addon' + def type_to_flag(type): switcher = { 'CoreProvider': CoreProviderFlag, @@ -236,7 +241,8 @@ def type_to_flag(type): 'ControlPlaneProvider': ControlPlaneProviderFlag, 'InfrastructureProvider': InfrastructureProviderFlag, 'IPAMProvider': IPAMProviderFlag, - 'RuntimeExtensionProvider': RuntimeExtensionProviderFlag + 'RuntimeExtensionProvider': RuntimeExtensionProviderFlag, + 'AddonProvider': AddonProviderFlag } func = switcher.get(type, lambda: 'Invalid type') return func() diff --git a/docs/book/src/clusterctl/provider-contract.md b/docs/book/src/clusterctl/provider-contract.md index a152d87593a3..3ff369c8515b 100644 --- a/docs/book/src/clusterctl/provider-contract.md +++ b/docs/book/src/clusterctl/provider-contract.md @@ -170,6 +170,7 @@ It is strongly recommended that: * Control plane providers release a file called `control-plane-components.yaml` * IPAM providers release a file called `ipam-components.yaml` * Runtime extensions providers release a file called `runtime-extension-components.yaml` +* Add-on providers release a file called `addon-components.yaml` #### Target namespace diff --git a/hack/tools/tilt-prepare/main.go b/hack/tools/tilt-prepare/main.go index 9869aa3c3660..de6a3d7a2945 100644 --- a/hack/tools/tilt-prepare/main.go +++ b/hack/tools/tilt-prepare/main.go @@ -936,6 +936,10 @@ func getProviderObj(version *string) func(prefix string, objs []unstructured.Uns providerType = string(clusterctlv1.RuntimeExtensionProviderType) providerName = manifestLabel[len("runtime-extension-"):] } + if strings.HasPrefix(manifestLabel, "addon-") { + providerType = string(clusterctlv1.AddonProviderType) + providerName = manifestLabel[len("addon-"):] + } provider := &clusterctlv1.Provider{ TypeMeta: metav1.TypeMeta{ diff --git a/test/e2e/clusterctl_upgrade.go b/test/e2e/clusterctl_upgrade.go index 8ef677152763..c1910931eeca 100644 --- a/test/e2e/clusterctl_upgrade.go +++ b/test/e2e/clusterctl_upgrade.go @@ -89,6 +89,9 @@ type ClusterctlUpgradeSpecInput struct { // InitWithRuntimeExtensionProviders specifies the runtime extension provider versions to add to the secondary management cluster, e.g. `test:v0.0.1`. // If not set, the runtime extension provider version is calculated based on the contract. InitWithRuntimeExtensionProviders []string + // InitWithAddonProviders specifies the add-on provider versions to add to the secondary management cluster, e.g. `helm:v0.0.1`. + // If not set, the add-on provider version is calculated based on the contract. + InitWithAddonProviders []string // UpgradeClusterctlVariables can be used to set additional variables for clusterctl upgrade. UpgradeClusterctlVariables map[string]string SkipCleanup bool @@ -112,6 +115,7 @@ type ClusterctlUpgradeSpecInput struct { InfrastructureProviders []string IPAMProviders []string RuntimeExtensionProviders []string + AddonProviders []string } // ClusterctlUpgradeSpec implements a test that verifies clusterctl upgrade of a management cluster. @@ -280,6 +284,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg infrastructureProviders []string ipamProviders []string runtimeExtensionProviders []string + addonProviders []string ) coreProvider = input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.ClusterAPIProviderName)[0] @@ -297,6 +302,8 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg input.E2EConfig.GetProviderLatestVersionsByContract(initContract, input.E2EConfig.IPAMProviders()...)) runtimeExtensionProviders = getValueOrFallback(input.InitWithRuntimeExtensionProviders, input.E2EConfig.GetProviderLatestVersionsByContract(initContract, input.E2EConfig.RuntimeExtensionProviders()...)) + addonProviders = getValueOrFallback(input.InitWithAddonProviders, + input.E2EConfig.GetProviderLatestVersionsByContract(initContract, input.E2EConfig.AddonProviders()...)) clusterctl.InitManagementClusterAndWatchControllerLogs(ctx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{ ClusterctlBinaryPath: clusterctlBinaryPath, // use older version of clusterctl to init the management cluster @@ -308,6 +315,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg InfrastructureProviders: infrastructureProviders, IPAMProviders: ipamProviders, RuntimeExtensionProviders: runtimeExtensionProviders, + AddonProviders: addonProviders, LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name), }, input.E2EConfig.GetIntervals(specName, "wait-controllers")...) @@ -401,7 +409,8 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg len(input.ControlPlaneProviders) > 0 || len(input.InfrastructureProviders) > 0 || len(input.IPAMProviders) > 0 || - len(input.RuntimeExtensionProviders) > 0 + len(input.RuntimeExtensionProviders) > 0 || + len(input.AddonProviders) > 0 if isCustomUpgrade { By("Upgrading providers to custom versions") @@ -415,6 +424,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg InfrastructureProviders: input.InfrastructureProviders, IPAMProviders: input.IPAMProviders, RuntimeExtensionProviders: input.RuntimeExtensionProviders, + AddonProviders: input.AddonProviders, LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name), }, input.E2EConfig.GetIntervals(specName, "wait-controllers")...) } else { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f0dce8fea018..d5bf3d59c4c4 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -249,6 +249,7 @@ func initBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, config * InfrastructureProviders: config.InfrastructureProviders(), IPAMProviders: config.IPAMProviders(), RuntimeExtensionProviders: config.RuntimeExtensionProviders(), + AddonProviders: config.AddonProviders(), LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) } diff --git a/test/e2e/self_hosted.go b/test/e2e/self_hosted.go index cf3f19d5a346..7bde5a23dc42 100644 --- a/test/e2e/self_hosted.go +++ b/test/e2e/self_hosted.go @@ -192,6 +192,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) InfrastructureProviders: input.E2EConfig.InfrastructureProviders(), IPAMProviders: input.E2EConfig.IPAMProviders(), RuntimeExtensionProviders: input.E2EConfig.RuntimeExtensionProviders(), + AddonProviders: input.E2EConfig.AddonProviders(), LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name), }, input.E2EConfig.GetIntervals(specName, "wait-controllers")...) diff --git a/test/framework/clusterctl/client.go b/test/framework/clusterctl/client.go index 1fc7c3128036..67cdbbf39203 100644 --- a/test/framework/clusterctl/client.go +++ b/test/framework/clusterctl/client.go @@ -57,6 +57,7 @@ type InitInput struct { InfrastructureProviders []string IPAMProviders []string RuntimeExtensionProviders []string + AddonProviders []string } // Init calls clusterctl init with the list of providers defined in the local repository. @@ -75,6 +76,7 @@ func Init(_ context.Context, input InitInput) { InfrastructureProviders: input.InfrastructureProviders, IPAMProviders: input.IPAMProviders, RuntimeExtensionProviders: input.RuntimeExtensionProviders, + AddonProviders: input.AddonProviders, LogUsageInstructions: true, WaitProviders: true, } @@ -124,6 +126,9 @@ func calculateClusterCtlInitArgs(input InitInput) []string { if len(input.RuntimeExtensionProviders) > 0 { args = append(args, "--runtime-extension", strings.Join(input.RuntimeExtensionProviders, ",")) } + if len(input.AddonProviders) > 0 { + args = append(args, "--addon", strings.Join(input.AddonProviders, ",")) + } return args } @@ -141,6 +146,7 @@ type UpgradeInput struct { InfrastructureProviders []string IPAMProviders []string RuntimeExtensionProviders []string + AddonProviders []string } // Upgrade calls clusterctl upgrade apply with the list of providers defined in the local repository. @@ -161,19 +167,21 @@ func Upgrade(ctx context.Context, input UpgradeInput) { len(input.ControlPlaneProviders) > 0 || len(input.InfrastructureProviders) > 0 || len(input.IPAMProviders) > 0 || - len(input.RuntimeExtensionProviders) > 0 + len(input.RuntimeExtensionProviders) > 0 || + len(input.AddonProviders) > 0 Expect((input.Contract != "" && !isCustomUpgrade) || (input.Contract == "" && isCustomUpgrade)).To(BeTrue(), `Invalid arguments. Either the input.Contract parameter or at least one of the following providers has to be set: - input.CoreProvider, input.BootstrapProviders, input.ControlPlaneProviders, input.InfrastructureProviders, input.IPAMProviders, input.RuntimeExtensionProviders`) + input.CoreProvider, input.BootstrapProviders, input.ControlPlaneProviders, input.InfrastructureProviders, input.IPAMProviders, input.RuntimeExtensionProviders, input.AddonProviders`) if isCustomUpgrade { - log.Logf("clusterctl upgrade apply --core %s --bootstrap %s --control-plane %s --infrastructure %s --ipam %s --runtime-extension %s --config %s --kubeconfig %s", + log.Logf("clusterctl upgrade apply --core %s --bootstrap %s --control-plane %s --infrastructure %s --ipam %s --runtime-extension %s --addon %s --config %s --kubeconfig %s", input.CoreProvider, strings.Join(input.BootstrapProviders, ","), strings.Join(input.ControlPlaneProviders, ","), strings.Join(input.InfrastructureProviders, ","), strings.Join(input.IPAMProviders, ","), strings.Join(input.RuntimeExtensionProviders, ","), + strings.Join(input.AddonProviders, ","), input.ClusterctlConfigPath, input.KubeconfigPath, ) @@ -197,6 +205,7 @@ func Upgrade(ctx context.Context, input UpgradeInput) { InfrastructureProviders: input.InfrastructureProviders, IPAMProviders: input.IPAMProviders, RuntimeExtensionProviders: input.RuntimeExtensionProviders, + AddonProviders: input.AddonProviders, WaitProviders: true, } diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index 677afaa0c745..00f5e8af2278 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -43,6 +43,7 @@ type InitManagementClusterAndWatchControllerLogsInput struct { InfrastructureProviders []string IPAMProviders []string RuntimeExtensionProviders []string + AddonProviders []string LogFolder string DisableMetricsCollection bool ClusterctlBinaryPath string @@ -85,6 +86,7 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init InfrastructureProviders: input.InfrastructureProviders, IPAMProviders: input.IPAMProviders, RuntimeExtensionProviders: input.RuntimeExtensionProviders, + AddonProviders: input.AddonProviders, // setup clusterctl logs folder LogFolder: input.LogFolder, } @@ -139,6 +141,7 @@ type UpgradeManagementClusterAndWaitInput struct { InfrastructureProviders []string IPAMProviders []string RuntimeExtensionProviders []string + AddonProviders []string LogFolder string } @@ -153,10 +156,11 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen len(input.ControlPlaneProviders) > 0 || len(input.InfrastructureProviders) > 0 || len(input.IPAMProviders) > 0 || - len(input.RuntimeExtensionProviders) > 0 + len(input.RuntimeExtensionProviders) > 0 || + len(input.AddonProviders) > 0 Expect((input.Contract != "" && !isCustomUpgrade) || (input.Contract == "" && isCustomUpgrade)).To(BeTrue(), `Invalid argument. Either the input.Contract parameter or at least one of the following providers has to be set: - input.CoreProvider, input.BootstrapProviders, input.ControlPlaneProviders, input.InfrastructureProviders, input.IPAMProviders, input.RuntimeExtensionProviders`) + input.CoreProvider, input.BootstrapProviders, input.ControlPlaneProviders, input.InfrastructureProviders, input.IPAMProviders, input.RuntimeExtensionProviders, input.AddonProviders`) Expect(os.MkdirAll(input.LogFolder, 0750)).To(Succeed(), "Invalid argument. input.LogFolder can't be created for UpgradeManagementClusterAndWait") @@ -172,6 +176,7 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen InfrastructureProviders: input.InfrastructureProviders, IPAMProviders: input.IPAMProviders, RuntimeExtensionProviders: input.RuntimeExtensionProviders, + AddonProviders: input.AddonProviders, LogFolder: input.LogFolder, }) diff --git a/test/framework/clusterctl/e2e_config.go b/test/framework/clusterctl/e2e_config.go index 3603b1bdf474..d5065719912a 100644 --- a/test/framework/clusterctl/e2e_config.go +++ b/test/framework/clusterctl/e2e_config.go @@ -401,6 +401,7 @@ func (c *E2EConfig) validateProviders() error { clusterctlv1.InfrastructureProviderType: nil, clusterctlv1.IPAMProviderType: nil, clusterctlv1.RuntimeExtensionProviderType: nil, + clusterctlv1.AddonProviderType: nil, } for i, providerConfig := range c.Providers { // Providers name should not be empty. @@ -410,7 +411,7 @@ func (c *E2EConfig) validateProviders() error { // Providers type should be one of the know types. providerType := clusterctlv1.ProviderType(providerConfig.Type) switch providerType { - case clusterctlv1.CoreProviderType, clusterctlv1.BootstrapProviderType, clusterctlv1.ControlPlaneProviderType, clusterctlv1.InfrastructureProviderType, clusterctlv1.IPAMProviderType, clusterctlv1.RuntimeExtensionProviderType: + case clusterctlv1.CoreProviderType, clusterctlv1.BootstrapProviderType, clusterctlv1.ControlPlaneProviderType, clusterctlv1.InfrastructureProviderType, clusterctlv1.IPAMProviderType, clusterctlv1.RuntimeExtensionProviderType, clusterctlv1.AddonProviderType: providersByType[providerType] = append(providersByType[providerType], providerConfig.Name) default: return errInvalidArg("Providers[%d].Type=%q", i, providerConfig.Type) @@ -519,6 +520,11 @@ func (c *E2EConfig) RuntimeExtensionProviders() []string { return c.getProviders(clusterctlv1.RuntimeExtensionProviderType) } +// AddonProviders returns the add-on provider selected for running this E2E test. +func (c *E2EConfig) AddonProviders() []string { + return c.getProviders(clusterctlv1.AddonProviderType) +} + func (c *E2EConfig) getProviders(t clusterctlv1.ProviderType) []string { InfraProviders := []string{} for _, provider := range c.Providers { diff --git a/test/framework/clusterctl/repository.go b/test/framework/clusterctl/repository.go index a6061ca227fd..ff20678286ca 100644 --- a/test/framework/clusterctl/repository.go +++ b/test/framework/clusterctl/repository.go @@ -121,7 +121,7 @@ func CreateRepository(ctx context.Context, input CreateRepositoryInput) string { Type: provider.Type, } providers = append(providers, p) - if !(clusterctlv1.ProviderType(provider.Type) == clusterctlv1.IPAMProviderType || clusterctlv1.ProviderType(provider.Type) == clusterctlv1.RuntimeExtensionProviderType) { + if !(clusterctlv1.ProviderType(provider.Type) == clusterctlv1.IPAMProviderType || clusterctlv1.ProviderType(provider.Type) == clusterctlv1.RuntimeExtensionProviderType || clusterctlv1.ProviderType(provider.Type) == clusterctlv1.AddonProviderType) { providersV1_2 = append(providersV1_2, p) } }