From bbbb31792e5e2250d31d8191ac81904dcd30d295 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 22 May 2024 00:48:19 +0000 Subject: [PATCH] Add CLI flag for ingress controllers * Add new --ingress-controller CLI flag * Refactor --ingress-controller and --cni flags to use common helper for disabling all unused charts * Wire first ingress controller name into global.systemDefaultIngressClass chart variable Signed-off-by: Brad Davidson --- pkg/bootstrap/bootstrap.go | 7 +- pkg/cli/cmds/server.go | 125 ++++++++++++++++++++----------- pkg/pebinaryexecutor/pebinary.go | 15 ++-- pkg/podexecutor/staticpod.go | 26 +++---- pkg/rke2/rke2.go | 27 ++++++- pkg/rke2/rke2_linux.go | 6 ++ pkg/rke2/rke2_windows.go | 28 ++++--- 7 files changed, 154 insertions(+), 80 deletions(-) diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go index 8aa723ce2dd..6e9597a36e1 100644 --- a/pkg/bootstrap/bootstrap.go +++ b/pkg/bootstrap/bootstrap.go @@ -157,7 +157,7 @@ func Stage(resolver *images.Resolver, nodeConfig *daemonconfig.Node, cfg cmds.Ag // UpdateManifests copies the staged manifests into the server's manifests dir, and applies // cluster configuration values to any HelmChart manifests found in the manifests directory. -func UpdateManifests(resolver *images.Resolver, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { +func UpdateManifests(resolver *images.Resolver, ingressController string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { ref, err := resolver.GetReference(images.Runtime) if err != nil { return err @@ -184,7 +184,7 @@ func UpdateManifests(resolver *images.Resolver, nodeConfig *daemonconfig.Node, c // Fix up HelmCharts to pass through configured values. // This needs to be done every time in order to sync values from the CLI - if err := setChartValues(manifestsDir, nodeConfig, cfg); err != nil { + if err := setChartValues(manifestsDir, ingressController, nodeConfig, cfg); err != nil { return errors.Wrap(err, "failed to rewrite HelmChart manifests to pass through CLI values") } return nil @@ -304,7 +304,7 @@ func copyFile(target, source string) error { // pass through settings to both the Helm job and the chart values. // NOTE: This will probably fail if any manifest contains multiple documents. This should // not matter for any of our packaged components, but may prevent this from working on user manifests. -func setChartValues(manifestsDir string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { +func setChartValues(manifestsDir, ingressController string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { chartValues := map[string]string{ "global.clusterCIDR": util.JoinIPNets(nodeConfig.AgentConfig.ClusterCIDRs), "global.clusterCIDRv4": util.JoinIP4Nets(nodeConfig.AgentConfig.ClusterCIDRs), @@ -313,6 +313,7 @@ func setChartValues(manifestsDir string, nodeConfig *daemonconfig.Node, cfg cmds "global.clusterDomain": nodeConfig.AgentConfig.ClusterDomain, "global.rke2DataDir": cfg.DataDir, "global.serviceCIDR": util.JoinIPNets(nodeConfig.AgentConfig.ServiceCIDRs), + "global.systemDefaultIngressClass": ingressController, "global.systemDefaultRegistry": nodeConfig.AgentConfig.SystemDefaultRegistry, "global.cattle.systemDefaultRegistry": nodeConfig.AgentConfig.SystemDefaultRegistry, } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index db977f86af5..bbe0f9d7605 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -1,6 +1,7 @@ package cmds import ( + "errors" "strings" "github.com/k3s-io/k3s/pkg/cli/cmds" @@ -9,6 +10,7 @@ import ( "github.com/rancher/wrangler/v3/pkg/slice" "github.com/sirupsen/logrus" "github.com/urfave/cli" + "k8s.io/apimachinery/pkg/util/sets" ) const ( @@ -16,22 +18,12 @@ const ( ) var ( - DisableItems = []string{"rke2-coredns", "rke2-ingress-nginx", "rke2-metrics-server"} - CNIItems = []string{"calico", "canal", "cilium", "flannel"} - config = rke2.Config{} serverFlag = []cli.Flag{ - &cli.StringSliceFlag{ - Name: "cni", - Usage: "(networking) CNI Plugins to deploy, one of none, " + strings.Join(CNIItems, ", ") + "; optionally with multus as the first value to enable the multus meta-plugin (default: canal)", - EnvVar: "RKE2_CNI", - }, - &cli.BoolFlag{ - Name: "enable-servicelb", - Usage: "(components) Enable rke2 default cloud controller manager's service controller", - EnvVar: "RKE2_ENABLE_SERVICELB", - }, + rke2.CNIFlag, + rke2.IngressControllerFlag, + rke2.ServiceLBFlag, } k3sServerBase = mustCmdFromK3S(cmds.NewServerCommand(ServerRun), K3SFlagSet{ @@ -80,7 +72,7 @@ var ( "kine-tls": dropFlag, "default-local-storage-path": dropFlag, "disable": { - Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(DisableItems, ", ") + ")", + Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(rke2.DisableItems, ", ") + ")", }, "disable-scheduler": copyFlag, "disable-cloud-controller": copyFlag, @@ -164,47 +156,90 @@ func ServerRun(clx *cli.Context) error { validateCloudProviderName(clx, Server) validateProfile(clx, Server) validateCNI(clx) + validateIngress(clx) return rke2.Server(clx, config) } +// validateCNI validates the CNI selection, and disables any un-selected CNI charts func validateCNI(clx *cli.Context) { - cnis := []string{} - for _, cni := range clx.StringSlice("cni") { - for _, v := range strings.Split(cni, ",") { - cnis = append(cnis, v) + disableExceptSelected(clx, rke2.CNIItems, rke2.CNIFlag, func(values cli.StringSlice) (cli.StringSlice, error) { + switch len(values) { + case 0: + values = append(values, "canal") + fallthrough + case 1: + if values[0] == "multus" { + return nil, errors.New("multus must be used alongside another primary cni selection") + } + clx.Set("disable", "rke2-multus") + case 2: + if values[0] == "multus" { + values = values[1:] + } else { + return nil, errors.New("may only provide multiple values if multus is the first value") + } + default: + return nil, errors.New("must specify 1 or 2 values") } - } + return values, nil + }) +} - switch len(cnis) { - case 0: - cnis = append(cnis, "canal") - fallthrough - case 1: - if cnis[0] == "multus" { - logrus.Fatal("invalid value provided for --cni flag: multus must be used alongside another primary cni selection") +// validateCNI validates the ingress controller selection, and disables any un-selected ingress controller charts +func validateIngress(clx *cli.Context) { + disableExceptSelected(clx, rke2.IngressItems, rke2.IngressControllerFlag, func(values cli.StringSlice) (cli.StringSlice, error) { + if len(values) == 0 { + values = append(values, "ingress-nginx") } - clx.Set("disable", "rke2-multus") - case 2: - if cnis[0] == "multus" { - cnis = cnis[1:] - } else { - logrus.Fatal("invalid values provided for --cni flag: may only provide multiple values if multus is the first value") + return values, nil + }) +} + +// disableExceptSelected takes a list of valid flag values, and a CLI StringSlice flag that holds the user's selected values. +// Selected values are split to support comma-separated lists, in addition to repeated use of the same flag. +// Once the list has been split, a validation function is called to allow for custom validation or defaulting of selected values. +// Finally, charts for any valid items not selected are added to the --disable list. +// A value of 'none' will cause all valid items to be disabled. +// Errors from the validation function, or selection of a value not in the valid list, will cause a fatal error to be logged. +func disableExceptSelected(clx *cli.Context, valid []string, flag *cli.StringSliceFlag, validateFunc func(cli.StringSlice) (cli.StringSlice, error)) { + // split comma-separated values + values := cli.StringSlice{} + if flag.Value != nil { + for _, value := range *flag.Value { + for _, v := range strings.Split(value, ",") { + values = append(values, v) + } } - default: - logrus.Fatal("invalid values provided for --cni flag: may not provide more than two values") } - switch { - case cnis[0] == "none": - fallthrough - case slice.ContainsString(CNIItems, cnis[0]): - for _, d := range CNIItems { - if cnis[0] != d { - clx.Set("disable", "rke2-"+d) - clx.Set("disable", "rke2-"+d+"-crd") - } + // validate the flag after splitting values + if v, err := validateFunc(values); err != nil { + logrus.Fatalf("Failed to validate --%s flag: %v", flag.Name, err) + } else { + flag.Value = &v + } + + // prepare a list of items to disable, based on all valid components. + // we have to use an intermediate set because the flag interface + // doesn't allow us to remove flag values once added. + disabledCharts := sets.Set[string]{} + for _, d := range valid { + disabledCharts.Insert("rke2-"+d, "rke2-"+d+"-crd") + } + + // re-enable components for any selected flag values + for _, d := range *flag.Value { + switch { + case d == "none": + break + case slice.ContainsString(valid, d): + disabledCharts.Delete("rke2-"+d, "rke2-"+d+"-crd") + default: + logrus.Fatalf("Invalid value %s for --%s flag: must be one of %s", d, flag.Name, strings.Join(valid, ",")) } - default: - logrus.Fatal("invalid value provided for --cni flag") + } + + for _, c := range disabledCharts.UnsortedList() { + clx.Set("disable", c) } } diff --git a/pkg/pebinaryexecutor/pebinary.go b/pkg/pebinaryexecutor/pebinary.go index ab1e9683b4e..059b90a6e64 100644 --- a/pkg/pebinaryexecutor/pebinary.go +++ b/pkg/pebinaryexecutor/pebinary.go @@ -46,19 +46,20 @@ var ( ) type PEBinaryConfig struct { - ManifestsDir string - ImagesDir string - Resolver *images.Resolver + CNIPlugin win.CNIPlugin CloudProvider *CloudProviderConfig - CISMode bool + Resolver *images.Resolver + ManifestsDir string DataDir string AuditPolicyFile string KubeletPath string + CNIName string + ImagesDir string KubeConfigKubeProxy string + IngressController string + CISMode bool DisableETCD bool IsServer bool - CNIName string - CNIPlugin win.CNIPlugin } type CloudProviderConfig struct { @@ -105,7 +106,7 @@ func (p *PEBinaryConfig) Bootstrap(ctx context.Context, nodeConfig *config.Node, } if p.IsServer { - return bootstrap.UpdateManifests(p.Resolver, nodeConfig, cfg) + return bootstrap.UpdateManifests(p.Resolver, p.IngressController, nodeConfig, cfg) } restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController) diff --git a/pkg/podexecutor/staticpod.go b/pkg/podexecutor/staticpod.go index ee74cfa154f..b120489e5d1 100644 --- a/pkg/podexecutor/staticpod.go +++ b/pkg/podexecutor/staticpod.go @@ -105,25 +105,25 @@ type ControlPlaneProbeConfs struct { } type StaticPodConfig struct { + Resolver *images.Resolver + stopKubelet context.CancelFunc + CloudProvider *CloudProviderConfig ControlPlaneResources - ControlPlaneProbeConfs + DataDir string + RuntimeEndpoint string + ManifestsDir string + IngressController string + ImagesDir string + AuditPolicyFile string + PSAConfigFile string + KubeletPath string ControlPlaneEnv ControlPlaneMounts - ManifestsDir string - ImagesDir string - Resolver *images.Resolver - CloudProvider *CloudProviderConfig - DataDir string - AuditPolicyFile string - PSAConfigFile string - KubeletPath string - RuntimeEndpoint string + ControlPlaneProbeConfs CISMode bool DisableETCD bool ExternalDatabase bool IsServer bool - - stopKubelet context.CancelFunc } type CloudProviderConfig struct { @@ -159,7 +159,7 @@ func (s *StaticPodConfig) Bootstrap(_ context.Context, nodeConfig *daemonconfig. return err } if s.IsServer { - return bootstrap.UpdateManifests(s.Resolver, nodeConfig, cfg) + return bootstrap.UpdateManifests(s.Resolver, s.IngressController, nodeConfig, cfg) } // Remove the kube-proxy static pod manifest before starting the agent. diff --git a/pkg/rke2/rke2.go b/pkg/rke2/rke2.go index 8d5c8327432..aecec1df286 100644 --- a/pkg/rke2/rke2.go +++ b/pkg/rke2/rke2.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" "github.com/rancher/rke2/pkg/controllers/cisnetworkpolicy" "github.com/rancher/rke2/pkg/images" + "github.com/rancher/wrangler/v3/pkg/slice" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -62,6 +63,30 @@ type ExtraEnv struct { CloudControllerManager cli.StringSlice } +var ( + DisableItems = []string{"rke2-coredns", "rke2-metrics-server", "rke2-snapshot-controller", "rke2-snapshot-controller-crd", "rke2-snapshot-validation-webhook"} + CNIItems = []string{"calico", "canal", "cilium", "flannel"} + IngressItems = []string{"ingress-nginx", "traefik"} + + CNIFlag = &cli.StringSliceFlag{ + Name: "cni", + Usage: "(networking) CNI Plugins to deploy, one of none, " + strings.Join(CNIItems, ", ") + "; optionally with multus as the first value to enable the multus meta-plugin (default: canal)", + EnvVar: "RKE2_CNI", + Value: &cli.StringSlice{}, + } + IngressControllerFlag = &cli.StringSliceFlag{ + Name: "ingress-controller", + Usage: "(networking) Ingress Controllers to deploy, one of none, " + strings.Join(IngressItems, ", ") + "; the first value will be set as the default ingress class (default: ingress-nginx)", + EnvVar: "RKE_INGRESS_CONTROLLER", + Value: &cli.StringSlice{}, + } + ServiceLBFlag = &cli.BoolFlag{ + Name: "enable-servicelb", + Usage: "(components) Enable rke2 default cloud controller manager's service controller", + EnvVar: "RKE2_ENABLE_SERVICELB", + } +) + // Valid CIS Profile versions const ( CISProfile123 = "cis-1.23" @@ -115,7 +140,7 @@ func Server(clx *cli.Context, cfg Config) error { var leaderControllers rawServer.CustomControllers - cnis := clx.StringSlice("cni") + cnis := *CNIFlag.Value if cisMode && (len(cnis) == 0 || slice.ContainsString(cnis, "canal")) { leaderControllers = append(leaderControllers, cisnetworkpolicy.Controller) } else { diff --git a/pkg/rke2/rke2_linux.go b/pkg/rke2/rke2_linux.go index 1fc77fbd723..7d92512d488 100644 --- a/pkg/rke2/rke2_linux.go +++ b/pkg/rke2/rke2_linux.go @@ -139,6 +139,11 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta containerRuntimeEndpoint = containerdSock } + var ingressControllerName string + if IngressControllerFlag.Value != nil && len(*IngressControllerFlag.Value) > 0 { + ingressControllerName = (*IngressControllerFlag.Value)[0] + } + return &podexecutor.StaticPodConfig{ Resolver: resolver, ImagesDir: agentImagesDir, @@ -153,6 +158,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta DisableETCD: clx.Bool("disable-etcd"), ExternalDatabase: ExternalDatabase, IsServer: isServer, + IngressController: ingressControllerName, ControlPlaneResources: *controlPlaneResources, ControlPlaneProbeConfs: *controlPlaneProbeConfs, ControlPlaneEnv: *extraEnv, diff --git a/pkg/rke2/rke2_windows.go b/pkg/rke2/rke2_windows.go index 54cb1969204..ba011275ff1 100644 --- a/pkg/rke2/rke2_windows.go +++ b/pkg/rke2/rke2_windows.go @@ -58,17 +58,23 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*pebinaryexecuto cfg.KubeletPath = "kubelet" } + var ingressControllerName string + if IngressControllerFlag.Value != nil && len(*IngressControllerFlag.Value) > 0 { + ingressControllerName = (*IngressControllerFlag.Value)[0] + } + return &pebinaryexecutor.PEBinaryConfig{ - Resolver: resolver, - ImagesDir: agentImagesDir, - ManifestsDir: agentManifestsDir, - CISMode: isCISMode(clx), - CloudProvider: cpConfig, - DataDir: dataDir, - AuditPolicyFile: clx.String("audit-policy-file"), - KubeletPath: cfg.KubeletPath, - DisableETCD: clx.Bool("disable-etcd"), - IsServer: isServer, - CNIName: "", + Resolver: resolver, + ImagesDir: agentImagesDir, + ManifestsDir: agentManifestsDir, + CISMode: isCISMode(clx), + CloudProvider: cpConfig, + DataDir: dataDir, + AuditPolicyFile: clx.String("audit-policy-file"), + KubeletPath: cfg.KubeletPath, + DisableETCD: clx.Bool("disable-etcd"), + IsServer: isServer, + IngressController: ingressControllerName, + CNIName: "", }, nil }