Add CLI flag for ingress controllers

* Add new --ingress-controller CLI flag
* Refactor --ingress-controller and --cni flags to use common helper for
  disabling all unused charts
* Wire first ingress controller name into global.systemDefaultIngressClass
  chart variable

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/bootstrap/bootstrap.go

@@ -157,7 +157,7 @@ func Stage(resolver *images.Resolver, nodeConfig *daemonconfig.Node, cfg cmds.Ag
 
 // UpdateManifests copies the staged manifests into the server's manifests dir, and applies
 // cluster configuration values to any HelmChart manifests found in the manifests directory.
-func UpdateManifests(resolver *images.Resolver, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
+func UpdateManifests(resolver *images.Resolver, ingressController string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
 	ref, err := resolver.GetReference(images.Runtime)
 	if err != nil {
 		return err
@@ -184,7 +184,7 @@ func UpdateManifests(resolver *images.Resolver, nodeConfig *daemonconfig.Node, c
 
 	// Fix up HelmCharts to pass through configured values.
 	// This needs to be done every time in order to sync values from the CLI
-	if err := setChartValues(manifestsDir, nodeConfig, cfg); err != nil {
+	if err := setChartValues(manifestsDir, ingressController, nodeConfig, cfg); err != nil {
 		return errors.Wrap(err, "failed to rewrite HelmChart manifests to pass through CLI values")
 	}
 	return nil
@@ -304,7 +304,7 @@ func copyFile(target, source string) error {
 // pass through settings to both the Helm job and the chart values.
 // NOTE: This will probably fail if any manifest contains multiple documents. This should
 // not matter for any of our packaged components, but may prevent this from working on user manifests.
-func setChartValues(manifestsDir string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
+func setChartValues(manifestsDir, ingressController string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error {
 	chartValues := map[string]string{
 		"global.clusterCIDR":                  util.JoinIPNets(nodeConfig.AgentConfig.ClusterCIDRs),
 		"global.clusterCIDRv4":                util.JoinIP4Nets(nodeConfig.AgentConfig.ClusterCIDRs),
@@ -313,6 +313,7 @@ func setChartValues(manifestsDir string, nodeConfig *daemonconfig.Node, cfg cmds
 		"global.clusterDomain":                nodeConfig.AgentConfig.ClusterDomain,
 		"global.rke2DataDir":                  cfg.DataDir,
 		"global.serviceCIDR":                  util.JoinIPNets(nodeConfig.AgentConfig.ServiceCIDRs),
+		"global.systemDefaultIngressClass":    ingressController,
 		"global.systemDefaultRegistry":        nodeConfig.AgentConfig.SystemDefaultRegistry,
 		"global.cattle.systemDefaultRegistry": nodeConfig.AgentConfig.SystemDefaultRegistry,
 	}

pkg/cli/cmds/server.go

@@ -1,6 +1,7 @@
 package cmds
 
 import (
+	"errors"
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
@@ -9,29 +10,20 @@ import (
 	"github.com/rancher/wrangler/v3/pkg/slice"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 const (
 	rke2Path = "/var/lib/rancher/rke2"
 )
 
 var (
-	DisableItems = []string{"rke2-coredns", "rke2-ingress-nginx", "rke2-metrics-server"}
-	CNIItems     = []string{"calico", "canal", "cilium", "flannel"}
-
 	config = rke2.Config{}
 
 	serverFlag = []cli.Flag{
-		&cli.StringSliceFlag{
-			Name:   "cni",
-			Usage:  "(networking) CNI Plugins to deploy, one of none, " + strings.Join(CNIItems, ", ") + "; optionally with multus as the first value to enable the multus meta-plugin (default: canal)",
-			EnvVar: "RKE2_CNI",
-		},
-		&cli.BoolFlag{
-			Name:   "enable-servicelb",
-			Usage:  "(components) Enable rke2 default cloud controller manager's service controller",
-			EnvVar: "RKE2_ENABLE_SERVICELB",
-		},
+		rke2.CNIFlag,
+		rke2.IngressControllerFlag,
+		rke2.ServiceLBFlag,
 	}
 
 	k3sServerBase = mustCmdFromK3S(cmds.NewServerCommand(ServerRun), K3SFlagSet{
@@ -80,7 +72,7 @@ var (
 		"kine-tls":                          dropFlag,
 		"default-local-storage-path":        dropFlag,
 		"disable": {
-			Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(DisableItems, ", ") + ")",
+			Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(rke2.DisableItems, ", ") + ")",
 		},
 		"disable-scheduler":                 copyFlag,
 		"disable-cloud-controller":          copyFlag,
@@ -164,47 +156,82 @@ func ServerRun(clx *cli.Context) error {
 	validateCloudProviderName(clx, Server)
 	validateProfile(clx, Server)
 	validateCNI(clx)
+	validateIngress(clx)
 	return rke2.Server(clx, config)
 }
 
 func validateCNI(clx *cli.Context) {
-	cnis := []string{}
-	for _, cni := range clx.StringSlice("cni") {
-		for _, v := range strings.Split(cni, ",") {
-			cnis = append(cnis, v)
+	disableExceptSelected(clx, rke2.CNIItems, rke2.CNIFlag, func(values cli.StringSlice) (cli.StringSlice, error) {
+		switch len(values) {
+		case 0:
+			values = append(values, "canal")
+			fallthrough
+		case 1:
+			if values[0] == "multus" {
+				return nil, errors.New("multus must be used alongside another primary cni selection")
+			}
+			clx.Set("disable", "rke2-multus")
+		case 2:
+			if values[0] == "multus" {
+				values = values[1:]
+			} else {
+				return nil, errors.New("may only provide multiple values if multus is the first value")
+			}
+		default:
+			return nil, errors.New("must specify 1 or 2 values")
 		}
-	}
+		return values, nil
+	})
+}
 
-	switch len(cnis) {
-	case 0:
-		cnis = append(cnis, "canal")
-		fallthrough
-	case 1:
-		if cnis[0] == "multus" {
-			logrus.Fatal("invalid value provided for --cni flag: multus must be used alongside another primary cni selection")
+func validateIngress(clx *cli.Context) {
+	disableExceptSelected(clx, rke2.IngressItems, rke2.IngressControllerFlag, func(values cli.StringSlice) (cli.StringSlice, error) {
+		if len(values) == 0 {
+			values = append(values, "ingress-nginx")
 		}
-		clx.Set("disable", "rke2-multus")
-	case 2:
-		if cnis[0] == "multus" {
-			cnis = cnis[1:]
-		} else {
-			logrus.Fatal("invalid values provided for --cni flag: may only provide multiple values if multus is the first value")
+		return values, nil
+	})
+}
+
+func disableExceptSelected(clx *cli.Context, valid []string, flag *cli.StringSliceFlag, validateFunc func(cli.StringSlice) (cli.StringSlice, error)) {
+	// split comma-separated values
+	values := cli.StringSlice{}
+	if flag.Value != nil {
+		for _, value := range *flag.Value {
+			for _, v := range strings.Split(value, ",") {
+				values = append(values, v)
+			}
 		}
-	default:
-		logrus.Fatal("invalid values provided for --cni flag: may not provide more than two values")
 	}
 
-	switch {
-	case cnis[0] == "none":
-		fallthrough
-	case slice.ContainsString(CNIItems, cnis[0]):
-		for _, d := range CNIItems {
-			if cnis[0] != d {
-				clx.Set("disable", "rke2-"+d)
-				clx.Set("disable", "rke2-"+d+"-crd")
-			}
+	// validate the flag after splitting values
+	if v, err := validateFunc(values); err != nil {
+		logrus.Fatalf("Failed to validate --%s flag: %v", flag.Name, err)
+	} else {
+		flag.Value = &v
+	}
+
+	// prepare a list of items to disable, based on all valid components.
+	// we have to use an intermediate set because the flag interface
+	// doesn't allow us to remove flag values once added.
+	disabledCharts := sets.Set[string]{}
+	for _, d := range valid {
+		disabledCharts.Insert("rke2-"+d, "rke2-"+d+"-crd")
+	}
+
+	// re-enable components for any selected flag values
+	for _, d := range *flag.Value {
+		switch {
+		case d == "none":
+			break
+		case slice.ContainsString(valid, d):
+			disabledCharts.Delete("rke2-"+d, "rke2-"+d+"-crd")
+		default:
+			logrus.Fatalf("Invalid value %s for --%s flag: must be one of %s", d, flag.Name, strings.Join(valid, ","))
 		}
-	default:
-		logrus.Fatal("invalid value provided for --cni flag")
+	}
+
+	for _, c := range disabledCharts.UnsortedList() {
+		clx.Set("disable", c)
 	}
 }

pkg/pebinaryexecutor/pebinary.go

@@ -46,19 +46,20 @@ var (
 )
 
 type PEBinaryConfig struct {
-	ManifestsDir        string
-	ImagesDir           string
-	Resolver            *images.Resolver
+	CNIPlugin           win.CNIPlugin
 	CloudProvider       *CloudProviderConfig
-	CISMode             bool
+	Resolver            *images.Resolver
+	ManifestsDir        string
 	DataDir             string
 	AuditPolicyFile     string
 	KubeletPath         string
+	CNIName             string
+	ImagesDir           string
 	KubeConfigKubeProxy string
+	IngressController   string
+	CISMode             bool
 	DisableETCD         bool
 	IsServer            bool
-	CNIName             string
-	CNIPlugin           win.CNIPlugin
 }
 
 type CloudProviderConfig struct {
@@ -105,7 +106,7 @@ func (p *PEBinaryConfig) Bootstrap(ctx context.Context, nodeConfig *config.Node,
 	}
 
 	if p.IsServer {
-		return bootstrap.UpdateManifests(p.Resolver, nodeConfig, cfg)
+		return bootstrap.UpdateManifests(p.Resolver, p.IngressController, nodeConfig, cfg)
 	}
 
 	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)

pkg/podexecutor/staticpod.go

@@ -105,25 +105,25 @@ type ControlPlaneProbeConfs struct {
 }
 
 type StaticPodConfig struct {
+	Resolver      *images.Resolver
+	stopKubelet   context.CancelFunc
+	CloudProvider *CloudProviderConfig
 	ControlPlaneResources
-	ControlPlaneProbeConfs
+	DataDir           string
+	RuntimeEndpoint   string
+	ManifestsDir      string
+	IngressController string
+	ImagesDir         string
+	AuditPolicyFile   string
+	PSAConfigFile     string
+	KubeletPath       string
 	ControlPlaneEnv
 	ControlPlaneMounts
-	ManifestsDir     string
-	ImagesDir        string
-	Resolver         *images.Resolver
-	CloudProvider    *CloudProviderConfig
-	DataDir          string
-	AuditPolicyFile  string
-	PSAConfigFile    string
-	KubeletPath      string
-	RuntimeEndpoint  string
+	ControlPlaneProbeConfs
 	CISMode          bool
 	DisableETCD      bool
 	ExternalDatabase bool
 	IsServer         bool
-
-	stopKubelet context.CancelFunc
 }
 
 type CloudProviderConfig struct {
@@ -159,7 +159,7 @@ func (s *StaticPodConfig) Bootstrap(_ context.Context, nodeConfig *daemonconfig.
 		return err
 	}
 	if s.IsServer {
-		return bootstrap.UpdateManifests(s.Resolver, nodeConfig, cfg)
+		return bootstrap.UpdateManifests(s.Resolver, s.IngressController, nodeConfig, cfg)
 	}
 
 	// Remove the kube-proxy static pod manifest before starting the agent.

pkg/rke2/rke2.go

@@ -22,6 +22,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rancher/rke2/pkg/controllers/cisnetworkpolicy"
 	"github.com/rancher/rke2/pkg/images"
+
 	"github.com/rancher/wrangler/v3/pkg/slice"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -62,6 +63,30 @@ type ExtraEnv struct {
 	CloudControllerManager cli.StringSlice
 }
 
+var (
+	DisableItems = []string{"rke2-coredns", "rke2-metrics-server", "rke2-snapshot-controller", "rke2-snapshot-controller-crd", "rke2-snapshot-validation-webhook"}
+	CNIItems     = []string{"calico", "canal", "cilium", "flannel"}
+	IngressItems = []string{"ingress-nginx", "traefik"}
+
+	CNIFlag = &cli.StringSliceFlag{
+		Name:   "cni",
+		Usage:  "(networking) CNI Plugins to deploy, one of none, " + strings.Join(CNIItems, ", ") + "; optionally with multus as the first value to enable the multus meta-plugin (default: canal)",
+		EnvVar: "RKE2_CNI",
+		Value:  &cli.StringSlice{},
+	}
+	IngressControllerFlag = &cli.StringSliceFlag{
+		Name:   "ingress-controller",
+		Usage:  "(networking) Ingress Controllers to deploy, one of none, " + strings.Join(IngressItems, ", ") + "; the first value will be set as the default ingress class (default: ingress-nginx)",
+		EnvVar: "RKE_INGRESS_CONTROLLER",
+		Value:  &cli.StringSlice{},
+	}
+	ServiceLBFlag = &cli.BoolFlag{
+		Name:   "enable-servicelb",
+		Usage:  "(components) Enable rke2 default cloud controller manager's service controller",
+		EnvVar: "RKE2_ENABLE_SERVICELB",
+	}
+)
+
 // Valid CIS Profile versions
 const (
 	CISProfile123          = "cis-1.23"
@@ -115,7 +140,7 @@ func Server(clx *cli.Context, cfg Config) error {
 
 	var leaderControllers rawServer.CustomControllers
 
-	cnis := clx.StringSlice("cni")
+	cnis := *CNIFlag.Value
 	if cisMode && (len(cnis) == 0 || slice.ContainsString(cnis, "canal")) {
 		leaderControllers = append(leaderControllers, cisnetworkpolicy.Controller)
 	}

pkg/rke2/rke2_linux.go

@@ -139,6 +139,11 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta
 		containerRuntimeEndpoint = containerdSock
 	}
 
+	var ingressControllerName string
+	if IngressControllerFlag.Value != nil && len(*IngressControllerFlag.Value) > 0 {
+		ingressControllerName = (*IngressControllerFlag.Value)[0]
+	}
+
 	return &podexecutor.StaticPodConfig{
 		Resolver:               resolver,
 		ImagesDir:              agentImagesDir,
@@ -153,6 +158,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta
 		DisableETCD:            clx.Bool("disable-etcd"),
 		ExternalDatabase:       ExternalDatabase,
 		IsServer:               isServer,
+		IngressController:      ingressControllerName,
 		ControlPlaneResources:  *controlPlaneResources,
 		ControlPlaneProbeConfs: *controlPlaneProbeConfs,
 		ControlPlaneEnv:        *extraEnv,

pkg/rke2/rke2_windows.go

@@ -58,17 +58,23 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*pebinaryexecuto
 		cfg.KubeletPath = "kubelet"
 	}
 
+	var ingressControllerName string
+	if IngressControllerFlag.Value != nil && len(*IngressControllerFlag.Value) > 0 {
+		ingressControllerName = (*IngressControllerFlag.Value)[0]
+	}
+
 	return &pebinaryexecutor.PEBinaryConfig{
-		Resolver:        resolver,
-		ImagesDir:       agentImagesDir,
-		ManifestsDir:    agentManifestsDir,
-		CISMode:         isCISMode(clx),
-		CloudProvider:   cpConfig,
-		DataDir:         dataDir,
-		AuditPolicyFile: clx.String("audit-policy-file"),
-		KubeletPath:     cfg.KubeletPath,
-		DisableETCD:     clx.Bool("disable-etcd"),
-		IsServer:        isServer,
-		CNIName:         "",
+		Resolver:          resolver,
+		ImagesDir:         agentImagesDir,
+		ManifestsDir:      agentManifestsDir,
+		CISMode:           isCISMode(clx),
+		CloudProvider:     cpConfig,
+		DataDir:           dataDir,
+		AuditPolicyFile:   clx.String("audit-policy-file"),
+		KubeletPath:       cfg.KubeletPath,
+		DisableETCD:       clx.Bool("disable-etcd"),
+		IsServer:          isServer,
+		IngressController: ingressControllerName,
+		CNIName:           "",
 	}, nil
 }