From 9b9194035b3b47152168ba7ef086abd9855a04ad Mon Sep 17 00:00:00 2001 From: maru Date: Tue, 8 Jul 2025 03:41:17 +0000 Subject: [PATCH 1/4] [tmpnet] Source tmpnet defaults from a configmap Previously the scheduling label and value required to enable exclusive scheduling were defined as flag defaults. To enable the cluster to define these defaults, the defaults are now sourced from a configmap in the target namespace. --- tests/fixture/tmpnet/flags/kube_runtime.go | 14 ++---- tests/fixture/tmpnet/kube_runtime.go | 53 ++++++++++++++++++++++ tests/fixture/tmpnet/network.go | 8 +++- tests/fixture/tmpnet/network_test.go | 2 +- 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/tests/fixture/tmpnet/flags/kube_runtime.go b/tests/fixture/tmpnet/flags/kube_runtime.go index cc5304cb7952..3eee81ea99d9 100644 --- a/tests/fixture/tmpnet/flags/kube_runtime.go +++ b/tests/fixture/tmpnet/flags/kube_runtime.go @@ -20,10 +20,9 @@ const ( ) var ( - errKubeNamespaceRequired = errors.New("--kube-namespace is required") - errKubeImageRequired = errors.New("--kube-image is required") - errKubeMinVolumeSizeRequired = fmt.Errorf("--kube-volume-size must be >= %d", tmpnet.MinimumVolumeSizeGB) - errKubeSchedulingLabelRequired = errors.New("--kube-scheduling-label-key and --kube-scheduling-label-value are required when --kube-use-exclusive-scheduling is enabled") + errKubeNamespaceRequired = errors.New("--kube-namespace is required") + errKubeImageRequired = errors.New("--kube-image is required") + errKubeMinVolumeSizeRequired = fmt.Errorf("--kube-volume-size must be >= %d", tmpnet.MinimumVolumeSizeGB) ) type kubeRuntimeVars struct { @@ -77,13 +76,13 @@ func (v *kubeRuntimeVars) register(stringVar varFunc[string], uintVar varFunc[ui stringVar( &v.schedulingLabelKey, "kube-scheduling-label-key", - "purpose", + "", kubeDocPrefix+"The label key to use for exclusive scheduling for node selection and toleration", ) stringVar( &v.schedulingLabelValue, "kube-scheduling-label-value", - "higher-spec", + "", kubeDocPrefix+"The label value to use for exclusive scheduling for node selection and toleration", ) } @@ -98,9 +97,6 @@ func (v *kubeRuntimeVars) getKubeRuntimeConfig() (*tmpnet.KubeRuntimeConfig, err if v.volumeSizeGB < tmpnet.MinimumVolumeSizeGB { return nil, errKubeMinVolumeSizeRequired } - if v.useExclusiveScheduling && (len(v.schedulingLabelKey) == 0 || len(v.schedulingLabelValue) == 0) { - return nil, errKubeSchedulingLabelRequired - } return &tmpnet.KubeRuntimeConfig{ ConfigPath: v.config.Path, ConfigContext: v.config.Context, diff --git a/tests/fixture/tmpnet/kube_runtime.go b/tests/fixture/tmpnet/kube_runtime.go index c3bb4e72e4ab..c6fb229e20ca 100644 --- a/tests/fixture/tmpnet/kube_runtime.go +++ b/tests/fixture/tmpnet/kube_runtime.go @@ -21,6 +21,7 @@ import ( "github.com/ava-labs/avalanchego/config" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/utils/logging" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -49,8 +50,13 @@ const ( // are never scheduled to the same nodes. antiAffinityLabelKey = "tmpnet-scheduling" antiAffinityLabelValue = "exclusive" + + // Name of config map containing tmpnet defaults + kubeRuntimeConfigMapName = "tmpnet" ) +var errMissingSchedulingLabels = errors.New("--kube-scheduling-label-key and --kube-scheduling-label-value are required when exclusive scheduling is enabled") + type KubeRuntimeConfig struct { // Path to the kubeconfig file identifying the target cluster ConfigPath string `json:"configPath,omitempty"` @@ -72,6 +78,53 @@ type KubeRuntimeConfig struct { SchedulingLabelValue string `json:"schedulingLabelValue,omitempty"` } +// ensureDefaults sets cluster-specific defaults for fields not already set by flags. +func (c *KubeRuntimeConfig) ensureDefaults(ctx context.Context, log logging.Logger) error { + // Only source defaults from the cluster if exclusive scheduling is enabled + if !c.UseExclusiveScheduling { + return nil + } + + clientset, err := GetClientset(log, c.ConfigPath, c.ConfigContext) + if err != nil { + return err + } + + log.Info("attempting to retrieve configmap containing tmpnet defaults", + zap.String("namespace", c.Namespace), + zap.String("configMap", kubeRuntimeConfigMapName), + ) + + configMap, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, kubeRuntimeConfigMapName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get ConfigMap: %w", err) + } + + var ( + schedulingLabelKey = configMap.Data["defaultSchedulingLabelKey"] + schedulingLabelValue = configMap.Data["defaultSchedulingLabelValue"] + ) + if len(c.SchedulingLabelKey) == 0 && len(schedulingLabelKey) > 0 { + log.Info("setting default value for SchedulingLabelKey", + zap.String("schedulingLabelKey", schedulingLabelKey), + ) + c.SchedulingLabelKey = schedulingLabelKey + } + if len(c.SchedulingLabelValue) == 0 && len(schedulingLabelValue) > 0 { + log.Info("setting default value for SchedulingLabelValue", + zap.String("schedulingLabelValue", schedulingLabelValue), + ) + c.SchedulingLabelValue = schedulingLabelValue + } + + // Validate that the scheduling labels are now set + if len(c.SchedulingLabelKey) == 0 || len(c.SchedulingLabelValue) == 0 { + return errMissingSchedulingLabels + } + + return nil +} + type KubeRuntime struct { node *Node } diff --git a/tests/fixture/tmpnet/network.go b/tests/fixture/tmpnet/network.go index fd54079049f4..b41c989635fa 100644 --- a/tests/fixture/tmpnet/network.go +++ b/tests/fixture/tmpnet/network.go @@ -177,7 +177,7 @@ func BootstrapNewNetwork( if err := checkVMBinaries(log, network.Subnets, network.DefaultRuntimeConfig.Process); err != nil { return err } - if err := network.EnsureDefaultConfig(log); err != nil { + if err := network.EnsureDefaultConfig(ctx, log); err != nil { return err } if err := network.Create(rootNetworkDir); err != nil { @@ -234,7 +234,7 @@ func ReadNetwork(ctx context.Context, log logging.Logger, dir string) (*Network, } // Initializes a new network with default configuration. -func (n *Network) EnsureDefaultConfig(log logging.Logger) error { +func (n *Network) EnsureDefaultConfig(ctx context.Context, log logging.Logger) error { log.Info("preparing configuration for new network", zap.Any("runtimeConfig", n.DefaultRuntimeConfig), ) @@ -281,6 +281,10 @@ func (n *Network) EnsureDefaultConfig(log logging.Logger) error { return errMissingRuntimeConfig } + if n.DefaultRuntimeConfig.Kube != nil { + return n.DefaultRuntimeConfig.Kube.ensureDefaults(ctx, log) + } + return nil } diff --git a/tests/fixture/tmpnet/network_test.go b/tests/fixture/tmpnet/network_test.go index fa789d7e9935..c632b41186b0 100644 --- a/tests/fixture/tmpnet/network_test.go +++ b/tests/fixture/tmpnet/network_test.go @@ -26,7 +26,7 @@ func TestNetworkSerialization(t *testing.T) { network.PrimarySubnetConfig = ConfigMap{ "validatorOnly": true, } - require.NoError(network.EnsureDefaultConfig(logging.NoLog{})) + require.NoError(network.EnsureDefaultConfig(ctx, logging.NoLog{})) require.NoError(network.Create(tmpDir)) // Ensure node runtime is initialized require.NoError(network.readNodes(ctx)) From 313e0c3805c24b6723d402a0db84752c375a55bb Mon Sep 17 00:00:00 2001 From: maru Date: Tue, 8 Jul 2025 04:32:26 +0000 Subject: [PATCH 2/4] fixup: Align with expected configmap --- tests/fixture/tmpnet/kube_runtime.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fixture/tmpnet/kube_runtime.go b/tests/fixture/tmpnet/kube_runtime.go index c6fb229e20ca..b6095a46caf5 100644 --- a/tests/fixture/tmpnet/kube_runtime.go +++ b/tests/fixture/tmpnet/kube_runtime.go @@ -80,8 +80,8 @@ type KubeRuntimeConfig struct { // ensureDefaults sets cluster-specific defaults for fields not already set by flags. func (c *KubeRuntimeConfig) ensureDefaults(ctx context.Context, log logging.Logger) error { - // Only source defaults from the cluster if exclusive scheduling is enabled - if !c.UseExclusiveScheduling { + requireSchedulingDefaults := c.UseExclusiveScheduling && (len(c.SchedulingLabelKey) == 0 || len(c.SchedulingLabelValue) == 0) + if !requireSchedulingDefaults { return nil } @@ -101,8 +101,8 @@ func (c *KubeRuntimeConfig) ensureDefaults(ctx context.Context, log logging.Logg } var ( - schedulingLabelKey = configMap.Data["defaultSchedulingLabelKey"] - schedulingLabelValue = configMap.Data["defaultSchedulingLabelValue"] + schedulingLabelKey = configMap.Data["schedulingLabelKey"] + schedulingLabelValue = configMap.Data["schedulingLabelValue"] ) if len(c.SchedulingLabelKey) == 0 && len(schedulingLabelKey) > 0 { log.Info("setting default value for SchedulingLabelKey", From ba934d99070ddf6b24bbf334317959cbe59f1310 Mon Sep 17 00:00:00 2001 From: maru Date: Tue, 8 Jul 2025 05:14:45 +0000 Subject: [PATCH 3/4] fixup: Set defaults before logging --- tests/fixture/tmpnet/network.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/fixture/tmpnet/network.go b/tests/fixture/tmpnet/network.go index b41c989635fa..b990ed114393 100644 --- a/tests/fixture/tmpnet/network.go +++ b/tests/fixture/tmpnet/network.go @@ -235,6 +235,13 @@ func ReadNetwork(ctx context.Context, log logging.Logger, dir string) (*Network, // Initializes a new network with default configuration. func (n *Network) EnsureDefaultConfig(ctx context.Context, log logging.Logger) error { + // Populate runtime defaults before logging it + if n.DefaultRuntimeConfig.Kube != nil { + if err := n.DefaultRuntimeConfig.Kube.ensureDefaults(ctx, log); err != nil { + return err + } + } + log.Info("preparing configuration for new network", zap.Any("runtimeConfig", n.DefaultRuntimeConfig), ) @@ -281,10 +288,6 @@ func (n *Network) EnsureDefaultConfig(ctx context.Context, log logging.Logger) e return errMissingRuntimeConfig } - if n.DefaultRuntimeConfig.Kube != nil { - return n.DefaultRuntimeConfig.Kube.ensureDefaults(ctx, log) - } - return nil } From 01f7270989b272ace1c66ef8d70ca68e165ae554 Mon Sep 17 00:00:00 2001 From: maru Date: Tue, 8 Jul 2025 05:33:18 +0000 Subject: [PATCH 4/4] fixup: Rename the configmap name --- tests/fixture/tmpnet/kube_runtime.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fixture/tmpnet/kube_runtime.go b/tests/fixture/tmpnet/kube_runtime.go index b6095a46caf5..4f5343ab84ed 100644 --- a/tests/fixture/tmpnet/kube_runtime.go +++ b/tests/fixture/tmpnet/kube_runtime.go @@ -52,7 +52,7 @@ const ( antiAffinityLabelValue = "exclusive" // Name of config map containing tmpnet defaults - kubeRuntimeConfigMapName = "tmpnet" + defaultsConfigMapName = "tmpnet-defaults" ) var errMissingSchedulingLabels = errors.New("--kube-scheduling-label-key and --kube-scheduling-label-value are required when exclusive scheduling is enabled") @@ -92,10 +92,10 @@ func (c *KubeRuntimeConfig) ensureDefaults(ctx context.Context, log logging.Logg log.Info("attempting to retrieve configmap containing tmpnet defaults", zap.String("namespace", c.Namespace), - zap.String("configMap", kubeRuntimeConfigMapName), + zap.String("configMap", defaultsConfigMapName), ) - configMap, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, kubeRuntimeConfigMapName, metav1.GetOptions{}) + configMap, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, defaultsConfigMapName, metav1.GetOptions{}) if err != nil { return fmt.Errorf("failed to get ConfigMap: %w", err) }