diff --git a/controllers/ingress/group_controller.go b/controllers/ingress/group_controller.go index 3355f0f311..30e4271b64 100644 --- a/controllers/ingress/group_controller.go +++ b/controllers/ingress/group_controller.go @@ -63,7 +63,7 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco cloud.EC2(), cloud.ELBV2(), cloud.ACM(), annotationParser, subnetsResolver, authConfigBuilder, enhancedBackendBuilder, trackingProvider, elbv2TaggingManager, controllerConfig.FeatureGates, - cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, + cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultSubnets, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger, metricsCollector) stackMarshaller := deploy.NewDefaultStackMarshaller() diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index dbadd0e1d6..14806f96be 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -49,7 +49,7 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName) serviceUtils := service.NewServiceUtils(annotationParser, shared_constants.ServiceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates) modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider, - elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, + elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultSubnets, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector) stackMarshaller := deploy.NewDefaultStackMarshaller() diff --git a/docs/deploy/configurations.md b/docs/deploy/configurations.md index 5722fc5f56..bfa919e1af 100644 --- a/docs/deploy/configurations.md +++ b/docs/deploy/configurations.md @@ -77,6 +77,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne | backend-security-group | string | | Backend security group id to use for the ingress rules on the worker node SG | | cluster-name | string | | Kubernetes cluster name | | default-ssl-policy | string | ELBSecurityPolicy-2016-08 | Default SSL Policy that will be applied to all Ingresses or Services that do not have the SSL Policy annotation | +| default-subnets | stringList | [] | Default subnets to be selected when not explicitly specified through annotations or other methods | | default-tags | stringMap | | AWS Tags that will be applied to all AWS resources managed by this controller. Specified Tags takes highest priority | | default-target-type | string | instance | Default target type for Ingresses and Services - ip, instance | | default-load-balancer-scheme | string | internal | Default scheme for ELBs - internal, internet-facing | diff --git a/docs/deploy/subnet_discovery.md b/docs/deploy/subnet_discovery.md index 9a1646224c..716a4c56c1 100644 --- a/docs/deploy/subnet_discovery.md +++ b/docs/deploy/subnet_discovery.md @@ -61,7 +61,8 @@ A subnet is classified as public if its route table contains a route to an Inter The controller selects one subnet per availability zone. When multiple subnets exist per Availability Zone, the following priority order applies: 1. Subnets with cluster tag for the current cluster (`kubernetes.io/cluster/`) are prioritized -2. Subnets with lower lexicographical order of subnet ID are prioritized +2. Subnets with the `--default-subnets` flag (prioritized in the order specified) +3. Subnets with lower lexicographical order of subnet ID are prioritized ## Minimum Subnet Requirements diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 3aa091c4fc..d0d837f756 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -16,6 +16,7 @@ import ( const ( flagLogLevel = "log-level" flagK8sClusterName = "cluster-name" + flagDefaultSubnets = "default-subnets" flagDefaultTags = "default-tags" flagDefaultTargetType = "default-target-type" flagDefaultLoadBalancerScheme = "default-load-balancer-scheme" @@ -78,6 +79,9 @@ type ControllerConfig struct { // Configurations for the Service controller ServiceConfig ServiceConfig + // Default subnets that will be used for all AWS resources managed by the networking controller. + DefaultSubnets []string + // Default AWS Tags that will be applied to all AWS resources managed by this controller. DefaultTags map[string]string @@ -137,6 +141,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { fs.StringVar(&cfg.LogLevel, flagLogLevel, defaultLogLevel, "Set the controller log level - info(default), debug") fs.StringVar(&cfg.ClusterName, flagK8sClusterName, "", "Kubernetes cluster name") + fs.StringSliceVar(&cfg.DefaultSubnets, flagDefaultSubnets, nil, + "Default subnets that will be used for all AWS resources managed by the networking controller") fs.StringToStringVar(&cfg.DefaultTags, flagDefaultTags, nil, "Default AWS Tags that will be applied to all AWS resources managed by this controller") fs.StringVar(&cfg.DefaultTargetType, flagDefaultTargetType, string(elbv2.TargetTypeInstance), @@ -186,7 +192,9 @@ func (cfg *ControllerConfig) Validate() error { if len(cfg.ClusterName) == 0 { return errors.New("kubernetes cluster name must be specified") } - + if err := cfg.validateDefaultSubnets(); err != nil { + return err + } if err := cfg.validateDefaultTagsCollisionWithTrackingTags(); err != nil { return err } @@ -211,6 +219,27 @@ func (cfg *ControllerConfig) Validate() error { return nil } +func (cfg *ControllerConfig) validateDefaultSubnets() error { + if len(cfg.DefaultSubnets) == 0 { + return nil + } + for _, subnetID := range cfg.DefaultSubnets { + if !strings.HasPrefix(subnetID, "subnet-") { + return errors.Errorf("invalid value %v for default subnet id", subnetID) + } + } + + //validate duplicate subnet ids + seen := make(map[string]bool) + for _, str := range cfg.DefaultSubnets { + if seen[str] { + return errors.Errorf("duplicate subnet id %v is specified in the --default-subnets flag", str) + } + seen[str] = true + } + return nil +} + func (cfg *ControllerConfig) validateDefaultTagsCollisionWithTrackingTags() error { for tagKey := range cfg.DefaultTags { if trackingTagKeys.Has(tagKey) { diff --git a/pkg/config/controller_config_test.go b/pkg/config/controller_config_test.go index 74ec7ae382..4893a55798 100644 --- a/pkg/config/controller_config_test.go +++ b/pkg/config/controller_config_test.go @@ -1,10 +1,11 @@ package config import ( + "testing" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" - "testing" ) func TestControllerConfig_validateDefaultTagsCollisionWithTrackingTags(t *testing.T) { @@ -219,3 +220,49 @@ func TestControllerConfig_validateManageBackendSecurityGroupRulesConfiguration(t }) } } + +func TestControllerConfig_validateDefaultSubnets(t *testing.T) { + type fields struct { + DefaultSubnets []string + } + tests := []struct { + name string + fields fields + wantErr error + }{ + { + name: "default subnets is empty", + fields: fields{ + DefaultSubnets: nil, + }, + wantErr: nil, + }, + { + name: "default subnets is not empty", + fields: fields{ + DefaultSubnets: []string{"subnet-1", "subnet-2"}, + }, + wantErr: nil, + }, + { + name: "default subnets is not empty and duplicate subnets are specified", + fields: fields{ + DefaultSubnets: []string{"subnet-1", "subnet-2", "subnet-1"}, + }, + wantErr: errors.New("duplicate subnet id subnet-1 is specified in the --default-subnets flag"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &ControllerConfig{ + DefaultSubnets: tt.fields.DefaultSubnets, + } + err := cfg.validateDefaultSubnets() + if tt.wantErr != nil { + assert.EqualError(t, err, tt.wantErr.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/pkg/ingress/model_build_load_balancer.go b/pkg/ingress/model_build_load_balancer.go index e83419fa8f..f97b127892 100644 --- a/pkg/ingress/model_build_load_balancer.go +++ b/pkg/ingress/model_build_load_balancer.go @@ -260,6 +260,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerSubnetMappings(ctx context.Cont chosenSubnets, err := t.subnetsResolver.ResolveViaDiscovery(ctx, networking.WithSubnetsResolveLBType(elbv2model.LoadBalancerTypeApplication), networking.WithSubnetsResolveLBScheme(scheme), + networking.WithDefaultSubnets(t.defaultSubnets), ) if err != nil { return nil, errors.Wrap(err, "couldn't auto-discover subnets") diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index 50cd258771..8e619f085e 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -3,9 +3,10 @@ package ingress import ( "context" "reflect" - "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strconv" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -46,7 +47,7 @@ func NewDefaultModelBuilder(k8sClient client.Client, eventRecorder record.EventR annotationParser annotations.Parser, subnetsResolver networkingpkg.SubnetsResolver, authConfigBuilder AuthConfigBuilder, enhancedBackendBuilder EnhancedBackendBuilder, trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, featureGates config.FeatureGates, - vpcID string, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, + vpcID string, clusterName string, defaultSubnets []string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, backendSGProvider networkingpkg.BackendSGProvider, sgResolver networkingpkg.SecurityGroupResolver, enableBackendSG bool, defaultEnableManageBackendSGRules bool, disableRestrictedSGRules bool, allowedCAARNs []string, enableIPTargetType bool, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *defaultModelBuilder { certDiscovery := NewACMCertDiscovery(acmClient, allowedCAARNs, logger) @@ -69,6 +70,7 @@ func NewDefaultModelBuilder(k8sClient client.Client, eventRecorder record.EventR trackingProvider: trackingProvider, elbv2TaggingManager: elbv2TaggingManager, featureGates: featureGates, + defaultSubnets: defaultSubnets, defaultTags: defaultTags, externalManagedTags: sets.NewString(externalManagedTags...), defaultSSLPolicy: defaultSSLPolicy, @@ -106,6 +108,7 @@ type defaultModelBuilder struct { trackingProvider tracking.Provider elbv2TaggingManager elbv2deploy.TaggingManager featureGates config.FeatureGates + defaultSubnets []string defaultTags map[string]string externalManagedTags sets.String defaultSSLPolicy string @@ -154,6 +157,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metrics stack: stack, frontendNlbTargetGroupDesiredState: frontendNlbTargetGroupDesiredState, + defaultSubnets: b.defaultSubnets, defaultTags: b.defaultTags, externalManagedTags: b.externalManagedTags, defaultIPAddressType: elbv2model.IPAddressTypeIPV4, @@ -213,6 +217,7 @@ type defaultModelBuildTask struct { disableRestrictedSGRules bool enableIPTargetType bool + defaultSubnets []string defaultTags map[string]string externalManagedTags sets.String defaultIPAddressType elbv2model.IPAddressType diff --git a/pkg/networking/subnet_resolver.go b/pkg/networking/subnet_resolver.go index c755846647..cad81993e4 100644 --- a/pkg/networking/subnet_resolver.go +++ b/pkg/networking/subnet_resolver.go @@ -53,6 +53,8 @@ type SubnetsResolveOptions struct { // The Load Balancer Scheme. // By default, it's internet-facing. LBScheme elbv2model.LoadBalancerScheme + // Subnets specified with --default-subnets + DefaultSubnets []string } // ApplyOptions applies slice of SubnetsResolveOption. @@ -86,6 +88,13 @@ func WithSubnetsResolveLBScheme(lbScheme elbv2model.LoadBalancerScheme) SubnetsR } } +// WithDefaultSubnets generates an option that configures DefaultSubnets. +func WithDefaultSubnets(defaultSubnets []string) SubnetsResolveOption { + return func(opts *SubnetsResolveOptions) { + opts.DefaultSubnets = defaultSubnets + } +} + // SubnetsResolver is responsible for resolve EC2 Subnets for Load Balancers. type SubnetsResolver interface { // ResolveViaDiscovery resolve subnets by auto discover matching subnets. @@ -412,7 +421,7 @@ func (r *defaultSubnetsResolver) validateSpecifiedSubnets(ctx context.Context, s // chooseAndValidateSubnetsPerAZ will choose one subnet per AZ from eligible subnets and then validate against chosen subnets. func (r *defaultSubnetsResolver) chooseAndValidateSubnetsPerAZ(ctx context.Context, subnets []ec2types.Subnet, resolveOpts SubnetsResolveOptions) ([]ec2types.Subnet, error) { categorizedSubnets := r.categorizeSubnetsByEligibility(subnets) - chosenSubnets := r.chooseSubnetsPerAZ(categorizedSubnets.eligible) + chosenSubnets := r.chooseSubnetsPerAZ(categorizedSubnets.eligible, resolveOpts.DefaultSubnets) if len(chosenSubnets) == 0 { return nil, fmt.Errorf("unable to resolve at least one subnet. Evaluated %d subnets: %d are tagged for other clusters, and %d have insufficient available IP addresses", len(subnets), len(categorizedSubnets.ineligibleClusterTag), len(categorizedSubnets.insufficientIPs)) @@ -452,7 +461,15 @@ func (r *defaultSubnetsResolver) categorizeSubnetsByEligibility(subnets []ec2typ // chooseSubnetsPerAZ will choose one subnet per AZ. // * subnets with current cluster tag will be prioritized. -func (r *defaultSubnetsResolver) chooseSubnetsPerAZ(subnets []ec2types.Subnet) []ec2types.Subnet { +func (r *defaultSubnetsResolver) chooseSubnetsPerAZ(subnets []ec2types.Subnet, defaultSubnets []string) []ec2types.Subnet { + + prioritySubnetMap := make(map[string]int) + + if len(defaultSubnets) > 0 { + for i, subnetID := range defaultSubnets { + prioritySubnetMap[subnetID] = i + } + } subnetsByAZ := mapSDKSubnetsByAZ(subnets) chosenSubnets := make([]ec2types.Subnet, 0, len(subnetsByAZ)) for az, azSubnets := range subnetsByAZ { @@ -467,9 +484,24 @@ func (r *defaultSubnetsResolver) chooseSubnetsPerAZ(subnets []ec2types.Subnet) [ } else if (!subnetIHasCurrentClusterTag) && subnetJHasCurrentClusterTag { return false } + + // When azSubnets are specified in --default-azSubnets, the azSubnets list will be sorted according to this order. + // Any azSubnets not specified in --default-azSubnets will be sorted in lexicographical order and placed after the prioritized azSubnets. + iVal, iExists := prioritySubnetMap[awssdk.ToString(azSubnets[i].SubnetId)] + jVal, jExists := prioritySubnetMap[awssdk.ToString(azSubnets[j].SubnetId)] + + if iExists && jExists { + return iVal < jVal + } + if iExists { + return true + } + if jExists { + return false + } return awssdk.ToString(azSubnets[i].SubnetId) < awssdk.ToString(azSubnets[j].SubnetId) }) - r.logger.V(1).Info("multiple subnets in the same AvailabilityZone", "AvailabilityZone", az, + r.logger.V(1).Info("multiple azSubnets in the same AvailabilityZone", "AvailabilityZone", az, "chosen", azSubnets[0].SubnetId, "ignored", extractSubnetIDs(azSubnets[1:])) chosenSubnets = append(chosenSubnets, azSubnets[0]) } diff --git a/pkg/networking/subnet_resolver_test.go b/pkg/networking/subnet_resolver_test.go index ff3cc3177e..59a4e81d77 100644 --- a/pkg/networking/subnet_resolver_test.go +++ b/pkg/networking/subnet_resolver_test.go @@ -1313,6 +1313,320 @@ func Test_defaultSubnetsResolver_ResolveViaDiscovery(t *testing.T) { }, wantErr: errors.New("failed to list subnets by reachability: some error"), }, + { + name: "When multiple subnets from the same Availability Zone are specified in the --default-subnets flag, the first written subnet will be selected", + fields: fields{ + clusterTagCheckEnabled: true, + albSingleSubnetEnabled: false, + discoveryByReachabilityEnabled: true, + describeSubnetsAsListCalls: []describeSubnetsAsListCall{ + { + input: &ec2sdk.DescribeSubnetsInput{ + Filters: []ec2types.Filter{ + { + Name: awssdk.String("vpc-id"), + Values: []string{"vpc-dummy"}, + }, + { + Name: awssdk.String("tag:kubernetes.io/role/elb"), + Values: []string{"", "1"}, + }, + }, + }, + output: []ec2types.Subnet{ + { + SubnetId: awssdk.String("subnet-1"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-3"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-4"), + AvailabilityZone: awssdk.String("us-west-2b"), + AvailabilityZoneId: awssdk.String("usw2-az2"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-2"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-5"), + AvailabilityZone: awssdk.String("us-west-2c"), + AvailabilityZoneId: awssdk.String("usw2-az3"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + }, + }, + }, + fetchAZInfosCalls: []fetchAZInfosCall{ + { + availabilityZoneIDs: []string{"usw2-az1"}, + azInfoByAZID: map[string]ec2types.AvailabilityZone{ + "usw2-az1": { + ZoneId: awssdk.String("usw2-az1"), + ZoneType: awssdk.String("availability-zone"), + }, + }, + }, + { + availabilityZoneIDs: []string{"usw2-az2"}, + azInfoByAZID: map[string]ec2types.AvailabilityZone{ + "usw2-az2": { + ZoneId: awssdk.String("usw2-az2"), + ZoneType: awssdk.String("availability-zone"), + }, + }, + }, + { + availabilityZoneIDs: []string{"usw2-az3"}, + azInfoByAZID: map[string]ec2types.AvailabilityZone{ + "usw2-az3": { + ZoneId: awssdk.String("usw2-az3"), + ZoneType: awssdk.String("availability-zone"), + }, + }, + }, + }, + }, + args: args{ + opts: []SubnetsResolveOption{ + WithSubnetsResolveLBType(elbv2model.LoadBalancerTypeApplication), + WithSubnetsResolveLBScheme(elbv2model.LoadBalancerSchemeInternetFacing), + WithDefaultSubnets([]string{"subnet-3", "subnet-1"}), + }, + }, + want: []ec2types.Subnet{ + { + SubnetId: awssdk.String("subnet-3"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-4"), + AvailabilityZone: awssdk.String("us-west-2b"), + AvailabilityZoneId: awssdk.String("usw2-az2"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-5"), + AvailabilityZone: awssdk.String("us-west-2c"), + AvailabilityZoneId: awssdk.String("usw2-az3"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + }, + }, + { + name: "When subnets from different AZs are specified in the --default-subnets flag, the specified subnet will be selected for each AZ.", + fields: fields{ + clusterTagCheckEnabled: true, + albSingleSubnetEnabled: false, + discoveryByReachabilityEnabled: true, + describeSubnetsAsListCalls: []describeSubnetsAsListCall{ + { + input: &ec2sdk.DescribeSubnetsInput{ + Filters: []ec2types.Filter{ + { + Name: awssdk.String("vpc-id"), + Values: []string{"vpc-dummy"}, + }, + { + Name: awssdk.String("tag:kubernetes.io/role/elb"), + Values: []string{"", "1"}, + }, + }, + }, + output: []ec2types.Subnet{ + { + SubnetId: awssdk.String("subnet-1"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-3"), + AvailabilityZone: awssdk.String("us-west-2b"), + AvailabilityZoneId: awssdk.String("usw2-az2"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-2"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-4"), + AvailabilityZone: awssdk.String("us-west-2b"), + AvailabilityZoneId: awssdk.String("usw2-az2"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + }, + }, + }, + fetchAZInfosCalls: []fetchAZInfosCall{ + { + availabilityZoneIDs: []string{"usw2-az1"}, + azInfoByAZID: map[string]ec2types.AvailabilityZone{ + "usw2-az1": { + ZoneId: awssdk.String("usw2-az1"), + ZoneType: awssdk.String("availability-zone"), + }, + }, + }, + { + availabilityZoneIDs: []string{"usw2-az2"}, + azInfoByAZID: map[string]ec2types.AvailabilityZone{ + "usw2-az2": { + ZoneId: awssdk.String("usw2-az2"), + ZoneType: awssdk.String("availability-zone"), + }, + }, + }, + }, + }, + args: args{ + opts: []SubnetsResolveOption{ + WithSubnetsResolveLBType(elbv2model.LoadBalancerTypeApplication), + WithSubnetsResolveLBScheme(elbv2model.LoadBalancerSchemeInternetFacing), + WithDefaultSubnets([]string{"subnet-4", "subnet-1"}), + }, + }, + want: []ec2types.Subnet{ + { + SubnetId: awssdk.String("subnet-1"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-4"), + AvailabilityZone: awssdk.String("us-west-2b"), + AvailabilityZoneId: awssdk.String("usw2-az2"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + }, + }, + { + name: "When non-existent subnets specified for the --default-subnets flag,", + fields: fields{ + clusterTagCheckEnabled: true, + albSingleSubnetEnabled: false, + discoveryByReachabilityEnabled: true, + describeSubnetsAsListCalls: []describeSubnetsAsListCall{ + { + input: &ec2sdk.DescribeSubnetsInput{ + Filters: []ec2types.Filter{ + { + Name: awssdk.String("vpc-id"), + Values: []string{"vpc-dummy"}, + }, + { + Name: awssdk.String("tag:kubernetes.io/role/elb"), + Values: []string{"", "1"}, + }, + }, + }, + output: []ec2types.Subnet{ + { + SubnetId: awssdk.String("subnet-1"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-3"), + AvailabilityZone: awssdk.String("us-west-2b"), + AvailabilityZoneId: awssdk.String("usw2-az2"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-2"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-4"), + AvailabilityZone: awssdk.String("us-west-2b"), + AvailabilityZoneId: awssdk.String("usw2-az2"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + }, + }, + }, + fetchAZInfosCalls: []fetchAZInfosCall{ + { + availabilityZoneIDs: []string{"usw2-az1"}, + azInfoByAZID: map[string]ec2types.AvailabilityZone{ + "usw2-az1": { + ZoneId: awssdk.String("usw2-az1"), + ZoneType: awssdk.String("availability-zone"), + }, + }, + }, + { + availabilityZoneIDs: []string{"usw2-az2"}, + azInfoByAZID: map[string]ec2types.AvailabilityZone{ + "usw2-az2": { + ZoneId: awssdk.String("usw2-az2"), + ZoneType: awssdk.String("availability-zone"), + }, + }, + }, + }, + }, + args: args{ + opts: []SubnetsResolveOption{ + WithSubnetsResolveLBType(elbv2model.LoadBalancerTypeApplication), + WithSubnetsResolveLBScheme(elbv2model.LoadBalancerSchemeInternetFacing), + WithDefaultSubnets([]string{"subnet-6", "subnet-1"}), + }, + }, + want: []ec2types.Subnet{ + { + SubnetId: awssdk.String("subnet-1"), + AvailabilityZone: awssdk.String("us-west-2a"), + AvailabilityZoneId: awssdk.String("usw2-az1"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + { + SubnetId: awssdk.String("subnet-3"), + AvailabilityZone: awssdk.String("us-west-2b"), + AvailabilityZoneId: awssdk.String("usw2-az2"), + AvailableIpAddressCount: awssdk.Int32(8), + VpcId: awssdk.String("vpc-dummy"), + }, + }, + }, } for _, tt := range tests { @@ -2584,7 +2898,7 @@ func Test_defaultSubnetsResolver_chooseSubnetsPerAZ(t *testing.T) { t.Run(tt.name, func(t *testing.T) { r := NewDefaultSubnetsResolver(nil, nil, "vpc-dummy", "cluster-dummy", true, false, true, logr.New(&log.NullLogSink{})) - got := r.chooseSubnetsPerAZ(tt.subnets) + got := r.chooseSubnetsPerAZ(tt.subnets, nil) assert.Equal(t, tt.want, got) }) } diff --git a/pkg/service/model_build_load_balancer.go b/pkg/service/model_build_load_balancer.go index 139f2b8233..3c9fb3d1d7 100644 --- a/pkg/service/model_build_load_balancer.go +++ b/pkg/service/model_build_load_balancer.go @@ -469,11 +469,13 @@ func (t *defaultModelBuildTask) buildLoadBalancerSubnets(ctx context.Context, sc return t.subnetsResolver.ResolveViaDiscovery(ctx, networking.WithSubnetsResolveLBType(elbv2model.LoadBalancerTypeNetwork), networking.WithSubnetsResolveLBScheme(scheme), + networking.WithDefaultSubnets(t.defaultSubnets), ) } return t.subnetsResolver.ResolveViaDiscovery(ctx, networking.WithSubnetsResolveLBType(elbv2model.LoadBalancerTypeNetwork), networking.WithSubnetsResolveLBScheme(scheme), + networking.WithDefaultSubnets(t.defaultSubnets), ) } diff --git a/pkg/service/model_builder.go b/pkg/service/model_builder.go index 3f01b7b253..07c05563ae 100644 --- a/pkg/service/model_builder.go +++ b/pkg/service/model_builder.go @@ -2,10 +2,11 @@ package service import ( "context" - "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strconv" "sync" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/go-logr/logr" @@ -42,7 +43,7 @@ type ModelBuilder interface { // NewDefaultModelBuilder construct a new defaultModelBuilder func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, vpcID string, trackingProvider tracking.Provider, - elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string, + elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultSubnets []string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, enableIPTargetType bool, serviceUtils ServiceUtils, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool, defaultEnableManageBackendSGRules bool, disableRestrictedSGRules bool, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *defaultModelBuilder { @@ -56,6 +57,7 @@ func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver serviceUtils: serviceUtils, clusterName: clusterName, vpcID: vpcID, + defaultSubnets: defaultSubnets, defaultTags: defaultTags, externalManagedTags: sets.NewString(externalManagedTags...), defaultSSLPolicy: defaultSSLPolicy, @@ -92,6 +94,7 @@ type defaultModelBuilder struct { clusterName string vpcID string + defaultSubnets []string defaultTags map[string]string externalManagedTags sets.String defaultSSLPolicy string @@ -128,6 +131,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service stack: stack, tgByResID: make(map[string]*elbv2model.TargetGroup), + defaultSubnets: b.defaultSubnets, defaultTags: b.defaultTags, externalManagedTags: b.externalManagedTags, defaultSSLPolicy: b.defaultSSLPolicy, @@ -198,6 +202,7 @@ type defaultModelBuildTask struct { fetchExistingLoadBalancerOnce sync.Once existingLoadBalancer *elbv2deploy.LoadBalancerWithTags + defaultSubnets []string defaultTags map[string]string externalManagedTags sets.String defaultSSLPolicy string diff --git a/pkg/service/model_builder_test.go b/pkg/service/model_builder_test.go index 0bc9c05f16..82d52f812f 100644 --- a/pkg/service/model_builder_test.go +++ b/pkg/service/model_builder_test.go @@ -6663,7 +6663,7 @@ func Test_defaultModelBuilderTask_Build(t *testing.T) { } mockMetricsCollector := lbcmetrics.NewMockCollector() builder := NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, "vpc-xxx", trackingProvider, elbv2TaggingManager, ec2Client, featureGates, - "my-cluster", nil, nil, "ELBSecurityPolicy-2016-08", defaultTargetType, defaultLoadBalancerScheme, enableIPTargetType, serviceUtils, + "my-cluster", nil, nil, nil, "ELBSecurityPolicy-2016-08", defaultTargetType, defaultLoadBalancerScheme, enableIPTargetType, serviceUtils, backendSGProvider, sgResolver, tt.enableBackendSG, tt.enableManageBackendSGRules, tt.disableRestrictedSGRules, logr.New(&log.NullLogSink{}), mockMetricsCollector) ctx := context.Background() stack, _, _, err := builder.Build(ctx, tt.svc, mockMetricsCollector)