From 3951a3af9514fb55688f58d0f772792260625db4 Mon Sep 17 00:00:00 2001 From: niclask25 Date: Thu, 10 Apr 2025 11:45:56 +0200 Subject: [PATCH 01/40] feat: add healthProbeBindAddr --- helm/aws-load-balancer-controller/README.md | 4 ++- .../templates/deployment.yaml | 27 ++++++++++++---- helm/aws-load-balancer-controller/test.yaml | 15 ++++----- helm/aws-load-balancer-controller/values.yaml | 32 +++++++++---------- 4 files changed, 46 insertions(+), 32 deletions(-) diff --git a/helm/aws-load-balancer-controller/README.md b/helm/aws-load-balancer-controller/README.md index d4f3f6272f..779d283b86 100644 --- a/helm/aws-load-balancer-controller/README.md +++ b/helm/aws-load-balancer-controller/README.md @@ -244,7 +244,8 @@ The default values set by the application itself can be confirmed [here](https:/ | `tolerateNonExistentBackendAction` | whether to allow rules that reference a backend action that does not exist. (When enabled, it will return 503 error if backend action not exist) | `true` | | `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | | `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | -| `livenessProbe` | Liveness probe settings for the controller | (see `values.yaml`) | +| `livenessProbe` | Liveness probe settings for the controller | `{}` (see `values.yaml`) | +| `readinessProbe` | Readiness probe settings for the controller | `{}` (see `values.yaml`) | (see `values.yaml`) | | `env` | Environment variables to set for aws-load-balancer-controller pod | None | | `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | | `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | @@ -285,3 +286,4 @@ The default values set by the application itself can be confirmed [here](https:/ | `loadBalancerClass` | Sets the AWS load balancer type to be used when the Kubernetes service requests an external load balancer | `service.k8s.aws/nlb` | | `creator` | if set to a `value!=helm`, it will disable the addition of default helm labels | `helm` | | `runtimeClassName` | Runtime class name for the controller pods , such as `gvisor` or `kata`. An unspecified `nil` or empty `""` RuntimeClassName is equivalent to the backwards-compatible default behavior as if the RuntimeClass feature is disabled. | "" | +| `healthProbeBindAddr` | The address the health probe endpoint binds to | `""` | \ No newline at end of file diff --git a/helm/aws-load-balancer-controller/templates/deployment.yaml b/helm/aws-load-balancer-controller/templates/deployment.yaml index 130ca61583..b75fe14357 100644 --- a/helm/aws-load-balancer-controller/templates/deployment.yaml +++ b/helm/aws-load-balancer-controller/templates/deployment.yaml @@ -178,6 +178,9 @@ spec: {{- if .Values.vpcTags }} - --aws-vpc-tags={{ include "aws-load-balancer-controller.convertMapToCsv" .Values.vpcTags | trimSuffix "," }} {{- end }} + {{- if .Values.healthProbeBindAddr }} + - --health-probe-bind-addr={{ .Values.healthProbeBindAddr }} + {{- end }} {{- if or .Values.env .Values.envSecretName }} env: {{- if .Values.env}} @@ -225,14 +228,26 @@ spec: protocol: TCP resources: {{- toYaml .Values.resources | nindent 10 }} - {{- with .Values.livenessProbe }} livenessProbe: - {{- toYaml . | nindent 10 }} - {{- end }} - {{- with .Values.readinessProbe }} + httpGet: + path: /healthz + port: {{ (split ":" .Values.healthProbeBindAddr)._1 | default 61779 }} + scheme: HTTP + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds | default 30 }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds | default 10 }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds | default 10 }} + successThreshold: {{ .Values.livenessProbe.successThreshold | default 1 }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold | default 2 }} readinessProbe: - {{- toYaml . | nindent 10 }} - {{- end }} + httpGet: + path: /readyz + port: {{ (split ":" .Values.healthProbeBindAddr)._1 | default 61779 }} + scheme: HTTP + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds | default 10 }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds | default 10 }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds | default 10 }} + successThreshold: {{ .Values.readinessProbe.successThreshold | default 1 }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold | default 2 }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} {{- with .Values.nodeSelector }} nodeSelector: diff --git a/helm/aws-load-balancer-controller/test.yaml b/helm/aws-load-balancer-controller/test.yaml index be074e92db..0acb577b18 100644 --- a/helm/aws-load-balancer-controller/test.yaml +++ b/helm/aws-load-balancer-controller/test.yaml @@ -161,6 +161,9 @@ enableWaf: # Enable WAF V2 addon for ALB (default true) enableWafv2: +# The address the health probe endpoint binds to. (default ":61779") +healthProbeBindAddr: "" + # Maximum number of concurrently running reconcile loops for ingress (default 3) ingressMaxConcurrentReconciles: @@ -220,14 +223,10 @@ tolerateNonExistentBackendAction: defaultSSLPolicy: # Liveness probe configuration for the controller -livenessProbe: - failureThreshold: 2 - httpGet: - path: /healthz - port: 61779 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 10 +livenessProbe: {} + +# readiness probe configuration for the controller +readinessProbe: {} # Environment variables to set for aws-load-balancer-controller pod. # We strongly discourage programming access credentials in the controller environment. You should setup IRSA or diff --git a/helm/aws-load-balancer-controller/values.yaml b/helm/aws-load-balancer-controller/values.yaml index 14261c8cb8..08c5395f19 100644 --- a/helm/aws-load-balancer-controller/values.yaml +++ b/helm/aws-load-balancer-controller/values.yaml @@ -201,6 +201,9 @@ enableWaf: # Enable WAF V2 addon for ALB (default true) enableWafv2: +# The address the health probe endpoint binds to. (default ":61779") +healthProbeBindAddr: "" + # Maximum number of concurrently running reconcile loops for ingress (default 3) ingressMaxConcurrentReconciles: @@ -267,25 +270,20 @@ tolerateNonExistentBackendAction: defaultSSLPolicy: # Liveness probe configuration for the controller -livenessProbe: - failureThreshold: 2 - httpGet: - path: /healthz - port: 61779 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 10 +livenessProbe: {} + # failureThreshold: 3 + # initialDelaySeconds: 30 + # periodSeconds: 10 + # successThreshold: 1 + # timeoutSeconds: 1 # readiness probe configuration for the controller -readinessProbe: - failureThreshold: 2 - httpGet: - path: /readyz - port: 61779 - scheme: HTTP - successThreshold: 1 - initialDelaySeconds: 10 - timeoutSeconds: 10 +readinessProbe: {} + # failureThreshold: 3 + # initialDelaySeconds: 10 + # periodSeconds: 10 + # successThreshold: 1 + # timeoutSeconds: 1 # Environment variables to set for aws-load-balancer-controller pod. # We strongly discourage programming access credentials in the controller environment. You should setup IRSA or From 9668a4b8ab42f536198b952763c4192912fb819e Mon Sep 17 00:00:00 2001 From: niclask25 Date: Thu, 10 Apr 2025 11:54:20 +0200 Subject: [PATCH 02/40] feat: add healthProbeBindAddr --- helm/aws-load-balancer-controller/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/aws-load-balancer-controller/README.md b/helm/aws-load-balancer-controller/README.md index 779d283b86..8b9aa07b65 100644 --- a/helm/aws-load-balancer-controller/README.md +++ b/helm/aws-load-balancer-controller/README.md @@ -245,7 +245,7 @@ The default values set by the application itself can be confirmed [here](https:/ | `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | | `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | | `livenessProbe` | Liveness probe settings for the controller | `{}` (see `values.yaml`) | -| `readinessProbe` | Readiness probe settings for the controller | `{}` (see `values.yaml`) | (see `values.yaml`) | +| `readinessProbe` | Readiness probe settings for the controller | `{}` (see `values.yaml`) | | `env` | Environment variables to set for aws-load-balancer-controller pod | None | | `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | | `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | From bf9e1f74938c1902ea5b0df8ca27e2be3421319f Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 27 Mar 2025 16:48:11 -0700 Subject: [PATCH 03/40] initial reconcilers for gateway --- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 270 +++++++++++ ...way.k8s.aws_targetgroupconfigurations.yaml | 456 ++++++++++++++++++ config/rbac/role.yaml | 66 +++ controllers/gateway/constants.go | 46 ++ controllers/gateway/gateway_controller.go | 290 +++++++++++ main.go | 18 + pkg/config/controller_config.go | 6 + pkg/config/feature_gates.go | 4 +- pkg/gateway/model/model_builder.go | 81 ++++ .../routeutils/listener_attachment_helper.go | 7 + pkg/gateway/routeutils/loader.go | 8 + pkg/gateway/routeutils/namespace_selector.go | 6 + .../routeutils/route_attachment_helper.go | 4 + .../routeutils/route_listener_mapper.go | 8 + 14 files changed, 1269 insertions(+), 1 deletion(-) create mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml create mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml create mode 100644 controllers/gateway/constants.go create mode 100644 controllers/gateway/gateway_controller.go create mode 100644 pkg/gateway/model/model_builder.go diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml new file mode 100644 index 0000000000..01dc98c189 --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -0,0 +1,270 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: loadbalancerconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: LoadBalancerConfiguration + listKind: LoadBalancerConfigurationList + plural: loadbalancerconfigurations + singular: loadbalancerconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LoadBalancerConfigurationSpec defines the desired state of + LoadBalancerConfiguration + properties: + customerOwnedIpv4Pool: + description: customerOwnedIpv4Pool is the ID of the customer-owned + address for Application Load Balancers on Outposts pool. + type: string + enablePrefixForIpv6SourceNat: + description: enablePrefixForIpv6SourceNat indicates whether to use + an IPv6 prefix from each subnet for source NAT for Network Load + Balancers with UDP listeners. + enum: + - "on" + - "off" + type: string + enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: + description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic + Indicates whether to evaluate inbound security group rules for traffic + sent to a Network Load Balancer through Amazon Web Services PrivateLink. + type: string + ipAddressType: + description: loadBalancerIPType defines what kind of load balancer + to provision (ipv4, dual stack) + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string + listenerConfigurations: + description: listenerConfigurations is an optional list of configurations + for each listener on LB + items: + properties: + alpnPolicy: + description: alpnPolicy an optional string that allows you to + configure ALPN policies on your Load Balancer + enum: + - HTTP1Only + - HTTP2Only + - HTTP2Optional + - HTTP2Preferred + - None + type: string + certificates: + description: certificates is the list of other certificates + to add to the listener. + items: + type: string + type: array + defaultCertificate: + description: |- + TODO: Add validation in admission webhook to make it required for secure protocols + defaultCertificate the cert arn to be used by default. + type: string + listenerAttributes: + description: listenerAttributes defines the attributes for the + listener + items: + description: ListenerAttribute defines listener attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + mutualAuthentication: + description: mutualAuthentication defines the mutual authentication + configuration information. + properties: + advertiseTrustStoreCaNames: + description: Indicates whether trust store CA certificate + names are advertised. + enum: + - "on" + - "off" + type: string + ignoreClientCertificateExpiry: + description: Indicates whether expired client certificates + are ignored. + type: boolean + mode: + description: The client certificate handling method. Options + are off , passthrough or verify + enum: + - "off" + - passthrough + - verify + type: string + trustStore: + description: The Name or ARN of the trust store. + type: string + required: + - mode + type: object + protocolPort: + description: protocolPort is identifier for the listener on + load balancer. It should be of the form PROTOCOL:PORT + pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ + type: string + sslPolicy: + description: sslPolicy is the security policy that defines which + protocols and ciphers are supported for secure listeners [HTTPS + or TLS listener]. + type: string + required: + - protocolPort + type: object + type: array + loadBalancerAttributes: + description: LoadBalancerAttributes defines the attribute of LB + items: + description: LoadBalancerAttribute defines LB attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + loadBalancerName: + description: loadBalancerName defines the name of the LB to provision. + If unspecified, it will be automatically generated. + maxLength: 32 + minLength: 1 + type: string + loadBalancerSubnets: + description: loadBalancerSubnets is an optional list of subnet configurations + to be used in the LB + items: + description: SubnetConfiguration defines the subnet settings for + a Load Balancer. + properties: + eipAllocation: + description: eipAllocation the EIP name for this subnet. + type: string + identifier: + description: identifier name or id for the subnet + type: string + privateIPv4Allocation: + description: privateIPv4Allocation the private ipv4 address + to assign to this subnet. + type: string + privateIPv6Allocation: + description: privateIPv6Allocation the private ipv6 address + to assign to this subnet. + type: string + required: + - identifier + type: object + type: array + scheme: + description: scheme defines the type of LB to provision. If unspecified, + it will be automatically inferred. + enum: + - internal + - internet-facing + type: string + securityGroupPrefixes: + description: securityGroupPrefixes an optional list of prefixes that + are allowed to access the LB. + items: + type: string + type: array + securityGroups: + description: securityGroups an optional list of security group ids + or names to apply to the LB + items: + type: string + type: array + sourceRanges: + description: sourceRanges an optional list of CIDRs that are allowed + to access the LB. + items: + type: string + type: array + tags: + description: Tags defines list of Tags on LB. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + vpcId: + description: vpcId is the ID of the VPC for the load balancer. + type: string + type: object + status: + description: LoadBalancerConfigurationStatus defines the observed state + of TargetGroupBinding + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml new file mode 100644 index 0000000000..c34856f29b --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -0,0 +1,456 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: targetgroupconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: TargetGroupConfiguration + listKind: TargetGroupConfigurationList + plural: targetgroupconfigurations + singular: targetgroupconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The Kubernetes Service's name + jsonPath: .spec.targetReference.name + name: SERVICE-NAME + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TargetGroupConfiguration is the Schema for defining TargetGroups + with an AWS ELB Gateway + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TargetGroupConfigurationSpec defines the TargetGroup properties + for a route. + properties: + defaultConfiguration: + description: defaultRouteConfiguration fallback configuration applied + to all routes, unless overridden by route-specific configurations. + properties: + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount of + time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health checks + on the targets. + type: string + healthCheckPort: + description: healthCheckPort The port to use to connect with + the target. + format: int32 + type: integer + healthCheckProtocol: + description: healthCheckProtocol The protocol to use to connect + with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols + are not supported for health checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, in seconds, + during which no response means a failed health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering an unhealthy + target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes to use + when checking for a successful response from a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering the target + unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target group + is of type IPv4 or IPv6. If unspecified, it will be automatically + inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups to + only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute of target + group + items: + description: TargetGroupAttribute defines target group attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetType: + description: targetType is the TargetType of TargetGroup. If unspecified, + it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + routeConfigurations: + description: routeConfigurations the route configuration for specific + routes + items: + description: RouteConfiguration defines the per route configuration + properties: + name: + description: name the name of the route, it should be in the + form of ROUTE:NAMESPACE:NAME + pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ + type: string + targetGroupProps: + description: targetGroupProps the target group specific properties + properties: + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount + of time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health + checks on the targets. + type: string + healthCheckPort: + description: healthCheckPort The port to use to connect + with the target. + format: int32 + type: integer + healthCheckProtocol: + description: healthCheckProtocol The protocol to use + to connect with the target. The GENEVE, TLS, UDP, + and TCP_UDP protocols are not supported for health + checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, + in seconds, during which no response means a failed + health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering + an unhealthy target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes + to use when checking for a successful response from + a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering + the target unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target + group is of type IPv4 or IPv6. If unspecified, it will + be automatically inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups + to only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute + of target group + items: + description: TargetGroupAttribute defines target group + attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetType: + description: targetType is the TargetType of TargetGroup. + If unspecified, it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + required: + - name + - targetGroupProps + type: object + type: array + targetReference: + description: targetReference the kubernetes object to attach the Target + Group settings to. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + + Defaults to "Service" when not specified. + type: string + name: + description: Name is the name of the referent. + type: string + required: + - name + type: object + required: + - targetReference + type: object + status: + description: TargetGroupConfigurationStatus defines the observed state + of TargetGroupConfiguration + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d21f142f44..9893463391 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -128,6 +128,72 @@ rules: verbs: - patch - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways/finalizers + verbs: + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes/finalizers + verbs: + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes/finalizers + verbs: + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes/status + verbs: + - get + - patch + - update - apiGroups: - networking.k8s.io resources: diff --git a/controllers/gateway/constants.go b/controllers/gateway/constants.go new file mode 100644 index 0000000000..852d7cd0a4 --- /dev/null +++ b/controllers/gateway/constants.go @@ -0,0 +1,46 @@ +package gateway + +/* +Common constants +*/ + +const ( + // GatewayResourceGroupVersion the groupVersion used by Gateway & GatewayClass resources. + GatewayResourceGroupVersion = "gateway.networking.k8s.io/v1" +) + +/* +NLB constants +*/ + +const ( + // nlbGatewayController gateway controller name for NLB + nlbGatewayController = "gateway.k8s.aws/nlb" + + // nlbGatewayTagPrefix the tag applied to all resources created by the NLB Gateway controller. + nlbGatewayTagPrefix = "gateway.k8s.aws.nlb" + + // nlbRouteResourceGroupVersion the groupVersion used by TCPRoute and UDPRoute + nlbRouteResourceGroupVersion = "gateway.networking.k8s.io/v1alpha2" + + // nlbGatewayFinalizer the finalizer we attach the NLB Gateway object + nlbGatewayFinalizer = "gateway.k8s.aws/nlb-finalizer" +) + +/* +ALB Constants +*/ + +const ( + // albGatewayController gateway controller name for ALB + albGatewayController = "gateway.k8s.aws/alb" + + // albGatewayTagPrefix the tag applied to all resources created by the ALB Gateway controller. + albGatewayTagPrefix = "gateway.k8s.aws.nlb" + + // albRouteResourceGroupVersion the groupVersion used by HTTPRoute and GRPCRoute + albRouteResourceGroupVersion = "gateway.networking.k8s.io/v1" + + // albGatewayFinalizer the finalizer we attach the ALB Gateway object + albGatewayFinalizer = "gateway.k8s.aws/alb-finalizer" +) diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go new file mode 100644 index 0000000000..5ac347f4f7 --- /dev/null +++ b/controllers/gateway/gateway_controller.go @@ -0,0 +1,290 @@ +package gateway + +import ( + "context" + "fmt" + "github.com/go-logr/logr" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy" + elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" + gatewaymodel "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/model" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + lbcmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/lbc" + metricsutil "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/util" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +type Reconciler interface { + Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) + SetupWithManager(mgr ctrl.Manager) error +} + +var _ Reconciler = &gatewayReconciler{} + +// NewNLBGatewayReconciler constructs a gateway reconciler to handle specifically for NLB gateways +func NewNLBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { + return newGatewayReconciler(nlbGatewayController, controllerConfig.NLBGatewayMaxConcurrentReconciles, nlbGatewayTagPrefix, nlbGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters) +} + +// NewALBGatewayReconciler constructs a gateway reconciler to handle specifically for ALB gateways +func NewALBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { + return newGatewayReconciler(albGatewayController, controllerConfig.ALBGatewayMaxConcurrentReconciles, albGatewayTagPrefix, albGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters) +} + +// newGatewayReconciler constructs a reconciler that responds to gateway object changes +func newGatewayReconciler(controllerName string, maxConcurrentReconciles int, gatewayTagPrefix string, finalizer string, routeLoader routeutils.Loader, routeFilter routeutils.LoadRouteFilter, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { + + trackingProvider := tracking.NewDefaultProvider(gatewayTagPrefix, controllerConfig.ClusterName) + modelBuilder := gatewaymodel.NewDefaultModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider, elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger) + + stackMarshaller := deploy.NewDefaultStackMarshaller() + stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName) + + return &gatewayReconciler{ + controllerName: controllerName, + maxConcurrentReconciles: maxConcurrentReconciles, + finalizer: finalizer, + gatewayLoader: routeLoader, + routeFilter: routeFilter, + k8sClient: k8sClient, + modelBuilder: modelBuilder, + backendSGProvider: backendSGProvider, + stackMarshaller: stackMarshaller, + stackDeployer: stackDeployer, + finalizerManager: finalizerManager, + eventRecorder: eventRecorder, + logger: logger, + metricsCollector: metricsCollector, + reconcileCounters: reconcileCounters, + } +} + +// gatewayReconciler reconciles a Gateway. +type gatewayReconciler struct { + controllerName string + finalizer string + maxConcurrentReconciles int + gatewayLoader routeutils.Loader + routeFilter routeutils.LoadRouteFilter + k8sClient client.Client + modelBuilder gatewaymodel.Builder + backendSGProvider networking.BackendSGProvider + stackMarshaller deploy.StackMarshaller + stackDeployer deploy.StackDeployer + finalizerManager k8s.FinalizerManager + eventRecorder record.EventRecorder + logger logr.Logger + + metricsCollector lbcmetrics.MetricCollector + reconcileCounters *metricsutil.ReconcileCounters +} + +// TODO - Add Gateway and TG configuration permissions + +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/finalizers,verbs=update + +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=udproutes,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=udproutes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=udproutes/finalizers,verbs=update + +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes/finalizers,verbs=update + +func (r *gatewayReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { + err := r.reconcileHelper(ctx, req) + if err != nil { + r.logger.Error(err, "Got this error!") + } + return runtime.HandleReconcileError(err, r.logger) +} + +func (r *gatewayReconciler) reconcileHelper(ctx context.Context, req reconcile.Request) error { + + gw := &gwv1.Gateway{} + if err := r.k8sClient.Get(ctx, req.NamespacedName, gw); err != nil { + return client.IgnoreNotFound(err) + } + + r.logger.Info("Got request for reconcile", "gw", *gw) + + gwClass := &gwv1.GatewayClass{} + + // Gateway Class is a cluster scoped resource, but the k8s client only accepts namespaced names. + gwClassNamespacedName := types.NamespacedName{ + Name: string(gw.Spec.GatewayClassName), + } + + if err := r.k8sClient.Get(ctx, gwClassNamespacedName, gwClass); err != nil { + r.logger.Info("Failed to get GatewayClass", "error", err, "gw-class", gwClassNamespacedName.Name) + return client.IgnoreNotFound(err) + } + + if string(gwClass.Spec.ControllerName) != r.controllerName { + // ignore this gateway event as the gateway belongs to a different controller. + return nil + } + + allRoutes, err := r.gatewayLoader.LoadRoutesForGateway(ctx, *gw, r.routeFilter) + + if err != nil { + return err + } + + stack, lb, backendSGRequired, err := r.buildModel(ctx, gw, gwClass, allRoutes) + + if err != nil { + return err + } + + if lb == nil { + err = r.reconcileDelete(ctx, gw, allRoutes) + if err != nil { + r.logger.Error(err, "Failed to process gateway delete") + } + return err + } + + return r.reconcileUpdate(ctx, gw, stack, lb, backendSGRequired) +} + +func (r *gatewayReconciler) reconcileDelete(ctx context.Context, gw *gwv1.Gateway, routes map[int][]routeutils.RouteDescriptor) error { + for _, routeList := range routes { + if len(routeList) != 0 { + // TODO - Better error messaging (e.g. tell user the routes that are still attached) + return errors.New("Gateway still has routes attached") + } + } + + return r.finalizerManager.RemoveFinalizers(ctx, gw, r.finalizer) +} + +func (r *gatewayReconciler) reconcileUpdate(ctx context.Context, gw *gwv1.Gateway, stack core.Stack, + lb *elbv2model.LoadBalancer, backendSGRequired bool) error { + + if err := r.finalizerManager.AddFinalizers(ctx, gw, r.finalizer); err != nil { + r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err)) + return err + } + err := r.deployModel(ctx, gw, stack) + if err != nil { + return err + } + lbDNS, err := lb.DNSName().Resolve(ctx) + if err != nil { + return err + } + + if !backendSGRequired { + if err := r.backendSGProvider.Release(ctx, networking.ResourceTypeService, []types.NamespacedName{k8s.NamespacedName(gw)}); err != nil { + return err + } + } + + if err = r.updateGatewayStatus(ctx, lbDNS, gw); err != nil { + r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err)) + return err + } + r.eventRecorder.Event(gw, corev1.EventTypeNormal, k8s.ServiceEventReasonSuccessfullyReconciled, "Successfully reconciled") + return nil +} + +func (r *gatewayReconciler) deployModel(ctx context.Context, gw *gwv1.Gateway, stack core.Stack) error { + if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, r.controllerName); err != nil { + var requeueNeededAfter *runtime.RequeueNeededAfter + if errors.As(err, &requeueNeededAfter) { + return err + } + r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err)) + return err + } + r.logger.Info("successfully deployed model", "gateway", k8s.NamespacedName(gw)) + return nil +} + +func (r *gatewayReconciler) buildModel(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass, listenerToRoute map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) { + stack, lb, backendSGRequired, err := r.modelBuilder.Build(ctx, gw, gwClass, listenerToRoute) + if err != nil { + r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err)) + return nil, nil, false, err + } + stackJSON, err := r.stackMarshaller.Marshal(stack) + if err != nil { + r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err)) + return nil, nil, false, err + } + r.logger.Info("successfully built model", "model", stackJSON) + return stack, lb, backendSGRequired, nil +} + +func (r *gatewayReconciler) updateGatewayStatus(ctx context.Context, lbDNS string, gw *gwv1.Gateway) error { + // TODO Consider LB ARN. + + // Gateway Address Status + if len(gw.Status.Addresses) != 1 || + gw.Status.Addresses[0].Value != "" || + gw.Status.Addresses[0].Value != lbDNS { + gwOld := gw.DeepCopy() + ipAddressType := gwv1.HostnameAddressType + gw.Status.Addresses = []gwv1.GatewayStatusAddress{ + { + Type: &ipAddressType, + Value: lbDNS, + }, + } + if err := r.k8sClient.Status().Patch(ctx, gw, client.MergeFrom(gwOld)); err != nil { + return errors.Wrapf(err, "failed to update gw status: %v", k8s.NamespacedName(gw)) + } + } + + // TODO: Listener status ListenerStatus + // https://github.com/aws/aws-application-networking-k8s/blob/main/pkg/controllers/gateway_controller.go#L350 + + return nil +} + +func (r *gatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { + /* + gatewayClassHandler := eventhandlers.NewEnqueueRequestsForGatewayClassEvent(r.logger, r.k8sClient, r.config) + tcpRouteHandler := eventhandlers.NewEnqueueRequestsForTCPRouteEvent(r.logger, r.k8sClient, r.config) + udpRouteHandler := eventhandlers.NewEnqueueRequestsForUDPRouteEvent(r.logger, r.k8sClient, r.config) + return ctrl.NewControllerManagedBy(mgr). + Named("nlbgateway"). + // Anything that influences a gateway object must be added here. + For(&gwv1.Gateway{}). + Watches(&gwv1.GatewayClass{}, gatewayClassHandler). + Watches(&gwalpha2.TCPRoute{}, tcpRouteHandler). + Watches(&gwalpha2.UDPRoute{}, udpRouteHandler). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.config.MaxConcurrentReconciles, + }). + Complete(r) + */ + + return ctrl.NewControllerManagedBy(mgr). + Named(r.controllerName). + // Anything that influences a gateway object must be added here. + For(&gwv1.Gateway{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.maxConcurrentReconciles, + }). + Complete(r) +} diff --git a/main.go b/main.go index d2ba702db1..4ac85474de 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,8 @@ package main import ( "os" + "sigs.k8s.io/aws-load-balancer-controller/controllers/gateway" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" "k8s.io/client-go/util/workqueue" @@ -171,6 +173,22 @@ func main() { os.Exit(1) } + if controllerCFG.FeatureGates.Enabled(config.GatewayAPI) { + routeLoader := routeutils.NewLoader(mgr.GetClient()) + nlbGatewayReconciler := gateway.NewNLBGatewayReconciler(routeLoader, cloud, mgr.GetClient(), mgr.GetEventRecorderFor("nlbgateway"), controllerCFG, finalizerManager, sgReconciler, sgManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("nlbgateway"), lbcMetricsCollector, reconcileCounters) + nlbControllerError := nlbGatewayReconciler.SetupWithManager(mgr) + if nlbControllerError != nil { + setupLog.Error(nlbControllerError, "Unable to create NLB Gateway controller") + os.Exit(1) + } + albGatewayReconciler := gateway.NewALBGatewayReconciler(routeLoader, cloud, mgr.GetClient(), mgr.GetEventRecorderFor("albgateway"), controllerCFG, finalizerManager, sgReconciler, sgManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("albgateway"), lbcMetricsCollector, reconcileCounters) + albControllerErr := albGatewayReconciler.SetupWithManager(mgr) + if albControllerErr != nil { + setupLog.Error(albControllerErr, "Unable to create ALB Gateway controller") + os.Exit(1) + } + } + // Add liveness probe err = mgr.AddHealthzCheck("health-ping", healthz.Ping) setupLog.Info("adding health check for controller") diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index c1d775b6f2..86f32eb83f 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -98,6 +98,12 @@ type ControllerConfig struct { // Max exponential backoff delay for reconcile failures of TargetGroupBinding TargetGroupBindingMaxExponentialBackoffDelay time.Duration + // ALBGatewayMaxConcurrentReconciles Max concurrent reconcile loops for ALB Gateway objects + ALBGatewayMaxConcurrentReconciles int + + // NLBGatewayMaxConcurrentReconciles Max concurrent reconcile loops for NLB Gateway objects + NLBGatewayMaxConcurrentReconciles int + // EnableBackendSecurityGroup specifies whether to use optimized security group rules EnableBackendSecurityGroup bool diff --git a/pkg/config/feature_gates.go b/pkg/config/feature_gates.go index 09576ae418..69dd61b133 100644 --- a/pkg/config/feature_gates.go +++ b/pkg/config/feature_gates.go @@ -22,8 +22,9 @@ const ( NLBHealthCheckAdvancedConfig Feature = "NLBHealthCheckAdvancedConfig" NLBSecurityGroup Feature = "NLBSecurityGroup" ALBSingleSubnet Feature = "ALBSingleSubnet" - SubnetDiscoveryByReachability Feature = "SubnetDiscoveryByReachability" LBCapacityReservation Feature = "LBCapacityReservation" + SubnetDiscoveryByReachability Feature = "SubnetDiscoveryByReachability" + GatewayAPI Feature = "ELBGatewayAPI" ) type FeatureGates interface { @@ -64,6 +65,7 @@ func NewFeatureGates() FeatureGates { ALBSingleSubnet: false, SubnetDiscoveryByReachability: true, LBCapacityReservation: true, + GatewayAPI: false, }, } } diff --git a/pkg/gateway/model/model_builder.go b/pkg/gateway/model/model_builder.go new file mode 100644 index 0000000000..66976d4b50 --- /dev/null +++ b/pkg/gateway/model/model_builder.go @@ -0,0 +1,81 @@ +package model + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// Builder builds the model stack for a Gateway resource. +type Builder interface { + // Build model stack for a gateway + Build(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass, routes map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) +} + +// NewDefaultModelBuilder construct a new defaultModelBuilder +func NewDefaultModelBuilder(subnetsResolver networking.SubnetsResolver, + vpcInfoProvider networking.VPCInfoProvider, vpcID string, trackingProvider tracking.Provider, + elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string, + externalManagedTags sets.Set[string], defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, + backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool, + disableRestrictedSGRules bool, logger logr.Logger) Builder { + return &defaultModelBuilder{ + subnetsResolver: subnetsResolver, + vpcInfoProvider: vpcInfoProvider, + backendSGProvider: backendSGProvider, + sgResolver: sgResolver, + trackingProvider: trackingProvider, + elbv2TaggingManager: elbv2TaggingManager, + featureGates: featureGates, + ec2Client: ec2Client, + enableBackendSG: enableBackendSG, + disableRestrictedSGRules: disableRestrictedSGRules, + + clusterName: clusterName, + vpcID: vpcID, + defaultTags: defaultTags, + externalManagedTags: externalManagedTags, + defaultSSLPolicy: defaultSSLPolicy, + defaultTargetType: elbv2model.TargetType(defaultTargetType), + defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), + logger: logger, + } +} + +var _ Builder = &defaultModelBuilder{} + +type defaultModelBuilder struct { + subnetsResolver networking.SubnetsResolver + vpcInfoProvider networking.VPCInfoProvider + backendSGProvider networking.BackendSGProvider + sgResolver networking.SecurityGroupResolver + trackingProvider tracking.Provider + elbv2TaggingManager elbv2deploy.TaggingManager + featureGates config.FeatureGates + ec2Client services.EC2 + enableBackendSG bool + disableRestrictedSGRules bool + + clusterName string + vpcID string + defaultTags map[string]string + externalManagedTags sets.Set[string] + defaultSSLPolicy string + defaultTargetType elbv2model.TargetType + defaultLoadBalancerScheme elbv2model.LoadBalancerScheme + logger logr.Logger +} + +func (d *defaultModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass, routes map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) { + //TODO implement me + panic("implement me") +} diff --git a/pkg/gateway/routeutils/listener_attachment_helper.go b/pkg/gateway/routeutils/listener_attachment_helper.go index 8d1baea813..1653e2e59a 100644 --- a/pkg/gateway/routeutils/listener_attachment_helper.go +++ b/pkg/gateway/routeutils/listener_attachment_helper.go @@ -3,6 +3,7 @@ package routeutils import ( "context" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -19,6 +20,12 @@ type listenerAttachmentHelperImpl struct { namespaceSelector namespaceSelector } +func newListenerAttachmentHelper(k8sClient client.Client) listenerAttachmentHelper { + return &listenerAttachmentHelperImpl{ + namespaceSelector: newNamespaceSelector(k8sClient), + } +} + // listenerAllowsAttachment utility method to determine if a listener will allow a route to connect using // Gateway API rules to determine compatibility between lister and route. func (attachmentHelper *listenerAttachmentHelperImpl) listenerAllowsAttachment(ctx context.Context, gw gwv1.Gateway, listener gwv1.Listener, route preLoadRouteDescriptor) (bool, error) { diff --git a/pkg/gateway/routeutils/loader.go b/pkg/gateway/routeutils/loader.go index 41ffbf1f9f..1f5f64b48a 100644 --- a/pkg/gateway/routeutils/loader.go +++ b/pkg/gateway/routeutils/loader.go @@ -57,6 +57,14 @@ type loaderImpl struct { allRouteLoaders map[string]func(context context.Context, client client.Client) ([]preLoadRouteDescriptor, error) } +func NewLoader(k8sClient client.Client) Loader { + return &loaderImpl{ + mapper: newListenerToRouteMapper(k8sClient), + k8sClient: k8sClient, + allRouteLoaders: allRoutes, + } +} + // LoadRoutesForGateway loads all relevant data for a single Gateway. func (l *loaderImpl) LoadRoutesForGateway(ctx context.Context, gw gwv1.Gateway, filter LoadRouteFilter) (map[int][]RouteDescriptor, error) { // 1. Load all relevant routes according to the filter diff --git a/pkg/gateway/routeutils/namespace_selector.go b/pkg/gateway/routeutils/namespace_selector.go index 0933d02a73..c54f1aeacb 100644 --- a/pkg/gateway/routeutils/namespace_selector.go +++ b/pkg/gateway/routeutils/namespace_selector.go @@ -21,6 +21,12 @@ type namespaceSelectorImpl struct { k8sClient client.Client } +func newNamespaceSelector(k8sClient client.Client) namespaceSelector { + return &namespaceSelectorImpl{ + k8sClient: k8sClient, + } +} + // getNamespacesFromSelector queries the Kubernetes API for all namespaces that match a selector. func (n *namespaceSelectorImpl) getNamespacesFromSelector(context context.Context, selector *metav1.LabelSelector) (sets.Set[string], error) { namespaceList := v1.NamespaceList{} diff --git a/pkg/gateway/routeutils/route_attachment_helper.go b/pkg/gateway/routeutils/route_attachment_helper.go index f1a18f8ba3..7644f45ffd 100644 --- a/pkg/gateway/routeutils/route_attachment_helper.go +++ b/pkg/gateway/routeutils/route_attachment_helper.go @@ -15,6 +15,10 @@ var _ routeAttachmentHelper = &routeAttachmentHelperImpl{} type routeAttachmentHelperImpl struct { } +func newRouteAttachmentHelper() routeAttachmentHelper { + return &routeAttachmentHelperImpl{} +} + // doesRouteAttachToGateway is responsible for determining if a route and gateway should be connected. // This function implements the Gateway API spec for determining Gateway -> Route attachment. func (rah *routeAttachmentHelperImpl) doesRouteAttachToGateway(gw gwv1.Gateway, route preLoadRouteDescriptor) bool { diff --git a/pkg/gateway/routeutils/route_listener_mapper.go b/pkg/gateway/routeutils/route_listener_mapper.go index 51afc1eb67..6e9d15de41 100644 --- a/pkg/gateway/routeutils/route_listener_mapper.go +++ b/pkg/gateway/routeutils/route_listener_mapper.go @@ -2,6 +2,7 @@ package routeutils import ( "context" + "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -18,6 +19,13 @@ type listenerToRouteMapperImpl struct { routeAttachmentHelper routeAttachmentHelper } +func newListenerToRouteMapper(k8sClient client.Client) listenerToRouteMapper { + return &listenerToRouteMapperImpl{ + listenerAttachmentHelper: newListenerAttachmentHelper(k8sClient), + routeAttachmentHelper: newRouteAttachmentHelper(), + } +} + // mapGatewayAndRoutes will map route to the corresponding listener ports using the Gateway API spec rules. func (ltr *listenerToRouteMapperImpl) mapGatewayAndRoutes(ctx context.Context, gw gwv1.Gateway, routes []preLoadRouteDescriptor) (map[int][]preLoadRouteDescriptor, error) { result := make(map[int][]preLoadRouteDescriptor) From cb4d00748ce12dfe74f9b4f21830f9dac569c229 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 31 Mar 2025 11:04:43 -0700 Subject: [PATCH 04/40] add reference grant and targetgroup config look up --- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 270 --------- ...way.k8s.aws_targetgroupconfigurations.yaml | 456 --------------- controllers/gateway/constants.go | 32 +- controllers/gateway/gateway_controller.go | 13 +- main.go | 11 +- pkg/config/feature_gates.go | 6 +- pkg/gateway/routeutils/backend.go | 135 ++++- pkg/gateway/routeutils/backend_test.go | 547 +++++++++++++++++- pkg/gateway/routeutils/test_utils.go | 5 + pkg/metrics/util/reconcile_counter.go | 25 +- 10 files changed, 706 insertions(+), 794 deletions(-) delete mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml delete mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml deleted file mode 100644 index 01dc98c189..0000000000 --- a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ /dev/null @@ -1,270 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: loadbalancerconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: LoadBalancerConfiguration - listKind: LoadBalancerConfigurationList - plural: loadbalancerconfigurations - singular: loadbalancerconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LoadBalancerConfigurationSpec defines the desired state of - LoadBalancerConfiguration - properties: - customerOwnedIpv4Pool: - description: customerOwnedIpv4Pool is the ID of the customer-owned - address for Application Load Balancers on Outposts pool. - type: string - enablePrefixForIpv6SourceNat: - description: enablePrefixForIpv6SourceNat indicates whether to use - an IPv6 prefix from each subnet for source NAT for Network Load - Balancers with UDP listeners. - enum: - - "on" - - "off" - type: string - enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: - description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic - Indicates whether to evaluate inbound security group rules for traffic - sent to a Network Load Balancer through Amazon Web Services PrivateLink. - type: string - ipAddressType: - description: loadBalancerIPType defines what kind of load balancer - to provision (ipv4, dual stack) - enum: - - ipv4 - - dualstack - - dualstack-without-public-ipv4 - type: string - listenerConfigurations: - description: listenerConfigurations is an optional list of configurations - for each listener on LB - items: - properties: - alpnPolicy: - description: alpnPolicy an optional string that allows you to - configure ALPN policies on your Load Balancer - enum: - - HTTP1Only - - HTTP2Only - - HTTP2Optional - - HTTP2Preferred - - None - type: string - certificates: - description: certificates is the list of other certificates - to add to the listener. - items: - type: string - type: array - defaultCertificate: - description: |- - TODO: Add validation in admission webhook to make it required for secure protocols - defaultCertificate the cert arn to be used by default. - type: string - listenerAttributes: - description: listenerAttributes defines the attributes for the - listener - items: - description: ListenerAttribute defines listener attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - mutualAuthentication: - description: mutualAuthentication defines the mutual authentication - configuration information. - properties: - advertiseTrustStoreCaNames: - description: Indicates whether trust store CA certificate - names are advertised. - enum: - - "on" - - "off" - type: string - ignoreClientCertificateExpiry: - description: Indicates whether expired client certificates - are ignored. - type: boolean - mode: - description: The client certificate handling method. Options - are off , passthrough or verify - enum: - - "off" - - passthrough - - verify - type: string - trustStore: - description: The Name or ARN of the trust store. - type: string - required: - - mode - type: object - protocolPort: - description: protocolPort is identifier for the listener on - load balancer. It should be of the form PROTOCOL:PORT - pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ - type: string - sslPolicy: - description: sslPolicy is the security policy that defines which - protocols and ciphers are supported for secure listeners [HTTPS - or TLS listener]. - type: string - required: - - protocolPort - type: object - type: array - loadBalancerAttributes: - description: LoadBalancerAttributes defines the attribute of LB - items: - description: LoadBalancerAttribute defines LB attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - loadBalancerName: - description: loadBalancerName defines the name of the LB to provision. - If unspecified, it will be automatically generated. - maxLength: 32 - minLength: 1 - type: string - loadBalancerSubnets: - description: loadBalancerSubnets is an optional list of subnet configurations - to be used in the LB - items: - description: SubnetConfiguration defines the subnet settings for - a Load Balancer. - properties: - eipAllocation: - description: eipAllocation the EIP name for this subnet. - type: string - identifier: - description: identifier name or id for the subnet - type: string - privateIPv4Allocation: - description: privateIPv4Allocation the private ipv4 address - to assign to this subnet. - type: string - privateIPv6Allocation: - description: privateIPv6Allocation the private ipv6 address - to assign to this subnet. - type: string - required: - - identifier - type: object - type: array - scheme: - description: scheme defines the type of LB to provision. If unspecified, - it will be automatically inferred. - enum: - - internal - - internet-facing - type: string - securityGroupPrefixes: - description: securityGroupPrefixes an optional list of prefixes that - are allowed to access the LB. - items: - type: string - type: array - securityGroups: - description: securityGroups an optional list of security group ids - or names to apply to the LB - items: - type: string - type: array - sourceRanges: - description: sourceRanges an optional list of CIDRs that are allowed - to access the LB. - items: - type: string - type: array - tags: - description: Tags defines list of Tags on LB. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - vpcId: - description: vpcId is the ID of the VPC for the load balancer. - type: string - type: object - status: - description: LoadBalancerConfigurationStatus defines the observed state - of TargetGroupBinding - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml deleted file mode 100644 index c34856f29b..0000000000 --- a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml +++ /dev/null @@ -1,456 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: targetgroupconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: TargetGroupConfiguration - listKind: TargetGroupConfigurationList - plural: targetgroupconfigurations - singular: targetgroupconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: The Kubernetes Service's name - jsonPath: .spec.targetReference.name - name: SERVICE-NAME - type: string - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: TargetGroupConfiguration is the Schema for defining TargetGroups - with an AWS ELB Gateway - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TargetGroupConfigurationSpec defines the TargetGroup properties - for a route. - properties: - defaultConfiguration: - description: defaultRouteConfiguration fallback configuration applied - to all routes, unless overridden by route-specific configurations. - properties: - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount of - time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health checks - on the targets. - type: string - healthCheckPort: - description: healthCheckPort The port to use to connect with - the target. - format: int32 - type: integer - healthCheckProtocol: - description: healthCheckProtocol The protocol to use to connect - with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols - are not supported for health checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, in seconds, - during which no response means a failed health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering an unhealthy - target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes to use - when checking for a successful response from a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering the target - unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target group - is of type IPv4 or IPv6. If unspecified, it will be automatically - inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups to - only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute of target - group - items: - description: TargetGroupAttribute defines target group attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetType: - description: targetType is the TargetType of TargetGroup. If unspecified, - it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - routeConfigurations: - description: routeConfigurations the route configuration for specific - routes - items: - description: RouteConfiguration defines the per route configuration - properties: - name: - description: name the name of the route, it should be in the - form of ROUTE:NAMESPACE:NAME - pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ - type: string - targetGroupProps: - description: targetGroupProps the target group specific properties - properties: - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount - of time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health - checks on the targets. - type: string - healthCheckPort: - description: healthCheckPort The port to use to connect - with the target. - format: int32 - type: integer - healthCheckProtocol: - description: healthCheckProtocol The protocol to use - to connect with the target. The GENEVE, TLS, UDP, - and TCP_UDP protocols are not supported for health - checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, - in seconds, during which no response means a failed - health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering - an unhealthy target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes - to use when checking for a successful response from - a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering - the target unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target - group is of type IPv4 or IPv6. If unspecified, it will - be automatically inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups - to only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute - of target group - items: - description: TargetGroupAttribute defines target group - attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetType: - description: targetType is the TargetType of TargetGroup. - If unspecified, it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - required: - - name - - targetGroupProps - type: object - type: array - targetReference: - description: targetReference the kubernetes object to attach the Target - Group settings to. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - - Defaults to "Service" when not specified. - type: string - name: - description: Name is the name of the referent. - type: string - required: - - name - type: object - required: - - targetReference - type: object - status: - description: TargetGroupConfigurationStatus defines the observed state - of TargetGroupConfiguration - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/controllers/gateway/constants.go b/controllers/gateway/constants.go index 852d7cd0a4..a0c3d5c46e 100644 --- a/controllers/gateway/constants.go +++ b/controllers/gateway/constants.go @@ -14,17 +14,17 @@ NLB constants */ const ( - // nlbGatewayController gateway controller name for NLB - nlbGatewayController = "gateway.k8s.aws/nlb" + // NLBGatewayController gateway controller name for NLB + NLBGatewayController = "gateway.k8s.aws/nlb" - // nlbGatewayTagPrefix the tag applied to all resources created by the NLB Gateway controller. - nlbGatewayTagPrefix = "gateway.k8s.aws.nlb" + // NLBGatewayTagPrefix the tag applied to all resources created by the NLB Gateway controller. + NLBGatewayTagPrefix = "gateway.k8s.aws.nlb" - // nlbRouteResourceGroupVersion the groupVersion used by TCPRoute and UDPRoute - nlbRouteResourceGroupVersion = "gateway.networking.k8s.io/v1alpha2" + // NLBRouteResourceGroupVersion the groupVersion used by TCPRoute and UDPRoute + NLBRouteResourceGroupVersion = "gateway.networking.k8s.io/v1alpha2" - // nlbGatewayFinalizer the finalizer we attach the NLB Gateway object - nlbGatewayFinalizer = "gateway.k8s.aws/nlb-finalizer" + // NLBGatewayFinalizer the finalizer we attach the NLB Gateway object + NLBGatewayFinalizer = "gateway.k8s.aws/nlb-finalizer" ) /* @@ -32,15 +32,15 @@ ALB Constants */ const ( - // albGatewayController gateway controller name for ALB - albGatewayController = "gateway.k8s.aws/alb" + // ALBGatewayController gateway controller name for ALB + ALBGatewayController = "gateway.k8s.aws/alb" - // albGatewayTagPrefix the tag applied to all resources created by the ALB Gateway controller. - albGatewayTagPrefix = "gateway.k8s.aws.nlb" + // ALBGatewayTagPrefix the tag applied to all resources created by the ALB Gateway controller. + ALBGatewayTagPrefix = "gateway.k8s.aws.alb" - // albRouteResourceGroupVersion the groupVersion used by HTTPRoute and GRPCRoute - albRouteResourceGroupVersion = "gateway.networking.k8s.io/v1" + // ALBRouteResourceGroupVersion the groupVersion used by HTTPRoute and GRPCRoute + ALBRouteResourceGroupVersion = "gateway.networking.k8s.io/v1" - // albGatewayFinalizer the finalizer we attach the ALB Gateway object - albGatewayFinalizer = "gateway.k8s.aws/alb-finalizer" + // ALBGatewayFinalizer the finalizer we attach the ALB Gateway object + ALBGatewayFinalizer = "gateway.k8s.aws/alb-finalizer" ) diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 5ac347f4f7..9349f86be0 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -39,16 +39,16 @@ var _ Reconciler = &gatewayReconciler{} // NewNLBGatewayReconciler constructs a gateway reconciler to handle specifically for NLB gateways func NewNLBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { - return newGatewayReconciler(nlbGatewayController, controllerConfig.NLBGatewayMaxConcurrentReconciles, nlbGatewayTagPrefix, nlbGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters) + return newGatewayReconciler(NLBGatewayController, controllerConfig.NLBGatewayMaxConcurrentReconciles, NLBGatewayTagPrefix, NLBGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementNLBGateway) } // NewALBGatewayReconciler constructs a gateway reconciler to handle specifically for ALB gateways func NewALBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { - return newGatewayReconciler(albGatewayController, controllerConfig.ALBGatewayMaxConcurrentReconciles, albGatewayTagPrefix, albGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters) + return newGatewayReconciler(ALBGatewayController, controllerConfig.ALBGatewayMaxConcurrentReconciles, ALBGatewayTagPrefix, ALBGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementALBGateway) } // newGatewayReconciler constructs a reconciler that responds to gateway object changes -func newGatewayReconciler(controllerName string, maxConcurrentReconciles int, gatewayTagPrefix string, finalizer string, routeLoader routeutils.Loader, routeFilter routeutils.LoadRouteFilter, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { +func newGatewayReconciler(controllerName string, maxConcurrentReconciles int, gatewayTagPrefix string, finalizer string, routeLoader routeutils.Loader, routeFilter routeutils.LoadRouteFilter, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileTracker func(namespaceName types.NamespacedName)) Reconciler { trackingProvider := tracking.NewDefaultProvider(gatewayTagPrefix, controllerConfig.ClusterName) modelBuilder := gatewaymodel.NewDefaultModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider, elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger) @@ -71,7 +71,7 @@ func newGatewayReconciler(controllerName string, maxConcurrentReconciles int, ga eventRecorder: eventRecorder, logger: logger, metricsCollector: metricsCollector, - reconcileCounters: reconcileCounters, + reconcileTracker: reconcileTracker, } } @@ -91,8 +91,8 @@ type gatewayReconciler struct { eventRecorder record.EventRecorder logger logr.Logger - metricsCollector lbcmetrics.MetricCollector - reconcileCounters *metricsutil.ReconcileCounters + metricsCollector lbcmetrics.MetricCollector + reconcileTracker func(namespaceName types.NamespacedName) } // TODO - Add Gateway and TG configuration permissions @@ -110,6 +110,7 @@ type gatewayReconciler struct { //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes/finalizers,verbs=update func (r *gatewayReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { + r.reconcileTracker(req.NamespacedName) err := r.reconcileHelper(ctx, req) if err != nil { r.logger.Error(err, "Got this error!") diff --git a/main.go b/main.go index 4ac85474de..0a06bdbaa1 100644 --- a/main.go +++ b/main.go @@ -173,14 +173,21 @@ func main() { os.Exit(1) } - if controllerCFG.FeatureGates.Enabled(config.GatewayAPI) { - routeLoader := routeutils.NewLoader(mgr.GetClient()) + var routeLoader routeutils.Loader + if controllerCFG.FeatureGates.Enabled(config.NLBGatewayAPI) { + routeLoader = routeutils.NewLoader(mgr.GetClient()) nlbGatewayReconciler := gateway.NewNLBGatewayReconciler(routeLoader, cloud, mgr.GetClient(), mgr.GetEventRecorderFor("nlbgateway"), controllerCFG, finalizerManager, sgReconciler, sgManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("nlbgateway"), lbcMetricsCollector, reconcileCounters) nlbControllerError := nlbGatewayReconciler.SetupWithManager(mgr) if nlbControllerError != nil { setupLog.Error(nlbControllerError, "Unable to create NLB Gateway controller") os.Exit(1) } + } + + if controllerCFG.FeatureGates.Enabled(config.ALBGatewayAPI) { + if routeLoader == nil { + routeLoader = routeutils.NewLoader(mgr.GetClient()) + } albGatewayReconciler := gateway.NewALBGatewayReconciler(routeLoader, cloud, mgr.GetClient(), mgr.GetEventRecorderFor("albgateway"), controllerCFG, finalizerManager, sgReconciler, sgManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("albgateway"), lbcMetricsCollector, reconcileCounters) albControllerErr := albGatewayReconciler.SetupWithManager(mgr) if albControllerErr != nil { diff --git a/pkg/config/feature_gates.go b/pkg/config/feature_gates.go index 69dd61b133..19b1d93a1c 100644 --- a/pkg/config/feature_gates.go +++ b/pkg/config/feature_gates.go @@ -24,7 +24,8 @@ const ( ALBSingleSubnet Feature = "ALBSingleSubnet" LBCapacityReservation Feature = "LBCapacityReservation" SubnetDiscoveryByReachability Feature = "SubnetDiscoveryByReachability" - GatewayAPI Feature = "ELBGatewayAPI" + NLBGatewayAPI Feature = "NLBGatewayAPI" + ALBGatewayAPI Feature = "ALBGatewayAPI" ) type FeatureGates interface { @@ -65,7 +66,8 @@ func NewFeatureGates() FeatureGates { ALBSingleSubnet: false, SubnetDiscoveryByReachability: true, LBCapacityReservation: true, - GatewayAPI: false, + NLBGatewayAPI: false, + ALBGatewayAPI: false, }, } } diff --git a/pkg/gateway/routeutils/backend.go b/pkg/gateway/routeutils/backend.go index 3db827191e..c34baa7aac 100644 --- a/pkg/gateway/routeutils/backend.go +++ b/pkg/gateway/routeutils/backend.go @@ -6,24 +6,26 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +const ( + serviceKind = "Service" ) // Backend an abstraction on the Gateway Backend, meant to hide the underlying backend type from consumers (unless they really want to see it :)) type Backend struct { - Service *corev1.Service - ServicePort *corev1.ServicePort - TypeSpecificBackend interface{} - Weight int - // Add TG config here // + Service *corev1.Service + ELBv2TargetGroupConfig *elbv2gw.TargetGroupConfiguration + ServicePort *corev1.ServicePort + TypeSpecificBackend interface{} + Weight int } -// TODOs: -// 1/ Add reference grant checking -// 2/ Add target group configuration resolution - -// NOTE: Currently routeKind is not used, however, we will need it to load TG specific configuration. // commonBackendLoader this function will load the services and target group configurations associated with this gateway backend. func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) { @@ -40,23 +42,37 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci return nil, errors.Errorf("Missing port in backend reference") } - var namespace string + var svcNamespace string if backendRef.Namespace == nil { - namespace = routeIdentifier.Namespace + svcNamespace = routeIdentifier.Namespace } else { - namespace = string(*backendRef.Namespace) + svcNamespace = string(*backendRef.Namespace) } - // TODO - Need to implement reference grant check here - - svcName := types.NamespacedName{ - Namespace: namespace, + svcIdentifier := types.NamespacedName{ + Namespace: svcNamespace, Name: string(backendRef.Name), } + + // Check for reference grant when performing crossname gateway -> route attachment + if svcNamespace != routeIdentifier.Namespace { + allowed, err := referenceGrantCheck(ctx, k8sClient, svcIdentifier, routeIdentifier, routeKind) + if err != nil { + return nil, errors.Wrapf(err, "Unable to perform reference grant check") + } + + // We should not give any hints about the existence of this resource, therefore, we return nil. + // That way, users can't infer if the route is missing because of a misconfigured service reference + // or the sentence grant is not allowing the connection. + if !allowed { + return nil, nil + } + } + svc := &corev1.Service{} - err := k8sClient.Get(ctx, svcName, svc) + err := k8sClient.Get(ctx, svcIdentifier, svc) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcName)) + return nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcIdentifier)) } var servicePort *corev1.ServicePort @@ -68,12 +84,16 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci } } + tgConfig, err := lookUpTargetGroupConfiguration(ctx, k8sClient, k8s.NamespacedName(svc)) + + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object")) + } + if servicePort == nil { return nil, errors.Errorf("Unable to find service port for port %d", *backendRef.Port) } - // TODO - Need to TG CRD look up here - // Weight specifies the proportion of requests forwarded to the referenced // backend. This is computed as weight/(sum of all weights in this // BackendRefs list). For non-zero values, there may be some epsilon from @@ -90,9 +110,74 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci weight = int(*backendRef.Weight) } return &Backend{ - Service: svc, - ServicePort: servicePort, - Weight: weight, - TypeSpecificBackend: typeSpecificBackend, + Service: svc, + ServicePort: servicePort, + Weight: weight, + TypeSpecificBackend: typeSpecificBackend, + ELBv2TargetGroupConfig: tgConfig, }, nil } + +// lookUpTargetGroupConfiguration given a service, lookup the target group configuration associated with the service. +// recall that target group configuration always lives within the same namespace as the service. +func lookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client, serviceMetadata types.NamespacedName) (*elbv2gw.TargetGroupConfiguration, error) { + tgConfigList := &elbv2gw.TargetGroupConfigurationList{} + + // TODO - Add index + if err := k8sClient.List(ctx, tgConfigList, client.InNamespace(serviceMetadata.Namespace)); err != nil { + return nil, err + } + + for _, tgConfig := range tgConfigList.Items { + if tgConfig.Spec.TargetReference.Kind != nil && *tgConfig.Spec.TargetReference.Kind != serviceKind { + continue + } + + // TODO - Add a webhook to validate that only one target group config references this service. + // TODO - Add an index for this + if tgConfig.Spec.TargetReference.Name == serviceMetadata.Name { + return &tgConfig, nil + } + } + return nil, nil +} + +// Implements the reference grant API +// https://gateway-api.sigs.k8s.io/api-types/referencegrant/ +func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdentifier types.NamespacedName, routeIdentifier types.NamespacedName, routeKind string) (bool, error) { + referenceGrantList := &gwbeta1.ReferenceGrantList{} + if err := k8sClient.List(ctx, referenceGrantList, client.InNamespace(svcIdentifier.Namespace)); err != nil { + return false, err + } + + for _, grant := range referenceGrantList.Items { + var routeAllowed bool + + for _, from := range grant.Spec.From { + // Kind check maybe? + if string(from.Kind) == routeKind && string(from.Namespace) == routeIdentifier.Namespace { + routeAllowed = true + break + } + } + + if routeAllowed { + for _, to := range grant.Spec.To { + // As this is a backend reference, we only care about the "Service" Kind. + if to.Kind != serviceKind { + continue + } + + // If name is specified, we need to ensure that svc name matches the "to" name. + if to.Name != nil && string(*to.Name) != svcIdentifier.Name { + continue + } + + return true, nil + } + + } + } + + return false, nil +} diff --git a/pkg/gateway/routeutils/backend_test.go b/pkg/gateway/routeutils/backend_test.go index 46473d1e7c..1a042fc4fe 100644 --- a/pkg/gateway/routeutils/backend_test.go +++ b/pkg/gateway/routeutils/backend_test.go @@ -2,12 +2,15 @@ package routeutils import ( "context" + "fmt" awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "testing" ) @@ -24,15 +27,57 @@ func TestCommonBackendLoader(t *testing.T) { return &pn } + tgConfigTargetSvcAndNs := elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: namespaceToUse, + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: svcNameToUse, + }, + }, + } + + tgConfigDifferentSvcAndTargetNs := elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tg2", + Namespace: namespaceToUse, + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: "other-svc-name", + }, + }, + } + + tgConfigTargetSvcAndDifferentNs := elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tg3", + Namespace: "differentNs", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: svcNameToUse, + }, + }, + } + testCases := []struct { - name string - storedService *corev1.Service - backendRef gwv1.BackendRef - routeIdentifier types.NamespacedName - weight int - servicePort int32 - expectErr bool - expectNoResult bool + name string + storedService *corev1.Service + storedTGConfigs []elbv2gw.TargetGroupConfiguration + referenceGrants []gwbeta1.ReferenceGrant + backendRef gwv1.BackendRef + routeIdentifier types.NamespacedName + weight int + servicePort int32 + expectErr bool + expectNoResult bool + expectedTargetGroup *elbv2gw.TargetGroupConfiguration }{ { name: "backend ref without namespace", @@ -46,6 +91,7 @@ func TestCommonBackendLoader(t *testing.T) { Port: portConverter(80), }, }, + expectedTargetGroup: nil, // namespace is wrong storedService: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "backend-ref-ns", @@ -76,6 +122,7 @@ func TestCommonBackendLoader(t *testing.T) { }, Weight: awssdk.Int32(100), }, + expectedTargetGroup: nil, // namespace is wrong storedService: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "backend-ref-ns", @@ -96,7 +143,8 @@ func TestCommonBackendLoader(t *testing.T) { { name: "backend ref with namespace", routeIdentifier: types.NamespacedName{ - Name: routeNameToUse, + Name: routeNameToUse, + Namespace: namespaceToUse, }, backendRef: gwv1.BackendRef{ BackendObjectReference: gwv1.BackendObjectReference{ @@ -105,6 +153,7 @@ func TestCommonBackendLoader(t *testing.T) { Port: portConverter(80), }, }, + expectedTargetGroup: &tgConfigTargetSvcAndNs, storedService: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespaceToUse, @@ -122,10 +171,92 @@ func TestCommonBackendLoader(t *testing.T) { weight: 1, servicePort: 80, }, + { + name: "route and service in different namespace (no reference grant)", + routeIdentifier: types.NamespacedName{ + Name: routeNameToUse, + Namespace: "route-ns", + }, + backendRef: gwv1.BackendRef{ + BackendObjectReference: gwv1.BackendObjectReference{ + Name: gwv1.ObjectName(svcNameToUse), + Namespace: (*gwv1.Namespace)(&namespaceToUse), + Port: portConverter(80), + }, + }, + expectNoResult: true, + storedService: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaceToUse, + Name: svcNameToUse, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "port-80", + Port: 80, + }, + }, + }, + }, + }, + { + name: "route and service in different namespace (with reference grant)", + routeIdentifier: types.NamespacedName{ + Name: routeNameToUse, + Namespace: "route-ns", + }, + backendRef: gwv1.BackendRef{ + BackendObjectReference: gwv1.BackendObjectReference{ + Name: gwv1.ObjectName(svcNameToUse), + Namespace: (*gwv1.Namespace)(&namespaceToUse), + Port: portConverter(80), + }, + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaceToUse, + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind(kind), + Namespace: "route-ns", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: serviceKind, + }, + }, + }, + }, + }, + storedService: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaceToUse, + Name: svcNameToUse, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "port-80", + Port: 80, + }, + }, + }, + }, + expectedTargetGroup: &tgConfigTargetSvcAndNs, + weight: 1, + servicePort: 80, + }, { name: "0 weight backend should return nil", routeIdentifier: types.NamespacedName{ - Name: routeNameToUse, + Name: routeNameToUse, + Namespace: namespaceToUse, }, backendRef: gwv1.BackendRef{ BackendObjectReference: gwv1.BackendObjectReference{ @@ -140,7 +271,8 @@ func TestCommonBackendLoader(t *testing.T) { { name: "non-service based backend should return nil", routeIdentifier: types.NamespacedName{ - Name: routeNameToUse, + Name: routeNameToUse, + Namespace: namespaceToUse, }, backendRef: gwv1.BackendRef{ BackendObjectReference: gwv1.BackendObjectReference{ @@ -155,7 +287,8 @@ func TestCommonBackendLoader(t *testing.T) { { name: "missing port in backend ref should result in an error", routeIdentifier: types.NamespacedName{ - Name: routeNameToUse, + Name: routeNameToUse, + Namespace: namespaceToUse, }, backendRef: gwv1.BackendRef{ BackendObjectReference: gwv1.BackendObjectReference{ @@ -172,7 +305,18 @@ func TestCommonBackendLoader(t *testing.T) { k8sClient := generateTestClient() if tc.storedService != nil { - k8sClient.Create(context.Background(), tc.storedService) + err := k8sClient.Create(context.Background(), tc.storedService) + assert.NoError(t, err) + } + + for _, c := range []elbv2gw.TargetGroupConfiguration{tgConfigTargetSvcAndNs, tgConfigDifferentSvcAndTargetNs, tgConfigTargetSvcAndDifferentNs} { + err := k8sClient.Create(context.Background(), &c) + assert.NoError(t, err, fmt.Sprintf("%+v", c)) + } + + for _, g := range tc.referenceGrants { + err := k8sClient.Create(context.Background(), &g) + assert.NoError(t, err, fmt.Sprintf("%+v", g)) } result, err := commonBackendLoader(context.Background(), k8sClient, tc.backendRef, tc.backendRef, tc.routeIdentifier, kind) @@ -193,7 +337,384 @@ func TestCommonBackendLoader(t *testing.T) { assert.Equal(t, tc.weight, result.Weight) assert.Equal(t, tc.servicePort, result.ServicePort.Port) assert.Equal(t, tc.backendRef, result.TypeSpecificBackend) + + if tc.expectedTargetGroup == nil { + assert.Nil(t, result.ELBv2TargetGroupConfig) + } else { + assert.Equal(t, tc.expectedTargetGroup.Name, result.ELBv2TargetGroupConfig.Name) + assert.Equal(t, tc.expectedTargetGroup.Namespace, result.ELBv2TargetGroupConfig.Namespace) + } + }) + } +} + +func Test_lookUpTargetGroupConfiguration(t *testing.T) { + testCases := []struct { + name string + allTargetGroupConfigurations []elbv2gw.TargetGroupConfiguration + serviceMetadata types.NamespacedName + expectErr bool + expectedTGConfiguration *elbv2gw.TargetGroupConfiguration + }{ + { + name: "happy path, exactly one tg config", + allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: "svc1", + }, + }, + }, + }, + serviceMetadata: types.NamespacedName{ + Namespace: "namespace", + Name: "svc1", + }, + expectedTGConfiguration: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: "svc1", + }, + }, + }, + }, + { + name: "happy path, exactly one tg config (kind not specified)", + allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "svc1", + }, + }, + }, + }, + serviceMetadata: types.NamespacedName{ + Namespace: "namespace", + Name: "svc1", + }, + expectedTGConfiguration: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "svc1", + }, + }, + }, + }, + { + name: "sad path, svc name different", + allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: "svc2", + }, + }, + }, + }, + serviceMetadata: types.NamespacedName{ + Namespace: "namespace", + Name: "svc1", + }, + }, + { + name: "sad path, kind not supported", + allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String("cat"), + Name: "svc2", + }, + }, + }, + }, + serviceMetadata: types.NamespacedName{ + Namespace: "namespace", + Name: "svc1", + }, + }, + { + name: "sad path, many tg none match", + allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: "foo", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg2", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: "baz", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg3", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(serviceKind), + Name: "bar", + }, + }, + }, + }, + serviceMetadata: types.NamespacedName{ + Namespace: "namespace", + Name: "svc1", + }, + expectedTGConfiguration: nil, + }, + { + name: "sad path, no tg none match", + serviceMetadata: types.NamespacedName{ + Namespace: "namespace", + Name: "svc1", + }, + expectedTGConfiguration: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + k8sClient := generateTestClient() + for _, c := range tc.allTargetGroupConfigurations { + err := k8sClient.Create(context.Background(), &c) + assert.NoError(t, err) + } + + result, err := lookUpTargetGroupConfiguration(context.Background(), k8sClient, tc.serviceMetadata) + + if tc.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + if result != nil { + // Reset resource version from the create call. + result.ResourceVersion = "" + } + assert.Equal(t, tc.expectedTGConfiguration, result) }) } +} +func Test_referenceGrantCheck(t *testing.T) { + kind := HTTPRouteKind + testCases := []struct { + name string + referenceGrants []gwbeta1.ReferenceGrant + svcIdentifier types.NamespacedName + routeIdentifier types.NamespacedName + expected bool + expectErr bool + }{ + { + name: "happy path", + svcIdentifier: types.NamespacedName{ + Namespace: "svc-namespace", + Name: "svc-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "svc-namespace", + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind(kind), + Namespace: "route-namespace", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: serviceKind, + Name: (*gwbeta1.ObjectName)(awssdk.String("svc-name")), + }, + }, + }, + }, + }, + expected: true, + }, + { + name: "happy path (no name equals wildcard)", + svcIdentifier: types.NamespacedName{ + Namespace: "svc-namespace", + Name: "svc-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "svc-namespace", + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind(kind), + Namespace: "route-namespace", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: serviceKind, + }, + }, + }, + }, + }, + expected: true, + }, + { + name: "no grants, should not allow", + svcIdentifier: types.NamespacedName{ + Namespace: "svc-namespace", + Name: "svc-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + expected: false, + }, + { + name: "from is allowed, but not to", + svcIdentifier: types.NamespacedName{ + Namespace: "svc-namespace", + Name: "svc-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "svc-namespace", + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind(kind), + Namespace: "route-namespace", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: serviceKind, + Name: (*gwbeta1.ObjectName)(awssdk.String("baz")), + }, + }, + }, + }, + }, + expected: false, + }, + { + name: "to is allowed, but not from", + svcIdentifier: types.NamespacedName{ + Namespace: "svc-namespace", + Name: "svc-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "svc-namespace", + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind("other kind"), + Namespace: "route-namespace", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: serviceKind, + }, + }, + }, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + k8sClient := generateTestClient() + for _, ref := range tc.referenceGrants { + err := k8sClient.Create(context.Background(), &ref) + assert.NoError(t, err) + } + + result, err := referenceGrantCheck(context.Background(), k8sClient, tc.svcIdentifier, tc.routeIdentifier, kind) + if tc.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, result) + }) + } } diff --git a/pkg/gateway/routeutils/test_utils.go b/pkg/gateway/routeutils/test_utils.go index 98c1fccd5f..de9b9fa6aa 100644 --- a/pkg/gateway/routeutils/test_utils.go +++ b/pkg/gateway/routeutils/test_utils.go @@ -4,10 +4,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) func generateTestClient() client.Client { @@ -16,5 +18,8 @@ func generateTestClient() client.Client { elbv2api.AddToScheme(k8sSchema) gwv1.AddToScheme(k8sSchema) gwalpha2.AddToScheme(k8sSchema) + elbv2gw.AddToScheme(k8sSchema) + gwbeta1.AddToScheme(k8sSchema) + return testclient.NewClientBuilder().WithScheme(k8sSchema).Build() } diff --git a/pkg/metrics/util/reconcile_counter.go b/pkg/metrics/util/reconcile_counter.go index 8d29dd3236..4aecdc17db 100644 --- a/pkg/metrics/util/reconcile_counter.go +++ b/pkg/metrics/util/reconcile_counter.go @@ -8,10 +8,12 @@ import ( ) type ReconcileCounters struct { - serviceReconciles map[types.NamespacedName]int - ingressReconciles map[types.NamespacedName]int - tgbReconciles map[types.NamespacedName]int - mutex sync.Mutex + serviceReconciles map[types.NamespacedName]int + ingressReconciles map[types.NamespacedName]int + tgbReconciles map[types.NamespacedName]int + nlbGatewayReconciles map[types.NamespacedName]int + albGatewayReconciles map[types.NamespacedName]int + mutex sync.Mutex } type ResourceReconcileCount struct { @@ -46,6 +48,17 @@ func (c *ReconcileCounters) IncrementTGB(namespaceName types.NamespacedName) { c.tgbReconciles[namespaceName]++ } +func (c *ReconcileCounters) IncrementNLBGateway(namespaceName types.NamespacedName) { + c.mutex.Lock() + defer c.mutex.Unlock() + c.nlbGatewayReconciles[namespaceName]++ +} + +func (c *ReconcileCounters) IncrementALBGateway(namespaceName types.NamespacedName) { + c.mutex.Lock() + defer c.mutex.Unlock() + c.albGatewayReconciles[namespaceName]++ +} func (c *ReconcileCounters) GetTopReconciles(n int) map[string][]ResourceReconcileCount { c.mutex.Lock() defer c.mutex.Unlock() @@ -68,6 +81,8 @@ func (c *ReconcileCounters) GetTopReconciles(n int) map[string][]ResourceReconci topReconciles["service"] = getTopN(c.serviceReconciles) topReconciles["ingress"] = getTopN(c.ingressReconciles) topReconciles["targetgroupbinding"] = getTopN(c.tgbReconciles) + topReconciles["nlbgateway"] = getTopN(c.nlbGatewayReconciles) + topReconciles["albgateway"] = getTopN(c.albGatewayReconciles) return topReconciles } @@ -78,4 +93,6 @@ func (c *ReconcileCounters) ResetCounter() { c.serviceReconciles = make(map[types.NamespacedName]int) c.ingressReconciles = make(map[types.NamespacedName]int) c.tgbReconciles = make(map[types.NamespacedName]int) + c.nlbGatewayReconciles = make(map[types.NamespacedName]int) + c.albGatewayReconciles = make(map[types.NamespacedName]int) } From 23ca9e81af895ea37abedc0668a3297006b6e277 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Tue, 8 Apr 2025 16:29:22 -0700 Subject: [PATCH 05/40] update to go 1.24.2 --- .go-version | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.go-version b/.go-version index f9e8384bb6..e4a973f913 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.24.1 +1.24.2 diff --git a/go.mod b/go.mod index 8a509e58fe..b3a91337d7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module sigs.k8s.io/aws-load-balancer-controller -go 1.24.1 +go 1.24.2 require ( github.com/aws/aws-sdk-go v1.55.5 From e7c6c07d9e200ea432ca387ee694525483e04712 Mon Sep 17 00:00:00 2001 From: Daniel Brown Date: Wed, 9 Apr 2025 12:55:53 +0100 Subject: [PATCH 06/40] chore: change tgb field to lowercase everywhere to avoid logs dropped due to conflict in OS/ES Signed-off-by: Daniel Brown --- .../elbv2/targetgroupbinding_deferred_reconciler.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/controllers/elbv2/targetgroupbinding_deferred_reconciler.go b/controllers/elbv2/targetgroupbinding_deferred_reconciler.go index ab7aac4356..2db94ea801 100644 --- a/controllers/elbv2/targetgroupbinding_deferred_reconciler.go +++ b/controllers/elbv2/targetgroupbinding_deferred_reconciler.go @@ -2,15 +2,16 @@ package controllers import ( "context" + "math/rand" + "time" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" - "math/rand" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/aws-load-balancer-controller/pkg/targetgroupbinding" "sigs.k8s.io/controller-runtime/pkg/client" - "time" ) const ( @@ -56,7 +57,7 @@ func (d *deferredTargetGroupBindingReconcilerImpl) Enqueue(tgb *elbv2api.TargetG nsn := k8s.NamespacedName(tgb) if d.isEligibleForDefer(tgb) { d.enqueue(nsn) - d.logger.Info("enqueued new deferred TGB", "TGB", nsn.Name) + d.logger.Info("enqueued new deferred TGB", "tgb", nsn.Name) } } @@ -88,7 +89,7 @@ func (d *deferredTargetGroupBindingReconcilerImpl) handleDeferredItem(nsn types. // Re-check that this tgb hasn't been updated since it was enqueued if !d.isEligibleForDefer(tgb) { - d.logger.Info("TGB not eligible for deferral", "TGB", nsn) + d.logger.Info("TGB not eligible for deferral", "tgb", nsn) return } @@ -99,13 +100,13 @@ func (d *deferredTargetGroupBindingReconcilerImpl) handleDeferredItem(nsn types. d.handleDeferredItemError(nsn, err, "Failed to reset TGB checkpoint") return } - d.logger.Info("TGB checkpoint reset", "TGB", nsn) + d.logger.Info("TGB checkpoint reset", "tgb", nsn) } func (d *deferredTargetGroupBindingReconcilerImpl) handleDeferredItemError(nsn types.NamespacedName, err error, msg string) { err = client.IgnoreNotFound(err) if err != nil { - d.logger.Error(err, msg, "TGB", nsn) + d.logger.Error(err, msg, "tgb", nsn) d.enqueue(nsn) } } From 5e0447efeebcfd4e0c73f8edea0926c4cbcaae58 Mon Sep 17 00:00:00 2001 From: shuqz <95371565+shuqz@users.noreply.github.com> Date: Tue, 15 Apr 2025 15:12:03 -0700 Subject: [PATCH 07/40] support cli flag to enable manage backend SG rules for ALB remove auto-generated file address comments support enable manage backend SG rules for NLB --- config/webhook/manifests.yaml | 1 - controllers/ingress/group_controller.go | 2 +- controllers/service/service_controller.go | 2 +- docs/deploy/configurations.md | 99 +++++----- docs/deploy/security_groups.md | 7 +- go.mod | 2 +- go.sum | 2 + helm/aws-load-balancer-controller/README.md | 165 ++++++++--------- .../templates/deployment.yaml | 3 + helm/aws-load-balancer-controller/values.yaml | 3 + pkg/config/controller_config.go | 17 ++ pkg/config/controller_config_test.go | 54 ++++++ pkg/ingress/model_builder.go | 171 ++++++++++-------- pkg/ingress/model_builder_test.go | 104 +++++++++++ pkg/service/model_build_load_balancer.go | 9 +- pkg/service/model_build_load_balancer_test.go | 80 ++++++++ pkg/service/model_builder.go | 138 +++++++------- pkg/service/model_builder_test.go | 3 +- 18 files changed, 576 insertions(+), 286 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 00793b4707..f901bc74c3 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -1,4 +1,3 @@ ---- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/controllers/ingress/group_controller.go b/controllers/ingress/group_controller.go index 13cd9234dc..75f16ad824 100644 --- a/controllers/ingress/group_controller.go +++ b/controllers/ingress/group_controller.go @@ -65,7 +65,7 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco authConfigBuilder, enhancedBackendBuilder, trackingProvider, elbv2TaggingManager, controllerConfig.FeatureGates, cloud.VpcID(), controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, - controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger, metricsCollector) + controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), logger, metricsCollector) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, ingressTagPrefix, logger, metricsCollector, controllerName) diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index dcdd5f8607..6aa7383dfd 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -51,7 +51,7 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider, elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils, - backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector) + backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger, metricsCollector, controllerName) return &serviceReconciler{ diff --git a/docs/deploy/configurations.md b/docs/deploy/configurations.md index 7c63741930..1c31f4a140 100644 --- a/docs/deploy/configurations.md +++ b/docs/deploy/configurations.md @@ -64,55 +64,56 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne !!!warning "" The --cluster-name flag is mandatory and the value must match the name of the kubernetes cluster. If you specify an incorrect name, the subnet auto-discovery will not work. -| Flag | Type | Default | Description | -|---------------------------------------------------------------------------------|---------------------------------|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| aws-api-endpoints | AWS API Endpoints Config | | AWS API endpoints mapping, format: serviceID1=URL1,serviceID2=URL2 | -| aws-api-throttle | AWS Throttle Config | [default value](#default-throttle-config ) | throttle settings for AWS APIs, format: serviceID1:operationRegex1=rate:burst,serviceID2:operationRegex2=rate:burst | -| aws-max-retries | int | 10 | Maximum retries for AWS APIs | -| aws-region | string | [instance metadata](#instance-metadata) | AWS Region for the kubernetes cluster | -| aws-vpc-id | string | [instance metadata](#instance-metadata) | AWS VPC ID for the Kubernetes cluster | -| aws-vpc-tags | stringMap | | Tags for the Kubernetes cluster VPC, When both flags `--aws-vpc-id` and `--aws-vpc-tags` are specified, the controller prioritizes `--aws-vpc-id` and ignores the other flag. -| aws-vpc-tag-key | string | Name | Optional tag key used with aws-vpc-tags add only if VPC name tag key is not the default value "Name" -| allowed-certificate-authority-arns | stringList | [] | Specify an optional list of CA ARNs to filter on in cert discovery (empty means all CAs are allowed) | -| backend-security-group | string | | Backend security group id to use for the ingress rules on the worker node SG | -| cluster-name | string | | Kubernetes cluster name | -| default-ssl-policy | string | ELBSecurityPolicy-2016-08 | Default SSL Policy that will be applied to all Ingresses or Services that do not have the SSL Policy annotation | -| default-tags | stringMap | | AWS Tags that will be applied to all AWS resources managed by this controller. Specified Tags takes highest priority | -| default-target-type | string | instance | Default target type for Ingresses and Services - ip, instance | -| default-load-balancer-scheme | string | internal | Default scheme for ELBs - internal, internet-facing | -| [disable-ingress-class-annotation](#disable-ingress-class-annotation) | boolean | false | Disable new usage of the `kubernetes.io/ingress.class` annotation | -| [disable-ingress-group-name-annotation](#disable-ingress-group-name-annotation) | boolean | false | Disallow new use of the `alb.ingress.kubernetes.io/group.name` annotation | -| disable-restricted-sg-rules | boolean | false | Disable the usage of restricted security group rules | -| enable-backend-security-group | boolean | true | Enable sharing of security groups for backend traffic | -| enable-endpoint-slices | boolean | false | Use EndpointSlices instead of Endpoints for pod endpoint and TargetGroupBinding resolution for load balancers with IP targets. | -| enable-leader-election | boolean | true | Enable leader election for the load balancer controller manager. Enabling this will ensure there is only one active controller manager | -| enable-pod-readiness-gate-inject | boolean | true | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | -| enable-shield | boolean | true | Enable Shield addon for ALB | -| [enable-waf](#waf-addons) | boolean | true | Enable WAF addon for ALB | -| [enable-wafv2](#waf-addons) | boolean | true | Enable WAF V2 addon for ALB | -| external-managed-tags | stringList | | AWS Tag keys that will be managed externally. Specified Tags are ignored during reconciliation | -| [feature-gates](#feature-gates) | stringMap | | A set of key=value pairs to enable or disable features | -| health-probe-bind-addr | string | :61779 | The address the health probes binds to | -| ingress-class | string | alb | Name of the ingress class this controller satisfies | -| ingress-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for ingress | -| kubeconfig | string | in-cluster config | Path to the kubeconfig file containing authorization and API server information | -| leader-election-id | string | aws-load-balancer-controller-leader | Name of the leader election ID to use for this controller | -| leader-election-namespace | string | | Name of the leader election ID to use for this controller | -| load-balancer-class | string | service.k8s.aws/nlb | Name of the load balancer class specified in service `spec.loadBalancerClass` reconciled by this controller | -| log-level | string | info | Set the controller log level - info, debug | -| metrics-bind-addr | string | :8080 | The address the metric endpoint binds to | -| service-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for service | -| [sync-period](#sync-period) | duration | 10h0m0s | Period at which the controller forces the repopulation of its local object stores | -| targetgroupbinding-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for targetGroupBinding | -| targetgroupbinding-max-exponential-backoff-delay | duration | 16m40s | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | -| [lb-stabilization-monitor-interval](#lb-stabilization-monitor-interval) | duration | 2m | Interval at which the controller monitors the state of load balancer after creation -| tolerate-non-existent-backend-service | boolean | true | Whether to allow rules which refer to backend services that do not exist (When enabled, it will return 503 error if backend service not exist) | -| tolerate-non-existent-backend-action | boolean | true | Whether to allow rules which refer to backend actions that do not exist (When enabled, it will return 503 error if backend action not exist) | -| watch-namespace | string | | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched. | -| webhook-bind-port | int | 9443 | The TCP port the Webhook server binds to | -| webhook-cert-dir | string | /tmp/k8s-webhook-server/serving-certs | The directory that contains the server key and certificate | -| webhook-cert-file | string | tls.crt | The server certificate name | -| webhook-key-file | string | tls.key | The server key name | +| Flag | Type | Default | Description | +|---------------------------------------------------------------------------------|---------------------------------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| aws-api-endpoints | AWS API Endpoints Config | | AWS API endpoints mapping, format: serviceID1=URL1,serviceID2=URL2 | +| aws-api-throttle | AWS Throttle Config | [default value](#default-throttle-config ) | throttle settings for AWS APIs, format: serviceID1:operationRegex1=rate:burst,serviceID2:operationRegex2=rate:burst | +| aws-max-retries | int | 10 | Maximum retries for AWS APIs | +| aws-region | string | [instance metadata](#instance-metadata) | AWS Region for the kubernetes cluster | +| aws-vpc-id | string | [instance metadata](#instance-metadata) | AWS VPC ID for the Kubernetes cluster | +| aws-vpc-tags | stringMap | | Tags for the Kubernetes cluster VPC, When both flags `--aws-vpc-id` and `--aws-vpc-tags` are specified, the controller prioritizes `--aws-vpc-id` and ignores the other flag. +| aws-vpc-tag-key | string | Name | Optional tag key used with aws-vpc-tags add only if VPC name tag key is not the default value "Name" +| allowed-certificate-authority-arns | stringList | [] | Specify an optional list of CA ARNs to filter on in cert discovery (empty means all CAs are allowed) | +| backend-security-group | string | | Backend security group id to use for the ingress rules on the worker node SG | +| cluster-name | string | | Kubernetes cluster name | +| default-ssl-policy | string | ELBSecurityPolicy-2016-08 | Default SSL Policy that will be applied to all Ingresses or Services that do not have the SSL Policy annotation | +| default-tags | stringMap | | AWS Tags that will be applied to all AWS resources managed by this controller. Specified Tags takes highest priority | +| default-target-type | string | instance | Default target type for Ingresses and Services - ip, instance | +| default-load-balancer-scheme | string | internal | Default scheme for ELBs - internal, internet-facing | +| [disable-ingress-class-annotation](#disable-ingress-class-annotation) | boolean | false | Disable new usage of the `kubernetes.io/ingress.class` annotation | +| [disable-ingress-group-name-annotation](#disable-ingress-group-name-annotation) | boolean | false | Disallow new use of the `alb.ingress.kubernetes.io/group.name` annotation | +| disable-restricted-sg-rules | boolean | false | Disable the usage of restricted security group rules | +| enable-backend-security-group | boolean | true | Enable sharing of security groups for backend traffic | +| enable-manage-backend-security-group-rules | boolean | false | Enable managing backend security group rules by controller | +| enable-endpoint-slices | boolean | false | Use EndpointSlices instead of Endpoints for pod endpoint and TargetGroupBinding resolution for load balancers with IP targets. | +| enable-leader-election | boolean | true | Enable leader election for the load balancer controller manager. Enabling this will ensure there is only one active controller manager | +| enable-pod-readiness-gate-inject | boolean | true | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | +| enable-shield | boolean | true | Enable Shield addon for ALB | +| [enable-waf](#waf-addons) | boolean | true | Enable WAF addon for ALB | +| [enable-wafv2](#waf-addons) | boolean | true | Enable WAF V2 addon for ALB | +| external-managed-tags | stringList | | AWS Tag keys that will be managed externally. Specified Tags are ignored during reconciliation | +| [feature-gates](#feature-gates) | stringMap | | A set of key=value pairs to enable or disable features | +| health-probe-bind-addr | string | :61779 | The address the health probes binds to | +| ingress-class | string | alb | Name of the ingress class this controller satisfies | +| ingress-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for ingress | +| kubeconfig | string | in-cluster config | Path to the kubeconfig file containing authorization and API server information | +| leader-election-id | string | aws-load-balancer-controller-leader | Name of the leader election ID to use for this controller | +| leader-election-namespace | string | | Name of the leader election ID to use for this controller | +| load-balancer-class | string | service.k8s.aws/nlb | Name of the load balancer class specified in service `spec.loadBalancerClass` reconciled by this controller | +| log-level | string | info | Set the controller log level - info, debug | +| metrics-bind-addr | string | :8080 | The address the metric endpoint binds to | +| service-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for service | +| [sync-period](#sync-period) | duration | 10h0m0s | Period at which the controller forces the repopulation of its local object stores | +| targetgroupbinding-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for targetGroupBinding | +| targetgroupbinding-max-exponential-backoff-delay | duration | 16m40s | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | +| [lb-stabilization-monitor-interval](#lb-stabilization-monitor-interval) | duration | 2m | Interval at which the controller monitors the state of load balancer after creation +| tolerate-non-existent-backend-service | boolean | true | Whether to allow rules which refer to backend services that do not exist (When enabled, it will return 503 error if backend service not exist) | +| tolerate-non-existent-backend-action | boolean | true | Whether to allow rules which refer to backend actions that do not exist (When enabled, it will return 503 error if backend action not exist) | +| watch-namespace | string | | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched. | +| webhook-bind-port | int | 9443 | The TCP port the Webhook server binds to | +| webhook-cert-dir | string | /tmp/k8s-webhook-server/serving-certs | The directory that contains the server key and certificate | +| webhook-cert-file | string | tls.crt | The server certificate name | +| webhook-key-file | string | tls.key | The server key name | ### disable-ingress-class-annotation diff --git a/docs/deploy/security_groups.md b/docs/deploy/security_groups.md index a92ca0a0b7..7c36985c0b 100644 --- a/docs/deploy/security_groups.md +++ b/docs/deploy/security_groups.md @@ -66,7 +66,12 @@ If `--backend-security-group` is left empty, a security group with the following - For Service resources, set the `service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules` annotation to `true`. - If management of backend security group rules is enabled with an annotation on a Service or Ingress, then `--enable-backend-security-group` must be set to true. - These annotations are ignored when using auto-generated frontend security groups. - +- To enable managing backend security group rules for all resources, using cli flag `--enable-manage-backend-security-group-rules` + - when set to `true`, the controller will automatically manage backend security group rules for all resources + - individual annotation takes precedence over cli flag, meaning it can be overridden with annotation `alb.ingress.kubernetes.io/manage-backend-security-group-rules: "false"` for ALB or `service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules: "false"` for NLB + - for this to take effect, `--enable-backend-security-group` needs to be true and user explicitly specify security group using annotation: `alb.ingress.kubernetes.io/security-groups` or `service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules` + - when set to `false` (default value) or not set, the controller takes the individual annotations + ### Port Range Restrictions From version v2.3.0 onwards, the controller restricts port ranges in the backend security group rules by default. This improves the security of the default configuration. The LBC should generate the necessary rules to permit traffic, based on the Service and Ingress resources. diff --git a/go.mod b/go.mod index b3a91337d7..c3f5237e1f 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 - github.com/onsi/ginkgo/v2 v2.21.0 + github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.35.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.20.4 diff --git a/go.sum b/go.sum index c4fdfffe77..2dbe1de464 100644 --- a/go.sum +++ b/go.sum @@ -347,6 +347,8 @@ github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= diff --git a/helm/aws-load-balancer-controller/README.md b/helm/aws-load-balancer-controller/README.md index 8b9aa07b65..e023d215cb 100644 --- a/helm/aws-load-balancer-controller/README.md +++ b/helm/aws-load-balancer-controller/README.md @@ -180,89 +180,90 @@ The following tables lists the configurable parameters of the chart and their de The default values set by the application itself can be confirmed [here](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/configurations/#controller-configuration-options). -| Parameter | Description | Default | -| ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | -| `image.repository` | image repository | `public.ecr.aws/eks/aws-load-balancer-controller` | -| `image.tag` | image tag | `` | -| `image.pullPolicy` | image pull policy | `IfNotPresent` | -| `clusterName` | Kubernetes cluster name | None | -| `cluster.dnsDomain` | DNS domain of the Kubernetes cluster, included in TLS certificate requests | `cluster.local` | -| `securityContext` | Set to security context for pod | `{}` | -| `resources` | Controller pod resource requests & limits | `{}` | -| `priorityClassName` | Controller pod priority class | system-cluster-critical | -| `nodeSelector` | Node labels for controller pod assignment | `{}` | -| `tolerations` | Controller pod toleration for taints | `{}` | -| `affinity` | Affinity for pod assignment | `{}` | -| `configureDefaultAffinity` | Configure soft pod anti-affinity if custom affinity is not configured | `true` | -| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment | `{}` | -| `deploymentAnnotations` | Annotations to add to deployment | `{}` | -| `podAnnotations` | Annotations to add to each pod | `{}` | -| `podLabels` | Labels to add to each pod | `{}` | -| `additionalLabels` | Labels to add to all components | `{}` | -| `rbac.create` | if `true`, create and use RBAC resources | `true` | -| `serviceAccount.annotations` | optional annotations to add to service account | None | -| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a Service Account | `true` | -| `serviceAccount.imagePullSecrets` | List of image pull secrets to add to the Service Account | `[]` | -| `serviceAccount.create` | If `true`, create a new service account | `true` | -| `serviceAccount.name` | Service account to be used | None | -| `terminationGracePeriodSeconds` | Time period for controller pod to do a graceful shutdown | 10 | -| `ingressClass` | The ingress class to satisfy | alb | -| `createIngressClassResource` | Create ingressClass resource | true | -| `ingressClassParams.name` | IngressClassParams resource's name, default to the aws load balancer controller's name | None | -| `ingressClassParams.create` | If `true`, create a new ingressClassParams | true | -| `ingressClassParams.spec` | IngressClassParams defined ingress specifications | {} | -| `region` | The AWS region for the kubernetes cluster | None | -| `vpcId` | The VPC ID for the Kubernetes cluster | None | -| `vpcTags` | This is alternative to vpcId. Set this when your pods are unable to use the metadata service to determine VPC automatically. | None -| `awsApiEndpoints` | Custom AWS API Endpoints | None | -| `awsApiThrottle` | Custom AWS API throttle settings | None | -| `awsMaxRetries` | Maximum retries for AWS APIs | None | -| `defaultTargetType` | Default target type. Used as the default value of the `alb.ingress.kubernetes.io/target-type` and `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" annotations.`Possible values are `ip` and `instance`. | `instance` | -| `enablePodReadinessGateInject` | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | None | -| `enableShield` | Enable Shield addon for ALB | None | -| `enableWaf` | Enable WAF addon for ALB | None | -| `enableWafv2` | Enable WAF V2 addon for ALB | None | -| `ingressMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for ingress | None | -| `logLevel` | Set the controller log level - info, debug | None | -| `metricsBindAddr` | The address the metric endpoint binds to | "" | -| `webhookConfig.disableIngressValidation` | Disables the validation of resources of kind Ingress | None | -| `webhookBindPort` | The TCP port the Webhook server binds to | None | -| `webhookTLS.caCert` | TLS CA certificate for webhook (auto-generated if not provided) | "" | -| `webhookTLS.cert` | TLS certificate for webhook (auto-generated if not provided) | "" | -| `webhookTLS.key` | TLS private key for webhook (auto-generated if not provided) | "" | -| `webhookNamespaceSelectors` | Namespace selectors for the wekbook | None | -| `keepTLSSecret` | Reuse existing TLS Secret during chart upgrade | `true` | -| `serviceAnnotations` | Annotations to be added to the provisioned webhook service resource | `{}` | -| `serviceMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for service | None | -| `targetgroupbindingMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for targetGroupBinding | None | -| `targetgroupbindingMaxExponentialBackoffDelay` | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | None | -| `syncPeriod` | Period at which the controller forces the repopulation of its local object stores | None | -| `watchNamespace` | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched | None | -| `disableIngressClassAnnotation` | Disables the usage of kubernetes.io/ingress.class annotation | None | -| `disableIngressGroupNameAnnotation` | Disables the usage of alb.ingress.kubernetes.io/group.name annotation | None | -| `tolerateNonExistentBackendService` | whether to allow rules that reference a backend service that does not exist. (When enabled, it will return 503 error if backend service not exist) | `true` | -| `tolerateNonExistentBackendAction` | whether to allow rules that reference a backend action that does not exist. (When enabled, it will return 503 error if backend action not exist) | `true` | -| `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | -| `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | -| `livenessProbe` | Liveness probe settings for the controller | `{}` (see `values.yaml`) | +| Parameter | Description | Default | +| ---------------------------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------| +| `image.repository` | image repository | `public.ecr.aws/eks/aws-load-balancer-controller` | +| `image.tag` | image tag | `` | +| `image.pullPolicy` | image pull policy | `IfNotPresent` | +| `clusterName` | Kubernetes cluster name | None | +| `cluster.dnsDomain` | DNS domain of the Kubernetes cluster, included in TLS certificate requests | `cluster.local` | +| `securityContext` | Set to security context for pod | `{}` | +| `resources` | Controller pod resource requests & limits | `{}` | +| `priorityClassName` | Controller pod priority class | system-cluster-critical | +| `nodeSelector` | Node labels for controller pod assignment | `{}` | +| `tolerations` | Controller pod toleration for taints | `{}` | +| `affinity` | Affinity for pod assignment | `{}` | +| `configureDefaultAffinity` | Configure soft pod anti-affinity if custom affinity is not configured | `true` | +| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment | `{}` | +| `deploymentAnnotations` | Annotations to add to deployment | `{}` | +| `podAnnotations` | Annotations to add to each pod | `{}` | +| `podLabels` | Labels to add to each pod | `{}` | +| `additionalLabels` | Labels to add to all components | `{}` | +| `rbac.create` | if `true`, create and use RBAC resources | `true` | +| `serviceAccount.annotations` | optional annotations to add to service account | None | +| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a Service Account | `true` | +| `serviceAccount.imagePullSecrets` | List of image pull secrets to add to the Service Account | `[]` | +| `serviceAccount.create` | If `true`, create a new service account | `true` | +| `serviceAccount.name` | Service account to be used | None | +| `terminationGracePeriodSeconds` | Time period for controller pod to do a graceful shutdown | 10 | +| `ingressClass` | The ingress class to satisfy | alb | +| `createIngressClassResource` | Create ingressClass resource | true | +| `ingressClassParams.name` | IngressClassParams resource's name, default to the aws load balancer controller's name | None | +| `ingressClassParams.create` | If `true`, create a new ingressClassParams | true | +| `ingressClassParams.spec` | IngressClassParams defined ingress specifications | {} | +| `region` | The AWS region for the kubernetes cluster | None | +| `vpcId` | The VPC ID for the Kubernetes cluster | None | +| `vpcTags` | This is alternative to vpcId. Set this when your pods are unable to use the metadata service to determine VPC automatically. | None +| `awsApiEndpoints` | Custom AWS API Endpoints | None | +| `awsApiThrottle` | Custom AWS API throttle settings | None | +| `awsMaxRetries` | Maximum retries for AWS APIs | None | +| `defaultTargetType` | Default target type. Used as the default value of the `alb.ingress.kubernetes.io/target-type` and `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" annotations.`Possible values are `ip` and `instance`. | `instance` | +| `enablePodReadinessGateInject` | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | None | +| `enableShield` | Enable Shield addon for ALB | None | +| `enableWaf` | Enable WAF addon for ALB | None | +| `enableWafv2` | Enable WAF V2 addon for ALB | None | +| `ingressMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for ingress | None | +| `logLevel` | Set the controller log level - info, debug | None | +| `metricsBindAddr` | The address the metric endpoint binds to | "" | +| `webhookConfig.disableIngressValidation` | Disables the validation of resources of kind Ingress | None | +| `webhookBindPort` | The TCP port the Webhook server binds to | None | +| `webhookTLS.caCert` | TLS CA certificate for webhook (auto-generated if not provided) | "" | +| `webhookTLS.cert` | TLS certificate for webhook (auto-generated if not provided) | "" | +| `webhookTLS.key` | TLS private key for webhook (auto-generated if not provided) | "" | +| `webhookNamespaceSelectors` | Namespace selectors for the wekbook | None | +| `keepTLSSecret` | Reuse existing TLS Secret during chart upgrade | `true` | +| `serviceAnnotations` | Annotations to be added to the provisioned webhook service resource | `{}` | +| `serviceMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for service | None | +| `targetgroupbindingMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for targetGroupBinding | None | +| `targetgroupbindingMaxExponentialBackoffDelay` | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | None | +| `syncPeriod` | Period at which the controller forces the repopulation of its local object stores | None | +| `watchNamespace` | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched | None | +| `disableIngressClassAnnotation` | Disables the usage of kubernetes.io/ingress.class annotation | None | +| `disableIngressGroupNameAnnotation` | Disables the usage of alb.ingress.kubernetes.io/group.name annotation | None | +| `tolerateNonExistentBackendService` | whether to allow rules that reference a backend service that does not exist. (When enabled, it will return 503 error if backend service not exist) | `true` | +| `tolerateNonExistentBackendAction` | whether to allow rules that reference a backend action that does not exist. (When enabled, it will return 503 error if backend action not exist) | `true` | +| `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | +| `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | +| `livenessProbe` | Liveness probe settings for the controller | `{}` (see `values.yaml`) | | `readinessProbe` | Readiness probe settings for the controller | `{}` (see `values.yaml`) | -| `env` | Environment variables to set for aws-load-balancer-controller pod | None | -| `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | -| `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | -| `hostNetwork` | If `true`, use hostNetwork | `false` | -| `dnsPolicy` | Set dnsPolicy if required | `ClusterFirst` | -| `extraVolumeMounts` | Extra volume mounts for the pod | `[]` | -| `extraVolumes` | Extra volumes for the pod | `[]` | -| `defaultTags` | Default tags to apply to all AWS resources managed by this controller | `{}` | -| `replicaCount` | Number of controller pods to run, only one will be active due to leader election | `2` | -| `revisionHistoryLimit` | Number of revisions to keep | `10` | -| `podDisruptionBudget` | Limit the disruption for controller pods. Require at least 2 controller replicas and 3 worker nodes | `{}` | -| `updateStrategy` | Defines the update strategy for the deployment | `{}` | -| `enableCertManager` | If enabled, cert-manager issues the webhook certificates instead of the helm template, requires cert-manager and it's CRDs to be installed | `false` | -| `certManager.duration` | Overrides the default expiry duration for the webhook certificates. defaults to `90d` | `""` | -| `certManager.renewBefore` | Overrides the renewal time period duration for the webhook certificates. defaults to `60d` | `""` | -| `enableEndpointSlices` | If enabled, controller uses k8s EndpointSlices instead of Endpoints for IP targets | `false` | -| `enableBackendSecurityGroup` | If enabled, controller uses shared security group for backend traffic | `true` | +| `env` | Environment variables to set for aws-load-balancer-controller pod | None | +| `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | +| `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | +| `hostNetwork` | If `true`, use hostNetwork | `false` | +| `dnsPolicy` | Set dnsPolicy if required | `ClusterFirst` | +| `extraVolumeMounts` | Extra volume mounts for the pod | `[]` | +| `extraVolumes` | Extra volumes for the pod | `[]` | +| `defaultTags` | Default tags to apply to all AWS resources managed by this controller | `{}` | +| `replicaCount` | Number of controller pods to run, only one will be active due to leader election | `2` | +| `revisionHistoryLimit` | Number of revisions to keep | `10` | +| `podDisruptionBudget` | Limit the disruption for controller pods. Require at least 2 controller replicas and 3 worker nodes | `{}` | +| `updateStrategy` | Defines the update strategy for the deployment | `{}` | +| `enableCertManager` | If enabled, cert-manager issues the webhook certificates instead of the helm template, requires cert-manager and it's CRDs to be installed | `false` | +| `certManager.duration` | Overrides the default expiry duration for the webhook certificates. defaults to `90d` | `""` | +| `certManager.renewBefore` | Overrides the renewal time period duration for the webhook certificates. defaults to `60d` | `""` | +| `enableEndpointSlices` | If enabled, controller uses k8s EndpointSlices instead of Endpoints for IP targets | `false` | +| `enableBackendSecurityGroup` | If enabled, controller uses shared security group for backend traffic | `true` | +| `enableManageBackendSecurityGroupRules` | If enabled, controller will manage security group rules | `false` | | `backendSecurityGroup` | Backend security group to use instead of auto created one if the feature is enabled | `` | | `disableRestrictedSecurityGroupRules` | If disabled, controller will not specify port range restriction in the backend security group rules | `false` | | `objectSelector.matchExpressions` | Webhook configuration to select specific pods by specifying the expression to be matched | None | diff --git a/helm/aws-load-balancer-controller/templates/deployment.yaml b/helm/aws-load-balancer-controller/templates/deployment.yaml index b75fe14357..c148098a32 100644 --- a/helm/aws-load-balancer-controller/templates/deployment.yaml +++ b/helm/aws-load-balancer-controller/templates/deployment.yaml @@ -154,6 +154,9 @@ spec: {{- if kindIs "bool" .Values.enableBackendSecurityGroup }} - --enable-backend-security-group={{ .Values.enableBackendSecurityGroup }} {{- end }} + {{- if kindIs "bool" .Values.enableManageBackendSecurityGroupRules }} + - --enable-manage-backend-security-group-rules={{ .Values.enableManageBackendSecurityGroupRules }} + {{- end }} {{- if .Values.backendSecurityGroup }} - --backend-security-group={{ .Values.backendSecurityGroup }} {{- end }} diff --git a/helm/aws-load-balancer-controller/values.yaml b/helm/aws-load-balancer-controller/values.yaml index 08c5395f19..5b2d34c5ee 100644 --- a/helm/aws-load-balancer-controller/values.yaml +++ b/helm/aws-load-balancer-controller/values.yaml @@ -349,6 +349,9 @@ enableEndpointSlices: # enableBackendSecurityGroup enables shared security group for backend traffic (default true) enableBackendSecurityGroup: +# enableManageBackendSecurityGroupRules enables controller manages security group rules (default false) +enableManageBackendSecurityGroupRules: + # backendSecurityGroup specifies backend security group id (default controller auto create backend security group) backendSecurityGroup: diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 86f32eb83f..6a9530cdc1 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -26,6 +26,7 @@ const ( flagLbStabilizationMonitorInterval = "lb-stabilization-monitor-interval" flagDefaultSSLPolicy = "default-ssl-policy" flagEnableBackendSG = "enable-backend-security-group" + flagEnableManageBackendSGRules = "enable-manage-backend-security-group-rules" flagBackendSecurityGroup = "backend-security-group" flagEnableEndpointSlices = "enable-endpoint-slices" flagDisableRestrictedSGRules = "disable-restricted-sg-rules" @@ -34,6 +35,7 @@ const ( defaultMaxExponentialBackoffDelay = time.Second * 1000 defaultSSLPolicy = "ELBSecurityPolicy-2016-08" defaultEnableBackendSG = true + defaultEnableManageBackendSGRules = false defaultEnableEndpointSlices = false defaultDisableRestrictedSGRules = false defaultLbStabilizationMonitorInterval = time.Second * 120 @@ -107,6 +109,9 @@ type ControllerConfig struct { // EnableBackendSecurityGroup specifies whether to use optimized security group rules EnableBackendSecurityGroup bool + // EnableManageBackendSecurityGroupRules specifies whether to have controller manages security group rules + EnableManageBackendSecurityGroupRules bool + // BackendSecurityGroups specifies the configured backend security group to use // for optimized security group rules BackendSecurityGroup string @@ -145,6 +150,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { "Default SSL policy for load balancers listeners") fs.BoolVar(&cfg.EnableBackendSecurityGroup, flagEnableBackendSG, defaultEnableBackendSG, "Enable sharing of security groups for backend traffic") + fs.BoolVar(&cfg.EnableManageBackendSecurityGroupRules, flagEnableManageBackendSGRules, defaultEnableManageBackendSGRules, + "Enable managing of backend security group rules by controller") fs.StringVar(&cfg.BackendSecurityGroup, flagBackendSecurityGroup, "", "Backend security group id to use for the ingress rules on the worker node SG") fs.BoolVar(&cfg.EnableEndpointSlices, flagEnableEndpointSlices, defaultEnableEndpointSlices, @@ -187,6 +194,9 @@ func (cfg *ControllerConfig) Validate() error { if err := cfg.validateBackendSecurityGroupConfiguration(); err != nil { return err } + if err := cfg.validateManageBackendSecurityGroupRulesConfiguration(); err != nil { + return err + } return nil } @@ -245,3 +255,10 @@ func (cfg *ControllerConfig) validateBackendSecurityGroupConfiguration() error { } return nil } + +func (cfg *ControllerConfig) validateManageBackendSecurityGroupRulesConfiguration() error { + if cfg.EnableManageBackendSecurityGroupRules && !cfg.EnableBackendSecurityGroup { + return errors.Errorf("backend security group must be enabled when manage backend security group rule is enabled") + } + return nil +} diff --git a/pkg/config/controller_config_test.go b/pkg/config/controller_config_test.go index 5b5fc8dbbb..0351ded53f 100644 --- a/pkg/config/controller_config_test.go +++ b/pkg/config/controller_config_test.go @@ -164,3 +164,57 @@ func TestControllerConfig_validateExternalManagedTagsCollisionWithDefaultTags(t }) } } + +func TestControllerConfig_validateManageBackendSecurityGroupRulesConfiguration(t *testing.T) { + tests := []struct { + name string + enableManageBackendSecurityGroupRules bool + enableBackendSecurityGroup bool + wantErr bool + errMsg string + }{ + { + name: "with enableManageBackendSecurityGroupRules=false and enableBackendSecurityGroup=false - should succeed", + enableManageBackendSecurityGroupRules: false, + enableBackendSecurityGroup: false, + wantErr: false, + }, + { + name: "with enableManageBackendSecurityGroupRules=true and enableBackendSecurityGroup=true - should succeed", + enableManageBackendSecurityGroupRules: true, + enableBackendSecurityGroup: true, + wantErr: false, + }, + { + name: "with enableBackendSecurityGroup=true - should succeed", + enableManageBackendSecurityGroupRules: false, + enableBackendSecurityGroup: true, + wantErr: false, + }, + { + name: "with enableManageBackendSecurityGroupRules=true and enableBackendSecurityGroup=false - expect error", + enableManageBackendSecurityGroupRules: true, + enableBackendSecurityGroup: false, + wantErr: true, + errMsg: "backend security group must be enabled when manage backend security group rule is enabled", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &ControllerConfig{ + EnableManageBackendSecurityGroupRules: tt.enableManageBackendSecurityGroupRules, + EnableBackendSecurityGroup: tt.enableBackendSecurityGroup, + } + + err := cfg.validateManageBackendSecurityGroupRulesConfiguration() + + if tt.wantErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errMsg) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index 9f0246d03e..1e341f24c1 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -48,37 +48,38 @@ func NewDefaultModelBuilder(k8sClient client.Client, eventRecorder record.EventR trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, featureGates config.FeatureGates, vpcID string, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, backendSGProvider networkingpkg.BackendSGProvider, sgResolver networkingpkg.SecurityGroupResolver, - enableBackendSG bool, disableRestrictedSGRules bool, allowedCAARNs []string, enableIPTargetType bool, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *defaultModelBuilder { + enableBackendSG bool, defaultEnableManageBackendSGRules bool, disableRestrictedSGRules bool, allowedCAARNs []string, enableIPTargetType bool, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *defaultModelBuilder { certDiscovery := NewACMCertDiscovery(acmClient, allowedCAARNs, logger) ruleOptimizer := NewDefaultRuleOptimizer(logger) return &defaultModelBuilder{ - k8sClient: k8sClient, - eventRecorder: eventRecorder, - ec2Client: ec2Client, - elbv2Client: elbv2Client, - vpcID: vpcID, - clusterName: clusterName, - annotationParser: annotationParser, - subnetsResolver: subnetsResolver, - backendSGProvider: backendSGProvider, - sgResolver: sgResolver, - certDiscovery: certDiscovery, - authConfigBuilder: authConfigBuilder, - enhancedBackendBuilder: enhancedBackendBuilder, - ruleOptimizer: ruleOptimizer, - trackingProvider: trackingProvider, - elbv2TaggingManager: elbv2TaggingManager, - featureGates: featureGates, - defaultTags: defaultTags, - externalManagedTags: sets.NewString(externalManagedTags...), - defaultSSLPolicy: defaultSSLPolicy, - defaultTargetType: elbv2model.TargetType(defaultTargetType), - defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), - enableBackendSG: enableBackendSG, - disableRestrictedSGRules: disableRestrictedSGRules, - enableIPTargetType: enableIPTargetType, - logger: logger, - metricsCollector: metricsCollector, + k8sClient: k8sClient, + eventRecorder: eventRecorder, + ec2Client: ec2Client, + elbv2Client: elbv2Client, + vpcID: vpcID, + clusterName: clusterName, + annotationParser: annotationParser, + subnetsResolver: subnetsResolver, + backendSGProvider: backendSGProvider, + sgResolver: sgResolver, + certDiscovery: certDiscovery, + authConfigBuilder: authConfigBuilder, + enhancedBackendBuilder: enhancedBackendBuilder, + ruleOptimizer: ruleOptimizer, + trackingProvider: trackingProvider, + elbv2TaggingManager: elbv2TaggingManager, + featureGates: featureGates, + defaultTags: defaultTags, + externalManagedTags: sets.NewString(externalManagedTags...), + defaultSSLPolicy: defaultSSLPolicy, + defaultTargetType: elbv2model.TargetType(defaultTargetType), + defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), + enableBackendSG: enableBackendSG, + enableManageBackendSGRules: defaultEnableManageBackendSGRules, + disableRestrictedSGRules: disableRestrictedSGRules, + enableIPTargetType: enableIPTargetType, + logger: logger, + metricsCollector: metricsCollector, } } @@ -94,25 +95,26 @@ type defaultModelBuilder struct { vpcID string clusterName string - annotationParser annotations.Parser - subnetsResolver networkingpkg.SubnetsResolver - backendSGProvider networkingpkg.BackendSGProvider - sgResolver networkingpkg.SecurityGroupResolver - certDiscovery CertDiscovery - authConfigBuilder AuthConfigBuilder - enhancedBackendBuilder EnhancedBackendBuilder - ruleOptimizer RuleOptimizer - trackingProvider tracking.Provider - elbv2TaggingManager elbv2deploy.TaggingManager - featureGates config.FeatureGates - defaultTags map[string]string - externalManagedTags sets.String - defaultSSLPolicy string - defaultTargetType elbv2model.TargetType - defaultLoadBalancerScheme elbv2model.LoadBalancerScheme - enableBackendSG bool - disableRestrictedSGRules bool - enableIPTargetType bool + annotationParser annotations.Parser + subnetsResolver networkingpkg.SubnetsResolver + backendSGProvider networkingpkg.BackendSGProvider + sgResolver networkingpkg.SecurityGroupResolver + certDiscovery CertDiscovery + authConfigBuilder AuthConfigBuilder + enhancedBackendBuilder EnhancedBackendBuilder + ruleOptimizer RuleOptimizer + trackingProvider tracking.Provider + elbv2TaggingManager elbv2deploy.TaggingManager + featureGates config.FeatureGates + defaultTags map[string]string + externalManagedTags sets.String + defaultSSLPolicy string + defaultTargetType elbv2model.TargetType + defaultLoadBalancerScheme elbv2model.LoadBalancerScheme + enableBackendSG bool + enableManageBackendSGRules bool + disableRestrictedSGRules bool + enableIPTargetType bool logger logr.Logger metricsCollector lbcmetrics.MetricCollector @@ -122,28 +124,29 @@ type defaultModelBuilder struct { func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, error) { stack := core.NewDefaultStack(core.StackID(ingGroup.ID)) task := &defaultModelBuildTask{ - k8sClient: b.k8sClient, - eventRecorder: b.eventRecorder, - ec2Client: b.ec2Client, - elbv2Client: b.elbv2Client, - vpcID: b.vpcID, - clusterName: b.clusterName, - annotationParser: b.annotationParser, - subnetsResolver: b.subnetsResolver, - certDiscovery: b.certDiscovery, - authConfigBuilder: b.authConfigBuilder, - enhancedBackendBuilder: b.enhancedBackendBuilder, - ruleOptimizer: b.ruleOptimizer, - trackingProvider: b.trackingProvider, - elbv2TaggingManager: b.elbv2TaggingManager, - featureGates: b.featureGates, - backendSGProvider: b.backendSGProvider, - sgResolver: b.sgResolver, - logger: b.logger, - enableBackendSG: b.enableBackendSG, - disableRestrictedSGRules: b.disableRestrictedSGRules, - enableIPTargetType: b.enableIPTargetType, - metricsCollector: b.metricsCollector, + k8sClient: b.k8sClient, + eventRecorder: b.eventRecorder, + ec2Client: b.ec2Client, + elbv2Client: b.elbv2Client, + vpcID: b.vpcID, + clusterName: b.clusterName, + annotationParser: b.annotationParser, + subnetsResolver: b.subnetsResolver, + certDiscovery: b.certDiscovery, + authConfigBuilder: b.authConfigBuilder, + enhancedBackendBuilder: b.enhancedBackendBuilder, + ruleOptimizer: b.ruleOptimizer, + trackingProvider: b.trackingProvider, + elbv2TaggingManager: b.elbv2TaggingManager, + featureGates: b.featureGates, + backendSGProvider: b.backendSGProvider, + sgResolver: b.sgResolver, + logger: b.logger, + enableBackendSG: b.enableBackendSG, + enableManageBackendSGRules: b.enableManageBackendSGRules, + disableRestrictedSGRules: b.disableRestrictedSGRules, + enableIPTargetType: b.enableIPTargetType, + metricsCollector: b.metricsCollector, ingGroup: ingGroup, stack: stack, @@ -196,14 +199,15 @@ type defaultModelBuildTask struct { featureGates config.FeatureGates logger logr.Logger - ingGroup Group - sslRedirectConfig *SSLRedirectConfig - stack core.Stack - backendSGIDToken core.StringToken - backendSGAllocated bool - enableBackendSG bool - disableRestrictedSGRules bool - enableIPTargetType bool + ingGroup Group + sslRedirectConfig *SSLRedirectConfig + stack core.Stack + backendSGIDToken core.StringToken + backendSGAllocated bool + enableBackendSG bool + enableManageBackendSGRules bool + disableRestrictedSGRules bool + enableIPTargetType bool defaultTags map[string]string externalManagedTags sets.String @@ -451,7 +455,10 @@ func (t *defaultModelBuildTask) getDeletionProtectionViaAnnotation(ing *networki func (t *defaultModelBuildTask) buildManageSecurityGroupRulesFlag(_ context.Context) (bool, error) { explicitManageSGRulesFlag := make(map[bool]struct{}) - manageSGRules := false + // default value from cli flag + manageSGRules := t.enableManageBackendSGRules + + // check annotation, annotation has a higher priority than cli flag for _, member := range t.ingGroup.Members { rawManageSGRule := false exists, err := t.annotationParser.ParseBoolAnnotation(annotations.IngressSuffixManageSecurityGroupRules, &rawManageSGRule, member.Ing.Annotations) @@ -460,11 +467,15 @@ func (t *defaultModelBuildTask) buildManageSecurityGroupRulesFlag(_ context.Cont } if exists { explicitManageSGRulesFlag[rawManageSGRule] = struct{}{} - manageSGRules = rawManageSGRule + if rawManageSGRule != manageSGRules { + manageSGRules = rawManageSGRule + t.logger.V(1).Info("Override enable manage backend security group rules flag with annotation", "value: ", rawManageSGRule, "for ingress", k8s.NamespacedName(member.Ing).String(), "in ingress yaml file") + } } } + // if annotation is not specified, take value from cli flag if len(explicitManageSGRulesFlag) == 0 { - return false, nil + return t.enableManageBackendSGRules, nil } if len(explicitManageSGRulesFlag) > 1 { return false, errors.New("conflicting manage backend security group rules settings") diff --git a/pkg/ingress/model_builder_test.go b/pkg/ingress/model_builder_test.go index 51f5706570..0c27e4743c 100644 --- a/pkg/ingress/model_builder_test.go +++ b/pkg/ingress/model_builder_test.go @@ -4621,3 +4621,107 @@ func Test_defaultModelBuildTask_buildSSLRedirectConfig(t *testing.T) { }) } } + +func Test_defaultModelBuildTask_buildManageSecurityGroupRulesFlag(t *testing.T) { + type fields struct { + ingGroup Group + enableManageBackendSGRules bool + enableBackendSGRules bool + } + + tests := []struct { + name string + fields fields + want bool + wantError string + }{ + { + name: "with enableManageBackendSGRules and manage-backend-security-group-rules annotation", + fields: fields{ + ingGroup: Group{ + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + }, + }, + }, + enableManageBackendSGRules: true, + }, + want: true, + wantError: "", + }, + { + name: "with no annotation - should take value from cli flag", + fields: fields{ + ingGroup: Group{ + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + }, + }, + }, + enableManageBackendSGRules: true, + }, + want: true, + wantError: "", + }, + { + name: "with multiple ingress group have conflict value", + fields: fields{ + ingGroup: Group{ + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "false", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + }, + }, + }, + enableManageBackendSGRules: true, + }, + wantError: "conflicting manage backend security group rules settings", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + task := &defaultModelBuildTask{ + annotationParser: annotationParser, + ingGroup: tt.fields.ingGroup, + enableManageBackendSGRules: tt.fields.enableManageBackendSGRules, + } + got, err := task.buildManageSecurityGroupRulesFlag(context.Background()) + if tt.wantError != "" { + assert.EqualError(t, err, tt.wantError) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } + +} diff --git a/pkg/service/model_build_load_balancer.go b/pkg/service/model_build_load_balancer.go index c97a9a8d09..139f2b8233 100644 --- a/pkg/service/model_build_load_balancer.go +++ b/pkg/service/model_build_load_balancer.go @@ -169,15 +169,20 @@ func (t *defaultModelBuildTask) buildLoadBalancerSecurityGroups(ctx context.Cont } func (t *defaultModelBuildTask) buildManageSecurityGroupRulesFlag(ctx context.Context) (bool, error) { + manageSGRules := t.enableManageBackendSGRules + var rawEnabled bool exists, err := t.annotationParser.ParseBoolAnnotation(annotations.SvcLBSuffixManageSGRules, &rawEnabled, t.service.Annotations) if err != nil { return false, err } if exists { - return rawEnabled, nil + if rawEnabled != manageSGRules { + manageSGRules = rawEnabled + t.logger.V(1).Info("Override enable manage backend security group rules flag with annotation", "value: ", rawEnabled, "for service", k8s.NamespacedName(t.service).String(), "in service yaml file") + } } - return false, nil + return manageSGRules, nil } func (t *defaultModelBuildTask) buildLoadBalancerIPAddressType(_ context.Context) (elbv2model.IPAddressType, error) { diff --git a/pkg/service/model_build_load_balancer_test.go b/pkg/service/model_build_load_balancer_test.go index 4e681c6db3..02a6cdfbb0 100644 --- a/pkg/service/model_build_load_balancer_test.go +++ b/pkg/service/model_build_load_balancer_test.go @@ -1879,3 +1879,83 @@ func Test_defaultModelBuilderTask_buildLbCapacity(t *testing.T) { }) } } + +func Test_defaultModelBuildTask_buildManageSecurityGroupRulesFlag(t *testing.T) { + tests := []struct { + name string + enableManageBackendSGRules bool + annotations map[string]string + wantManageSGRules bool + wantErr bool + }{ + { + name: "with no annotation and enableManageBackendSGRules=false - expect enable manage security group rules to be false", + enableManageBackendSGRules: false, + annotations: map[string]string{}, + wantManageSGRules: false, + wantErr: false, + }, + { + name: "with no annotation and enableManageBackendSGRules=true - expect enable manage security group rules to be true", + enableManageBackendSGRules: true, + annotations: map[string]string{}, + wantManageSGRules: true, + wantErr: false, + }, + { + name: "with annotation true and enableManageBackendSGRules=false - expect override and enable manage security group rules to be true", + enableManageBackendSGRules: false, + annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules": "true", + }, + wantManageSGRules: true, + wantErr: false, + }, + { + name: "with annotation false and enableManageBackendSGRules=true - expect override and enable manage security group rules to be false", + enableManageBackendSGRules: true, + annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules": "false", + }, + wantManageSGRules: false, + wantErr: false, + }, + { + name: "with invalid annotation - expect error", + enableManageBackendSGRules: false, + annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules": "invalid", + }, + wantManageSGRules: false, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + annotationParser := annotations.NewSuffixAnnotationParser("service.beta.kubernetes.io") + + task := &defaultModelBuildTask{ + enableManageBackendSGRules: tt.enableManageBackendSGRules, + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: tt.annotations, + }, + }, + annotationParser: annotationParser, + } + + got, err := task.buildManageSecurityGroupRulesFlag(context.Background()) + if (err != nil) != tt.wantErr { + t.Errorf("buildManageSecurityGroupRulesFlag() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.wantManageSGRules { + t.Errorf("buildManageSecurityGroupRulesFlag() got = %v, want %v", got, tt.wantManageSGRules) + } + }) + } +} diff --git a/pkg/service/model_builder.go b/pkg/service/model_builder.go index 028a723eda..759a957554 100644 --- a/pkg/service/model_builder.go +++ b/pkg/service/model_builder.go @@ -44,49 +44,51 @@ func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver vpcInfoProvider networking.VPCInfoProvider, vpcID string, trackingProvider tracking.Provider, elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string, externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, enableIPTargetType bool, serviceUtils ServiceUtils, - backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool, + backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool, defaultEnableManageBackendSGRules bool, disableRestrictedSGRules bool, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector) *defaultModelBuilder { return &defaultModelBuilder{ - annotationParser: annotationParser, - subnetsResolver: subnetsResolver, - vpcInfoProvider: vpcInfoProvider, - trackingProvider: trackingProvider, - elbv2TaggingManager: elbv2TaggingManager, - featureGates: featureGates, - serviceUtils: serviceUtils, - clusterName: clusterName, - vpcID: vpcID, - defaultTags: defaultTags, - externalManagedTags: sets.NewString(externalManagedTags...), - defaultSSLPolicy: defaultSSLPolicy, - defaultTargetType: elbv2model.TargetType(defaultTargetType), - defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), - enableIPTargetType: enableIPTargetType, - backendSGProvider: backendSGProvider, - sgResolver: sgResolver, - ec2Client: ec2Client, - enableBackendSG: enableBackendSG, - disableRestrictedSGRules: disableRestrictedSGRules, - logger: logger, - metricsCollector: metricsCollector, + annotationParser: annotationParser, + subnetsResolver: subnetsResolver, + vpcInfoProvider: vpcInfoProvider, + trackingProvider: trackingProvider, + elbv2TaggingManager: elbv2TaggingManager, + featureGates: featureGates, + serviceUtils: serviceUtils, + clusterName: clusterName, + vpcID: vpcID, + defaultTags: defaultTags, + externalManagedTags: sets.NewString(externalManagedTags...), + defaultSSLPolicy: defaultSSLPolicy, + defaultTargetType: elbv2model.TargetType(defaultTargetType), + defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), + enableIPTargetType: enableIPTargetType, + backendSGProvider: backendSGProvider, + sgResolver: sgResolver, + ec2Client: ec2Client, + enableBackendSG: enableBackendSG, + enableManageBackendSGRules: defaultEnableManageBackendSGRules, + disableRestrictedSGRules: disableRestrictedSGRules, + logger: logger, + metricsCollector: metricsCollector, } } var _ ModelBuilder = &defaultModelBuilder{} type defaultModelBuilder struct { - annotationParser annotations.Parser - subnetsResolver networking.SubnetsResolver - vpcInfoProvider networking.VPCInfoProvider - backendSGProvider networking.BackendSGProvider - sgResolver networking.SecurityGroupResolver - trackingProvider tracking.Provider - elbv2TaggingManager elbv2deploy.TaggingManager - featureGates config.FeatureGates - serviceUtils ServiceUtils - ec2Client services.EC2 - enableBackendSG bool - disableRestrictedSGRules bool + annotationParser annotations.Parser + subnetsResolver networking.SubnetsResolver + vpcInfoProvider networking.VPCInfoProvider + backendSGProvider networking.BackendSGProvider + sgResolver networking.SecurityGroupResolver + trackingProvider tracking.Provider + elbv2TaggingManager elbv2deploy.TaggingManager + featureGates config.FeatureGates + serviceUtils ServiceUtils + ec2Client services.EC2 + enableBackendSG bool + enableManageBackendSGRules bool + disableRestrictedSGRules bool clusterName string vpcID string @@ -103,23 +105,24 @@ type defaultModelBuilder struct { func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, bool, error) { stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(service))) task := &defaultModelBuildTask{ - clusterName: b.clusterName, - vpcID: b.vpcID, - annotationParser: b.annotationParser, - subnetsResolver: b.subnetsResolver, - backendSGProvider: b.backendSGProvider, - sgResolver: b.sgResolver, - vpcInfoProvider: b.vpcInfoProvider, - trackingProvider: b.trackingProvider, - elbv2TaggingManager: b.elbv2TaggingManager, - featureGates: b.featureGates, - serviceUtils: b.serviceUtils, - enableIPTargetType: b.enableIPTargetType, - ec2Client: b.ec2Client, - enableBackendSG: b.enableBackendSG, - disableRestrictedSGRules: b.disableRestrictedSGRules, - logger: b.logger, - metricsCollector: b.metricsCollector, + clusterName: b.clusterName, + vpcID: b.vpcID, + annotationParser: b.annotationParser, + subnetsResolver: b.subnetsResolver, + backendSGProvider: b.backendSGProvider, + sgResolver: b.sgResolver, + vpcInfoProvider: b.vpcInfoProvider, + trackingProvider: b.trackingProvider, + elbv2TaggingManager: b.elbv2TaggingManager, + featureGates: b.featureGates, + serviceUtils: b.serviceUtils, + enableIPTargetType: b.enableIPTargetType, + ec2Client: b.ec2Client, + enableBackendSG: b.enableBackendSG, + enableManageBackendSGRules: b.enableManageBackendSGRules, + disableRestrictedSGRules: b.disableRestrictedSGRules, + logger: b.logger, + metricsCollector: b.metricsCollector, service: service, stack: stack, @@ -163,21 +166,22 @@ func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service } type defaultModelBuildTask struct { - clusterName string - vpcID string - annotationParser annotations.Parser - subnetsResolver networking.SubnetsResolver - vpcInfoProvider networking.VPCInfoProvider - backendSGProvider networking.BackendSGProvider - sgResolver networking.SecurityGroupResolver - trackingProvider tracking.Provider - elbv2TaggingManager elbv2deploy.TaggingManager - featureGates config.FeatureGates - serviceUtils ServiceUtils - enableIPTargetType bool - ec2Client services.EC2 - logger logr.Logger - metricsCollector lbcmetrics.MetricCollector + clusterName string + vpcID string + annotationParser annotations.Parser + subnetsResolver networking.SubnetsResolver + vpcInfoProvider networking.VPCInfoProvider + backendSGProvider networking.BackendSGProvider + sgResolver networking.SecurityGroupResolver + trackingProvider tracking.Provider + elbv2TaggingManager elbv2deploy.TaggingManager + featureGates config.FeatureGates + serviceUtils ServiceUtils + enableIPTargetType bool + enableManageBackendSGRules bool + ec2Client services.EC2 + logger logr.Logger + metricsCollector lbcmetrics.MetricCollector service *corev1.Service diff --git a/pkg/service/model_builder_test.go b/pkg/service/model_builder_test.go index 0e865bd618..0bc9c05f16 100644 --- a/pkg/service/model_builder_test.go +++ b/pkg/service/model_builder_test.go @@ -118,6 +118,7 @@ func Test_defaultModelBuilderTask_Build(t *testing.T) { resolveSGViaNameOrIDCall []resolveSGViaNameOrIDCall backendSecurityGroup string enableBackendSG bool + enableManageBackendSGRules bool disableRestrictedSGRules bool svc *corev1.Service wantError bool @@ -6663,7 +6664,7 @@ func Test_defaultModelBuilderTask_Build(t *testing.T) { mockMetricsCollector := lbcmetrics.NewMockCollector() builder := NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, "vpc-xxx", trackingProvider, elbv2TaggingManager, ec2Client, featureGates, "my-cluster", nil, nil, "ELBSecurityPolicy-2016-08", defaultTargetType, defaultLoadBalancerScheme, enableIPTargetType, serviceUtils, - backendSGProvider, sgResolver, tt.enableBackendSG, tt.disableRestrictedSGRules, logr.New(&log.NullLogSink{}), mockMetricsCollector) + backendSGProvider, sgResolver, tt.enableBackendSG, tt.enableManageBackendSGRules, tt.disableRestrictedSGRules, logr.New(&log.NullLogSink{}), mockMetricsCollector) ctx := context.Background() stack, _, _, err := builder.Build(ctx, tt.svc, mockMetricsCollector) if tt.wantError { From 6b8c78a83c7ad51140ba9b02f06860315bcbf7cd Mon Sep 17 00:00:00 2001 From: Leah Tucker <57272433+tucktuck9@users.noreply.github.com> Date: Tue, 1 Apr 2025 11:10:20 -0600 Subject: [PATCH 08/40] Update security_groups.md Updates the Backend Security Groups configuration documentation to clarify important requirements for using Custom Shared Backend Security Groups. --- docs/deploy/security_groups.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/deploy/security_groups.md b/docs/deploy/security_groups.md index 7c36985c0b..903bdc4966 100644 --- a/docs/deploy/security_groups.md +++ b/docs/deploy/security_groups.md @@ -38,10 +38,14 @@ Backend Security Groups control traffic between AWS Load Balancers and their tar **Enable or Disable:** Use `--enable-backend-security-group` (default `true`) to enable/disable the shared backend security group. -You can turn off the shared backend security group feature by setting it to `false`. However, if you have a high number of Ingress resources with frontend security groups auto-generated by the controller, you might run into security group rule limits on the instance/ENI security groups. +Note that while you can turn off the shared backend security group feature by setting it to `false`, if you have a high number of Ingress resources with frontend security groups auto-generated by the controller, you might run into security group rule limits on the instance/ENI security groups. **Specification:** Use `--backend-security-group` to pass in a security group ID to use as a custom shared backend security group. +**Important Notes:** +* The Custom Shared Backend Security Group (`--backend-security-group` option) only works when the automatic addition of Inbound Rules to the Node/ENI Security Group is enabled. +* If a Custom Frontend Security Group is configured, you must set the annotation `service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules: "true"` for the Custom Shared Backend Security Group to work correctly. + If `--backend-security-group` is left empty, a security group with the following attributes will be created: ```yaml From 93a600d29b67d4d744423cf13219d59895c0d506 Mon Sep 17 00:00:00 2001 From: Leah Tucker <57272433+tucktuck9@users.noreply.github.com> Date: Tue, 1 Apr 2025 10:47:03 -0600 Subject: [PATCH 09/40] Update configurations.md We received feedback about this page, expressing the need to clarify the controller configuration options content and to direct users to the actual supported Helm chart options. --- docs/deploy/configurations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deploy/configurations.md b/docs/deploy/configurations.md index 1c31f4a140..5722fc5f56 100644 --- a/docs/deploy/configurations.md +++ b/docs/deploy/configurations.md @@ -1,5 +1,5 @@ # Controller configuration options -This document covers configuration of the AWS Load Balancer controller +This document primarily covers the runtime configuration options for the AWS Load Balancer Controller. For installation-specific configuration options, see the [Helm chart values documentation](https://github.com/aws/eks-charts/tree/master/stable/aws-load-balancer-controller#configuration). !!!warning "limitation" The v2.0.0+ version of AWSLoadBalancerController currently only support one controller deployment(with one or multiple replicas) per cluster. @@ -183,4 +183,4 @@ There are a set of key=value pairs that describe AWS load balancer controller fe | NLBHealthCheckAdvancedConfiguration | string | true | Enable or disable advanced health check configuration for NLB, for example health check timeout | | ALBSingleSubnet | string | false | If enabled, controller will allow using only 1 subnet for provisioning ALB, which need to get whitelisted by ELB in advance | | NLBSecurityGroup | string | true | Enable or disable all NLB security groups actions including frontend sg creation, backend sg creation, and backend sg modifications | -| LBCapacityReservation | string | true | Enable or disable the capacity reservation feature on ALB and NLB \ No newline at end of file +| LBCapacityReservation | string | true | Enable or disable the capacity reservation feature on ALB and NLB From 7cba6b9319651e7deb33b1aa80d3d0c532a3ebf7 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Tue, 8 Apr 2025 16:37:52 -0700 Subject: [PATCH 10/40] [feat: gw api] subnet discovery that works for both ALB / NLB --- .../v1beta1/loadbalancerconfig_types.go | 35 +- apis/gateway/v1beta1/zz_generated.deepcopy.go | 34 +- config/crd/gateway/gateway-crds.yaml | 48 +- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 48 +- controllers/gateway/gateway_controller.go | 11 +- main.go | 8 +- pkg/gateway/model/base_model_builder.go | 88 +++ pkg/gateway/model/constants.go | 5 + pkg/gateway/model/model_build_loadbalancer.go | 97 +++ pkg/gateway/model/model_build_subnet.go | 221 ++++++ pkg/gateway/model/model_build_subnet_test.go | 663 ++++++++++++++++++ pkg/gateway/model/model_builder.go | 81 --- pkg/gateway/model/subnet/eip_mutator.go | 34 + pkg/gateway/model/subnet/eip_mutator_test.go | 104 +++ pkg/gateway/model/subnet/ipv6_mutator.go | 65 ++ pkg/gateway/model/subnet/ipv6_mutator_test.go | 214 ++++++ pkg/gateway/model/subnet/mutator.go | 11 + .../model/subnet/private_ipv4_mutator.go | 63 ++ .../model/subnet/private_ipv4_mutator_test.go | 214 ++++++ .../model/subnet/source_nat_mutator.go | 40 ++ .../model/subnet/source_nat_mutator_test.go | 119 ++++ pkg/networking/utils.go | 66 +- 22 files changed, 2085 insertions(+), 184 deletions(-) create mode 100644 pkg/gateway/model/base_model_builder.go create mode 100644 pkg/gateway/model/constants.go create mode 100644 pkg/gateway/model/model_build_loadbalancer.go create mode 100644 pkg/gateway/model/model_build_subnet.go create mode 100644 pkg/gateway/model/model_build_subnet_test.go delete mode 100644 pkg/gateway/model/model_builder.go create mode 100644 pkg/gateway/model/subnet/eip_mutator.go create mode 100644 pkg/gateway/model/subnet/eip_mutator_test.go create mode 100644 pkg/gateway/model/subnet/ipv6_mutator.go create mode 100644 pkg/gateway/model/subnet/ipv6_mutator_test.go create mode 100644 pkg/gateway/model/subnet/mutator.go create mode 100644 pkg/gateway/model/subnet/private_ipv4_mutator.go create mode 100644 pkg/gateway/model/subnet/private_ipv4_mutator_test.go create mode 100644 pkg/gateway/model/subnet/source_nat_mutator.go create mode 100644 pkg/gateway/model/subnet/source_nat_mutator_test.go diff --git a/apis/gateway/v1beta1/loadbalancerconfig_types.go b/apis/gateway/v1beta1/loadbalancerconfig_types.go index 15e658e6ea..076561bc59 100644 --- a/apis/gateway/v1beta1/loadbalancerconfig_types.go +++ b/apis/gateway/v1beta1/loadbalancerconfig_types.go @@ -26,15 +26,6 @@ const ( LoadBalancerIpAddressTypeDualstackWithoutPublicIpv4 LoadBalancerIpAddressType = "dualstack-without-public-ipv4" ) -// +kubebuilder:validation:Enum=on;off -// EnablePrefixForIpv6SourceNatEnum defines the enum values for EnablePrefixForIpv6SourceNat -type EnablePrefixForIpv6SourceNatEnum string - -const ( - EnablePrefixForIpv6SourceNatEnumOn EnablePrefixForIpv6SourceNatEnum = "on" - EnablePrefixForIpv6SourceNatEnumOff EnablePrefixForIpv6SourceNatEnum = "off" -) - // LoadBalancerAttribute defines LB attribute. type LoadBalancerAttribute struct { // The key of the attribute. @@ -64,20 +55,25 @@ type LoadBalancerTag struct { // SubnetConfiguration defines the subnet settings for a Load Balancer. type SubnetConfiguration struct { - // identifier name or id for the subnet + // identifier [Application LoadBalancer / Network LoadBalancer] name or id for the subnet + // +optional Identifier string `json:"identifier"` - // eipAllocation the EIP name for this subnet. + // eipAllocation [Network LoadBalancer] the EIP name for this subnet. // +optional EIPAllocation *string `json:"eipAllocation,omitempty"` - // privateIPv4Allocation the private ipv4 address to assign to this subnet. + // privateIPv4Allocation [Network LoadBalancer] the private ipv4 address to assign to this subnet. // +optional PrivateIPv4Allocation *string `json:"privateIPv4Allocation,omitempty"` - // privateIPv6Allocation the private ipv6 address to assign to this subnet. + // IPv6Allocation [Network LoadBalancer] the ipv6 address to assign to this subnet. // +optional - PrivateIPv6Allocation *string `json:"privateIPv6Allocation,omitempty"` + IPv6Allocation *string `json:"ipv6Allocation,omitempty"` + + // SourceNatIPv6Prefix [Network LoadBalancer] The IPv6 prefix to use for source NAT. Specify an IPv6 prefix (/80 netmask) from the subnet CIDR block or auto_assigned to use an IPv6 prefix selected at random from the subnet CIDR block. + // +optional + SourceNatIPv6Prefix *string `json:"sourceNatIPv6Prefix,omitempty"` } // +kubebuilder:validation:Enum=HTTP1Only;HTTP2Only;HTTP2Optional;HTTP2Preferred;None @@ -183,10 +179,6 @@ type LoadBalancerConfigurationSpec struct { // +optional IpAddressType *LoadBalancerIpAddressType `json:"ipAddressType,omitempty"` - // enablePrefixForIpv6SourceNat indicates whether to use an IPv6 prefix from each subnet for source NAT for Network Load Balancers with UDP listeners. - // +optional - EnablePrefixForIpv6SourceNat *EnablePrefixForIpv6SourceNatEnum `json:"enablePrefixForIpv6SourceNat,omitempty"` - // enforceSecurityGroupInboundRulesOnPrivateLinkTraffic Indicates whether to evaluate inbound security group rules for traffic sent to a Network Load Balancer through Amazon Web Services PrivateLink. // +optional EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic *string `json:"enforceSecurityGroupInboundRulesOnPrivateLinkTraffic,omitempty"` @@ -196,9 +188,16 @@ type LoadBalancerConfigurationSpec struct { CustomerOwnedIpv4Pool *string `json:"customerOwnedIpv4Pool,omitempty"` // loadBalancerSubnets is an optional list of subnet configurations to be used in the LB + // This value takes precedence over loadBalancerSubnetsSelector if both are selected. // +optional LoadBalancerSubnets *[]SubnetConfiguration `json:"loadBalancerSubnets,omitempty"` + // LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each + // tag specified in the map key contains one of the values in the corresponding + // value list. + // +optional + LoadBalancerSubnetsSelector *map[string][]string `json:"loadBalancerSubnetsSelector,omitempty"` + // listenerConfigurations is an optional list of configurations for each listener on LB // +optional ListenerConfigurations *[]ListenerConfiguration `json:"listenerConfigurations,omitempty"` diff --git a/apis/gateway/v1beta1/zz_generated.deepcopy.go b/apis/gateway/v1beta1/zz_generated.deepcopy.go index cc5daa8c7b..41bb66a02f 100644 --- a/apis/gateway/v1beta1/zz_generated.deepcopy.go +++ b/apis/gateway/v1beta1/zz_generated.deepcopy.go @@ -263,11 +263,6 @@ func (in *LoadBalancerConfigurationSpec) DeepCopyInto(out *LoadBalancerConfigura *out = new(LoadBalancerIpAddressType) **out = **in } - if in.EnablePrefixForIpv6SourceNat != nil { - in, out := &in.EnablePrefixForIpv6SourceNat, &out.EnablePrefixForIpv6SourceNat - *out = new(EnablePrefixForIpv6SourceNatEnum) - **out = **in - } if in.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic != nil { in, out := &in.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic, &out.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic *out = new(string) @@ -289,6 +284,26 @@ func (in *LoadBalancerConfigurationSpec) DeepCopyInto(out *LoadBalancerConfigura } } } + if in.LoadBalancerSubnetsSelector != nil { + in, out := &in.LoadBalancerSubnetsSelector, &out.LoadBalancerSubnetsSelector + *out = new(map[string][]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + } if in.ListenerConfigurations != nil { in, out := &in.ListenerConfigurations, &out.ListenerConfigurations *out = new([]ListenerConfiguration) @@ -478,8 +493,13 @@ func (in *SubnetConfiguration) DeepCopyInto(out *SubnetConfiguration) { *out = new(string) **out = **in } - if in.PrivateIPv6Allocation != nil { - in, out := &in.PrivateIPv6Allocation, &out.PrivateIPv6Allocation + if in.IPv6Allocation != nil { + in, out := &in.IPv6Allocation, &out.IPv6Allocation + *out = new(string) + **out = **in + } + if in.SourceNatIPv6Prefix != nil { + in, out := &in.SourceNatIPv6Prefix, &out.SourceNatIPv6Prefix *out = new(string) **out = **in } diff --git a/config/crd/gateway/gateway-crds.yaml b/config/crd/gateway/gateway-crds.yaml index 41df93daf0..735b106b5e 100644 --- a/config/crd/gateway/gateway-crds.yaml +++ b/config/crd/gateway/gateway-crds.yaml @@ -48,14 +48,6 @@ spec: description: customerOwnedIpv4Pool is the ID of the customer-owned address for Application Load Balancers on Outposts pool. type: string - enablePrefixForIpv6SourceNat: - description: enablePrefixForIpv6SourceNat indicates whether to use - an IPv6 prefix from each subnet for source NAT for Network Load - Balancers with UDP listeners. - enum: - - "on" - - "off" - type: string enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic Indicates whether to evaluate inbound security group rules for traffic @@ -178,30 +170,48 @@ spec: minLength: 1 type: string loadBalancerSubnets: - description: loadBalancerSubnets is an optional list of subnet configurations - to be used in the LB + description: |- + loadBalancerSubnets is an optional list of subnet configurations to be used in the LB + This value takes precedence over loadBalancerSubnetsSelector if both are selected. items: description: SubnetConfiguration defines the subnet settings for a Load Balancer. properties: eipAllocation: - description: eipAllocation the EIP name for this subnet. + description: eipAllocation [Network LoadBalancer] the EIP name + for this subnet. type: string identifier: - description: identifier name or id for the subnet + description: identifier [Application LoadBalancer / Network + LoadBalancer] name or id for the subnet + type: string + ipv6Allocation: + description: IPv6Allocation [Network LoadBalancer] the ipv6 + address to assign to this subnet. type: string privateIPv4Allocation: - description: privateIPv4Allocation the private ipv4 address - to assign to this subnet. + description: privateIPv4Allocation [Network LoadBalancer] the + private ipv4 address to assign to this subnet. type: string - privateIPv6Allocation: - description: privateIPv6Allocation the private ipv6 address - to assign to this subnet. + sourceNatIPv6Prefix: + description: SourceNatIPv6Prefix [Network LoadBalancer] The + IPv6 prefix to use for source NAT. Specify an IPv6 prefix + (/80 netmask) from the subnet CIDR block or auto_assigned + to use an IPv6 prefix selected at random from the subnet CIDR + block. type: string - required: - - identifier type: object type: array + loadBalancerSubnetsSelector: + additionalProperties: + items: + type: string + type: array + description: |- + LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each + tag specified in the map key contains one of the values in the corresponding + value list. + type: object scheme: description: scheme defines the type of LB to provision. If unspecified, it will be automatically inferred. diff --git a/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml index 01dc98c189..c858e905ce 100644 --- a/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ b/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -49,14 +49,6 @@ spec: description: customerOwnedIpv4Pool is the ID of the customer-owned address for Application Load Balancers on Outposts pool. type: string - enablePrefixForIpv6SourceNat: - description: enablePrefixForIpv6SourceNat indicates whether to use - an IPv6 prefix from each subnet for source NAT for Network Load - Balancers with UDP listeners. - enum: - - "on" - - "off" - type: string enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic Indicates whether to evaluate inbound security group rules for traffic @@ -179,30 +171,48 @@ spec: minLength: 1 type: string loadBalancerSubnets: - description: loadBalancerSubnets is an optional list of subnet configurations - to be used in the LB + description: |- + loadBalancerSubnets is an optional list of subnet configurations to be used in the LB + This value takes precedence over loadBalancerSubnetsSelector if both are selected. items: description: SubnetConfiguration defines the subnet settings for a Load Balancer. properties: eipAllocation: - description: eipAllocation the EIP name for this subnet. + description: eipAllocation [Network LoadBalancer] the EIP name + for this subnet. type: string identifier: - description: identifier name or id for the subnet + description: identifier [Application LoadBalancer / Network + LoadBalancer] name or id for the subnet + type: string + ipv6Allocation: + description: IPv6Allocation [Network LoadBalancer] the ipv6 + address to assign to this subnet. type: string privateIPv4Allocation: - description: privateIPv4Allocation the private ipv4 address - to assign to this subnet. + description: privateIPv4Allocation [Network LoadBalancer] the + private ipv4 address to assign to this subnet. type: string - privateIPv6Allocation: - description: privateIPv6Allocation the private ipv6 address - to assign to this subnet. + sourceNatIPv6Prefix: + description: SourceNatIPv6Prefix [Network LoadBalancer] The + IPv6 prefix to use for source NAT. Specify an IPv6 prefix + (/80 netmask) from the subnet CIDR block or auto_assigned + to use an IPv6 prefix selected at random from the subnet CIDR + block. type: string - required: - - identifier type: object type: array + loadBalancerSubnetsSelector: + additionalProperties: + items: + type: string + type: array + description: |- + LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each + tag specified in the map key contains one of the values in the corresponding + value list. + type: object scheme: description: scheme defines the type of LB to provision. If unspecified, it will be automatically inferred. diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 9349f86be0..21d4dba8a9 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy" @@ -39,19 +40,19 @@ var _ Reconciler = &gatewayReconciler{} // NewNLBGatewayReconciler constructs a gateway reconciler to handle specifically for NLB gateways func NewNLBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { - return newGatewayReconciler(NLBGatewayController, controllerConfig.NLBGatewayMaxConcurrentReconciles, NLBGatewayTagPrefix, NLBGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementNLBGateway) + return newGatewayReconciler(NLBGatewayController, elbv2model.LoadBalancerTypeNetwork, controllerConfig.NLBGatewayMaxConcurrentReconciles, NLBGatewayTagPrefix, NLBGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementNLBGateway) } // NewALBGatewayReconciler constructs a gateway reconciler to handle specifically for ALB gateways func NewALBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { - return newGatewayReconciler(ALBGatewayController, controllerConfig.ALBGatewayMaxConcurrentReconciles, ALBGatewayTagPrefix, ALBGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementALBGateway) + return newGatewayReconciler(ALBGatewayController, elbv2model.LoadBalancerTypeApplication, controllerConfig.ALBGatewayMaxConcurrentReconciles, ALBGatewayTagPrefix, ALBGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementALBGateway) } // newGatewayReconciler constructs a reconciler that responds to gateway object changes -func newGatewayReconciler(controllerName string, maxConcurrentReconciles int, gatewayTagPrefix string, finalizer string, routeLoader routeutils.Loader, routeFilter routeutils.LoadRouteFilter, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileTracker func(namespaceName types.NamespacedName)) Reconciler { +func newGatewayReconciler(controllerName string, lbType elbv2model.LoadBalancerType, maxConcurrentReconciles int, gatewayTagPrefix string, finalizer string, routeLoader routeutils.Loader, routeFilter routeutils.LoadRouteFilter, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileTracker func(namespaceName types.NamespacedName)) Reconciler { trackingProvider := tracking.NewDefaultProvider(gatewayTagPrefix, controllerConfig.ClusterName) - modelBuilder := gatewaymodel.NewDefaultModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider, elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger) + modelBuilder := gatewaymodel.NewModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), lbType, trackingProvider, elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName) @@ -222,7 +223,7 @@ func (r *gatewayReconciler) deployModel(ctx context.Context, gw *gwv1.Gateway, s } func (r *gatewayReconciler) buildModel(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass, listenerToRoute map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) { - stack, lb, backendSGRequired, err := r.modelBuilder.Build(ctx, gw, gwClass, listenerToRoute) + stack, lb, backendSGRequired, err := r.modelBuilder.Build(ctx, gw, &elbv2gw.LoadBalancerConfiguration{}, listenerToRoute) if err != nil { r.eventRecorder.Event(gw, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err)) return nil, nil, false, err diff --git a/main.go b/main.go index 0a06bdbaa1..4e30beecb2 100644 --- a/main.go +++ b/main.go @@ -173,9 +173,8 @@ func main() { os.Exit(1) } - var routeLoader routeutils.Loader if controllerCFG.FeatureGates.Enabled(config.NLBGatewayAPI) { - routeLoader = routeutils.NewLoader(mgr.GetClient()) + routeLoader := routeutils.NewLoader(mgr.GetClient()) nlbGatewayReconciler := gateway.NewNLBGatewayReconciler(routeLoader, cloud, mgr.GetClient(), mgr.GetEventRecorderFor("nlbgateway"), controllerCFG, finalizerManager, sgReconciler, sgManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("nlbgateway"), lbcMetricsCollector, reconcileCounters) nlbControllerError := nlbGatewayReconciler.SetupWithManager(mgr) if nlbControllerError != nil { @@ -183,11 +182,8 @@ func main() { os.Exit(1) } } - if controllerCFG.FeatureGates.Enabled(config.ALBGatewayAPI) { - if routeLoader == nil { - routeLoader = routeutils.NewLoader(mgr.GetClient()) - } + routeLoader := routeutils.NewLoader(mgr.GetClient()) albGatewayReconciler := gateway.NewALBGatewayReconciler(routeLoader, cloud, mgr.GetClient(), mgr.GetEventRecorderFor("albgateway"), controllerCFG, finalizerManager, sgReconciler, sgManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("albgateway"), lbcMetricsCollector, reconcileCounters) albControllerErr := albGatewayReconciler.SetupWithManager(mgr) if albControllerErr != nil { diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go new file mode 100644 index 0000000000..83f22b266e --- /dev/null +++ b/pkg/gateway/model/base_model_builder.go @@ -0,0 +1,88 @@ +package model + +import ( + "context" + "github.com/go-logr/logr" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/sets" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strconv" +) + +// Builder builds the model stack for a Gateway resource. +type Builder interface { + // Build model stack for a gateway + Build(ctx context.Context, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) +} + +// NewModelBuilder construct a new baseModelBuilder +func NewModelBuilder(subnetsResolver networking.SubnetsResolver, + vpcInfoProvider networking.VPCInfoProvider, vpcID string, loadBalancerType elbv2model.LoadBalancerType, trackingProvider tracking.Provider, + elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string, + externalManagedTags sets.Set[string], defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, + backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool, + disableRestrictedSGRules bool, logger logr.Logger) Builder { + + subnetBuilder := newSubnetModelBuilder(loadBalancerType, trackingProvider, subnetsResolver, elbv2TaggingManager) + + return &baseModelBuilder{ + lbBuilder: newLoadBalancerBuilder(subnetBuilder, defaultLoadBalancerScheme), + } +} + +var _ Builder = &baseModelBuilder{} + +type baseModelBuilder struct { + lbBuilder loadBalancerBuilder + logger logr.Logger +} + +func (baseBuilder *baseModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) { + if gw.DeletionTimestamp != nil && !gw.DeletionTimestamp.IsZero() { + if baseBuilder.isDeleteProtected(lbConf) { + return nil, nil, false, errors.Errorf("Unable to delete gateway %+v because deletion protection is enabled.", k8s.NamespacedName(gw)) + } + } + + stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(gw))) + + // TODO - Fix + _, err := baseBuilder.lbBuilder.buildLoadBalancerSpec(ctx, gw, stack, lbConf, routes) + + if err != nil { + return nil, nil, false, err + } + + return stack, nil, false, nil +} + +func (baseBuilder *baseModelBuilder) isDeleteProtected(lbConf *elbv2gw.LoadBalancerConfiguration) bool { + if lbConf == nil { + return false + } + + for _, attr := range lbConf.Spec.LoadBalancerAttributes { + if attr.Key == deletionProtectionAttributeKey { + deletionProtectionEnabled, err := strconv.ParseBool(attr.Value) + + if err != nil { + baseBuilder.logger.Error(err, "Unable to parse deletion protection value, assuming false.") + return false + } + + return deletionProtectionEnabled + } + } + + return false +} diff --git a/pkg/gateway/model/constants.go b/pkg/gateway/model/constants.go new file mode 100644 index 0000000000..327cd6f28a --- /dev/null +++ b/pkg/gateway/model/constants.go @@ -0,0 +1,5 @@ +package model + +const ( + deletionProtectionAttributeKey = "deletion_protection.enabled" +) diff --git a/pkg/gateway/model/model_build_loadbalancer.go b/pkg/gateway/model/model_build_loadbalancer.go new file mode 100644 index 0000000000..98945e4d8d --- /dev/null +++ b/pkg/gateway/model/model_build_loadbalancer.go @@ -0,0 +1,97 @@ +package model + +import ( + "context" + "github.com/pkg/errors" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +type loadBalancerBuilder interface { + buildLoadBalancerSpec(ctx context.Context, gw *gwv1.Gateway, stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor) (elbv2model.LoadBalancerSpec, error) +} + +type loadBalancerBuilderImpl struct { + subnetBuilder subnetModelBuilder + + defaultLoadBalancerScheme elbv2model.LoadBalancerScheme + defaultIPType elbv2model.IPAddressType +} + +func newLoadBalancerBuilder(subnetBuilder subnetModelBuilder, defaultLoadBalancerScheme string) loadBalancerBuilder { + + return &loadBalancerBuilderImpl{ + subnetBuilder: subnetBuilder, + defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), + defaultIPType: elbv2model.IPAddressTypeIPV4, + } +} + +func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerSpec(ctx context.Context, gw *gwv1.Gateway, stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor) (elbv2model.LoadBalancerSpec, error) { + scheme, err := lbModelBuilder.buildLoadBalancerScheme(lbConf) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } + ipAddressType, err := lbModelBuilder.buildLoadBalancerIPAddressType(lbConf) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } + configuredSubnets, sourcePrefixEnabled, err := lbModelBuilder.subnetBuilder.buildLoadBalancerSubnets(ctx, lbConf.Spec.LoadBalancerSubnets, lbConf.Spec.LoadBalancerSubnetsSelector, scheme, ipAddressType, stack) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } + + return elbv2model.LoadBalancerSpec{ + Type: elbv2model.LoadBalancerTypeApplication, + Scheme: scheme, + IPAddressType: ipAddressType, + SubnetMappings: configuredSubnets, + EnablePrefixForIpv6SourceNat: lbModelBuilder.translateSourcePrefixEnabled(sourcePrefixEnabled), + }, nil +} + +func (lbModelBuilder *loadBalancerBuilderImpl) translateSourcePrefixEnabled(b bool) elbv2model.EnablePrefixForIpv6SourceNat { + if b { + return elbv2model.EnablePrefixForIpv6SourceNatOn + } + return elbv2model.EnablePrefixForIpv6SourceNatOff + +} + +func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerScheme(lbConf *elbv2gw.LoadBalancerConfiguration) (elbv2model.LoadBalancerScheme, error) { + scheme := lbConf.Spec.Scheme + + if scheme == nil { + return lbModelBuilder.defaultLoadBalancerScheme, nil + } + switch *scheme { + case elbv2gw.LoadBalancerScheme(elbv2model.LoadBalancerSchemeInternetFacing): + return elbv2model.LoadBalancerSchemeInternetFacing, nil + case elbv2gw.LoadBalancerScheme(elbv2model.LoadBalancerSchemeInternal): + return elbv2model.LoadBalancerSchemeInternal, nil + default: + return "", errors.Errorf("unknown scheme: %v", *scheme) + } +} + +// buildLoadBalancerIPAddressType builds the LoadBalancer IPAddressType. +func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerIPAddressType(lbConf *elbv2gw.LoadBalancerConfiguration) (elbv2model.IPAddressType, error) { + + if lbConf.Spec.IpAddressType == nil { + return lbModelBuilder.defaultIPType, nil + } + + switch *lbConf.Spec.IpAddressType { + case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeIPV4): + return elbv2model.IPAddressTypeIPV4, nil + case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeDualStack): + return elbv2model.IPAddressTypeDualStack, nil + case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeDualStackWithoutPublicIPV4): + return elbv2model.IPAddressTypeDualStackWithoutPublicIPV4, nil + default: + return "", errors.Errorf("unknown IPAddressType: %v", *lbConf.Spec.IpAddressType) + } +} diff --git a/pkg/gateway/model/model_build_subnet.go b/pkg/gateway/model/model_build_subnet.go new file mode 100644 index 0000000000..71ffbbb9ac --- /dev/null +++ b/pkg/gateway/model/model_build_subnet.go @@ -0,0 +1,221 @@ +package model + +import ( + "context" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/model/subnet" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" +) + +type subnetModelBuilder interface { + buildLoadBalancerSubnets(ctx context.Context, gwSubnetConfig *[]elbv2gw.SubnetConfiguration, gwSubnetTagSelectors *map[string][]string, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, stack core.Stack) ([]elbv2model.SubnetMapping, bool, error) +} + +type subnetModelBuilderImpl struct { + loadBalancerType elbv2model.LoadBalancerType + subnetMutatorChain []subnet.Mutator + + trackingProvider tracking.Provider + subnetsResolver networking.SubnetsResolver + elbv2TaggingManager elbv2deploy.TaggingManager +} + +func newSubnetModelBuilder(loadBalancerType elbv2model.LoadBalancerType, trackingProvider tracking.Provider, subnetsResolver networking.SubnetsResolver, elbv2TaggingManager elbv2deploy.TaggingManager) subnetModelBuilder { + var subnetMutatorChain []subnet.Mutator + + if loadBalancerType == elbv2model.LoadBalancerTypeNetwork { + subnetMutatorChain = []subnet.Mutator{ + subnet.NewEIPMutator(), + subnet.NewPrivateIPv4Mutator(), + subnet.NewIPv6Mutator(), + subnet.NewSourceNATMutator(), + } + } + + return &subnetModelBuilderImpl{ + loadBalancerType: loadBalancerType, + subnetMutatorChain: subnetMutatorChain, + + trackingProvider: trackingProvider, + subnetsResolver: subnetsResolver, + elbv2TaggingManager: elbv2TaggingManager, + } +} + +func (subnetBuilder *subnetModelBuilderImpl) buildLoadBalancerSubnets(ctx context.Context, gwSubnetConfig *[]elbv2gw.SubnetConfiguration, gwSubnetTagSelectors *map[string][]string, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, stack core.Stack) ([]elbv2model.SubnetMapping, bool, error) { + sourceNATEnabled, err := subnetBuilder.validateSubnetsInput(gwSubnetConfig, scheme, ipAddressType) + + if err != nil { + return nil, false, err + } + + resolvedEC2Subnets, err := subnetBuilder.resolveEC2Subnets(ctx, stack, gwSubnetConfig, gwSubnetTagSelectors, scheme) + + if err != nil { + return nil, false, err + } + + resultPtrs := make([]*elbv2model.SubnetMapping, 0) + + for _, ec2Subnet := range resolvedEC2Subnets { + resultPtrs = append(resultPtrs, &elbv2model.SubnetMapping{ + SubnetID: *ec2Subnet.SubnetId, + }) + } + + var subnetConfig []elbv2gw.SubnetConfiguration + + if gwSubnetConfig != nil { + subnetConfig = *gwSubnetConfig + } + + for _, mutator := range subnetBuilder.subnetMutatorChain { + err := mutator.Mutate(resultPtrs, resolvedEC2Subnets, subnetConfig) + if err != nil { + return nil, false, err + } + } + + result := make([]elbv2model.SubnetMapping, 0, len(resultPtrs)) + + for _, v := range resultPtrs { + result = append(result, *v) + } + + return result, sourceNATEnabled, nil +} + +func (subnetBuilder *subnetModelBuilderImpl) validateSubnetsInput(subnetConfigsPtr *[]elbv2gw.SubnetConfiguration, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType) (bool, error) { + if subnetConfigsPtr == nil || len(*subnetConfigsPtr) == 0 { + return false, nil + } + + subnetsConfig := *subnetConfigsPtr + identifierSpecified := subnetsConfig[0].Identifier != "" + eipAllocationSpecified := subnetsConfig[0].EIPAllocation != nil + ipv6AllocationSpecified := subnetsConfig[0].IPv6Allocation != nil + privateIPv4AllocationSpecified := subnetsConfig[0].PrivateIPv4Allocation != nil + sourceNATSpecified := subnetsConfig[0].SourceNatIPv6Prefix != nil + + for _, subnetConfig := range subnetsConfig { + if (subnetConfig.Identifier != "") != identifierSpecified { + return false, errors.Errorf("Either specify all subnet identifiers or none.") + } + + if (subnetConfig.EIPAllocation != nil) != eipAllocationSpecified { + return false, errors.Errorf("Either specify all eip allocations or none.") + } + + if (subnetConfig.IPv6Allocation != nil) != ipv6AllocationSpecified { + return false, errors.Errorf("Either specify all ipv6 allocations or none.") + } + + if (subnetConfig.PrivateIPv4Allocation != nil) != privateIPv4AllocationSpecified { + return false, errors.Errorf("Either specify all private ipv4 allocations or none.") + } + + if (subnetConfig.SourceNatIPv6Prefix != nil) != sourceNATSpecified { + return false, errors.Errorf("Either specify all source nat prefixes or none.") + } + } + + if eipAllocationSpecified { + if subnetBuilder.loadBalancerType != elbv2model.LoadBalancerTypeNetwork { + return false, errors.Errorf("EIP Allocation is only allowed for Network LoadBalancers") + } + + if scheme != elbv2model.LoadBalancerSchemeInternetFacing { + return false, errors.Errorf("EIPAllocation can only be set for internet facing load balancers") + } + } + + if ipv6AllocationSpecified { + if subnetBuilder.loadBalancerType != elbv2model.LoadBalancerTypeNetwork { + return false, errors.Errorf("IPv6Allocation is only supported for Network LoadBalancers") + } + + if ipAddressType != elbv2model.IPAddressTypeDualStack { + return false, errors.Errorf("IPv6Allocation can only be set for dualstack load balancers") + } + } + + if privateIPv4AllocationSpecified { + if subnetBuilder.loadBalancerType != elbv2model.LoadBalancerTypeNetwork { + return false, errors.Errorf("PrivateIPv4Allocation is only supported for Network LoadBalancers") + } + + if scheme != elbv2model.LoadBalancerSchemeInternal { + return false, errors.Errorf("PrivateIPv4Allocation can only be set for internal load balancers") + } + } + + if sourceNATSpecified { + if subnetBuilder.loadBalancerType != elbv2model.LoadBalancerTypeNetwork { + return false, errors.Errorf("SourceNatIPv6Prefix is only supported for Network LoadBalancers") + } + } + + return sourceNATSpecified, nil +} + +func (subnetBuilder *subnetModelBuilderImpl) resolveEC2Subnets(ctx context.Context, stack core.Stack, subnetConfigsPtr *[]elbv2gw.SubnetConfiguration, subnetTagSelector *map[string][]string, scheme elbv2model.LoadBalancerScheme) ([]ec2types.Subnet, error) { + // if we have identifiers, query directly by them. + // this assumes that validateSubnetsInput() was already ran on the input. + if subnetConfigsPtr != nil && len(*subnetConfigsPtr) != 0 && (*subnetConfigsPtr)[0].Identifier != "" { + nameOrIds := make([]string, 0) + + for _, configuredSubnet := range *subnetConfigsPtr { + nameOrIds = append(nameOrIds, configuredSubnet.Identifier) + } + + return subnetBuilder.subnetsResolver.ResolveViaNameOrIDSlice(ctx, nameOrIds, + networking.WithSubnetsResolveLBType(subnetBuilder.loadBalancerType), + networking.WithSubnetsResolveLBScheme(scheme), + ) + } + + if subnetTagSelector != nil && len(*subnetTagSelector) != 0 { + selectorTags := *subnetTagSelector + selector := elbv2api.SubnetSelector{ + Tags: selectorTags, + } + + return subnetBuilder.subnetsResolver.ResolveViaSelector(ctx, selector, + networking.WithSubnetsResolveLBType(subnetBuilder.loadBalancerType), + networking.WithSubnetsResolveLBScheme(scheme), + ) + } + + stackTags := subnetBuilder.trackingProvider.StackTags(stack) + + sdkLBs, err := subnetBuilder.elbv2TaggingManager.ListLoadBalancers(ctx, tracking.TagsAsTagFilter(stackTags)) + if err != nil { + return nil, err + } + + if len(sdkLBs) == 0 || (string(scheme) != string(sdkLBs[0].LoadBalancer.Scheme)) { + return subnetBuilder.subnetsResolver.ResolveViaDiscovery(ctx, + networking.WithSubnetsResolveLBType(subnetBuilder.loadBalancerType), + networking.WithSubnetsResolveLBScheme(scheme), + ) + } + + storedSubnetIds := make([]string, 0) + for _, availabilityZone := range sdkLBs[0].LoadBalancer.AvailabilityZones { + subnetID := awssdk.ToString(availabilityZone.SubnetId) + storedSubnetIds = append(storedSubnetIds, subnetID) + } + return subnetBuilder.subnetsResolver.ResolveViaNameOrIDSlice(ctx, storedSubnetIds, + networking.WithSubnetsResolveLBType(subnetBuilder.loadBalancerType), + networking.WithSubnetsResolveLBScheme(scheme), + ) + +} diff --git a/pkg/gateway/model/model_build_subnet_test.go b/pkg/gateway/model/model_build_subnet_test.go new file mode 100644 index 0000000000..d59fce9513 --- /dev/null +++ b/pkg/gateway/model/model_build_subnet_test.go @@ -0,0 +1,663 @@ +package model + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/model/subnet" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + "testing" +) + +func Test_NewSubnetModelBuilder(t *testing.T) { + + trackingProvider := tracking.NewDefaultProvider("", "") + subnetResolver := networking.NewDefaultSubnetsResolver(nil, nil, "", "", false, false, false, logr.Discard()) + taggingManager := elbv2deploy.NewDefaultTaggingManager(nil, "", nil, nil, logr.Discard()) + + builderNLB := newSubnetModelBuilder(elbv2model.LoadBalancerTypeNetwork, trackingProvider, subnetResolver, taggingManager) + subnetBuilderNLB := builderNLB.(*subnetModelBuilderImpl) + + assert.Equal(t, trackingProvider, subnetBuilderNLB.trackingProvider) + assert.Equal(t, subnetResolver, subnetBuilderNLB.subnetsResolver) + assert.Equal(t, taggingManager, subnetBuilderNLB.elbv2TaggingManager) + assert.Equal(t, 4, len(subnetBuilderNLB.subnetMutatorChain)) + + builderALB := newSubnetModelBuilder(elbv2model.LoadBalancerTypeApplication, trackingProvider, subnetResolver, taggingManager) + subnetBuilderALB := builderALB.(*subnetModelBuilderImpl) + + assert.Equal(t, trackingProvider, subnetBuilderALB.trackingProvider) + assert.Equal(t, subnetResolver, subnetBuilderALB.subnetsResolver) + assert.Equal(t, taggingManager, subnetBuilderALB.elbv2TaggingManager) + assert.Equal(t, 0, len(subnetBuilderALB.subnetMutatorChain)) +} + +type mockMutator struct { + called int +} + +func (m *mockMutator) Mutate(_ []*elbv2model.SubnetMapping, _ []ec2types.Subnet, _ []elbv2gw.SubnetConfiguration) error { + m.called++ + return nil +} + +// Test Basic flow here, in depth validation goes in helper function tests. +func Test_BuildLoadBalancerSubnets(t *testing.T) { + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetsResolver := networking.NewMockSubnetsResolver(ctrl) + subnetsResolver.EXPECT().ResolveViaNameOrIDSlice(gomock.Any(), gomock.Any(), gomock.Any()).Return([]ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, nil) + + mm := &mockMutator{} + + builder := subnetModelBuilderImpl{ + loadBalancerType: elbv2model.LoadBalancerTypeNetwork, + subnetMutatorChain: []subnet.Mutator{ + mm, + }, + subnetsResolver: subnetsResolver, + } + + gwSubnetConfig := []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo", + }, + { + Identifier: "bar", + }, + } + + expectedMappings := []elbv2model.SubnetMapping{ + { + SubnetID: "foo", + }, + { + SubnetID: "bar", + }, + } + + subnetMappings, ipv6SourceNatEnabled, err := builder.buildLoadBalancerSubnets(context.Background(), &gwSubnetConfig, nil, elbv2model.LoadBalancerSchemeInternal, elbv2model.IPAddressTypeIPV4, nil) + + assert.NoError(t, err) + assert.Equal(t, expectedMappings, subnetMappings) + assert.False(t, ipv6SourceNatEnabled) + assert.Equal(t, 1, mm.called) +} + +func Test_ValidateSubnetsInput(t *testing.T) { + testCases := []struct { + name string + lbType elbv2model.LoadBalancerType + lbScheme elbv2model.LoadBalancerScheme + ipAddressType elbv2model.IPAddressType + + subnetConfig []elbv2gw.SubnetConfiguration + + sourceNatEnabled bool + expectErr bool + }{ + { + name: "no config to validate", + lbType: elbv2model.LoadBalancerTypeApplication, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeIPV4, + }, + { + name: "EIPAllocation specified", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + EIPAllocation: awssdk.String("foo"), + }, + { + EIPAllocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeIPV4, + }, + { + name: "IPv6 allocation specified", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + IPv6Allocation: awssdk.String("foo"), + }, + { + IPv6Allocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeDualStack, + }, + { + name: "PrivateIPv4Allocation specified", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + PrivateIPv4Allocation: awssdk.String("foo"), + }, + { + PrivateIPv4Allocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternal, + ipAddressType: elbv2model.IPAddressTypeDualStack, + }, + { + name: "SourceNatIPv6Prefix not specified in all", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + SourceNatIPv6Prefix: awssdk.String("foo"), + }, + { + SourceNatIPv6Prefix: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeIPV4, + sourceNatEnabled: true, + }, + { + name: "EIPAllocation not specified in all", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + EIPAllocation: awssdk.String("foo"), + }, + {}, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeIPV4, + expectErr: true, + }, + { + name: "IPv6 allocation not specified in all", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + IPv6Allocation: awssdk.String("foo"), + }, + {}, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeDualStack, + expectErr: true, + }, + { + name: "PrivateIPv4Allocation not specified in all", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + PrivateIPv4Allocation: awssdk.String("foo"), + }, + {}, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternal, + ipAddressType: elbv2model.IPAddressTypeDualStack, + expectErr: true, + }, + { + name: "SourceNatIPv6Prefix not specified in all", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + SourceNatIPv6Prefix: awssdk.String("foo"), + }, + {}, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeIPV4, + expectErr: true, + }, + { + name: "EIPAllocation specified for alb", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + EIPAllocation: awssdk.String("foo"), + }, + { + EIPAllocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeApplication, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeIPV4, + expectErr: true, + }, + { + name: "EIPAllocation specified for non-internet facing nlb", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + EIPAllocation: awssdk.String("foo"), + }, + { + EIPAllocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternal, + ipAddressType: elbv2model.IPAddressTypeIPV4, + expectErr: true, + }, + { + name: "IPv6 allocation specified for alb", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + IPv6Allocation: awssdk.String("foo"), + }, + { + IPv6Allocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeApplication, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeDualStack, + expectErr: true, + }, + { + name: "IPv6 allocation specified for non-dualstack nlb", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + IPv6Allocation: awssdk.String("foo"), + }, + { + IPv6Allocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeIPV4, + expectErr: true, + }, + { + name: "PrivateIPv4Allocation specified for internet facing nlb", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + PrivateIPv4Allocation: awssdk.String("foo"), + }, + { + PrivateIPv4Allocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeNetwork, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeDualStack, + expectErr: true, + }, + { + name: "PrivateIPv4Allocation specified for alb", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + PrivateIPv4Allocation: awssdk.String("foo"), + }, + { + PrivateIPv4Allocation: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeApplication, + lbScheme: elbv2model.LoadBalancerSchemeInternal, + ipAddressType: elbv2model.IPAddressTypeDualStack, + expectErr: true, + }, + { + name: "SourceNatIPv6Prefix specified for alb", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + SourceNatIPv6Prefix: awssdk.String("foo"), + }, + { + SourceNatIPv6Prefix: awssdk.String("bar"), + }, + }, + lbType: elbv2model.LoadBalancerTypeApplication, + lbScheme: elbv2model.LoadBalancerSchemeInternetFacing, + ipAddressType: elbv2model.IPAddressTypeIPV4, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := subnetModelBuilderImpl{ + loadBalancerType: tc.lbType, + } + + ipv6SourceNatEnabled, err := builder.validateSubnetsInput(&tc.subnetConfig, tc.lbScheme, tc.ipAddressType) + + if tc.expectErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tc.sourceNatEnabled, ipv6SourceNatEnabled) + }) + } +} + +type mockProvider struct { +} + +func (m *mockProvider) ResourceIDTagKey() string { + //TODO implement me + panic("implement me") +} + +func (m *mockProvider) StackTags(stack core.Stack) map[string]string { + return make(map[string]string) +} + +func (m *mockProvider) ResourceTags(stack core.Stack, res core.Resource, additionalTags map[string]string) map[string]string { + //TODO implement me + panic("implement me") +} + +func (m *mockProvider) StackLabels(stack core.Stack) map[string]string { + //TODO implement me + panic("implement me") +} + +func (m *mockProvider) StackTagsLegacy(stack core.Stack) map[string]string { + //TODO implement me + panic("implement me") +} + +func (m *mockProvider) LegacyTagKeys() []string { + //TODO implement me + panic("implement me") +} + +func Test_ResolveEC2Subnets(t *testing.T) { + + type subnetCall struct { + subnets []ec2types.Subnet + err error + } + + type listLoadBalancersCall struct { + sdkLBs []elbv2deploy.LoadBalancerWithTags + err error + } + + testCases := []struct { + name string + + subnetConfig *[]elbv2gw.SubnetConfiguration + selector *map[string][]string + idOrNameResolutionCall *subnetCall + selectorCall *subnetCall + discoveryCall *subnetCall + + listLoadBalancersCall *listLoadBalancersCall + + expected []ec2types.Subnet + expectErr bool + }{ + { + name: "resolve static ids", + idOrNameResolutionCall: &subnetCall{ + subnets: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + subnetConfig: &[]elbv2gw.SubnetConfiguration{ + { + Identifier: "foo", + }, + { + Identifier: "bar", + }, + }, + expected: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + { + name: "resolve static ids - fail", + idOrNameResolutionCall: &subnetCall{ + err: errors.New("bad thing"), + }, + subnetConfig: &[]elbv2gw.SubnetConfiguration{ + { + Identifier: "foo", + }, + { + Identifier: "bar", + }, + }, + expectErr: true, + }, + { + name: "resolve via selector", + selectorCall: &subnetCall{ + subnets: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + selector: &map[string][]string{ + "key1": {"v1", "v2"}, + }, + expected: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + { + name: "resolve via selector - fail", + selectorCall: &subnetCall{ + err: errors.New("bad thing"), + }, + selector: &map[string][]string{ + "key1": {"v1", "v2"}, + }, + expectErr: true, + }, + { + name: "no lbs triggers discovery", + discoveryCall: &subnetCall{ + subnets: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + listLoadBalancersCall: &listLoadBalancersCall{ + sdkLBs: []elbv2deploy.LoadBalancerWithTags{}, + }, + expected: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + { + name: "no lbs triggers discovery - fail", + discoveryCall: &subnetCall{ + err: errors.New("bad thing"), + }, + listLoadBalancersCall: &listLoadBalancersCall{ + sdkLBs: []elbv2deploy.LoadBalancerWithTags{}, + }, + expectErr: true, + }, + { + name: "wrong scheme triggers discovery", + discoveryCall: &subnetCall{ + subnets: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + listLoadBalancersCall: &listLoadBalancersCall{ + sdkLBs: []elbv2deploy.LoadBalancerWithTags{ + { + LoadBalancer: &elbv2types.LoadBalancer{ + Scheme: elbv2types.LoadBalancerSchemeEnumInternetFacing, + }, + }, + }, + }, + expected: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + { + name: "reuse lb subnet settings for static ids", + idOrNameResolutionCall: &subnetCall{ + subnets: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + listLoadBalancersCall: &listLoadBalancersCall{ + sdkLBs: []elbv2deploy.LoadBalancerWithTags{ + { + LoadBalancer: &elbv2types.LoadBalancer{ + Scheme: elbv2types.LoadBalancerSchemeEnumInternal, + AvailabilityZones: []elbv2types.AvailabilityZone{ + { + SubnetId: awssdk.String("foo"), + }, + { + SubnetId: awssdk.String("bar"), + }, + }, + }, + }, + }, + }, + expected: []ec2types.Subnet{ + { + SubnetId: awssdk.String("foo"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + { + SubnetId: awssdk.String("bar"), + CidrBlock: awssdk.String("192.168.0.0/19"), + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetsResolver := networking.NewMockSubnetsResolver(ctrl) + + if tc.idOrNameResolutionCall != nil { + subnetsResolver.EXPECT().ResolveViaNameOrIDSlice(gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.idOrNameResolutionCall.subnets, tc.idOrNameResolutionCall.err) + } + + if tc.discoveryCall != nil { + subnetsResolver.EXPECT().ResolveViaDiscovery(gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.discoveryCall.subnets, tc.discoveryCall.err) + } + + if tc.selectorCall != nil { + subnetsResolver.EXPECT().ResolveViaSelector(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.selectorCall.subnets, tc.selectorCall.err) + } + + elbv2TaggingManager := elbv2deploy.NewMockTaggingManager(ctrl) + if tc.listLoadBalancersCall != nil { + elbv2TaggingManager.EXPECT().ListLoadBalancers(gomock.Any(), gomock.Any()).Return(tc.listLoadBalancersCall.sdkLBs, tc.listLoadBalancersCall.err) + } + + builder := subnetModelBuilderImpl{ + loadBalancerType: elbv2model.LoadBalancerTypeNetwork, + subnetsResolver: subnetsResolver, + trackingProvider: &mockProvider{}, + elbv2TaggingManager: elbv2TaggingManager, + } + + subnets, err := builder.resolveEC2Subnets(context.Background(), nil, tc.subnetConfig, tc.selector, elbv2model.LoadBalancerSchemeInternal) + + if tc.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, subnets) + + }) + } +} diff --git a/pkg/gateway/model/model_builder.go b/pkg/gateway/model/model_builder.go deleted file mode 100644 index 66976d4b50..0000000000 --- a/pkg/gateway/model/model_builder.go +++ /dev/null @@ -1,81 +0,0 @@ -package model - -import ( - "context" - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" - "sigs.k8s.io/aws-load-balancer-controller/pkg/config" - elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" - "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" - "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" - "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" - elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" - "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" - gwv1 "sigs.k8s.io/gateway-api/apis/v1" -) - -// Builder builds the model stack for a Gateway resource. -type Builder interface { - // Build model stack for a gateway - Build(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass, routes map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) -} - -// NewDefaultModelBuilder construct a new defaultModelBuilder -func NewDefaultModelBuilder(subnetsResolver networking.SubnetsResolver, - vpcInfoProvider networking.VPCInfoProvider, vpcID string, trackingProvider tracking.Provider, - elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string, - externalManagedTags sets.Set[string], defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, - backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool, - disableRestrictedSGRules bool, logger logr.Logger) Builder { - return &defaultModelBuilder{ - subnetsResolver: subnetsResolver, - vpcInfoProvider: vpcInfoProvider, - backendSGProvider: backendSGProvider, - sgResolver: sgResolver, - trackingProvider: trackingProvider, - elbv2TaggingManager: elbv2TaggingManager, - featureGates: featureGates, - ec2Client: ec2Client, - enableBackendSG: enableBackendSG, - disableRestrictedSGRules: disableRestrictedSGRules, - - clusterName: clusterName, - vpcID: vpcID, - defaultTags: defaultTags, - externalManagedTags: externalManagedTags, - defaultSSLPolicy: defaultSSLPolicy, - defaultTargetType: elbv2model.TargetType(defaultTargetType), - defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), - logger: logger, - } -} - -var _ Builder = &defaultModelBuilder{} - -type defaultModelBuilder struct { - subnetsResolver networking.SubnetsResolver - vpcInfoProvider networking.VPCInfoProvider - backendSGProvider networking.BackendSGProvider - sgResolver networking.SecurityGroupResolver - trackingProvider tracking.Provider - elbv2TaggingManager elbv2deploy.TaggingManager - featureGates config.FeatureGates - ec2Client services.EC2 - enableBackendSG bool - disableRestrictedSGRules bool - - clusterName string - vpcID string - defaultTags map[string]string - externalManagedTags sets.Set[string] - defaultSSLPolicy string - defaultTargetType elbv2model.TargetType - defaultLoadBalancerScheme elbv2model.LoadBalancerScheme - logger logr.Logger -} - -func (d *defaultModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway, gwClass *gwv1.GatewayClass, routes map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) { - //TODO implement me - panic("implement me") -} diff --git a/pkg/gateway/model/subnet/eip_mutator.go b/pkg/gateway/model/subnet/eip_mutator.go new file mode 100644 index 0000000000..40de5de456 --- /dev/null +++ b/pkg/gateway/model/subnet/eip_mutator.go @@ -0,0 +1,34 @@ +package subnet + +import ( + "fmt" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" +) + +type eipMutator struct { +} + +func NewEIPMutator() Mutator { + return &eipMutator{} +} + +func (mutator *eipMutator) Mutate(elbSubnets []*elbv2model.SubnetMapping, _ []ec2types.Subnet, subnetConfig []elbv2gw.SubnetConfiguration) error { + + // We've already validated it's all or none for EIP Allocations. + if elbSubnets == nil || len(subnetConfig) == 0 || subnetConfig[0].EIPAllocation == nil { + return nil + } + + if len(elbSubnets) != len(subnetConfig) { + return errors.Errorf("Unable to assign EIP allocations we have %+v subnets and %v EIP allocations", len(elbSubnets), len(subnetConfig)) + } + + for i, elbSubnet := range elbSubnets { + fmt.Println("HERE!") + elbSubnet.AllocationID = subnetConfig[i].EIPAllocation + } + return nil +} diff --git a/pkg/gateway/model/subnet/eip_mutator_test.go b/pkg/gateway/model/subnet/eip_mutator_test.go new file mode 100644 index 0000000000..164988fdf1 --- /dev/null +++ b/pkg/gateway/model/subnet/eip_mutator_test.go @@ -0,0 +1,104 @@ +package subnet + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "testing" +) + +func Test_EIPMutator(t *testing.T) { + testCases := []struct { + name string + subnetConfig []elbv2gw.SubnetConfiguration + + expectedEIPAllocations []*string + expectErr bool + }{ + { + name: "no subnet config", + expectedEIPAllocations: []*string{ + nil, nil, nil, + }, + }, + { + name: "subnet config but no eip allocation", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + }, + { + Identifier: "foo2", + }, + { + Identifier: "foo3", + }, + }, + expectedEIPAllocations: []*string{ + nil, nil, nil, + }, + }, + { + name: "subnet config with eip allocation", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + EIPAllocation: awssdk.String("alloc1"), + }, + { + Identifier: "foo2", + EIPAllocation: awssdk.String("alloc2"), + }, + { + Identifier: "foo3", + EIPAllocation: awssdk.String("alloc3"), + }, + }, + expectedEIPAllocations: []*string{ + awssdk.String("alloc1"), awssdk.String("alloc2"), awssdk.String("alloc3"), + }, + }, + { + name: "mismatch subnet config length should trigger error", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + EIPAllocation: awssdk.String("alloc1"), + }, + }, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + subnets := []*elbv2model.SubnetMapping{ + { + SubnetID: "foo1", + }, + { + SubnetID: "foo2", + }, + { + SubnetID: "foo3", + }, + } + + m := eipMutator{} + + err := m.Mutate(subnets, nil, tc.subnetConfig) + if tc.expectErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + + for i, expected := range tc.expectedEIPAllocations { + assert.Equal(t, expected, subnets[i].AllocationID) + } + }) + } +} diff --git a/pkg/gateway/model/subnet/ipv6_mutator.go b/pkg/gateway/model/subnet/ipv6_mutator.go new file mode 100644 index 0000000000..34a48c5a7e --- /dev/null +++ b/pkg/gateway/model/subnet/ipv6_mutator.go @@ -0,0 +1,65 @@ +package subnet + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" + "net/netip" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" +) + +type ipv6Mutator struct { + // networking.GetSubnetAssociatedIPv6CIDRs(ec2Subnet) + prefixResolver func(subnet ec2types.Subnet) ([]netip.Prefix, error) + + // networking.FilterIPsWithinCIDRs([]netip.Addr{ipv4Address}, subnetIPv4CIDRs) + ipCidrFilter func(ips []netip.Addr, cidrs []netip.Prefix) []netip.Addr +} + +func NewIPv6Mutator() Mutator { + return &ipv6Mutator{ + prefixResolver: networking.GetSubnetAssociatedIPv6CIDRs, + ipCidrFilter: networking.FilterIPsWithinCIDRs, + } +} + +func (mutator *ipv6Mutator) Mutate(elbSubnets []*elbv2model.SubnetMapping, ec2Subnets []ec2types.Subnet, subnetConfig []elbv2gw.SubnetConfiguration) error { + + // We've already validated it's already all or nothing for ipv6 allocations. + if elbSubnets == nil || len(subnetConfig) == 0 || subnetConfig[0].IPv6Allocation == nil { + return nil + } + + if len(elbSubnets) != len(subnetConfig) { + return errors.Errorf("Unable to assign IPv6 addresses we have %+v subnets and %v addresses", len(elbSubnets), len(subnetConfig)) + } + + ipv6Addrs := make([]netip.Addr, 0) + for _, cfg := range subnetConfig { + ipv6Address, err := netip.ParseAddr(*cfg.IPv6Allocation) + if err != nil { + return errors.Errorf("IPv6 addresses must be valid IP address: %v", *cfg.IPv6Allocation) + } + if !ipv6Address.Is6() { + return errors.Errorf("IPv6 addresses must be valid IPv6 address: %v", *cfg.IPv6Allocation) + } + + ipv6Addrs = append(ipv6Addrs, ipv6Address) + } + + for i, elbSubnet := range elbSubnets { + ec2Subnet := ec2Subnets[i] + subnetIPv6CIDRs, err := mutator.prefixResolver(ec2Subnet) + if err != nil { + return err + } + ipv6AddressesWithinSubnet := mutator.ipCidrFilter(ipv6Addrs, subnetIPv6CIDRs) + if len(ipv6AddressesWithinSubnet) != 1 { + return errors.Errorf("expect one IPv6 address configured for subnet: %v", awssdk.ToString(ec2Subnet.SubnetId)) + } + elbSubnet.IPv6Address = awssdk.String(ipv6AddressesWithinSubnet[0].String()) + } + return nil +} diff --git a/pkg/gateway/model/subnet/ipv6_mutator_test.go b/pkg/gateway/model/subnet/ipv6_mutator_test.go new file mode 100644 index 0000000000..90f935ec73 --- /dev/null +++ b/pkg/gateway/model/subnet/ipv6_mutator_test.go @@ -0,0 +1,214 @@ +package subnet + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "net/netip" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "testing" +) + +func Test_IPv6Mutator(t *testing.T) { + testCases := []struct { + name string + subnetConfig []elbv2gw.SubnetConfiguration + + resolvedCidrs []netip.Prefix + resolvedCidrsErr error + + filteredIPs []netip.Addr + + returnNoValidIPs bool + + expectedIPv6Allocation []*string + expectErr bool + }{ + { + name: "no subnet config", + expectedIPv6Allocation: []*string{ + nil, nil, nil, + }, + }, + { + name: "subnet config but no ipv6 addrs", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + }, + { + Identifier: "foo2", + }, + { + Identifier: "foo3", + }, + }, + expectedIPv6Allocation: []*string{ + nil, nil, nil, + }, + }, + { + name: "subnet config with ipv6 addrs", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::1"), + }, + { + Identifier: "foo2", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::2"), + }, + { + Identifier: "foo3", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::3"), + }, + }, + resolvedCidrs: []netip.Prefix{}, + expectedIPv6Allocation: []*string{ + awssdk.String("2600:1f13:837:8504::1"), awssdk.String("2600:1f13:837:8504::2"), awssdk.String("2600:1f13:837:8504::3"), + }, + }, + { + name: "ip doesnt belong to cidr", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::1"), + }, + { + Identifier: "foo2", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::2"), + }, + { + Identifier: "foo3", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::3"), + }, + }, + returnNoValidIPs: true, + expectErr: true, + }, + { + name: "cidr resolver error", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::1"), + }, + { + Identifier: "foo2", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::2"), + }, + { + Identifier: "foo3", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::3"), + }, + }, + resolvedCidrsErr: errors.New("bad thing"), + expectErr: true, + }, + { + name: "subnet config with ipv6 addrs - allocation is not an ip address", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + IPv6Allocation: awssdk.String("foo"), + }, + { + Identifier: "foo2", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::2"), + }, + { + Identifier: "foo3", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::3"), + }, + }, + expectErr: true, + }, + { + name: "subnet config with ipv6 addrs - allocation is not an ipv6 address", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + IPv6Allocation: awssdk.String("127.0.0.1"), + }, + { + Identifier: "foo2", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::2"), + }, + { + Identifier: "foo3", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::3"), + }, + }, + expectErr: true, + }, + { + name: "mismatch subnet config length should trigger error", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + IPv6Allocation: awssdk.String("2600:1f13:837:8504::1"), + }, + }, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + subnets := []*elbv2model.SubnetMapping{ + { + SubnetID: "foo1", + }, + { + SubnetID: "foo2", + }, + { + SubnetID: "foo3", + }, + } + + ec2Subnets := make([]ec2types.Subnet, 0) + + for _, s := range subnets { + ec2Subnets = append(ec2Subnets, ec2types.Subnet{ + SubnetId: &s.SubnetID, + }) + } + + filterIndx := 0 + + m := ipv6Mutator{ + prefixResolver: func(subnet ec2types.Subnet) ([]netip.Prefix, error) { + return tc.resolvedCidrs, tc.resolvedCidrsErr + }, + ipCidrFilter: func(ips []netip.Addr, cidrs []netip.Prefix) []netip.Addr { + + if tc.returnNoValidIPs { + return []netip.Addr{} + } + + assert.Equal(t, tc.resolvedCidrs, cidrs) + ip := tc.expectedIPv6Allocation[filterIndx] + filterIndx += 1 + return []netip.Addr{netip.MustParseAddr(*ip)} + }, + } + + err := m.Mutate(subnets, ec2Subnets, tc.subnetConfig) + if tc.expectErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + + for i, expected := range tc.expectedIPv6Allocation { + assert.Equal(t, expected, subnets[i].IPv6Address) + } + }) + } +} diff --git a/pkg/gateway/model/subnet/mutator.go b/pkg/gateway/model/subnet/mutator.go new file mode 100644 index 0000000000..11c4a8ff49 --- /dev/null +++ b/pkg/gateway/model/subnet/mutator.go @@ -0,0 +1,11 @@ +package subnet + +import ( + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" +) + +type Mutator interface { + Mutate(elbSubnets []*elbv2model.SubnetMapping, ec2Subnets []ec2types.Subnet, subnetConfig []elbv2gw.SubnetConfiguration) error +} diff --git a/pkg/gateway/model/subnet/private_ipv4_mutator.go b/pkg/gateway/model/subnet/private_ipv4_mutator.go new file mode 100644 index 0000000000..7a2d3a3cc5 --- /dev/null +++ b/pkg/gateway/model/subnet/private_ipv4_mutator.go @@ -0,0 +1,63 @@ +package subnet + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" + "net/netip" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" +) + +type privateIPv4Mutator struct { + // networking.GetSubnetAssociatedIPv4CIDRs(ec2Subnet) + prefixResolver func(subnet ec2types.Subnet) ([]netip.Prefix, error) + + // networking.FilterIPsWithinCIDRs([]netip.Addr{ipv4Address}, subnetIPv4CIDRs) + ipCidrFilter func(ips []netip.Addr, cidrs []netip.Prefix) []netip.Addr +} + +func NewPrivateIPv4Mutator() Mutator { + return &privateIPv4Mutator{ + prefixResolver: networking.GetSubnetAssociatedIPv4CIDRs, + ipCidrFilter: networking.FilterIPsWithinCIDRs, + } +} + +func (mutator *privateIPv4Mutator) Mutate(elbSubnets []*elbv2model.SubnetMapping, ec2Subnets []ec2types.Subnet, subnetConfig []elbv2gw.SubnetConfiguration) error { + // We've already validated it's already all or nothing for private ipv4 allocations. + if elbSubnets == nil || len(subnetConfig) == 0 || subnetConfig[0].PrivateIPv4Allocation == nil { + return nil + } + + if len(elbSubnets) != len(subnetConfig) { + return errors.Errorf("Unable to assign private IPv4 addresses we have %+v subnets and %v addresses", len(elbSubnets), len(subnetConfig)) + } + + ipv4Addrs := make([]netip.Addr, 0) + for _, cfg := range subnetConfig { + ipv4Address, err := netip.ParseAddr(*cfg.PrivateIPv4Allocation) + if err != nil { + return errors.Errorf("private IPv4 addresses must be valid IP address: %v", *cfg.PrivateIPv4Allocation) + } + if !ipv4Address.Is4() { + return errors.Errorf("private IPv4 addresses must be valid IPv4 address: %v", *cfg.PrivateIPv4Allocation) + } + ipv4Addrs = append(ipv4Addrs, ipv4Address) + } + + for i, elbSubnet := range elbSubnets { + ec2Subnet := ec2Subnets[i] + subnetIPv4CIDRs, err := mutator.prefixResolver(ec2Subnet) + if err != nil { + return err + } + ipv4AddressesWithinSubnet := mutator.ipCidrFilter(ipv4Addrs, subnetIPv4CIDRs) + if len(ipv4AddressesWithinSubnet) != 1 { + return errors.Errorf("expect one private IPv4 address configured for subnet: %v", awssdk.ToString(ec2Subnet.SubnetId)) + } + elbSubnet.PrivateIPv4Address = awssdk.String(ipv4AddressesWithinSubnet[0].String()) + } + return nil +} diff --git a/pkg/gateway/model/subnet/private_ipv4_mutator_test.go b/pkg/gateway/model/subnet/private_ipv4_mutator_test.go new file mode 100644 index 0000000000..38d6be4402 --- /dev/null +++ b/pkg/gateway/model/subnet/private_ipv4_mutator_test.go @@ -0,0 +1,214 @@ +package subnet + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "net/netip" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "testing" +) + +func Test_PrivateIPv4Mutator(t *testing.T) { + testCases := []struct { + name string + subnetConfig []elbv2gw.SubnetConfiguration + + resolvedCidrs []netip.Prefix + resolvedCidrsErr error + + filteredIPs []netip.Addr + + returnNoValidIPs bool + + expectedPrivateIPv4 []*string + expectErr bool + }{ + { + name: "no subnet config", + expectedPrivateIPv4: []*string{ + nil, nil, nil, + }, + }, + { + name: "subnet config but no ipv4 addrs", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + }, + { + Identifier: "foo2", + }, + { + Identifier: "foo3", + }, + }, + expectedPrivateIPv4: []*string{ + nil, nil, nil, + }, + }, + { + name: "subnet config with ipv4 addrs", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + PrivateIPv4Allocation: awssdk.String("127.0.0.1"), + }, + { + Identifier: "foo2", + PrivateIPv4Allocation: awssdk.String("127.0.0.2"), + }, + { + Identifier: "foo3", + PrivateIPv4Allocation: awssdk.String("127.0.0.3"), + }, + }, + resolvedCidrs: []netip.Prefix{}, + expectedPrivateIPv4: []*string{ + awssdk.String("127.0.0.1"), awssdk.String("127.0.0.2"), awssdk.String("127.0.0.3"), + }, + }, + { + name: "ip doesnt belong to cidr", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + PrivateIPv4Allocation: awssdk.String("127.0.0.1"), + }, + { + Identifier: "foo2", + PrivateIPv4Allocation: awssdk.String("127.0.0.2"), + }, + { + Identifier: "foo3", + PrivateIPv4Allocation: awssdk.String("127.0.0.3"), + }, + }, + returnNoValidIPs: true, + expectErr: true, + }, + { + name: "cidr resolver error", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + PrivateIPv4Allocation: awssdk.String("127.0.0.1"), + }, + { + Identifier: "foo2", + PrivateIPv4Allocation: awssdk.String("127.0.0.2"), + }, + { + Identifier: "foo3", + PrivateIPv4Allocation: awssdk.String("127.0.0.3"), + }, + }, + resolvedCidrsErr: errors.New("bad thing"), + expectErr: true, + }, + { + name: "subnet config with ipv4 addrs - allocation is not an ip address", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + PrivateIPv4Allocation: awssdk.String("foo"), + }, + { + Identifier: "foo2", + PrivateIPv4Allocation: awssdk.String("127.0.0.2"), + }, + { + Identifier: "foo3", + PrivateIPv4Allocation: awssdk.String("127.0.0.3"), + }, + }, + expectErr: true, + }, + { + name: "subnet config with ipv6 addrs - allocation is not an ipv4 address", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + PrivateIPv4Allocation: awssdk.String("2600:1f13:837:8504::2"), + }, + { + Identifier: "foo2", + PrivateIPv4Allocation: awssdk.String("127.0.0.2"), + }, + { + Identifier: "foo3", + PrivateIPv4Allocation: awssdk.String("127.0.0.3"), + }, + }, + expectErr: true, + }, + { + name: "mismatch subnet config length should trigger error", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + PrivateIPv4Allocation: awssdk.String("127.0.0.2"), + }, + }, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + subnets := []*elbv2model.SubnetMapping{ + { + SubnetID: "foo1", + }, + { + SubnetID: "foo2", + }, + { + SubnetID: "foo3", + }, + } + + ec2Subnets := make([]ec2types.Subnet, 0) + + for _, s := range subnets { + ec2Subnets = append(ec2Subnets, ec2types.Subnet{ + SubnetId: &s.SubnetID, + }) + } + + filterIndx := 0 + + m := privateIPv4Mutator{ + prefixResolver: func(subnet ec2types.Subnet) ([]netip.Prefix, error) { + return tc.resolvedCidrs, tc.resolvedCidrsErr + }, + ipCidrFilter: func(ips []netip.Addr, cidrs []netip.Prefix) []netip.Addr { + + if tc.returnNoValidIPs { + return []netip.Addr{} + } + + assert.Equal(t, tc.resolvedCidrs, cidrs) + ip := tc.expectedPrivateIPv4[filterIndx] + filterIndx += 1 + return []netip.Addr{netip.MustParseAddr(*ip)} + }, + } + + err := m.Mutate(subnets, ec2Subnets, tc.subnetConfig) + if tc.expectErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + + for i, expected := range tc.expectedPrivateIPv4 { + assert.Equal(t, expected, subnets[i].PrivateIPv4Address) + } + }) + } +} diff --git a/pkg/gateway/model/subnet/source_nat_mutator.go b/pkg/gateway/model/subnet/source_nat_mutator.go new file mode 100644 index 0000000000..d072e97b8c --- /dev/null +++ b/pkg/gateway/model/subnet/source_nat_mutator.go @@ -0,0 +1,40 @@ +package subnet + +import ( + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" +) + +type sourceNATMutator struct { + // networking.ValidateSourceNatPrefixForSubnetPair(sourceNatPrefix, ec2Subnet) + validator func(sourceNatIpv6Prefix string, subnet ec2types.Subnet) error +} + +func NewSourceNATMutator() Mutator { + return &sourceNATMutator{ + validator: networking.ValidateSourceNatPrefixForSubnetPair, + } +} + +func (mutator *sourceNATMutator) Mutate(elbSubnets []*elbv2model.SubnetMapping, ec2Subnet []ec2types.Subnet, subnetConfig []elbv2gw.SubnetConfiguration) error { + // We've already validated it's already all or nothing for source nat prefixes + if elbSubnets == nil || len(subnetConfig) == 0 || subnetConfig[0].SourceNatIPv6Prefix == nil { + return nil + } + + if len(elbSubnets) != len(subnetConfig) { + return errors.Errorf("Unable to assign Source NAT prefix because we have %+v subnets and %v prefixes", len(elbSubnets), len(subnetConfig)) + } + + for i, elbSubnet := range elbSubnets { + validationErr := mutator.validator(*subnetConfig[i].SourceNatIPv6Prefix, ec2Subnet[i]) + if validationErr != nil { + return validationErr + } + elbSubnet.SourceNatIpv6Prefix = subnetConfig[i].SourceNatIPv6Prefix + } + return nil +} diff --git a/pkg/gateway/model/subnet/source_nat_mutator_test.go b/pkg/gateway/model/subnet/source_nat_mutator_test.go new file mode 100644 index 0000000000..711767caa9 --- /dev/null +++ b/pkg/gateway/model/subnet/source_nat_mutator_test.go @@ -0,0 +1,119 @@ +package subnet + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/stretchr/testify/assert" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "testing" +) + +func Test_SourceNATMutator(t *testing.T) { + testCases := []struct { + name string + subnetConfig []elbv2gw.SubnetConfiguration + + validationError error + + expectedSourceNATPrefixes []*string + expectErr bool + }{ + { + name: "no subnet config", + expectedSourceNATPrefixes: []*string{ + nil, nil, nil, + }, + }, + { + name: "subnet config but no source nat config", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + }, + { + Identifier: "foo2", + }, + { + Identifier: "foo3", + }, + }, + expectedSourceNATPrefixes: []*string{ + nil, nil, nil, + }, + }, + { + name: "subnet config with source nat config", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + SourceNatIPv6Prefix: awssdk.String("alloc1"), + }, + { + Identifier: "foo2", + SourceNatIPv6Prefix: awssdk.String("alloc2"), + }, + { + Identifier: "foo3", + SourceNatIPv6Prefix: awssdk.String("alloc3"), + }, + }, + expectedSourceNATPrefixes: []*string{ + awssdk.String("alloc1"), awssdk.String("alloc2"), awssdk.String("alloc3"), + }, + }, + { + name: "mismatch subnet config length should trigger error", + subnetConfig: []elbv2gw.SubnetConfiguration{ + { + Identifier: "foo1", + SourceNatIPv6Prefix: awssdk.String("alloc1"), + }, + }, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + subnets := []*elbv2model.SubnetMapping{ + { + SubnetID: "foo1", + }, + { + SubnetID: "foo2", + }, + { + SubnetID: "foo3", + }, + } + + ec2Subnets := make([]ec2types.Subnet, 0) + + for _, s := range subnets { + ec2Subnets = append(ec2Subnets, ec2types.Subnet{ + SubnetId: &s.SubnetID, + }) + } + + m := sourceNATMutator{ + validator: func(sourceNatIpv6Prefix string, subnet ec2types.Subnet) error { + return tc.validationError + }, + } + + err := m.Mutate(subnets, ec2Subnets, tc.subnetConfig) + if tc.expectErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + + for i, expected := range tc.expectedSourceNATPrefixes { + assert.Equal(t, expected, subnets[i].SourceNatIpv6Prefix) + } + }) + } +} diff --git a/pkg/networking/utils.go b/pkg/networking/utils.go index c87eb53cf3..3aaaacc7cd 100644 --- a/pkg/networking/utils.go +++ b/pkg/networking/utils.go @@ -10,6 +10,8 @@ import ( elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" ) +const requiredPrefixLengthForSourceNatCidr = "80" + // ParseCIDRs will parse CIDRs in string format into parsed IPPrefix func ParseCIDRs(cidrs []string) ([]netip.Prefix, error) { var ipPrefixes []netip.Prefix @@ -111,7 +113,6 @@ func ValidateEnablePrefixForIpv6SourceNat(EnablePrefixForIpv6SourceNat string, i // ValidateSourceNatPrefixes function returns the validation error if error exists for sourceNatIpv6Prefixes annotation value func ValidateSourceNatPrefixes(sourceNatIpv6Prefixes []string, ipAddressType elbv2model.IPAddressType, isPrefixForIpv6SourceNatEnabled bool, ec2Subnets []ec2types.Subnet) error { - const requiredPrefixLengthForSourceNatCidr = "80" if ipAddressType != elbv2model.IPAddressTypeDualStack { return errors.Errorf("source-nat-ipv6-prefixes annotation can only be set for Network Load Balancers using Dualstack IP address type.") } @@ -123,36 +124,43 @@ func ValidateSourceNatPrefixes(sourceNatIpv6Prefixes []string, ipAddressType elb return errors.Errorf("Number of values in source-nat-ipv6-prefixes (%d) must match the number of subnets (%d).", len(sourceNatIpv6Prefixes), len(ec2Subnets)) } for idx, sourceNatIpv6Prefix := range sourceNatIpv6Prefixes { - var subnet = ec2Subnets[idx] - var sourceNatIpv6PrefixParsedList []netip.Addr - if sourceNatIpv6Prefix != elbv2model.SourceNatIpv6PrefixAutoAssigned { - subStrings := strings.Split(sourceNatIpv6Prefix, "/") - if len(subStrings) < 2 { - return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v.", sourceNatIpv6Prefix) - } - var ipAddressPart = subStrings[0] - var prefixLengthPart = subStrings[1] - if prefixLengthPart != requiredPrefixLengthForSourceNatCidr { - return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v. Prefix length must be %v, but %v is specified.", sourceNatIpv6Prefix, requiredPrefixLengthForSourceNatCidr, prefixLengthPart) - } - sourceNatIpv6PrefixNetIpParsed, err := netip.ParseAddr(ipAddressPart) - if err != nil { - return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v. Value must be a valid IPv6 CIDR.", sourceNatIpv6Prefix) - } - sourceNatIpv6PrefixParsedList = append(sourceNatIpv6PrefixParsedList, sourceNatIpv6PrefixNetIpParsed) - if !sourceNatIpv6PrefixNetIpParsed.Is6() { - return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v. Value must be a valid IPv6 CIDR.", sourceNatIpv6Prefix) - } - subnetIPv6CIDRs, err := GetSubnetAssociatedIPv6CIDRs(subnet) - if err != nil { - return errors.Errorf("Subnet has invalid IPv6 CIDRs: %v. Subnet must have valid IPv6 CIDRs.", subnetIPv6CIDRs) - } - sourceNatIpv6PrefixWithinSubnet := FilterIPsWithinCIDRs(sourceNatIpv6PrefixParsedList, subnetIPv6CIDRs) - if len(sourceNatIpv6PrefixWithinSubnet) != 1 { - return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v. Value must be within subnet CIDR range: %v.", sourceNatIpv6Prefix, subnetIPv6CIDRs) - } + err := ValidateSourceNatPrefixForSubnetPair(sourceNatIpv6Prefix, ec2Subnets[idx]) + if err != nil { + return err } } return nil } + +// ValidateSourceNatPrefixForSubnetPair validates the input for one source nat ipv6 prefix -> ec2 subnet pair +func ValidateSourceNatPrefixForSubnetPair(sourceNatIpv6Prefix string, subnet ec2types.Subnet) error { + if sourceNatIpv6Prefix == elbv2model.SourceNatIpv6PrefixAutoAssigned { + return nil + } + subStrings := strings.Split(sourceNatIpv6Prefix, "/") + if len(subStrings) < 2 { + return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v.", sourceNatIpv6Prefix) + } + var ipAddressPart = subStrings[0] + var prefixLengthPart = subStrings[1] + if prefixLengthPart != requiredPrefixLengthForSourceNatCidr { + return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v. Prefix length must be %v, but %v is specified.", sourceNatIpv6Prefix, requiredPrefixLengthForSourceNatCidr, prefixLengthPart) + } + sourceNatIpv6PrefixNetIpParsed, err := netip.ParseAddr(ipAddressPart) + if err != nil { + return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v. Value must be a valid IPv6 CIDR.", sourceNatIpv6Prefix) + } + if !sourceNatIpv6PrefixNetIpParsed.Is6() { + return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v. Value must be a valid IPv6 CIDR.", sourceNatIpv6Prefix) + } + subnetIPv6CIDRs, err := GetSubnetAssociatedIPv6CIDRs(subnet) + if err != nil { + return errors.Errorf("Subnet has invalid IPv6 CIDRs: %v. Subnet must have valid IPv6 CIDRs.", subnetIPv6CIDRs) + } + sourceNatIpv6PrefixWithinSubnet := FilterIPsWithinCIDRs([]netip.Addr{sourceNatIpv6PrefixNetIpParsed}, subnetIPv6CIDRs) + if len(sourceNatIpv6PrefixWithinSubnet) != 1 { + return errors.Errorf("Invalid value in source-nat-ipv6-prefixes: %v. Value must be within subnet CIDR range: %v.", sourceNatIpv6Prefix, subnetIPv6CIDRs) + } + return nil +} From 6f70423092b44cfd620bfea7f5de7c292b582789 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Tue, 15 Apr 2025 14:37:18 -0700 Subject: [PATCH 11/40] refactor validation --- go.mod | 4 +-- go.sum | 4 +-- pkg/gateway/model/model_build_subnet.go | 44 ++++++++++++------------- pkg/gateway/model/subnet/eip_mutator.go | 2 -- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index c3f5237e1f..85adea5665 100644 --- a/go.mod +++ b/go.mod @@ -32,9 +32,10 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.10.0 go.uber.org/zap v1.27.0 + golang.org/x/net v0.37.0 golang.org/x/time v0.7.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - helm.sh/helm/v3 v3.17.2 + helm.sh/helm/v3 v3.17.3 k8s.io/api v0.32.2 k8s.io/apimachinery v0.32.2 k8s.io/cli-runtime v0.32.2 @@ -172,7 +173,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.37.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect diff --git a/go.sum b/go.sum index 2dbe1de464..773185bdc8 100644 --- a/go.sum +++ b/go.sum @@ -595,8 +595,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -helm.sh/helm/v3 v3.17.2 h1:agYQ5ew2jq5vdx2K7q5W44KyKQrnSubUMCQsjkiv3/o= -helm.sh/helm/v3 v3.17.2/go.mod h1:+uJKMH/UiMzZQOALR3XUf3BLIoczI2RKKD6bMhPh4G8= +helm.sh/helm/v3 v3.17.3 h1:3n5rW3D0ArjFl0p4/oWO8IbY/HKaNNwJtOQFdH2AZHg= +helm.sh/helm/v3 v3.17.3/go.mod h1:+uJKMH/UiMzZQOALR3XUf3BLIoczI2RKKD6bMhPh4G8= k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= k8s.io/apiextensions-apiserver v0.32.2 h1:2YMk285jWMk2188V2AERy5yDwBYrjgWYggscghPCvV4= diff --git a/pkg/gateway/model/model_build_subnet.go b/pkg/gateway/model/model_build_subnet.go index 71ffbbb9ac..9b2c13301d 100644 --- a/pkg/gateway/model/model_build_subnet.go +++ b/pkg/gateway/model/model_build_subnet.go @@ -105,28 +105,6 @@ func (subnetBuilder *subnetModelBuilderImpl) validateSubnetsInput(subnetConfigsP privateIPv4AllocationSpecified := subnetsConfig[0].PrivateIPv4Allocation != nil sourceNATSpecified := subnetsConfig[0].SourceNatIPv6Prefix != nil - for _, subnetConfig := range subnetsConfig { - if (subnetConfig.Identifier != "") != identifierSpecified { - return false, errors.Errorf("Either specify all subnet identifiers or none.") - } - - if (subnetConfig.EIPAllocation != nil) != eipAllocationSpecified { - return false, errors.Errorf("Either specify all eip allocations or none.") - } - - if (subnetConfig.IPv6Allocation != nil) != ipv6AllocationSpecified { - return false, errors.Errorf("Either specify all ipv6 allocations or none.") - } - - if (subnetConfig.PrivateIPv4Allocation != nil) != privateIPv4AllocationSpecified { - return false, errors.Errorf("Either specify all private ipv4 allocations or none.") - } - - if (subnetConfig.SourceNatIPv6Prefix != nil) != sourceNATSpecified { - return false, errors.Errorf("Either specify all source nat prefixes or none.") - } - } - if eipAllocationSpecified { if subnetBuilder.loadBalancerType != elbv2model.LoadBalancerTypeNetwork { return false, errors.Errorf("EIP Allocation is only allowed for Network LoadBalancers") @@ -163,6 +141,28 @@ func (subnetBuilder *subnetModelBuilderImpl) validateSubnetsInput(subnetConfigsP } } + for _, subnetConfig := range subnetsConfig { + if (subnetConfig.Identifier != "") != identifierSpecified { + return false, errors.Errorf("Either specify all subnet identifiers or none.") + } + + if (subnetConfig.EIPAllocation != nil) != eipAllocationSpecified { + return false, errors.Errorf("Either specify all eip allocations or none.") + } + + if (subnetConfig.IPv6Allocation != nil) != ipv6AllocationSpecified { + return false, errors.Errorf("Either specify all ipv6 allocations or none.") + } + + if (subnetConfig.PrivateIPv4Allocation != nil) != privateIPv4AllocationSpecified { + return false, errors.Errorf("Either specify all private ipv4 allocations or none.") + } + + if (subnetConfig.SourceNatIPv6Prefix != nil) != sourceNATSpecified { + return false, errors.Errorf("Either specify all source nat prefixes or none.") + } + } + return sourceNATSpecified, nil } diff --git a/pkg/gateway/model/subnet/eip_mutator.go b/pkg/gateway/model/subnet/eip_mutator.go index 40de5de456..3457e9470f 100644 --- a/pkg/gateway/model/subnet/eip_mutator.go +++ b/pkg/gateway/model/subnet/eip_mutator.go @@ -1,7 +1,6 @@ package subnet import ( - "fmt" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/pkg/errors" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" @@ -27,7 +26,6 @@ func (mutator *eipMutator) Mutate(elbSubnets []*elbv2model.SubnetMapping, _ []ec } for i, elbSubnet := range elbSubnets { - fmt.Println("HERE!") elbSubnet.AllocationID = subnetConfig[i].EIPAllocation } return nil From 52a28532e1b4a2c42174a019fb9a51924c55f22c Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 17 Apr 2025 14:54:30 -0700 Subject: [PATCH 12/40] fix crds --- config/webhook/manifests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index f901bc74c3..00793b4707 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -1,3 +1,4 @@ +--- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: From 0ef1444704ab72f03a00ed1116265b6e73a22865 Mon Sep 17 00:00:00 2001 From: Shraddha Bang <18206078+shraddhabang@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:48:44 -0700 Subject: [PATCH 13/40] [feat: gw api] Add eventhandlers for all the gateway resources (#4148) --- config/rbac/role.yaml | 132 +++ .../eventhandlers/gateway_class_events.go | 80 ++ .../eventhandlers/grpc_route_events.go | 71 ++ .../eventhandlers/http_route_events.go | 71 ++ .../load_balancer_configuration_events.go | 83 ++ .../gateway/eventhandlers/service_events.go | 156 ++++ .../target_group_configuration_events.go | 78 ++ .../gateway/eventhandlers/tcp_route_events.go | 71 ++ .../gateway/eventhandlers/tls_route_events.go | 71 ++ .../gateway/eventhandlers/udp_route_events.go | 71 ++ controllers/gateway/eventhandlers/utils.go | 130 +++ .../gateway/eventhandlers/utils_test.go | 769 ++++++++++++++++++ controllers/gateway/gateway_controller.go | 212 ++++- main.go | 140 +++- pkg/config/controller_config.go | 6 + .../gateway/constants/controller_constants.go | 9 +- pkg/gateway/routeutils/backend_test.go | 7 +- pkg/gateway/routeutils/descriptor.go | 1 + pkg/gateway/routeutils/grpc.go | 12 + pkg/gateway/routeutils/grpc_test.go | 26 +- pkg/gateway/routeutils/http.go | 12 + pkg/gateway/routeutils/http_test.go | 26 +- pkg/gateway/routeutils/loader_test.go | 5 + .../routeutils/namespace_selector_test.go | 3 +- pkg/gateway/routeutils/tcp.go | 10 + pkg/gateway/routeutils/tcp_test.go | 34 +- pkg/gateway/routeutils/test_utils.go | 25 - pkg/gateway/routeutils/tls.go | 10 + pkg/gateway/routeutils/tls_test.go | 26 +- pkg/gateway/routeutils/udp.go | 10 + pkg/gateway/routeutils/udp_test.go | 34 +- pkg/gateway/routeutils/utils.go | 89 ++ pkg/gateway/routeutils/utils_test.go | 436 ++++++++++ pkg/metrics/util/reconcile_counter.go | 10 +- pkg/testutils/client_test_utils.go | 20 + 35 files changed, 2855 insertions(+), 91 deletions(-) create mode 100644 controllers/gateway/eventhandlers/gateway_class_events.go create mode 100644 controllers/gateway/eventhandlers/grpc_route_events.go create mode 100644 controllers/gateway/eventhandlers/http_route_events.go create mode 100644 controllers/gateway/eventhandlers/load_balancer_configuration_events.go create mode 100644 controllers/gateway/eventhandlers/service_events.go create mode 100644 controllers/gateway/eventhandlers/target_group_configuration_events.go create mode 100644 controllers/gateway/eventhandlers/tcp_route_events.go create mode 100644 controllers/gateway/eventhandlers/tls_route_events.go create mode 100644 controllers/gateway/eventhandlers/udp_route_events.go create mode 100644 controllers/gateway/eventhandlers/utils.go create mode 100644 controllers/gateway/eventhandlers/utils_test.go rename controllers/gateway/constants.go => pkg/gateway/constants/controller_constants.go (86%) delete mode 100644 pkg/gateway/routeutils/test_utils.go create mode 100644 pkg/gateway/routeutils/utils.go create mode 100644 pkg/gateway/routeutils/utils_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9893463391..313f45d488 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -128,6 +128,72 @@ rules: verbs: - patch - update +- apiGroups: + - gateway.k8s.aws + resources: + - loadbalancerconfigurations + verbs: + - get + - list + - watch +- apiGroups: + - gateway.k8s.aws + resources: + - loadbalancerconfigurations/finalizers + verbs: + - update +- apiGroups: + - gateway.k8s.aws + resources: + - loadbalancerconfigurations/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.k8s.aws + resources: + - targetgroupconfigurations + verbs: + - get + - list + - watch +- apiGroups: + - gateway.k8s.aws + resources: + - targetgroupconfigurations/finalizers + verbs: + - update +- apiGroups: + - gateway.k8s.aws + resources: + - targetgroupconfigurations/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/finalizers + verbs: + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + verbs: + - get + - patch + - update - apiGroups: - gateway.networking.k8s.io resources: @@ -150,6 +216,50 @@ rules: - get - patch - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes/finalizers + verbs: + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/finalizers + verbs: + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + verbs: + - get + - patch + - update - apiGroups: - gateway.networking.k8s.io resources: @@ -172,6 +282,28 @@ rules: - get - patch - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/finalizers + verbs: + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - patch + - update - apiGroups: - gateway.networking.k8s.io resources: diff --git a/controllers/gateway/eventhandlers/gateway_class_events.go b/controllers/gateway/eventhandlers/gateway_class_events.go new file mode 100644 index 0000000000..1cebc4f6c3 --- /dev/null +++ b/controllers/gateway/eventhandlers/gateway_class_events.go @@ -0,0 +1,80 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// NewEnqueueRequestsForGatewayClassEvent creates handler for GatewayClass resources +func NewEnqueueRequestsForGatewayClassEvent( + k8sClient client.Client, eventRecorder record.EventRecorder, gwController string, logger logr.Logger) handler.TypedEventHandler[*gatewayv1.GatewayClass, reconcile.Request] { + return &enqueueRequestsForGatewayClassEvent{ + k8sClient: k8sClient, + eventRecorder: eventRecorder, + gwController: gwController, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*gatewayv1.GatewayClass, reconcile.Request] = (*enqueueRequestsForGatewayClassEvent)(nil) + +// enqueueRequestsForGatewayClassEvent handles GatewayClass events +type enqueueRequestsForGatewayClassEvent struct { + k8sClient client.Client + eventRecorder record.EventRecorder + gwController string + logger logr.Logger +} + +func (h *enqueueRequestsForGatewayClassEvent) Create(ctx context.Context, e event.TypedCreateEvent[*gatewayv1.GatewayClass], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gwClassNew := e.Object + h.logger.V(1).Info("enqueue gatewayclass create event", "gatewayclass", gwClassNew.Name) + h.enqueueImpactedGateways(ctx, gwClassNew, queue) +} + +func (h *enqueueRequestsForGatewayClassEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*gatewayv1.GatewayClass], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gwClassOld := e.ObjectOld + gwClassNew := e.ObjectNew + + // we only care below update event: + // 1. GatewayClass spec updates + // 3. GatewayClass deletions + if equality.Semantic.DeepEqual(gwClassOld.Spec, gwClassNew.Spec) && + equality.Semantic.DeepEqual(gwClassOld.DeletionTimestamp.IsZero(), gwClassNew.DeletionTimestamp.IsZero()) { + return + } + + h.logger.V(1).Info("enqueue gatewayclass update event", "gatewayclass", gwClassNew.Name) + h.enqueueImpactedGateways(ctx, gwClassNew, queue) +} + +// Delete is not implemented for this handler as GatewayClass deletion should be finalized and is prevented while referenced by Gateways +func (h *enqueueRequestsForGatewayClassEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*gatewayv1.GatewayClass], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { +} + +func (h *enqueueRequestsForGatewayClassEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gatewayv1.GatewayClass], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gwClass := e.Object + h.enqueueImpactedGateways(ctx, gwClass, queue) +} + +func (h *enqueueRequestsForGatewayClassEvent) enqueueImpactedGateways(ctx context.Context, gwClass *gatewayv1.GatewayClass, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gwList := GetGatewaysManagedByLBController(ctx, h.k8sClient, h.gwController) + + for _, gw := range gwList { + if string(gw.Spec.GatewayClassName) == gwClass.Name { + h.logger.V(1).Info("enqueue gateway for gatewayclass event", + "gatewayclass", gwClass.GetName(), + "gateway", k8s.NamespacedName(gw)) + queue.Add(reconcile.Request{NamespacedName: k8s.NamespacedName(gw)}) + } + } +} diff --git a/controllers/gateway/eventhandlers/grpc_route_events.go b/controllers/gateway/eventhandlers/grpc_route_events.go new file mode 100644 index 0000000000..496d60e199 --- /dev/null +++ b/controllers/gateway/eventhandlers/grpc_route_events.go @@ -0,0 +1,71 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// NewEnqueueRequestsForGRPCRouteEvent creates handler for GRPCRoute resources +func NewEnqueueRequestsForGRPCRouteEvent( + k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger) handler.TypedEventHandler[*gatewayv1.GRPCRoute, reconcile.Request] { + return &enqueueRequestsForGRPCRouteEvent{ + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*gatewayv1.GRPCRoute, reconcile.Request] = (*enqueueRequestsForGRPCRouteEvent)(nil) + +// enqueueRequestsForGRPCRouteEvent handles GRPCRoute events +type enqueueRequestsForGRPCRouteEvent struct { + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger +} + +func (h *enqueueRequestsForGRPCRouteEvent) Create(ctx context.Context, e event.TypedCreateEvent[*gatewayv1.GRPCRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.Object + h.logger.V(1).Info("enqueue grpcroute create event", "grpcroute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForGRPCRouteEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*gatewayv1.GRPCRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.ObjectNew + h.logger.V(1).Info("enqueue grpcroute update event", "grpcroute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForGRPCRouteEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*gatewayv1.GRPCRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue grpcroute delete event", "grpcroute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForGRPCRouteEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gatewayv1.GRPCRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue grpcroute generic event", "grpcroute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForGRPCRouteEvent) enqueueImpactedGateways(ctx context.Context, route *gatewayv1.GRPCRoute, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gateways, err := GetImpactedGatewaysFromParentRefs(ctx, h.k8sClient, route.Spec.ParentRefs, route.Namespace, constants.ALBGatewayController) + if err != nil { + h.logger.V(1).Info("ignoring unknown gateways referred by", "grpcroute", route.Name, "error", err) + } + for _, gw := range gateways { + h.logger.V(1).Info("enqueue gateway for grpcroute event", + "grpcroute", k8s.NamespacedName(route), + "gateway", gw) + queue.Add(reconcile.Request{NamespacedName: gw}) + } +} diff --git a/controllers/gateway/eventhandlers/http_route_events.go b/controllers/gateway/eventhandlers/http_route_events.go new file mode 100644 index 0000000000..61cd4f3013 --- /dev/null +++ b/controllers/gateway/eventhandlers/http_route_events.go @@ -0,0 +1,71 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// NewEnqueueRequestsForHTTPRouteEvent creates handler for HTTPRoute resources +func NewEnqueueRequestsForHTTPRouteEvent( + k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger) handler.TypedEventHandler[*gatewayv1.HTTPRoute, reconcile.Request] { + return &enqueueRequestsForHTTPRouteEvent{ + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*gatewayv1.HTTPRoute, reconcile.Request] = (*enqueueRequestsForHTTPRouteEvent)(nil) + +// enqueueRequestsForHTTPRouteEvent handles HTTPRoute events +type enqueueRequestsForHTTPRouteEvent struct { + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger +} + +func (h *enqueueRequestsForHTTPRouteEvent) Create(ctx context.Context, e event.TypedCreateEvent[*gatewayv1.HTTPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.Object + h.logger.V(1).Info("enqueue httproute create event", "httproute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForHTTPRouteEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*gatewayv1.HTTPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.ObjectNew + h.logger.V(1).Info("enqueue httproute update event", "httproute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForHTTPRouteEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*gatewayv1.HTTPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue httproute delete event", "httproute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForHTTPRouteEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gatewayv1.HTTPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue grpcroute generic event", "grpcroute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForHTTPRouteEvent) enqueueImpactedGateways(ctx context.Context, route *gatewayv1.HTTPRoute, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gateways, err := GetImpactedGatewaysFromParentRefs(ctx, h.k8sClient, route.Spec.ParentRefs, route.Namespace, constants.ALBGatewayController) + if err != nil { + h.logger.V(1).Info("ignoring unknown gateways referred by", "httproute", route.Name, "error", err) + } + for _, gw := range gateways { + h.logger.V(1).Info("enqueue gateway for httproute event", + "httproute", k8s.NamespacedName(route), + "gateway", gw) + queue.Add(reconcile.Request{NamespacedName: gw}) + } +} diff --git a/controllers/gateway/eventhandlers/load_balancer_configuration_events.go b/controllers/gateway/eventhandlers/load_balancer_configuration_events.go new file mode 100644 index 0000000000..ccbef72e6c --- /dev/null +++ b/controllers/gateway/eventhandlers/load_balancer_configuration_events.go @@ -0,0 +1,83 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// NewEnqueueRequestsForLoadBalancerConfigurationEvent creates handler for LoadBalancerConfiguration resources +func NewEnqueueRequestsForLoadBalancerConfigurationEvent(gwClassEventChan chan<- event.TypedGenericEvent[*gatewayv1.GatewayClass], + k8sClient client.Client, eventRecorder record.EventRecorder, gwController string, logger logr.Logger) handler.TypedEventHandler[*elbv2gw.LoadBalancerConfiguration, reconcile.Request] { + return &enqueueRequestsForLoadBalancerConfigurationEvent{ + gwClassEventChan: gwClassEventChan, + k8sClient: k8sClient, + eventRecorder: eventRecorder, + gwController: gwController, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*elbv2gw.LoadBalancerConfiguration, reconcile.Request] = (*enqueueRequestsForLoadBalancerConfigurationEvent)(nil) + +// enqueueRequestsForLoadBalancerConfigurationEvent handles LoadBalancerConfiguration events +type enqueueRequestsForLoadBalancerConfigurationEvent struct { + gwClassEventChan chan<- event.TypedGenericEvent[*gatewayv1.GatewayClass] + k8sClient client.Client + eventRecorder record.EventRecorder + gwController string + logger logr.Logger +} + +func (h *enqueueRequestsForLoadBalancerConfigurationEvent) Create(ctx context.Context, e event.TypedCreateEvent[*elbv2gw.LoadBalancerConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + lbconfigNew := e.Object + h.logger.V(1).Info("enqueue loadbalancerconfiguration create event", "loadbalancerconfiguration", lbconfigNew.Name) + h.enqueueImpactedService(ctx, lbconfigNew, queue) +} + +func (h *enqueueRequestsForLoadBalancerConfigurationEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*elbv2gw.LoadBalancerConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + lbconfigNew := e.ObjectNew + h.logger.V(1).Info("enqueue loadbalancerconfiguration update event", "loadbalancerconfiguration", lbconfigNew.Name) + h.enqueueImpactedService(ctx, lbconfigNew, queue) +} + +func (h *enqueueRequestsForLoadBalancerConfigurationEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*elbv2gw.LoadBalancerConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + lbconfig := e.Object + h.logger.V(1).Info("enqueue loadbalancerconfiguration delete event", "loadbalancerconfiguration", lbconfig.Name) + h.enqueueImpactedService(ctx, lbconfig, queue) +} + +func (h *enqueueRequestsForLoadBalancerConfigurationEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*elbv2gw.LoadBalancerConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + lbconfig := e.Object + h.logger.V(1).Info("enqueue loadbalancerconfiguration generic event", "loadbalancerconfiguration", lbconfig.Name) + h.enqueueImpactedService(ctx, lbconfig, queue) +} + +func (h *enqueueRequestsForLoadBalancerConfigurationEvent) enqueueImpactedService(ctx context.Context, lbconfig *elbv2gw.LoadBalancerConfiguration, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gwClasses := GetImpactedGatewayClassesFromLbConfig(ctx, h.k8sClient, lbconfig, h.gwController) + for _, gwClass := range gwClasses { + h.logger.V(1).Info("enqueue gatewayClass for loadbalancerconfiguration event", + "loadbalancerconfiguration", k8s.NamespacedName(lbconfig), + "gatewayclass", k8s.NamespacedName(gwClass)) + h.gwClassEventChan <- event.TypedGenericEvent[*gatewayv1.GatewayClass]{ + Object: gwClass, + } + } + gateways := GetImpactedGatewaysFromLbConfig(ctx, h.k8sClient, lbconfig, h.gwController) + for _, gw := range gateways { + if _, isAlreadyEnqueued := gwClasses[string(gw.Spec.GatewayClassName)]; !isAlreadyEnqueued { + h.logger.V(1).Info("enqueue gateway for loadbalancerconfiguration event", + "loadbalancerconfiguration", k8s.NamespacedName(lbconfig), + "gateway", k8s.NamespacedName(gw)) + queue.Add(reconcile.Request{NamespacedName: k8s.NamespacedName(gw)}) + } + } +} diff --git a/controllers/gateway/eventhandlers/service_events.go b/controllers/gateway/eventhandlers/service_events.go new file mode 100644 index 0000000000..2fa7573c6f --- /dev/null +++ b/controllers/gateway/eventhandlers/service_events.go @@ -0,0 +1,156 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// NewenqueueRequestsForServiceEvent detects changes to TargetGroupConfiguration and enqueues all gateway classes and gateways that +// would effected by a change in the TargetGroupConfiguration +func NewEnqueueRequestsForServiceEvent(httpRouteEventChan chan<- event.TypedGenericEvent[*gatewayv1.HTTPRoute], + grpcRouteEventChan chan<- event.TypedGenericEvent[*gatewayv1.GRPCRoute], + tcpRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.TCPRoute], + udpRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.UDPRoute], + tlsRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.TLSRoute], k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger, gwController string) handler.TypedEventHandler[*corev1.Service, reconcile.Request] { + return &enqueueRequestsForServiceEvent{ + httpRouteEventChan: httpRouteEventChan, + grpcRouteEventChan: grpcRouteEventChan, + tcpRouteEventChan: tcpRouteEventChan, + udpRouteEventChan: udpRouteEventChan, + tlsRouteEventChan: tlsRouteEventChan, + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, + gwController: gwController, + } +} + +var _ handler.TypedEventHandler[*corev1.Service, reconcile.Request] = (*enqueueRequestsForServiceEvent)(nil) + +type enqueueRequestsForServiceEvent struct { + httpRouteEventChan chan<- event.TypedGenericEvent[*gatewayv1.HTTPRoute] + grpcRouteEventChan chan<- event.TypedGenericEvent[*gatewayv1.GRPCRoute] + tcpRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.TCPRoute] + udpRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.UDPRoute] + tlsRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.TLSRoute] + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger + gwController string +} + +func (h *enqueueRequestsForServiceEvent) Create(ctx context.Context, e event.TypedCreateEvent[*corev1.Service], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + svcNew := e.Object + h.logger.V(1).Info("enqueue service create event", "service", svcNew.Name) + h.enqueueImpactedRoutes(ctx, svcNew) +} + +func (h *enqueueRequestsForServiceEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*corev1.Service], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + svcNew := e.ObjectNew + h.logger.V(1).Info("enqueue service update event", "service", svcNew.Name) + h.enqueueImpactedRoutes(ctx, svcNew) +} + +func (h *enqueueRequestsForServiceEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*corev1.Service], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + svc := e.Object + h.logger.V(1).Info("enqueue service delete event", "service", svc.Name) + h.enqueueImpactedRoutes(ctx, svc) +} + +func (h *enqueueRequestsForServiceEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*corev1.Service], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + svc := e.Object + h.logger.V(1).Info("enqueue service generic event", "service", svc.Name) + h.enqueueImpactedRoutes(ctx, svc) +} + +func (h *enqueueRequestsForServiceEvent) enqueueImpactedRoutes( + ctx context.Context, + svc *corev1.Service, +) { + if h.gwController == constants.NLBGatewayController { + h.enqueueImpactedL4Routes(ctx, svc) + return + } + h.enqueueImpactedL7Routes(ctx, svc) + +} + +func (h *enqueueRequestsForServiceEvent) enqueueImpactedL4Routes( + ctx context.Context, + svc *corev1.Service, +) { + l4Routes, err := routeutils.ListL4Routes(ctx, h.k8sClient) + if err != nil { + h.logger.V(1).Info("ignoring to enqueue L4 impacted routes ", "error: ", err) + } + filteredRoutesBySvc := routeutils.FilterRoutesBySvc(l4Routes, svc) + for _, route := range filteredRoutesBySvc { + routeType := route.GetRouteKind() + switch routeType { + case routeutils.TCPRouteKind: + h.logger.V(1).Info("enqueue tcproute for service event", + "service", svc.Name, + "tcproute", route.GetRouteNamespacedName()) + h.tcpRouteEventChan <- event.TypedGenericEvent[*gwalpha2.TCPRoute]{ + Object: route.GetRawRoute().(*gwalpha2.TCPRoute), + } + case routeutils.UDPRouteKind: + h.logger.V(1).Info("enqueue updroute for service event", + "service", svc.Name, + "udproute", route.GetRouteNamespacedName()) + h.udpRouteEventChan <- event.TypedGenericEvent[*gwalpha2.UDPRoute]{ + Object: route.GetRawRoute().(*gwalpha2.UDPRoute), + } + case routeutils.TLSRouteKind: + h.logger.V(1).Info("enqueue tlsroute for service event", + "service", svc.Name, + "tlsroute", route.GetRouteNamespacedName()) + h.tlsRouteEventChan <- event.TypedGenericEvent[*gwalpha2.TLSRoute]{ + Object: route.GetRawRoute().(*gwalpha2.TLSRoute), + } + } + } + return +} + +func (h *enqueueRequestsForServiceEvent) enqueueImpactedL7Routes( + ctx context.Context, + svc *corev1.Service, +) { + l7Routes, err := routeutils.ListL7Routes(ctx, h.k8sClient) + if err != nil { + h.logger.V(1).Info("ignoring to enqueue impacted L7 routes ", "error: ", err) + } + filteredRoutesBySvc := routeutils.FilterRoutesBySvc(l7Routes, svc) + for _, route := range filteredRoutesBySvc { + routeType := route.GetRouteKind() + switch routeType { + case routeutils.HTTPRouteKind: + h.logger.V(1).Info("enqueue httproute for service event", + "service", svc.Name, + "httproute", route.GetRouteNamespacedName()) + h.httpRouteEventChan <- event.TypedGenericEvent[*gatewayv1.HTTPRoute]{ + Object: route.GetRawRoute().(*gatewayv1.HTTPRoute), + } + case routeutils.GRPCRouteKind: + h.logger.V(1).Info("enqueue grpcroute for service event", + "service", svc.Name, + "grpcroute", route.GetRouteNamespacedName()) + h.grpcRouteEventChan <- event.TypedGenericEvent[*gatewayv1.GRPCRoute]{ + Object: route.GetRawRoute().(*gatewayv1.GRPCRoute), + } + } + } + return +} diff --git a/controllers/gateway/eventhandlers/target_group_configuration_events.go b/controllers/gateway/eventhandlers/target_group_configuration_events.go new file mode 100644 index 0000000000..5867b090bc --- /dev/null +++ b/controllers/gateway/eventhandlers/target_group_configuration_events.go @@ -0,0 +1,78 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// NewEnqueueRequestsForTargetGroupConfigurationEvent creates handler for TargetGroupConfiguration resources +func NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan chan<- event.TypedGenericEvent[*corev1.Service], + k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger) handler.TypedEventHandler[*elbv2gw.TargetGroupConfiguration, reconcile.Request] { + return &enqueueRequestsForTargetGroupConfigurationEvent{ + svcEventChan: svcEventChan, + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*elbv2gw.TargetGroupConfiguration, reconcile.Request] = (*enqueueRequestsForTargetGroupConfigurationEvent)(nil) + +// enqueueRequestsForTargetGroupConfigurationEvent handles TargetGroupConfiguration events +type enqueueRequestsForTargetGroupConfigurationEvent struct { + svcEventChan chan<- event.TypedGenericEvent[*corev1.Service] + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger +} + +func (h *enqueueRequestsForTargetGroupConfigurationEvent) Create(ctx context.Context, e event.TypedCreateEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + tgconfigNew := e.Object + h.logger.V(1).Info("enqueue targetgroupconfiguration create event", "targetgroupconfiguration", tgconfigNew.Name) + h.enqueueImpactedService(ctx, tgconfigNew, queue) +} + +func (h *enqueueRequestsForTargetGroupConfigurationEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + tgconfigNew := e.ObjectNew + h.logger.V(1).Info("enqueue targetgroupconfiguration update event", "targetgroupconfiguration", tgconfigNew.Name) + h.enqueueImpactedService(ctx, tgconfigNew, queue) +} + +func (h *enqueueRequestsForTargetGroupConfigurationEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + tgconfig := e.Object + h.logger.V(1).Info("enqueue targetgroupconfiguration delete event", "targetgroupconfiguration", tgconfig.Name) + h.enqueueImpactedService(ctx, tgconfig, queue) +} + +func (h *enqueueRequestsForTargetGroupConfigurationEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + tgconfig := e.Object + h.logger.V(1).Info("enqueue targetgroupconfiguration generic event", "targetgroupconfiguration", tgconfig.Name) + h.enqueueImpactedService(ctx, tgconfig, queue) +} + +func (h *enqueueRequestsForTargetGroupConfigurationEvent) enqueueImpactedService(ctx context.Context, tgconfig *elbv2gw.TargetGroupConfiguration, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + svcName := types.NamespacedName{Namespace: tgconfig.Namespace, Name: tgconfig.Spec.TargetReference.Name} + svc := &corev1.Service{} + if err := h.k8sClient.Get(ctx, svcName, svc); err != nil { + h.logger.V(1).Info("ignoring targetgroupconfiguration event for unknown service", + "targetgroupconfiguration", k8s.NamespacedName(tgconfig), + "service", k8s.NamespacedName(svc)) + return + } + h.logger.V(1).Info("enqueue service for targetgroupconfiguration event", + "targetgroupconfiguration", k8s.NamespacedName(tgconfig), + "service", k8s.NamespacedName(svc)) + h.svcEventChan <- event.TypedGenericEvent[*corev1.Service]{ + Object: svc, + } +} diff --git a/controllers/gateway/eventhandlers/tcp_route_events.go b/controllers/gateway/eventhandlers/tcp_route_events.go new file mode 100644 index 0000000000..23e69f5b82 --- /dev/null +++ b/controllers/gateway/eventhandlers/tcp_route_events.go @@ -0,0 +1,71 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// NewEnqueueRequestsForTCPRouteEvent creates handler for TCPRoute resources +func NewEnqueueRequestsForTCPRouteEvent( + k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger) handler.TypedEventHandler[*gwalpha2.TCPRoute, reconcile.Request] { + return &enqueueRequestsForTCPRouteEvent{ + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*gwalpha2.TCPRoute, reconcile.Request] = (*enqueueRequestsForTCPRouteEvent)(nil) + +// enqueueRequestsForTCPRouteEvent handles TCPRoute events +type enqueueRequestsForTCPRouteEvent struct { + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger +} + +func (h *enqueueRequestsForTCPRouteEvent) Create(ctx context.Context, e event.TypedCreateEvent[*gwalpha2.TCPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.Object + h.logger.V(1).Info("enqueue tcproute create event", "tcproute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForTCPRouteEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*gwalpha2.TCPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.ObjectNew + h.logger.V(1).Info("enqueue tcproute update event", "tcproute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForTCPRouteEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*gwalpha2.TCPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue tcproute delete event", "tcproute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForTCPRouteEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gwalpha2.TCPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue tcproute generic event", "tcproute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForTCPRouteEvent) enqueueImpactedGateways(ctx context.Context, route *gwalpha2.TCPRoute, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gateways, err := GetImpactedGatewaysFromParentRefs(ctx, h.k8sClient, route.Spec.ParentRefs, route.Namespace, constants.NLBGatewayController) + if err != nil { + h.logger.V(1).Info("ignoring unknown gateways referred by", "tcproute", route.Name, "error", err) + } + for _, gw := range gateways { + h.logger.V(1).Info("enqueue gateway for tcproute event", + "tcproute", k8s.NamespacedName(route), + "gateway", gw) + queue.Add(reconcile.Request{NamespacedName: gw}) + } +} diff --git a/controllers/gateway/eventhandlers/tls_route_events.go b/controllers/gateway/eventhandlers/tls_route_events.go new file mode 100644 index 0000000000..6b49e20689 --- /dev/null +++ b/controllers/gateway/eventhandlers/tls_route_events.go @@ -0,0 +1,71 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// NewEnqueueRequestsForTLSRouteEvent creates handler for TLSRoute resources +func NewEnqueueRequestsForTLSRouteEvent( + k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger) handler.TypedEventHandler[*gwalpha2.TLSRoute, reconcile.Request] { + return &enqueueRequestsForTLSRouteEvent{ + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*gwalpha2.TLSRoute, reconcile.Request] = (*enqueueRequestsForTLSRouteEvent)(nil) + +// enqueueRequestsForTLSRouteEvent handles TLSRoute events +type enqueueRequestsForTLSRouteEvent struct { + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger +} + +func (h *enqueueRequestsForTLSRouteEvent) Create(ctx context.Context, e event.TypedCreateEvent[*gwalpha2.TLSRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.Object + h.logger.V(1).Info("enqueue tlsroute create event", "tlsroute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForTLSRouteEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*gwalpha2.TLSRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.ObjectNew + h.logger.V(1).Info("enqueue tlsroute update event", "tlsroute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForTLSRouteEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*gwalpha2.TLSRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue tlsroute delete event", "tlsroute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForTLSRouteEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gwalpha2.TLSRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue tlsroute generic event", "tlsroute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForTLSRouteEvent) enqueueImpactedGateways(ctx context.Context, route *gwalpha2.TLSRoute, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gateways, err := GetImpactedGatewaysFromParentRefs(ctx, h.k8sClient, route.Spec.ParentRefs, route.Namespace, constants.NLBGatewayController) + if err != nil { + h.logger.V(1).Info("ignoring unknown gateways referred by", "tlsroute", route.Name, "error", err) + } + for _, gw := range gateways { + h.logger.V(1).Info("enqueue gateway for tlsroute event", + "tlsroute", k8s.NamespacedName(route), + "gateway", gw) + queue.Add(reconcile.Request{NamespacedName: gw}) + } +} diff --git a/controllers/gateway/eventhandlers/udp_route_events.go b/controllers/gateway/eventhandlers/udp_route_events.go new file mode 100644 index 0000000000..894eb93532 --- /dev/null +++ b/controllers/gateway/eventhandlers/udp_route_events.go @@ -0,0 +1,71 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// NewEnqueueRequestsForUDPRouteEvent creates handler for UDPRoute resources +func NewEnqueueRequestsForUDPRouteEvent( + k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger) handler.TypedEventHandler[*gwalpha2.UDPRoute, reconcile.Request] { + return &enqueueRequestsForUDPRouteEvent{ + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*gwalpha2.UDPRoute, reconcile.Request] = (*enqueueRequestsForUDPRouteEvent)(nil) + +// enqueueRequestsForUDPRouteEvent handles UDPRoute events +type enqueueRequestsForUDPRouteEvent struct { + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger +} + +func (h *enqueueRequestsForUDPRouteEvent) Create(ctx context.Context, e event.TypedCreateEvent[*gwalpha2.UDPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.Object + h.logger.V(1).Info("enqueue udproute create event", "udproute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForUDPRouteEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*gwalpha2.UDPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + routeNew := e.ObjectNew + h.logger.V(1).Info("enqueue udproute update event", "udproute", routeNew.Name) + h.enqueueImpactedGateways(ctx, routeNew, queue) +} + +func (h *enqueueRequestsForUDPRouteEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*gwalpha2.UDPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue udproute delete event", "udproute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForUDPRouteEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gwalpha2.UDPRoute], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + route := e.Object + h.logger.V(1).Info("enqueue udproute generic event", "udproute", route.Name) + h.enqueueImpactedGateways(ctx, route, queue) +} + +func (h *enqueueRequestsForUDPRouteEvent) enqueueImpactedGateways(ctx context.Context, route *gwalpha2.UDPRoute, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gateways, err := GetImpactedGatewaysFromParentRefs(ctx, h.k8sClient, route.Spec.ParentRefs, route.Namespace, constants.NLBGatewayController) + if err != nil { + h.logger.V(1).Info("ignoring unknown gateways referred by", "udproute", route.Name, "error", err) + } + for _, gw := range gateways { + h.logger.V(1).Info("enqueue gateway for udproute event", + "udproute", k8s.NamespacedName(route), + "gateway", gw) + queue.Add(reconcile.Request{NamespacedName: gw}) + } +} diff --git a/controllers/gateway/eventhandlers/utils.go b/controllers/gateway/eventhandlers/utils.go new file mode 100644 index 0000000000..8fd985ff77 --- /dev/null +++ b/controllers/gateway/eventhandlers/utils.go @@ -0,0 +1,130 @@ +package eventhandlers + +import ( + "context" + "fmt" + "k8s.io/apimachinery/pkg/types" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// IsGatewayManagedByLBController checks if a Gateway is managed by the ALB/NLB Gateway Controller +// by verifying its associated GatewayClass controller name. +func IsGatewayManagedByLBController(ctx context.Context, k8sClient client.Client, gw *gwv1.Gateway, gwController string) bool { + if gw == nil { + return false + } + + gwClass := &gwv1.GatewayClass{} + if err := k8sClient.Get(ctx, client.ObjectKey{Name: string(gw.Spec.GatewayClassName)}, gwClass); err != nil { + return false + } + return string(gwClass.Spec.ControllerName) == gwController +} + +// GetGatewayClassesManagedByLBController retrieves all GatewayClasses managed by the ALB/NLB Gateway Controller. +func GetGatewayClassesManagedByLBController(ctx context.Context, k8sClient client.Client, gwController string) []*gwv1.GatewayClass { + managedGatewayClasses := make([]*gwv1.GatewayClass, 0) + gwClassList := &gwv1.GatewayClassList{} + if err := k8sClient.List(ctx, gwClassList); err != nil { + return managedGatewayClasses + } + managedGatewayClasses = make([]*gwv1.GatewayClass, 0, len(gwClassList.Items)) + + for i := range gwClassList.Items { + if string(gwClassList.Items[i].Spec.ControllerName) == gwController { + managedGatewayClasses = append(managedGatewayClasses, &gwClassList.Items[i]) + } + } + return managedGatewayClasses +} + +// GetGatewaysManagedByLBController retrieves all Gateways managed by the ALB/NLB Gateway Controller. +func GetGatewaysManagedByLBController(ctx context.Context, k8sClient client.Client, gwController string) []*gwv1.Gateway { + managedGateways := make([]*gwv1.Gateway, 0) + gwList := &gwv1.GatewayList{} + + if err := k8sClient.List(ctx, gwList); err != nil { + return managedGateways + } + + managedGateways = make([]*gwv1.Gateway, 0, len(gwList.Items)) + + for i := range gwList.Items { + if IsGatewayManagedByLBController(ctx, k8sClient, &gwList.Items[i], gwController) { + managedGateways = append(managedGateways, &gwList.Items[i]) + } + } + return managedGateways +} + +// GetImpactedGatewaysFromParentRefs identifies Gateways affected by changes in parent references. +// Returns Gateways that are impacted and managed by the LB controller. +func GetImpactedGatewaysFromParentRefs(ctx context.Context, k8sClient client.Client, parentRefs []gwv1.ParentReference, resourceNamespace string, gwController string) ([]types.NamespacedName, error) { + if len(parentRefs) == 0 { + return nil, nil + } + impactedGateways := make([]types.NamespacedName, 0, len(parentRefs)) + unknownGateways := make([]types.NamespacedName, 0, len(parentRefs)) + var err error + for _, parent := range parentRefs { + gwNamespace := resourceNamespace + if parent.Namespace != nil { + gwNamespace = string(*parent.Namespace) + } + + gwName := types.NamespacedName{ + Namespace: gwNamespace, + Name: string(parent.Name), + } + + gw := &gwv1.Gateway{} + if err := k8sClient.Get(ctx, gwName, gw); err != nil { + // Ignore and continue processing other refs + unknownGateways = append(unknownGateways, gwName) + continue + } + + if IsGatewayManagedByLBController(ctx, k8sClient, gw, gwController) { + impactedGateways = append(impactedGateways, gwName) + } + } + if len(unknownGateways) > 0 { + err = fmt.Errorf("failed to list gateways, %s", unknownGateways) + } + return impactedGateways, err +} + +// GetImpactedGatewayClassesFromLbConfig identifies GatewayClasses affected by LoadBalancer configuration changes. +// Returns GatewayClasses that reference the specified LoadBalancer configuration. +func GetImpactedGatewayClassesFromLbConfig(ctx context.Context, k8sClient client.Client, lbconfig *elbv2gw.LoadBalancerConfiguration, gwController string) map[string]*gwv1.GatewayClass { + if lbconfig == nil { + return nil + } + managedGwClasses := GetGatewayClassesManagedByLBController(ctx, k8sClient, gwController) + impactedGatewayClasses := make(map[string]*gwv1.GatewayClass, len(managedGwClasses)) + for _, gwClass := range managedGwClasses { + if gwClass.Spec.ParametersRef != nil && string(gwClass.Spec.ParametersRef.Kind) == constants.LoadBalancerConfiguration && string(*gwClass.Spec.ParametersRef.Namespace) == lbconfig.Namespace && gwClass.Spec.ParametersRef.Name == lbconfig.Name { + impactedGatewayClasses[gwClass.Name] = gwClass + } + } + return impactedGatewayClasses +} + +// GetImpactedGatewaysFromLbConfig identifies Gateways affected by LoadBalancer configuration changes. +// Returns Gateways that reference the specified LoadBalancer configuration. +func GetImpactedGatewaysFromLbConfig(ctx context.Context, k8sClient client.Client, lbconfig *elbv2gw.LoadBalancerConfiguration, gwController string) []*gwv1.Gateway { + if lbconfig == nil { + return nil + } + managedGateways := GetGatewaysManagedByLBController(ctx, k8sClient, gwController) + impactedGateways := make([]*gwv1.Gateway, 0, len(managedGateways)) + for _, gw := range managedGateways { + if gw.Spec.Infrastructure != nil && gw.Spec.Infrastructure.ParametersRef != nil && string(gw.Spec.Infrastructure.ParametersRef.Kind) == constants.LoadBalancerConfiguration && gw.Spec.Infrastructure.ParametersRef.Name == lbconfig.Name { + impactedGateways = append(impactedGateways, gw) + } + } + return impactedGateways +} diff --git a/controllers/gateway/eventhandlers/utils_test.go b/controllers/gateway/eventhandlers/utils_test.go new file mode 100644 index 0000000000..ad02ac491b --- /dev/null +++ b/controllers/gateway/eventhandlers/utils_test.go @@ -0,0 +1,769 @@ +package eventhandlers + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "testing" +) + +func Test_IsGatewayManagedByLBController(t *testing.T) { + type args struct { + gateway *gwv1.Gateway + gwClass *gwv1.GatewayClass + gwController string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "gateway with valid NLB controller", + args: args{ + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nlb-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "nlb-class", + }, + }, + gwClass: &gwv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nlb-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + }, + }, + gwController: constants.NLBGatewayController, + }, + want: true, + }, + { + name: "gateway with valid ALB controller", + args: args{ + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-alb-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "alb-class", + }, + }, + gwClass: &gwv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "alb-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + }, + }, + gwController: constants.ALBGatewayController, + }, + want: true, + }, + { + name: "gateway with invalid controller", + args: args{ + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-invalid-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "invalid-class", + }, + }, + gwClass: &gwv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: "some.other.controller", + }, + }, + gwController: constants.ALBGatewayController, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := testutils.GenerateTestClient() + k8sClient.Create(context.Background(), tt.args.gwClass) + k8sClient.Create(context.Background(), tt.args.gateway) + got := IsGatewayManagedByLBController(context.Background(), k8sClient, tt.args.gateway, tt.args.gwController) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_GetGatewayClassesManagedByLBController(t *testing.T) { + type args struct { + gwClasses []*gwv1.GatewayClass + gwController string + } + tests := []struct { + name string + args args + want int + }{ + { + name: "multiple gateway classes for NLB Gateway controller", + args: args{ + gwClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nlb-class-1", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nlb-class-2", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "alb-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: "some.other.controller", + }, + }, + }, + gwController: constants.NLBGatewayController, + }, + want: 2, + }, + { + name: "multiple gateway classes for ALB Gateway controller", + args: args{ + gwClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nlb-class-1", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "alb-class-1", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "alb-class-2", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: "some.other.controller", + }, + }, + }, + gwController: constants.ALBGatewayController, + }, + want: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := testutils.GenerateTestClient() + + for _, gwClass := range tt.args.gwClasses { + k8sClient.Create(context.Background(), gwClass) + } + + got := GetGatewayClassesManagedByLBController(context.Background(), k8sClient, tt.args.gwController) + assert.Equal(t, tt.want, len(got)) + }) + } +} + +func Test_GetImpactedGatewaysFromParentRefs(t *testing.T) { + type args struct { + parentRefs []gwv1.ParentReference + resourceNS string + gateways []*gwv1.Gateway + gatewayClasses []*gwv1.GatewayClass + gwController string + } + tests := []struct { + name string + args args + want []types.NamespacedName + wantErr error + }{ + { + name: "valid parent refs with managed gateways", + args: args{ + parentRefs: []gwv1.ParentReference{ + { + Name: "test-gw", + Namespace: (*gwv1.Namespace)(ptr.To("test-ns")), + }, + }, + resourceNS: "test-ns", + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "nlb-class", + }, + }, + }, + gatewayClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nlb-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + }, + }, + }, + gwController: constants.NLBGatewayController, + }, + want: []types.NamespacedName{ + { + Namespace: "test-ns", + Name: "test-gw", + }, + }, + wantErr: nil, + }, + { + name: "valid parent refs with unmanaged gateways", + args: args{ + parentRefs: []gwv1.ParentReference{ + { + Name: "test-gw", + Namespace: (*gwv1.Namespace)(ptr.To("test-ns")), + }, + }, + resourceNS: "test-ns", + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "alb-class", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unmanaged-gw", + Namespace: "test-ns", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "unmanaged-class", + }, + }, + }, + gatewayClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "alb-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "unmanaged-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: "some.other.controller", + }, + }, + }, + gwController: constants.ALBGatewayController, + }, + want: []types.NamespacedName{ + { + Namespace: "test-ns", + Name: "test-gw", + }, + }, + wantErr: nil, + }, + { + name: "valid parent refs with unmanaged and unimpacted gateways", + args: args{ + parentRefs: []gwv1.ParentReference{ + { + Name: "test-gw", + Namespace: (*gwv1.Namespace)(ptr.To("test-ns")), + }, + }, + resourceNS: "test-ns", + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "alb-class", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unimpacted-gw", + Namespace: "another-ns", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "alb-class", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unmanaged-gw", + Namespace: "test-ns", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "unmanaged-class", + }, + }, + }, + gatewayClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "alb-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "unmanaged-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: "some.other.controller", + }, + }, + }, + gwController: constants.ALBGatewayController, + }, + want: []types.NamespacedName{ + { + Namespace: "test-ns", + Name: "test-gw", + }, + }, + wantErr: nil, + }, + { + name: "valid parent refs with managed gateways and unknown gateways", + args: args{ + parentRefs: []gwv1.ParentReference{ + { + Name: "test-gw", + Namespace: (*gwv1.Namespace)(ptr.To("test-ns")), + }, + { + Name: "unknown-gw", + Namespace: (*gwv1.Namespace)(ptr.To("test-ns")), + }, + }, + resourceNS: "test-ns", + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gw", + Namespace: "test-ns", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "nlb-class", + }, + }, + }, + gatewayClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nlb-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + }, + }, + }, + gwController: constants.NLBGatewayController, + }, + want: []types.NamespacedName{ + { + Namespace: "test-ns", + Name: "test-gw", + }, + }, + wantErr: fmt.Errorf("failed to list gateways, [%s]", types.NamespacedName{Namespace: "test-ns", Name: "unknown-gw"}), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := testutils.GenerateTestClient() + + for _, gw := range tt.args.gateways { + k8sClient.Create(context.Background(), gw) + } + for _, gwClass := range tt.args.gatewayClasses { + k8sClient.Create(context.Background(), gwClass) + } + + got, err := GetImpactedGatewaysFromParentRefs(context.Background(), k8sClient, tt.args.parentRefs, tt.args.resourceNS, tt.args.gwController) + + assert.Equal(t, err, tt.wantErr) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_GetImpactedGatewayClassesFromLbConfig(t *testing.T) { + defaultNamespace := gwv1.Namespace("default") + anotherNamespace := gwv1.Namespace("another-namespace") + type args struct { + lbConfig *elbv2gw.LoadBalancerConfiguration + gwClasses []*gwv1.GatewayClass + gwController string + } + tests := []struct { + name string + args args + want int + }{ + { + name: "matching and non-matching lb config reference for ALB Gateway Controller", + args: args{ + lbConfig: &elbv2gw.LoadBalancerConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-config", + Namespace: "default", + }, + }, + gwClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-class-1", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + ParametersRef: &gwv1.ParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + Namespace: &defaultNamespace, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-class-2", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + ParametersRef: &gwv1.ParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + Namespace: &defaultNamespace, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + ParametersRef: &gwv1.ParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + Namespace: &anotherNamespace, + }, + }, + }, + }, + gwController: constants.ALBGatewayController, + }, + want: 1, + }, + { + name: "matching and non-matching lb config reference for NLB Gateway Controller", + args: args{ + lbConfig: &elbv2gw.LoadBalancerConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-config", + Namespace: "default", + }, + }, + gwClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-class-1", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + ParametersRef: &gwv1.ParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + Namespace: &defaultNamespace, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-class-2", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + ParametersRef: &gwv1.ParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + Namespace: &defaultNamespace, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + ParametersRef: &gwv1.ParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + Namespace: &anotherNamespace, + }, + }, + }, + }, + gwController: constants.NLBGatewayController, + }, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := testutils.GenerateTestClient() + for _, gwClass := range tt.args.gwClasses { + k8sClient.Create(context.Background(), gwClass) + } + + got := GetImpactedGatewayClassesFromLbConfig(context.Background(), k8sClient, tt.args.lbConfig, tt.args.gwController) + assert.Equal(t, tt.want, len(got)) + }) + } +} + +func Test_GetImpactedGatewaysFromLbConfig(t *testing.T) { + type args struct { + lbConfig *elbv2gw.LoadBalancerConfiguration + gateways []*gwv1.Gateway + gatewayClasses []*gwv1.GatewayClass + gwController string + } + tests := []struct { + name string + args args + want int + }{ + { + name: "matching and unmanaged lb config reference for ALB Gateway Controller", + args: args{ + lbConfig: &elbv2gw.LoadBalancerConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-config", + }, + }, + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-managed-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "test-managed-class", + Infrastructure: &gwv1.GatewayInfrastructure{ + ParametersRef: &gwv1.LocalParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unmatched-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "test-managed-class", + Infrastructure: &gwv1.GatewayInfrastructure{ + ParametersRef: nil, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unmanaged-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "test-unmanaged-class", + Infrastructure: &gwv1.GatewayInfrastructure{ + ParametersRef: &gwv1.LocalParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + }, + }, + }, + }, + }, + gatewayClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-managed-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unmanaged-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + }, + }, + }, + gwController: constants.ALBGatewayController, + }, + want: 1, + }, + { + name: "matching and unmanaged lb config reference for NLB Gateway Controller", + args: args{ + lbConfig: &elbv2gw.LoadBalancerConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-config", + }, + }, + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-managed-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "test-managed-class", + Infrastructure: &gwv1.GatewayInfrastructure{ + ParametersRef: &gwv1.LocalParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unmatched-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "test-managed-class", + Infrastructure: &gwv1.GatewayInfrastructure{ + ParametersRef: nil, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unmanaged-gw", + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "test-unmanaged-class", + Infrastructure: &gwv1.GatewayInfrastructure{ + ParametersRef: &gwv1.LocalParametersReference{ + Kind: "LoadBalancerConfiguration", + Name: "test-config", + }, + }, + }, + }, + }, + gatewayClasses: []*gwv1.GatewayClass{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-managed-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.NLBGatewayController, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unmanaged-class", + }, + Spec: gwv1.GatewayClassSpec{ + ControllerName: constants.ALBGatewayController, + }, + }, + }, + gwController: constants.NLBGatewayController, + }, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := testutils.GenerateTestClient() + for _, gwClass := range tt.args.gatewayClasses { + k8sClient.Create(context.Background(), gwClass) + } + for _, gw := range tt.args.gateways { + k8sClient.Create(context.Background(), gw) + } + got := GetImpactedGatewaysFromLbConfig(context.Background(), k8sClient, tt.args.lbConfig, tt.args.gwController) + assert.Equal(t, tt.want, len(got)) + }) + } +} diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 21d4dba8a9..6c91111124 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -10,11 +10,13 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/controllers/gateway/eventhandlers" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy" elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" gatewaymodel "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/model" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" @@ -27,25 +29,29 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) type Reconciler interface { Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) - SetupWithManager(mgr ctrl.Manager) error + SetupWithManager(ctx context.Context, mgr ctrl.Manager) (controller.Controller, error) + SetupWatches(ctx context.Context, controller controller.Controller, mgr ctrl.Manager) error } var _ Reconciler = &gatewayReconciler{} // NewNLBGatewayReconciler constructs a gateway reconciler to handle specifically for NLB gateways func NewNLBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { - return newGatewayReconciler(NLBGatewayController, elbv2model.LoadBalancerTypeNetwork, controllerConfig.NLBGatewayMaxConcurrentReconciles, NLBGatewayTagPrefix, NLBGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementNLBGateway) + return newGatewayReconciler(constants.NLBGatewayController, elbv2model.LoadBalancerTypeNetwork, controllerConfig.NLBGatewayMaxConcurrentReconciles, constants.NLBGatewayTagPrefix, constants.NLBGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementNLBGateway) } // NewALBGatewayReconciler constructs a gateway reconciler to handle specifically for ALB gateways func NewALBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { - return newGatewayReconciler(ALBGatewayController, elbv2model.LoadBalancerTypeApplication, controllerConfig.ALBGatewayMaxConcurrentReconciles, ALBGatewayTagPrefix, ALBGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementALBGateway) + return newGatewayReconciler(constants.ALBGatewayController, elbv2model.LoadBalancerTypeApplication, controllerConfig.ALBGatewayMaxConcurrentReconciles, constants.ALBGatewayTagPrefix, constants.ALBGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementALBGateway) } // newGatewayReconciler constructs a reconciler that responds to gateway object changes @@ -59,6 +65,7 @@ func newGatewayReconciler(controllerName string, lbType elbv2model.LoadBalancerT return &gatewayReconciler{ controllerName: controllerName, + lbType: lbType, maxConcurrentReconciles: maxConcurrentReconciles, finalizer: finalizer, gatewayLoader: routeLoader, @@ -79,6 +86,7 @@ func newGatewayReconciler(controllerName string, lbType elbv2model.LoadBalancerT // gatewayReconciler reconciles a Gateway. type gatewayReconciler struct { controllerName string + lbType elbv2model.LoadBalancerType finalizer string maxConcurrentReconciles int gatewayLoader routeutils.Loader @@ -102,6 +110,18 @@ type gatewayReconciler struct { //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/status,verbs=get;update;patch //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/finalizers,verbs=update +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/finalizers,verbs=update + +//+kubebuilder:rbac:groups=gateway.k8s.aws,resources=loadbalancerconfigurations,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.k8s.aws,resources=loadbalancerconfigurations/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.k8s.aws,resources=loadbalancerconfigurations/finalizers,verbs=update + +//+kubebuilder:rbac:groups=gateway.k8s.aws,resources=targetgroupconfigurations,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.k8s.aws,resources=targetgroupconfigurations/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.k8s.aws,resources=targetgroupconfigurations/finalizers,verbs=update + //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=udproutes,verbs=get;list;watch //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=udproutes/status,verbs=get;update;patch //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=udproutes/finalizers,verbs=update @@ -110,6 +130,18 @@ type gatewayReconciler struct { //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes/status,verbs=get;update;patch //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes/finalizers,verbs=update +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes/finalizers,verbs=update + +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/finalizers,verbs=update + +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes,verbs=get;list;watch +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes/finalizers,verbs=update + func (r *gatewayReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { r.reconcileTracker(req.NamespacedName) err := r.reconcileHelper(ctx, req) @@ -263,30 +295,152 @@ func (r *gatewayReconciler) updateGatewayStatus(ctx context.Context, lbDNS strin return nil } -func (r *gatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { - /* - gatewayClassHandler := eventhandlers.NewEnqueueRequestsForGatewayClassEvent(r.logger, r.k8sClient, r.config) - tcpRouteHandler := eventhandlers.NewEnqueueRequestsForTCPRouteEvent(r.logger, r.k8sClient, r.config) - udpRouteHandler := eventhandlers.NewEnqueueRequestsForUDPRouteEvent(r.logger, r.k8sClient, r.config) - return ctrl.NewControllerManagedBy(mgr). - Named("nlbgateway"). - // Anything that influences a gateway object must be added here. - For(&gwv1.Gateway{}). - Watches(&gwv1.GatewayClass{}, gatewayClassHandler). - Watches(&gwalpha2.TCPRoute{}, tcpRouteHandler). - Watches(&gwalpha2.UDPRoute{}, udpRouteHandler). - WithOptions(controller.Options{ - MaxConcurrentReconciles: r.config.MaxConcurrentReconciles, - }). - Complete(r) - */ - - return ctrl.NewControllerManagedBy(mgr). - Named(r.controllerName). - // Anything that influences a gateway object must be added here. - For(&gwv1.Gateway{}). - WithOptions(controller.Options{ - MaxConcurrentReconciles: r.maxConcurrentReconciles, - }). - Complete(r) +func (r *gatewayReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) (controller.Controller, error) { + c, err := controller.New(r.controllerName, mgr, controller.Options{ + MaxConcurrentReconciles: r.maxConcurrentReconciles, + Reconciler: r, + }) + if err != nil { + return nil, err + } + return c, nil +} + +func (r *gatewayReconciler) SetupWatches(ctx context.Context, c controller.Controller, mgr ctrl.Manager) error { + if err := r.setupCommonGatewayControllerWatches(c, mgr); err != nil { + return err + } + switch r.controllerName { + case constants.ALBGatewayController: + if err := r.setupALBGatewayControllerWatches(c, mgr); err != nil { + return err + } + break + case constants.NLBGatewayController: + if err := r.setupNLBGatewayControllerWatches(c, mgr); err != nil { + return err + } + break + default: + return fmt.Errorf("unknown controller %v", r.controllerName) + } + return nil +} + +func (r *gatewayReconciler) setupCommonGatewayControllerWatches(ctrl controller.Controller, mgr ctrl.Manager) error { + loggerPrefix := r.logger.WithName("eventHandlers") + gwClassEventChan := make(chan event.TypedGenericEvent[*gwv1.GatewayClass]) + lbConfigEventChan := make(chan event.TypedGenericEvent[*elbv2gw.LoadBalancerConfiguration]) + + gwClassEventHandler := eventhandlers.NewEnqueueRequestsForGatewayClassEvent(r.k8sClient, r.eventRecorder, r.controllerName, + loggerPrefix.WithName("GatewayClass")) + lbConfigEventHandler := eventhandlers.NewEnqueueRequestsForLoadBalancerConfigurationEvent(gwClassEventChan, r.k8sClient, r.eventRecorder, r.controllerName, + loggerPrefix.WithName("LoadBalancerConfiguration")) + + if err := ctrl.Watch(source.Channel(gwClassEventChan, gwClassEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Channel(lbConfigEventChan, lbConfigEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &gwv1.GatewayClass{}, gwClassEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &elbv2gw.LoadBalancerConfiguration{}, lbConfigEventHandler)); err != nil { + return err + } + return nil + +} + +func (r *gatewayReconciler) setupALBGatewayControllerWatches(ctrl controller.Controller, mgr ctrl.Manager) error { + loggerPrefix := r.logger.WithName("eventHandlers") + tbConfigEventChan := make(chan event.TypedGenericEvent[*elbv2gw.TargetGroupConfiguration]) + httpRouteEventChan := make(chan event.TypedGenericEvent[*gwv1.HTTPRoute]) + grpcRouteEventChan := make(chan event.TypedGenericEvent[*gwv1.GRPCRoute]) + svcEventChan := make(chan event.TypedGenericEvent[*corev1.Service]) + tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("TargetGroupConfiguration")) + grpcRouteEventHandler := eventhandlers.NewEnqueueRequestsForGRPCRouteEvent(r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("GRPCRoute")) + httpRouteEventHandler := eventhandlers.NewEnqueueRequestsForHTTPRouteEvent(r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("HTTPRoute")) + svcEventHandler := eventhandlers.NewEnqueueRequestsForServiceEvent(httpRouteEventChan, grpcRouteEventChan, nil, nil, nil, r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("Service"), constants.ALBGatewayController) + if err := ctrl.Watch(source.Channel(tbConfigEventChan, tgConfigEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Channel(httpRouteEventChan, httpRouteEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Channel(grpcRouteEventChan, grpcRouteEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Channel(svcEventChan, svcEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &elbv2gw.TargetGroupConfiguration{}, tgConfigEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &corev1.Service{}, svcEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &gwv1.HTTPRoute{}, httpRouteEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &gwv1.GRPCRoute{}, grpcRouteEventHandler)); err != nil { + return err + } + return nil +} + +func (r *gatewayReconciler) setupNLBGatewayControllerWatches(ctrl controller.Controller, mgr ctrl.Manager) error { + loggerPrefix := r.logger.WithName("eventHandlers") + tbConfigEventChan := make(chan event.TypedGenericEvent[*elbv2gw.TargetGroupConfiguration]) + tcpRouteEventChan := make(chan event.TypedGenericEvent[*gwalpha2.TCPRoute]) + udpRouteEventChan := make(chan event.TypedGenericEvent[*gwalpha2.UDPRoute]) + tlsRouteEventChan := make(chan event.TypedGenericEvent[*gwalpha2.TLSRoute]) + svcEventChan := make(chan event.TypedGenericEvent[*corev1.Service]) + tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("TargetGroupConfiguration")) + tcpRouteEventHandler := eventhandlers.NewEnqueueRequestsForTCPRouteEvent(r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("TCPRoute")) + udpRouteEventHandler := eventhandlers.NewEnqueueRequestsForUDPRouteEvent(r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("UDPRoute")) + tlsRouteEventHandler := eventhandlers.NewEnqueueRequestsForTLSRouteEvent(r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("TLSRoute")) + svcEventHandler := eventhandlers.NewEnqueueRequestsForServiceEvent(nil, nil, tcpRouteEventChan, udpRouteEventChan, tlsRouteEventChan, r.k8sClient, r.eventRecorder, + loggerPrefix.WithName("Service"), constants.NLBGatewayController) + if err := ctrl.Watch(source.Channel(tbConfigEventChan, tgConfigEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Channel(tcpRouteEventChan, tcpRouteEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Channel(udpRouteEventChan, udpRouteEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Channel(tlsRouteEventChan, tlsRouteEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Channel(svcEventChan, svcEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &elbv2gw.TargetGroupConfiguration{}, tgConfigEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &corev1.Service{}, svcEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &gwalpha2.TCPRoute{}, tcpRouteEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &gwalpha2.UDPRoute{}, udpRouteEventHandler)); err != nil { + return err + } + if err := ctrl.Watch(source.Kind(mgr.GetCache(), &gwalpha2.TLSRoute{}, tlsRouteEventHandler)); err != nil { + return err + } + return nil + } diff --git a/main.go b/main.go index 4e30beecb2..f0f1250dfc 100644 --- a/main.go +++ b/main.go @@ -17,9 +17,17 @@ limitations under the License. package main import ( + "context" + "fmt" "os" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/controllers/gateway" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "k8s.io/client-go/util/workqueue" @@ -68,9 +76,30 @@ func init() { _ = clientgoscheme.AddToScheme(scheme) _ = elbv2api.AddToScheme(scheme) + _ = elbv2gw.AddToScheme(scheme) + _ = gwv1.AddToScheme(scheme) + _ = gwalpha2.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } +// Define a struct to hold common gateway controller dependencies +type gatewayControllerConfig struct { + routeLoader routeutils.Loader + cloud services.Cloud + k8sClient client.Client + controllerCFG config.ControllerConfig + finalizerManager k8s.FinalizerManager + sgReconciler networking.SecurityGroupReconciler + sgManager networking.SecurityGroupManager + elbv2TaggingManager elbv2deploy.TaggingManager + subnetResolver networking.SubnetsResolver + vpcInfoProvider networking.VPCInfoProvider + backendSGProvider networking.BackendSGProvider + sgResolver networking.SecurityGroupResolver + metricsCollector lbcmetrics.MetricCollector + reconcileCounters *metricsutil.ReconcileCounters +} + func main() { infoLogger := getLoggerWithLogLevel("info") infoLogger.Info("version", @@ -173,25 +202,44 @@ func main() { os.Exit(1) } - if controllerCFG.FeatureGates.Enabled(config.NLBGatewayAPI) { - routeLoader := routeutils.NewLoader(mgr.GetClient()) - nlbGatewayReconciler := gateway.NewNLBGatewayReconciler(routeLoader, cloud, mgr.GetClient(), mgr.GetEventRecorderFor("nlbgateway"), controllerCFG, finalizerManager, sgReconciler, sgManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("nlbgateway"), lbcMetricsCollector, reconcileCounters) - nlbControllerError := nlbGatewayReconciler.SetupWithManager(mgr) - if nlbControllerError != nil { - setupLog.Error(nlbControllerError, "Unable to create NLB Gateway controller") - os.Exit(1) + // Initialize common gateway configuration + if controllerCFG.FeatureGates.Enabled(config.NLBGatewayAPI) || controllerCFG.FeatureGates.Enabled(config.ALBGatewayAPI) { + gwControllerConfig := &gatewayControllerConfig{ + cloud: cloud, + k8sClient: mgr.GetClient(), + controllerCFG: controllerCFG, + finalizerManager: finalizerManager, + sgReconciler: sgReconciler, + sgManager: sgManager, + elbv2TaggingManager: elbv2TaggingManager, + subnetResolver: subnetResolver, + vpcInfoProvider: vpcInfoProvider, + backendSGProvider: backendSGProvider, + sgResolver: sgResolver, + metricsCollector: lbcMetricsCollector, + reconcileCounters: reconcileCounters, } - } - if controllerCFG.FeatureGates.Enabled(config.ALBGatewayAPI) { - routeLoader := routeutils.NewLoader(mgr.GetClient()) - albGatewayReconciler := gateway.NewALBGatewayReconciler(routeLoader, cloud, mgr.GetClient(), mgr.GetEventRecorderFor("albgateway"), controllerCFG, finalizerManager, sgReconciler, sgManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("albgateway"), lbcMetricsCollector, reconcileCounters) - albControllerErr := albGatewayReconciler.SetupWithManager(mgr) - if albControllerErr != nil { - setupLog.Error(albControllerErr, "Unable to create ALB Gateway controller") - os.Exit(1) + + // Setup NLB Gateway controller if enabled + if controllerCFG.FeatureGates.Enabled(config.NLBGatewayAPI) { + gwControllerConfig.routeLoader = routeutils.NewLoader(mgr.GetClient()) + if err := setupGatewayController(ctx, mgr, gwControllerConfig, constants.NLBGatewayController); err != nil { + setupLog.Error(err, "failed to setup NLB Gateway controller") + os.Exit(1) + } } - } + // Setup ALB Gateway controller if enabled + if controllerCFG.FeatureGates.Enabled(config.ALBGatewayAPI) { + if gwControllerConfig.routeLoader == nil { + gwControllerConfig.routeLoader = routeutils.NewLoader(mgr.GetClient()) + } + if err := setupGatewayController(ctx, mgr, gwControllerConfig, constants.ALBGatewayController); err != nil { + setupLog.Error(err, "failed to setup ALB Gateway controller") + os.Exit(1) + } + } + } // Add liveness probe err = mgr.AddHealthzCheck("health-ping", healthz.Ping) setupLog.Info("adding health check for controller") @@ -252,6 +300,66 @@ func main() { } } +// setupGatewayController handles the setup of both NLB and ALB gateway controllers +func setupGatewayController(ctx context.Context, mgr ctrl.Manager, cfg *gatewayControllerConfig, controllerType string) error { + logger := ctrl.Log.WithName("controllers").WithName(controllerType) + + var reconciler gateway.Reconciler + switch controllerType { + case constants.NLBGatewayController: + reconciler = gateway.NewNLBGatewayReconciler( + cfg.routeLoader, + cfg.cloud, + cfg.k8sClient, + mgr.GetEventRecorderFor(controllerType), + cfg.controllerCFG, + cfg.finalizerManager, + cfg.sgReconciler, + cfg.sgManager, + cfg.elbv2TaggingManager, + cfg.subnetResolver, + cfg.vpcInfoProvider, + cfg.backendSGProvider, + cfg.sgResolver, + logger, + cfg.metricsCollector, + cfg.reconcileCounters, + ) + case constants.ALBGatewayController: + reconciler = gateway.NewALBGatewayReconciler( + cfg.routeLoader, + cfg.cloud, + cfg.k8sClient, + mgr.GetEventRecorderFor(controllerType), + cfg.controllerCFG, + cfg.finalizerManager, + cfg.sgReconciler, + cfg.sgManager, + cfg.elbv2TaggingManager, + cfg.subnetResolver, + cfg.vpcInfoProvider, + cfg.backendSGProvider, + cfg.sgResolver, + logger, + cfg.metricsCollector, + cfg.reconcileCounters, + ) + default: + return fmt.Errorf("unknown controller type: %s", controllerType) + } + + controller, err := reconciler.SetupWithManager(ctx, mgr) + if err != nil { + return fmt.Errorf("unable to create %s controller: %w", controllerType, err) + } + + if err := reconciler.SetupWatches(ctx, controller, mgr); err != nil { + return fmt.Errorf("unable to setup watches for %s controller: %w", controllerType, err) + } + + return nil +} + // loadControllerConfig loads the controller configuration. func loadControllerConfig() (config.ControllerConfig, error) { defaultAWSThrottleCFG := throttle.NewDefaultServiceOperationsThrottleConfig() diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 6a9530cdc1..1f0903af8a 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -22,6 +22,8 @@ const ( flagServiceTargetENISGTags = "service-target-eni-security-group-tags" flagServiceMaxConcurrentReconciles = "service-max-concurrent-reconciles" flagTargetGroupBindingMaxConcurrentReconciles = "targetgroupbinding-max-concurrent-reconciles" + flagALBGatewayMaxConcurrentReconciles = "alb-gateway-max-concurrent-reconciles" + flagNLBGatewayMaxConcurrentReconciles = "nlb-gateway-max-concurrent-reconciles" flagTargetGroupBindingMaxExponentialBackoffDelay = "targetgroupbinding-max-exponential-backoff-delay" flagLbStabilizationMonitorInterval = "lb-stabilization-monitor-interval" flagDefaultSSLPolicy = "default-ssl-policy" @@ -142,6 +144,10 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { "Maximum number of concurrently running reconcile loops for service") fs.IntVar(&cfg.TargetGroupBindingMaxConcurrentReconciles, flagTargetGroupBindingMaxConcurrentReconciles, defaultMaxConcurrentReconciles, "Maximum number of concurrently running reconcile loops for targetGroupBinding") + fs.IntVar(&cfg.ALBGatewayMaxConcurrentReconciles, flagALBGatewayMaxConcurrentReconciles, defaultMaxConcurrentReconciles, + "Maximum number of concurrently running reconcile loops for alb gateway") + fs.IntVar(&cfg.NLBGatewayMaxConcurrentReconciles, flagNLBGatewayMaxConcurrentReconciles, defaultMaxConcurrentReconciles, + "Maximum number of concurrently running reconcile loops for nlb gateway") fs.DurationVar(&cfg.TargetGroupBindingMaxExponentialBackoffDelay, flagTargetGroupBindingMaxExponentialBackoffDelay, defaultMaxExponentialBackoffDelay, "Maximum duration of exponential backoff for targetGroupBinding reconcile failures") fs.DurationVar(&cfg.LBStabilizationMonitorInterval, flagLbStabilizationMonitorInterval, defaultLbStabilizationMonitorInterval, diff --git a/controllers/gateway/constants.go b/pkg/gateway/constants/controller_constants.go similarity index 86% rename from controllers/gateway/constants.go rename to pkg/gateway/constants/controller_constants.go index a0c3d5c46e..194a0373fa 100644 --- a/controllers/gateway/constants.go +++ b/pkg/gateway/constants/controller_constants.go @@ -1,4 +1,4 @@ -package gateway +package constants /* Common constants @@ -7,10 +7,13 @@ Common constants const ( // GatewayResourceGroupVersion the groupVersion used by Gateway & GatewayClass resources. GatewayResourceGroupVersion = "gateway.networking.k8s.io/v1" + + // LoadBalancerConfiguration the CRD name of LoadBalancerConfiguration + LoadBalancerConfiguration = "LoadBalancerConfiguration" ) /* -NLB constants + NLB Gateway constants */ const ( @@ -28,7 +31,7 @@ const ( ) /* -ALB Constants + ALB Gateway Constants */ const ( diff --git a/pkg/gateway/routeutils/backend_test.go b/pkg/gateway/routeutils/backend_test.go index 1a042fc4fe..1ec9b4f4f8 100644 --- a/pkg/gateway/routeutils/backend_test.go +++ b/pkg/gateway/routeutils/backend_test.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "testing" @@ -302,7 +303,7 @@ func TestCommonBackendLoader(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() if tc.storedService != nil { err := k8sClient.Create(context.Background(), tc.storedService) @@ -520,7 +521,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() for _, c := range tc.allTargetGroupConfigurations { err := k8sClient.Create(context.Background(), &c) assert.NoError(t, err) @@ -702,7 +703,7 @@ func Test_referenceGrantCheck(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() for _, ref := range tc.referenceGrants { err := k8sClient.Create(context.Background(), &ref) assert.NoError(t, err) diff --git a/pkg/gateway/routeutils/descriptor.go b/pkg/gateway/routeutils/descriptor.go index 36c8743eb5..59e43db82d 100644 --- a/pkg/gateway/routeutils/descriptor.go +++ b/pkg/gateway/routeutils/descriptor.go @@ -16,6 +16,7 @@ type routeMetadataDescriptor interface { GetHostnames() []gwv1.Hostname GetParentRefs() []gwv1.ParentReference GetRawRoute() interface{} + GetBackendRefs() []gwv1.BackendRef } // preLoadRouteDescriptor this object is used to represent a route description that has not loaded its child data (services, tg config) diff --git a/pkg/gateway/routeutils/grpc.go b/pkg/gateway/routeutils/grpc.go index 1cd65bd98a..27e693cde6 100644 --- a/pkg/gateway/routeutils/grpc.go +++ b/pkg/gateway/routeutils/grpc.go @@ -99,6 +99,18 @@ func (grpcRoute *grpcRouteDescription) GetRawRoute() interface{} { return grpcRoute.route } +func (grpcRoute *grpcRouteDescription) GetBackendRefs() []gwv1.BackendRef { + backendRefs := make([]gwv1.BackendRef, 0) + if grpcRoute.route.Spec.Rules != nil { + for _, rule := range grpcRoute.route.Spec.Rules { + for _, grpcBackendRef := range rule.BackendRefs { + backendRefs = append(backendRefs, grpcBackendRef.BackendRef) + } + } + } + return backendRefs +} + var _ RouteDescriptor = &grpcRouteDescription{} // Can we use an indexer here to query more efficiently? diff --git a/pkg/gateway/routeutils/grpc_test.go b/pkg/gateway/routeutils/grpc_test.go index 2ba3f48de4..a00bb6e3a3 100644 --- a/pkg/gateway/routeutils/grpc_test.go +++ b/pkg/gateway/routeutils/grpc_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "testing" @@ -32,7 +33,7 @@ func Test_ConvertGRPCRuleToRouteRule(t *testing.T) { } func Test_ListGRPCRoutes(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() k8sClient.Create(context.Background(), &gwv1.GRPCRoute{ ObjectMeta: metav1.ObjectMeta{ @@ -43,7 +44,25 @@ func Test_ListGRPCRoutes(t *testing.T) { Hostnames: []gwv1.Hostname{ "host1", }, - Rules: nil, + Rules: []gwv1.GRPCRouteRule{ + { + BackendRefs: []gwv1.GRPCBackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwv1.GRPCBackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwv1.GRPCBackendRef{}, + }, + }, }, }) @@ -82,16 +101,19 @@ func Test_ListGRPCRoutes(t *testing.T) { assert.Equal(t, []gwv1.Hostname{ "host1", }, v.GetHostnames()) + assert.Equal(t, 6, len(v.GetBackendRefs())) } if routeNsn.Name == "foo2" { assert.Equal(t, []gwv1.Hostname{ "host2", }, v.GetHostnames()) + assert.Equal(t, 0, len(v.GetBackendRefs())) } if routeNsn.Name == "foo3" { assert.Equal(t, 0, len(v.GetHostnames())) + assert.Equal(t, 0, len(v.GetBackendRefs())) } } diff --git a/pkg/gateway/routeutils/http.go b/pkg/gateway/routeutils/http.go index 1288ed56c3..4dab813022 100644 --- a/pkg/gateway/routeutils/http.go +++ b/pkg/gateway/routeutils/http.go @@ -92,6 +92,18 @@ func (httpRoute *httpRouteDescription) GetRouteNamespacedName() types.Namespaced return k8s.NamespacedName(httpRoute.route) } +func (httpRoute *httpRouteDescription) GetBackendRefs() []gwv1.BackendRef { + backendRefs := make([]gwv1.BackendRef, 0) + if httpRoute.route.Spec.Rules != nil { + for _, rule := range httpRoute.route.Spec.Rules { + for _, httpBackendRef := range rule.BackendRefs { + backendRefs = append(backendRefs, httpBackendRef.BackendRef) + } + } + } + return backendRefs +} + func convertHTTPRoute(r gwv1.HTTPRoute) *httpRouteDescription { return &httpRouteDescription{route: &r, backendLoader: commonBackendLoader} } diff --git a/pkg/gateway/routeutils/http_test.go b/pkg/gateway/routeutils/http_test.go index 27f95dfe79..9d0d98f868 100644 --- a/pkg/gateway/routeutils/http_test.go +++ b/pkg/gateway/routeutils/http_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "testing" @@ -32,7 +33,7 @@ func Test_ConvertHTTPRuleToRouteRule(t *testing.T) { } func Test_ListHTTPRoutes(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() k8sClient.Create(context.Background(), &gwv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ @@ -43,7 +44,25 @@ func Test_ListHTTPRoutes(t *testing.T) { Hostnames: []gwv1.Hostname{ "host1", }, - Rules: nil, + Rules: []gwv1.HTTPRouteRule{ + { + BackendRefs: []gwv1.HTTPBackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwv1.HTTPBackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwv1.HTTPBackendRef{}, + }, + }, }, }) @@ -82,16 +101,19 @@ func Test_ListHTTPRoutes(t *testing.T) { assert.Equal(t, []gwv1.Hostname{ "host1", }, v.GetHostnames()) + assert.Equal(t, 6, len(v.GetBackendRefs())) } if routeNsn.Name == "foo2" { assert.Equal(t, []gwv1.Hostname{ "host2", }, v.GetHostnames()) + assert.Equal(t, 0, len(v.GetBackendRefs())) } if routeNsn.Name == "foo3" { assert.Equal(t, 0, len(v.GetHostnames())) + assert.Equal(t, 0, len(v.GetBackendRefs())) } } diff --git a/pkg/gateway/routeutils/loader_test.go b/pkg/gateway/routeutils/loader_test.go index 0f58c29a69..6a8a1145e5 100644 --- a/pkg/gateway/routeutils/loader_test.go +++ b/pkg/gateway/routeutils/loader_test.go @@ -50,6 +50,11 @@ func (m *mockRoute) GetParentRefs() []gwv1.ParentReference { panic("implement me") } +func (m *mockRoute) GetBackendRefs() []gwv1.BackendRef { + //TODO implement me + panic("implement me") +} + func (m *mockRoute) GetRawRoute() interface{} { //TODO implement me panic("implement me") diff --git a/pkg/gateway/routeutils/namespace_selector_test.go b/pkg/gateway/routeutils/namespace_selector_test.go index 421c6997ae..c591e6fb25 100644 --- a/pkg/gateway/routeutils/namespace_selector_test.go +++ b/pkg/gateway/routeutils/namespace_selector_test.go @@ -6,6 +6,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" "testing" ) @@ -92,7 +93,7 @@ func Test_getNamespacesFromSelector(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() nsSelector := namespaceSelectorImpl{ k8sClient: k8sClient, } diff --git a/pkg/gateway/routeutils/tcp.go b/pkg/gateway/routeutils/tcp.go index 677abc3ff5..2425f7fae9 100644 --- a/pkg/gateway/routeutils/tcp.go +++ b/pkg/gateway/routeutils/tcp.go @@ -102,6 +102,16 @@ func (tcpRoute *tcpRouteDescription) GetParentRefs() []gwv1.ParentReference { return tcpRoute.route.Spec.ParentRefs } +func (tcpRoute *tcpRouteDescription) GetBackendRefs() []gwv1.BackendRef { + backendRefs := make([]gwv1.BackendRef, 0) + if tcpRoute.route.Spec.Rules != nil { + for _, rule := range tcpRoute.route.Spec.Rules { + backendRefs = append(backendRefs, rule.BackendRefs...) + } + } + return backendRefs +} + var _ RouteDescriptor = &tcpRouteDescription{} // Can we use an indexer here to query more efficiently? diff --git a/pkg/gateway/routeutils/tcp_test.go b/pkg/gateway/routeutils/tcp_test.go index d9eda7640a..0ceef13f4b 100644 --- a/pkg/gateway/routeutils/tcp_test.go +++ b/pkg/gateway/routeutils/tcp_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -30,7 +31,7 @@ func Test_ConvertTCPRuleToRouteRule(t *testing.T) { } func Test_ListTCPRoutes(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() k8sClient.Create(context.Background(), &gwalpha2.TCPRoute{ ObjectMeta: metav1.ObjectMeta{ @@ -38,7 +39,25 @@ func Test_ListTCPRoutes(t *testing.T) { Namespace: "bar1", }, Spec: gwalpha2.TCPRouteSpec{ - Rules: nil, + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{}, + }, + }, }, }) @@ -70,6 +89,17 @@ func Test_ListTCPRoutes(t *testing.T) { assert.Equal(t, TCPRouteKind, v.GetRouteKind()) assert.NotNil(t, v.GetRawRoute()) assert.Equal(t, 0, len(v.GetHostnames())) + if routeNsn.Name == "foo1" { + assert.Equal(t, 6, len(v.GetBackendRefs())) + } + + if routeNsn.Name == "foo2" { + assert.Equal(t, 0, len(v.GetBackendRefs())) + } + + if routeNsn.Name == "foo3" { + assert.Equal(t, 0, len(v.GetBackendRefs())) + } } assert.Equal(t, "foo1", itemMap["bar1"]) diff --git a/pkg/gateway/routeutils/test_utils.go b/pkg/gateway/routeutils/test_utils.go deleted file mode 100644 index de9b9fa6aa..0000000000 --- a/pkg/gateway/routeutils/test_utils.go +++ /dev/null @@ -1,25 +0,0 @@ -package routeutils - -import ( - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" - elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" - testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - gwv1 "sigs.k8s.io/gateway-api/apis/v1" - gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" - gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" -) - -func generateTestClient() client.Client { - k8sSchema := runtime.NewScheme() - clientgoscheme.AddToScheme(k8sSchema) - elbv2api.AddToScheme(k8sSchema) - gwv1.AddToScheme(k8sSchema) - gwalpha2.AddToScheme(k8sSchema) - elbv2gw.AddToScheme(k8sSchema) - gwbeta1.AddToScheme(k8sSchema) - - return testclient.NewClientBuilder().WithScheme(k8sSchema).Build() -} diff --git a/pkg/gateway/routeutils/tls.go b/pkg/gateway/routeutils/tls.go index e3d0596c3d..f665c25b9e 100644 --- a/pkg/gateway/routeutils/tls.go +++ b/pkg/gateway/routeutils/tls.go @@ -102,6 +102,16 @@ func (tlsRoute *tlsRouteDescription) GetRawRoute() interface{} { return tlsRoute.route } +func (tlsRoute *tlsRouteDescription) GetBackendRefs() []gwv1.BackendRef { + backendRefs := make([]gwv1.BackendRef, 0) + if tlsRoute.route.Spec.Rules != nil { + for _, rule := range tlsRoute.route.Spec.Rules { + backendRefs = append(backendRefs, rule.BackendRefs...) + } + } + return backendRefs +} + var _ RouteDescriptor = &tlsRouteDescription{} func ListTLSRoutes(context context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error) { diff --git a/pkg/gateway/routeutils/tls_test.go b/pkg/gateway/routeutils/tls_test.go index 375feeb014..801480a5a6 100644 --- a/pkg/gateway/routeutils/tls_test.go +++ b/pkg/gateway/routeutils/tls_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -30,7 +31,7 @@ func Test_ConvertTLSRuleToRouteRule(t *testing.T) { } func Test_ListTLSRoutes(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() k8sClient.Create(context.Background(), &gwalpha2.TLSRoute{ ObjectMeta: metav1.ObjectMeta{ @@ -41,7 +42,25 @@ func Test_ListTLSRoutes(t *testing.T) { Hostnames: []gwv1.Hostname{ "host1", }, - Rules: nil, + Rules: []gwalpha2.TLSRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{}, + }, + }, }, }) @@ -80,16 +99,19 @@ func Test_ListTLSRoutes(t *testing.T) { assert.Equal(t, []gwv1.Hostname{ "host1", }, v.GetHostnames()) + assert.Equal(t, 6, len(v.GetBackendRefs())) } if routeNsn.Name == "foo2" { assert.Equal(t, []gwv1.Hostname{ "host2", }, v.GetHostnames()) + assert.Equal(t, 0, len(v.GetBackendRefs())) } if routeNsn.Name == "foo3" { assert.Equal(t, 0, len(v.GetHostnames())) + assert.Equal(t, 0, len(v.GetBackendRefs())) } } diff --git a/pkg/gateway/routeutils/udp.go b/pkg/gateway/routeutils/udp.go index fa7b122f75..818dd770a1 100644 --- a/pkg/gateway/routeutils/udp.go +++ b/pkg/gateway/routeutils/udp.go @@ -101,6 +101,16 @@ func (udpRoute *udpRouteDescription) GetRawRoute() interface{} { return udpRoute.route } +func (udpRoute *udpRouteDescription) GetBackendRefs() []gwv1.BackendRef { + backendRefs := make([]gwv1.BackendRef, 0) + if udpRoute.route.Spec.Rules != nil { + for _, rule := range udpRoute.route.Spec.Rules { + backendRefs = append(backendRefs, rule.BackendRefs...) + } + } + return backendRefs +} + var _ RouteDescriptor = &udpRouteDescription{} func ListUDPRoutes(context context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error) { diff --git a/pkg/gateway/routeutils/udp_test.go b/pkg/gateway/routeutils/udp_test.go index 69b82d1cd9..c3ae71dde8 100644 --- a/pkg/gateway/routeutils/udp_test.go +++ b/pkg/gateway/routeutils/udp_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -30,7 +31,7 @@ func Test_ConvertUDPRuleToRouteRule(t *testing.T) { } func Test_ListUDPRoutes(t *testing.T) { - k8sClient := generateTestClient() + k8sClient := testutils.GenerateTestClient() k8sClient.Create(context.Background(), &gwalpha2.UDPRoute{ ObjectMeta: metav1.ObjectMeta{ @@ -38,7 +39,25 @@ func Test_ListUDPRoutes(t *testing.T) { Namespace: "bar1", }, Spec: gwalpha2.UDPRouteSpec{ - Rules: nil, + Rules: []gwalpha2.UDPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{}, + }, + }, }, }) @@ -70,6 +89,17 @@ func Test_ListUDPRoutes(t *testing.T) { assert.Equal(t, UDPRouteKind, v.GetRouteKind()) assert.NotNil(t, v.GetRawRoute()) assert.Equal(t, 0, len(v.GetHostnames())) + if routeNsn.Name == "foo1" { + assert.Equal(t, 6, len(v.GetBackendRefs())) + } + + if routeNsn.Name == "foo2" { + assert.Equal(t, 0, len(v.GetBackendRefs())) + } + + if routeNsn.Name == "foo3" { + assert.Equal(t, 0, len(v.GetBackendRefs())) + } } assert.Equal(t, "foo1", itemMap["bar1"]) diff --git a/pkg/gateway/routeutils/utils.go b/pkg/gateway/routeutils/utils.go new file mode 100644 index 0000000000..75e002b270 --- /dev/null +++ b/pkg/gateway/routeutils/utils.go @@ -0,0 +1,89 @@ +package routeutils + +import ( + "context" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ListL4Routes retrieves all Layer 4 routes (TCP, UDP, TLS) from the cluster. +func ListL4Routes(ctx context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error) { + l4Routes := make([]preLoadRouteDescriptor, 0) + routekinds := []string{} + tcpRoutes, err := ListTCPRoutes(ctx, k8sClient) + if err != nil { + routekinds = append(routekinds, TCPRouteKind) + } + l4Routes = append(l4Routes, tcpRoutes...) + udpRoutes, err := ListUDPRoutes(ctx, k8sClient) + if err != nil { + routekinds = append(routekinds, UDPRouteKind) + } + l4Routes = append(l4Routes, udpRoutes...) + tlsRoutes, err := ListTLSRoutes(ctx, k8sClient) + if err != nil { + routekinds = append(routekinds, TLSRouteKind) + } + l4Routes = append(l4Routes, tlsRoutes...) + if len(routekinds) > 0 { + err = fmt.Errorf("failed to list L4 routes, %s", routekinds) + } + return l4Routes, err +} + +// ListL7Routes retrieves all Layer 7 routes (HTTP, gRPC) from the cluster. +func ListL7Routes(ctx context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error) { + l7Routes := make([]preLoadRouteDescriptor, 0) + routekinds := []string{} + httpRoutes, err := ListHTTPRoutes(ctx, k8sClient) + if err != nil { + routekinds = append(routekinds, HTTPRouteKind) + } + l7Routes = append(l7Routes, httpRoutes...) + grpcRoutes, err := ListGRPCRoutes(ctx, k8sClient) + if err != nil { + routekinds = append(routekinds, GRPCRouteKind) + } + l7Routes = append(l7Routes, grpcRoutes...) + if len(routekinds) > 0 { + err = fmt.Errorf("failed to list L7 routes, %s", routekinds) + } + return l7Routes, err +} + +// FilterRoutesBySvc filters a slice of routes based on service reference. +// Returns a new slice containing only routes that reference the specified service. +func FilterRoutesBySvc(routes []preLoadRouteDescriptor, svc *corev1.Service) []preLoadRouteDescriptor { + if svc == nil || len(routes) == 0 { + return []preLoadRouteDescriptor{} + } + filteredRoutes := make([]preLoadRouteDescriptor, 0, len(routes)) + svcID := types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + } + for _, route := range routes { + if isServiceReferredByRoute(route, svcID) { + filteredRoutes = append(filteredRoutes, route) + } + } + return filteredRoutes +} + +// isServiceReferredByRoute checks if a route references a specific service. +// Assuming we are only supporting services as backendRefs on Routes +func isServiceReferredByRoute(route preLoadRouteDescriptor, svcID types.NamespacedName) bool { + for _, backendRef := range route.GetBackendRefs() { + namespace := route.GetRouteNamespacedName().Namespace + if backendRef.Namespace != nil { + namespace = string(*backendRef.Namespace) + } + + if string(backendRef.Name) == svcID.Name && namespace == svcID.Namespace { + return true + } + } + return false +} diff --git a/pkg/gateway/routeutils/utils_test.go b/pkg/gateway/routeutils/utils_test.go new file mode 100644 index 0000000000..e82cea4ea7 --- /dev/null +++ b/pkg/gateway/routeutils/utils_test.go @@ -0,0 +1,436 @@ +package routeutils + +import ( + "context" + "fmt" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/testutils" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + mock_client "sigs.k8s.io/aws-load-balancer-controller/mocks/controller-runtime/client" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +var _ RouteDescriptor = &mockPreLoadRouteDescriptor{} + +// Mock implementations +type mockPreLoadRouteDescriptor struct { + backendRefs []gwv1.BackendRef + namespacedName types.NamespacedName +} + +func (m mockPreLoadRouteDescriptor) GetAttachedRules() []RouteRule { + //TODO implement me + panic("implement me") +} + +func (m mockPreLoadRouteDescriptor) GetRouteNamespacedName() types.NamespacedName { + return m.namespacedName +} + +func (m mockPreLoadRouteDescriptor) GetRouteKind() string { + //TODO implement me + panic("implement me") +} + +func (m mockPreLoadRouteDescriptor) GetHostnames() []gwv1.Hostname { + //TODO implement me + panic("implement me") +} + +func (m mockPreLoadRouteDescriptor) GetParentRefs() []gwv1.ParentReference { + //TODO implement me + panic("implement me") +} + +func (m mockPreLoadRouteDescriptor) GetRawRoute() interface{} { + //TODO implement me + panic("implement me") +} + +func (m mockPreLoadRouteDescriptor) GetBackendRefs() []gwv1.BackendRef { + return m.backendRefs +} + +func (m mockPreLoadRouteDescriptor) loadAttachedRules(context context.Context, k8sClient client.Client) (RouteDescriptor, error) { + //TODO implement me + panic("implement me") +} + +// Test ListL4Routes +func Test_ListL4Routes(t *testing.T) { + tests := []struct { + name string + mockSetup func(*gomock.Controller) client.Client + expectedRoutes int + expectedErr error + }{ + { + name: "Successfully lists all L4 routes", + mockSetup: func(ctrl *gomock.Controller) client.Client { + k8sClient := testutils.GenerateTestClient() + k8sClient.Create(context.Background(), &gwalpha2.TCPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo1", + Namespace: "bar1", + }, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{}, + }, + }, + }, + }) + k8sClient.Create(context.Background(), &gwalpha2.UDPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo1", + Namespace: "bar1", + }, + Spec: gwalpha2.UDPRouteSpec{ + Rules: []gwalpha2.UDPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{}, + }, + }, + }, + }) + k8sClient.Create(context.Background(), &gwalpha2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo1", + Namespace: "bar1", + }, + Spec: gwalpha2.TLSRouteSpec{ + Hostnames: []gwv1.Hostname{ + "host1", + }, + Rules: []gwalpha2.TLSRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwalpha2.BackendRef{}, + }, + }, + }, + }) + return k8sClient + }, + expectedRoutes: 3, + expectedErr: nil, + }, + { + name: "Handles error in TCP routes", + mockSetup: func(ctrl *gomock.Controller) client.Client { + mockClient := mock_client.NewMockClient(ctrl) + // Setup mock responses for TCP, UDP, and TLS routes + mockClient.EXPECT().List(gomock.Any(), &gwalpha2.TCPRouteList{}).Return(fmt.Errorf("TCP error")) + mockClient.EXPECT().List(gomock.Any(), &gwalpha2.UDPRouteList{}).Return(nil) + mockClient.EXPECT().List(gomock.Any(), &gwalpha2.TLSRouteList{}).Return(nil) + return mockClient + }, + expectedRoutes: 0, + expectedErr: fmt.Errorf("failed to list L4 routes, [TCPRoute]"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + client := tt.mockSetup(ctrl) + routes, err := ListL4Routes(context.Background(), client) + + assert.Equal(t, err, tt.expectedErr) + assert.Len(t, routes, tt.expectedRoutes) + + }) + } +} + +// Test ListL7Routes +func Test_ListL7Routes(t *testing.T) { + tests := []struct { + name string + mockSetup func(*gomock.Controller) client.Client + expectedRoutes int + expectedErr error + }{ + { + name: "Successfully lists all L7 routes", + mockSetup: func(ctrl *gomock.Controller) client.Client { + k8sClient := testutils.GenerateTestClient() + k8sClient.Create(context.Background(), &gwv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo1", + Namespace: "bar1", + }, + Spec: gwv1.HTTPRouteSpec{ + Hostnames: []gwv1.Hostname{ + "host1", + }, + Rules: []gwv1.HTTPRouteRule{ + { + BackendRefs: []gwv1.HTTPBackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwv1.HTTPBackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwv1.HTTPBackendRef{}, + }, + }, + }, + }) + k8sClient.Create(context.Background(), &gwv1.GRPCRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo1", + Namespace: "bar1", + }, + Spec: gwv1.GRPCRouteSpec{ + Hostnames: []gwv1.Hostname{ + "host1", + }, + Rules: []gwv1.GRPCRouteRule{ + { + BackendRefs: []gwv1.GRPCBackendRef{ + {}, + {}, + }, + }, + { + BackendRefs: []gwv1.GRPCBackendRef{ + {}, + {}, + {}, + {}, + }, + }, + { + BackendRefs: []gwv1.GRPCBackendRef{}, + }, + }, + }, + }) + return k8sClient + }, + expectedRoutes: 2, + expectedErr: nil, + }, + { + name: "Handles error in HTTP routes", + mockSetup: func(ctrl *gomock.Controller) client.Client { + mockClient := mock_client.NewMockClient(ctrl) + mockClient.EXPECT().List(gomock.Any(), &gwv1.HTTPRouteList{}).Return(fmt.Errorf("HTTP error")) + mockClient.EXPECT().List(gomock.Any(), &gwv1.GRPCRouteList{}).Return(nil) + return mockClient + }, + expectedRoutes: 0, + expectedErr: fmt.Errorf("failed to list L7 routes, [HTTPRoute]"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + client := tt.mockSetup(ctrl) + routes, err := ListL7Routes(context.Background(), client) + + assert.Equal(t, err, tt.expectedErr) + assert.Len(t, routes, tt.expectedRoutes) + + }) + } +} + +// Test FilterRoutesBySvc +func Test_FilterRoutesBySvc(t *testing.T) { + namespace := "test-ns" + svcName := "test-svc" + + tests := []struct { + name string + routes []preLoadRouteDescriptor + service *corev1.Service + expectedCount int + }{ + { + name: "Nil service returns nil", + routes: []preLoadRouteDescriptor{}, + service: nil, + expectedCount: 0, + }, + { + name: "Empty routes returns nil", + routes: []preLoadRouteDescriptor{}, + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: namespace, + }, + }, + expectedCount: 0, + }, + { + name: "Filters matching routes", + routes: []preLoadRouteDescriptor{ + mockPreLoadRouteDescriptor{ + backendRefs: []gwv1.BackendRef{ + { + BackendObjectReference: gwv1.BackendObjectReference{ + Name: gwv1.ObjectName(svcName), + }, + }, + }, + namespacedName: types.NamespacedName{ + Namespace: namespace, + Name: "route-1", + }, + }, + &mockPreLoadRouteDescriptor{ + backendRefs: []gwv1.BackendRef{ + { + BackendObjectReference: gwv1.BackendObjectReference{ + Name: gwv1.ObjectName("other-svc"), + }, + }, + }, + namespacedName: types.NamespacedName{ + Namespace: namespace, + Name: "route-2", + }, + }, + }, + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: namespace, + }, + }, + expectedCount: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filtered := FilterRoutesBySvc(tt.routes, tt.service) + assert.Len(t, filtered, tt.expectedCount) + }) + } +} + +// Test isServiceReferredByRoute +func Test_IsServiceReferredByRoute(t *testing.T) { + tests := []struct { + name string + route preLoadRouteDescriptor + svcID types.NamespacedName + expected bool + }{ + { + name: "Route refers to service", + route: mockPreLoadRouteDescriptor{ + backendRefs: []gwv1.BackendRef{ + { + BackendObjectReference: gwv1.BackendObjectReference{ + Name: gwv1.ObjectName("test-svc"), + }, + }, + }, + namespacedName: types.NamespacedName{ + Namespace: "test-ns", + Name: "route-1", + }, + }, + svcID: types.NamespacedName{ + Namespace: "test-ns", + Name: "test-svc", + }, + expected: true, + }, + { + name: "Route does not refer to service", + route: mockPreLoadRouteDescriptor{ + backendRefs: []gwv1.BackendRef{ + { + BackendObjectReference: gwv1.BackendObjectReference{ + Name: gwv1.ObjectName("other-svc"), + }, + }, + }, + namespacedName: types.NamespacedName{ + Namespace: "test-ns", + Name: "route-1", + }, + }, + svcID: types.NamespacedName{ + Namespace: "test-ns", + Name: "test-svc", + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isServiceReferredByRoute(tt.route, tt.svcID) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/metrics/util/reconcile_counter.go b/pkg/metrics/util/reconcile_counter.go index 4aecdc17db..25910aa654 100644 --- a/pkg/metrics/util/reconcile_counter.go +++ b/pkg/metrics/util/reconcile_counter.go @@ -23,10 +23,12 @@ type ResourceReconcileCount struct { func NewReconcileCounters() *ReconcileCounters { return &ReconcileCounters{ - serviceReconciles: make(map[types.NamespacedName]int), - ingressReconciles: make(map[types.NamespacedName]int), - tgbReconciles: make(map[types.NamespacedName]int), - mutex: sync.Mutex{}, + serviceReconciles: make(map[types.NamespacedName]int), + ingressReconciles: make(map[types.NamespacedName]int), + tgbReconciles: make(map[types.NamespacedName]int), + nlbGatewayReconciles: make(map[types.NamespacedName]int), + albGatewayReconciles: make(map[types.NamespacedName]int), + mutex: sync.Mutex{}, } } diff --git a/pkg/testutils/client_test_utils.go b/pkg/testutils/client_test_utils.go index 003f45e8d1..6ced1c3f71 100644 --- a/pkg/testutils/client_test_utils.go +++ b/pkg/testutils/client_test_utils.go @@ -2,8 +2,16 @@ package testutils import ( "github.com/golang/mock/gomock" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "reflect" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) // NewListOptionEquals constructs new goMock matcher for client's ListOption @@ -34,3 +42,15 @@ func (m *listOptionEquals) Matches(x interface{}) bool { func (m *listOptionEquals) String() string { return "list option equals" } + +func GenerateTestClient() client.Client { + k8sSchema := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sSchema) + elbv2api.AddToScheme(k8sSchema) + gwv1.AddToScheme(k8sSchema) + gwalpha2.AddToScheme(k8sSchema) + elbv2gw.AddToScheme(k8sSchema) + gwbeta1.AddToScheme(k8sSchema) + + return testclient.NewClientBuilder().WithScheme(k8sSchema).Build() +} From d0f5cad90c3367b9ab5d5d74ae33dd91e82d926c Mon Sep 17 00:00:00 2001 From: wweiwei-li <79778352+wweiwei-li@users.noreply.github.com> Date: Fri, 18 Apr 2025 14:15:46 -0700 Subject: [PATCH 14/40] Enable frontend NLB (#4126) --- controllers/gateway/gateway_controller.go | 3 +- controllers/ingress/group_controller.go | 85 +- controllers/service/service_controller.go | 2 +- docs/guide/ingress/annotations.md | 128 +++ pkg/annotations/constants.go | 105 +- .../elbv2/frontend_nlb_target_synthesizer.go | 166 +++ .../elbv2/frontend_nlb_targets_manager.go | 99 ++ pkg/deploy/elbv2/target_group_synthesizer.go | 2 + pkg/deploy/stack_deployer.go | 20 +- pkg/ingress/model_build_frontend_nlb.go | 768 ++++++++++++++ pkg/ingress/model_build_frontend_nlb_test.go | 966 ++++++++++++++++++ pkg/ingress/model_builder.go | 35 +- pkg/ingress/model_builder_test.go | 2 +- pkg/model/core/frontend_nlb_target_group.go | 31 + pkg/model/elbv2/target_group.go | 2 + test/e2e/ingress/utils.go | 21 +- test/e2e/ingress/vanilla_ingress_test.go | 78 ++ 17 files changed, 2434 insertions(+), 79 deletions(-) create mode 100644 pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go create mode 100644 pkg/deploy/elbv2/frontend_nlb_targets_manager.go create mode 100644 pkg/ingress/model_build_frontend_nlb.go create mode 100644 pkg/ingress/model_build_frontend_nlb_test.go create mode 100644 pkg/model/core/frontend_nlb_target_group.go diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 6c91111124..76b80a0db2 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -3,6 +3,7 @@ package gateway import ( "context" "fmt" + "github.com/go-logr/logr" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -242,7 +243,7 @@ func (r *gatewayReconciler) reconcileUpdate(ctx context.Context, gw *gwv1.Gatewa } func (r *gatewayReconciler) deployModel(ctx context.Context, gw *gwv1.Gateway, stack core.Stack) error { - if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, r.controllerName); err != nil { + if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, r.controllerName, nil); err != nil { var requeueNeededAfter *runtime.RequeueNeededAfter if errors.As(err, &requeueNeededAfter) { return err diff --git a/controllers/ingress/group_controller.go b/controllers/ingress/group_controller.go index 75f16ad824..3355f0f311 100644 --- a/controllers/ingress/group_controller.go +++ b/controllers/ingress/group_controller.go @@ -151,7 +151,7 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request) return errmetrics.NewErrorWithMetrics(controllerName, "add_group_finalizer_error", err, r.metricsCollector) } - _, lb, err := r.buildAndDeployModel(ctx, ingGroup) + _, lb, frontendNlb, err := r.buildAndDeployModel(ctx, ingGroup) if err != nil { return err } @@ -164,7 +164,14 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request) if statusErr != nil { return } - statusErr = r.updateIngressGroupStatus(ctx, ingGroup, lbDNS) + var frontendNlbDNS string + if frontendNlb != nil { + frontendNlbDNS, statusErr = frontendNlb.DNSName().Resolve(ctx) + if statusErr != nil { + return + } + } + statusErr = r.updateIngressGroupStatus(ctx, ingGroup, lbDNS, frontendNlbDNS) if statusErr != nil { r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", statusErr)) @@ -191,38 +198,40 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request) return nil } -func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingress.Group) (core.Stack, *elbv2model.LoadBalancer, error) { +func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingress.Group) (core.Stack, *elbv2model.LoadBalancer, *elbv2model.LoadBalancer, error) { var stack core.Stack var lb *elbv2model.LoadBalancer var secrets []types.NamespacedName var backendSGRequired bool var err error + var frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState + var frontendNlb *elbv2model.LoadBalancer buildModelFn := func() { - stack, lb, secrets, backendSGRequired, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector) + stack, lb, secrets, backendSGRequired, frontendNlbTargetGroupDesiredState, frontendNlb, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector) } r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "build_model", buildModelFn) if err != nil { r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err)) - return nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "build_model_error", err, r.metricsCollector) + return nil, nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "build_model_error", err, r.metricsCollector) } stackJSON, err := r.stackMarshaller.Marshal(stack) if err != nil { r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err)) - return nil, nil, err + return nil, nil, nil, err } r.logger.Info("successfully built model", "model", stackJSON) deployModelFn := func() { - err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress") + err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress", frontendNlbTargetGroupDesiredState) } r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "deploy_model", deployModelFn) if err != nil { var requeueNeededAfter *runtime.RequeueNeededAfter if errors.As(err, &requeueNeededAfter) { - return nil, nil, err + return nil, nil, nil, err } r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err)) - return nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "deploy_model_error", err, r.metricsCollector) + return nil, nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "deploy_model_error", err, r.metricsCollector) } r.logger.Info("successfully deployed model", "ingressGroup", ingGroup.ID) r.secretsManager.MonitorSecrets(ingGroup.ID.String(), secrets) @@ -232,9 +241,9 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr inactiveResources = append(inactiveResources, k8s.ToSliceOfNamespacedNames(ingGroup.Members)...) } if err := r.backendSGProvider.Release(ctx, networkingpkg.ResourceTypeIngress, inactiveResources); err != nil { - return nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "release_auto_generated_backend_sg_error", err, r.metricsCollector) + return nil, nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "release_auto_generated_backend_sg_error", err, r.metricsCollector) } - return stack, lb, nil + return stack, lb, frontendNlb, nil } func (r *groupReconciler) recordIngressGroupEvent(_ context.Context, ingGroup ingress.Group, eventType string, reason string, message string) { @@ -243,29 +252,41 @@ func (r *groupReconciler) recordIngressGroupEvent(_ context.Context, ingGroup in } } -func (r *groupReconciler) updateIngressGroupStatus(ctx context.Context, ingGroup ingress.Group, lbDNS string) error { +func (r *groupReconciler) updateIngressGroupStatus(ctx context.Context, ingGroup ingress.Group, lbDNS string, frontendNLBDNS string) error { for _, member := range ingGroup.Members { - if err := r.updateIngressStatus(ctx, lbDNS, member.Ing); err != nil { + if err := r.updateIngressStatus(ctx, lbDNS, frontendNLBDNS, member.Ing); err != nil { return err } } return nil } -func (r *groupReconciler) updateIngressStatus(ctx context.Context, lbDNS string, ing *networking.Ingress) error { +func (r *groupReconciler) updateIngressStatus(ctx context.Context, lbDNS string, frontendNlbDNS string, ing *networking.Ingress) error { + ingOld := ing.DeepCopy() if len(ing.Status.LoadBalancer.Ingress) != 1 || ing.Status.LoadBalancer.Ingress[0].IP != "" || ing.Status.LoadBalancer.Ingress[0].Hostname != lbDNS { - ingOld := ing.DeepCopy() ing.Status.LoadBalancer.Ingress = []networking.IngressLoadBalancerIngress{ { Hostname: lbDNS, }, } + } + + // Ensure frontendNLBDNS is appended if it is not already added + if frontendNlbDNS != "" && !hasFrontendNlbHostName(ing.Status.LoadBalancer.Ingress, frontendNlbDNS) { + ing.Status.LoadBalancer.Ingress = append(ing.Status.LoadBalancer.Ingress, networking.IngressLoadBalancerIngress{ + Hostname: frontendNlbDNS, + }) + } + + if !isIngressStatusEqual(ingOld.Status.LoadBalancer.Ingress, ing.Status.LoadBalancer.Ingress) { if err := r.k8sClient.Status().Patch(ctx, ing, client.MergeFrom(ingOld)); err != nil { return errors.Wrapf(err, "failed to update ingress status: %v", k8s.NamespacedName(ing)) } + } + return nil } @@ -387,3 +408,37 @@ func isResourceKindAvailable(resList *metav1.APIResourceList, kind string) bool } return false } + +func isIngressStatusEqual(a, b []networking.IngressLoadBalancerIngress) bool { + if len(a) != len(b) { + return false + } + + setA := make(map[string]struct{}, len(a)) + setB := make(map[string]struct{}, len(b)) + + for _, ingress := range a { + setA[ingress.Hostname] = struct{}{} + } + + for _, ingress := range b { + setB[ingress.Hostname] = struct{}{} + } + + for key := range setA { + if _, exists := setB[key]; !exists { + return false + } + } + return true +} + +func hasFrontendNlbHostName(ingressList []networking.IngressLoadBalancerIngress, frontendNlbDNS string) bool { + for _, ingress := range ingressList { + if ingress.Hostname == frontendNlbDNS { + return true + } + + } + return false +} diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index 6aa7383dfd..e89bc546c5 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -152,7 +152,7 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) } func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error { - if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "service"); err != nil { + if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "service", nil); err != nil { var requeueNeededAfter *runtime.RequeueNeededAfter if errors.As(err, &requeueNeededAfter) { return err diff --git a/docs/guide/ingress/annotations.md b/docs/guide/ingress/annotations.md index c1c6c7fcd1..af72f5a599 100644 --- a/docs/guide/ingress/annotations.md +++ b/docs/guide/ingress/annotations.md @@ -63,6 +63,19 @@ You can add annotations to kubernetes Ingress and Service objects to customize t | [alb.ingress.kubernetes.io/listener-attributes.${Protocol}-${Port}](#listener-attributes) | stringMap |N/A| Ingress |Merge| | [alb.ingress.kubernetes.io/minimum-load-balancer-capacity](#load-balancer-capacity-reservation) | stringMap |N/A| Ingress | Exclusive | | [alb.ingress.kubernetes.io/ipam-ipv4-pool-id](#ipam-ipv4-pool-id) | string |N/A| Ingress | Exclusive | +| [alb.ingress.kubernetes.io/enable-frontend-nlb](#enable-frontend-nlb) | boolean |false | Ingress | Exclusive | +| [alb.ingress.kubernetes.io/frontend-nlb-scheme](#frontend-nlb-scheme) | internal \| internet-facing |internal| Ingress | Exclusive | +| [alb.ingress.kubernetes.io/frontend-nlb-subnets](#frontend-nlb-subnets) | stringList |N/A| Ingress | Exclusive | +| [alb.ingress.kubernetes.io/frontend-nlb-security-groups](#frontend-nlb-security-groups) | stringList |N/A| Ingress | Exclusive | +| [alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping](#frontend-nlb-listener-port-mapping) | stringMap |N/A| Ingress | Merge | +| [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-port](#frontend-nlb-healthcheck-port) | integer \| traffic-port |traffic-port| Ingress | N/A | +| [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-protocol](#frontend-nlb-healthcheck-protocol) | HTTP \| HTTPS |HTTP| Ingress | N/A | +| [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-path](#frontend-nlb-healthcheck-path) | string |/| Ingress | N/A | +| [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-interval-seconds](#frontend-nlb-healthcheck-interval-seconds) | integer |15| Ingress | N/A | +| [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-timeout-seconds](#frontend-nlb-healthcheck-timeout-seconds) | integer |5| Ingress | N/A | +| [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-healthy-threshold-count](#frontend-nlb-healthcheck-healthy-threshold-count) | integer |3| Ingress | N/A | +| [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-unhealthy-threshold-count](#frontend-nlb-healthcheck-unhealthy-threshold-count) | integer |3| Ingress | N/A | +| [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-success-codes](#frontend-nlb-healthcheck-success-codes) | string |200| Ingress | N/A | ## IngressGroup IngressGroup feature enables you to group multiple Ingress resources together. @@ -1024,3 +1037,118 @@ Load balancer capacity unit reservation can be configured via following annotati - disable shield protection ```alb.ingress.kubernetes.io/shield-advanced-protection: 'false' ``` + + +## Enable frontend NLB +When this option is set to true, the controller will automatically provision a Network Load Balancer and register the Application Load Balancer as its target. Additional annotations are available to customize the NLB configurations, including options for scheme, security groups, subnets, and health check. The ingress resource will have two status entries, one for the NLB DNS and one for the ALB DNS. This allows users to combine the benefits of NLB and ALB into a single solution, leveraging NLB features like static IP address and PrivateLink, while retaining the rich routing capabilities of ALB. + +!!!warning + - If you need to change the ALB [scheme](#scheme), make sure to disable this feature first. Changing the scheme will create a new ALB, which could interfere with the current configuration. + - If you create ingress and enable the feature at once, provisioning the NLB and registering the ALB as target can take up to 3-4 mins to complete. + +- `alb.ingress.kubernetes.io/enable-frontend-nlb` enables frontend Network Load Balancer functionality. + + !!!example + - Enable frontend nlb + ``` + alb.ingress.kubernetes.io/enable-frontend-nlb: "true" + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-scheme` specifies the scheme for the Network Load Balancer. + + !!!example + - Set NLB scheme to internet-facing + ``` + alb.ingress.kubernetes.io/frontend-nlb-scheme: internet-facing + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-subnets` specifies the subnets for the Network Load Balancer. + + !!!example + - Specify subnets for NLB + ``` + alb.ingress.kubernetes.io/frontend-nlb-subnets: subnet-xxxx1,subnet-xxxx2 + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-security-groups` specifies the security groups for the Network Load Balancer. + + !!!example + - Specify security groups for NLB + ``` + alb.ingress.kubernetes.io/frontend-nlb-security-groups: sg-xxxx1,sg-xxxx2 + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping` specifies the port mapping configuration for the Network Load Balancer listeners. + + !!!note "Default" + - The port defaults to match the ALB listener port, based on whether `alb.ingress.kubernetes.io/listen-ports`(#listen-ports) is specified. + + !!!example + - Forward TCP traffic from NLB:80 to ALB:443 + ``` + alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping: 80=443 + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-healthcheck-port` specifies the port used for health checks. + + !!!example + - Set health check port + ``` + alb.ingress.kubernetes.io/frontend-nlb-healthcheck-port: traffic-port + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-healthcheck-protocol` specifies the protocol used for health checks. + + !!!example + - Set health check protocol + ``` + alb.ingress.kubernetes.io/frontend-nlb-healthcheck-protocol: HTTP + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-healthcheck-path` specifies the destination path for health checks. + + !!!example + - Set health check path + ``` + alb.ingress.kubernetes.io/frontend-nlb-healthcheck-path: /health + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-healthcheck-interval-seconds` specifies the interval between consecutive health checks. + + !!!example + - Set health check interval + ``` + alb.ingress.kubernetes.io/frontend-nlb-healthcheck-interval-seconds: '15' + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-healthcheck-timeout-seconds` specifies the target group health check timeout. + + !!!example + - Set health check timeout + ``` + alb.ingress.kubernetes.io/frontend-nlb-healthcheck-timeout-seconds: '5' + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-healthcheck-healthy-threshold-count` specifies the consecutive health check successes required before a target is considered healthy. + + !!!example + - Set healthy threshold count + ``` + alb.ingress.kubernetes.io/frontend-nlb-healthcheck-healthy-threshold-count: '3' + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-healthcheck-unhealthy-threshold-count` specifies the consecutive health check failures before a target gets marked unhealthy. + + !!!example + - Set unhealthy threshold count + ``` + alb.ingress.kubernetes.io/frontend-nlb-healthcheck-unhealthy-threshold-count: '3' + ``` + +- `alb.ingress.kubernetes.io/frontend-nlb-healthcheck-success-codes` specifies the HTTP codes that indicate a successful health check. + + !!!example + - Set success codes for health check + ``` + alb.ingress.kubernetes.io/frontend-nlb-healthcheck-success-codes: '200' + ``` diff --git a/pkg/annotations/constants.go b/pkg/annotations/constants.go index c9e178a2f5..870ccc39c4 100644 --- a/pkg/annotations/constants.go +++ b/pkg/annotations/constants.go @@ -13,52 +13,65 @@ const ( AnnotationPrefixIngress = "alb.ingress.kubernetes.io" // Ingress annotation suffixes - IngressSuffixLoadBalancerName = "load-balancer-name" - IngressSuffixGroupName = "group.name" - IngressSuffixGroupOrder = "group.order" - IngressSuffixTags = "tags" - IngressSuffixIPAddressType = "ip-address-type" - IngressSuffixScheme = "scheme" - IngressSuffixSubnets = "subnets" - IngressSuffixCustomerOwnedIPv4Pool = "customer-owned-ipv4-pool" - IngressSuffixLoadBalancerAttributes = "load-balancer-attributes" - IngressSuffixWAFv2ACLARN = "wafv2-acl-arn" - IngressSuffixWAFACLID = "waf-acl-id" - IngressSuffixWebACLID = "web-acl-id" // deprecated, use "waf-acl-id" instead. - IngressSuffixShieldAdvancedProtection = "shield-advanced-protection" - IngressSuffixSecurityGroups = "security-groups" - IngressSuffixListenPorts = "listen-ports" - IngressSuffixSSLRedirect = "ssl-redirect" - IngressSuffixInboundCIDRs = "inbound-cidrs" - IngressSuffixCertificateARN = "certificate-arn" - IngressSuffixSSLPolicy = "ssl-policy" - IngressSuffixTargetType = "target-type" - IngressSuffixBackendProtocol = "backend-protocol" - IngressSuffixBackendProtocolVersion = "backend-protocol-version" - IngressSuffixTargetGroupAttributes = "target-group-attributes" - IngressSuffixHealthCheckPort = "healthcheck-port" - IngressSuffixHealthCheckProtocol = "healthcheck-protocol" - IngressSuffixHealthCheckPath = "healthcheck-path" - IngressSuffixHealthCheckIntervalSeconds = "healthcheck-interval-seconds" - IngressSuffixHealthCheckTimeoutSeconds = "healthcheck-timeout-seconds" - IngressSuffixHealthyThresholdCount = "healthy-threshold-count" - IngressSuffixUnhealthyThresholdCount = "unhealthy-threshold-count" - IngressSuffixSuccessCodes = "success-codes" - IngressSuffixAuthType = "auth-type" - IngressSuffixAuthIDPCognito = "auth-idp-cognito" - IngressSuffixAuthIDPOIDC = "auth-idp-oidc" - IngressSuffixAuthOnUnauthenticatedRequest = "auth-on-unauthenticated-request" - IngressSuffixAuthScope = "auth-scope" - IngressSuffixAuthSessionCookie = "auth-session-cookie" - IngressSuffixAuthSessionTimeout = "auth-session-timeout" - IngressSuffixTargetNodeLabels = "target-node-labels" - IngressSuffixManageSecurityGroupRules = "manage-backend-security-group-rules" - IngressSuffixMutualAuthentication = "mutual-authentication" - IngressSuffixSecurityGroupPrefixLists = "security-group-prefix-lists" - IngressSuffixlsAttsAnnotationPrefix = "listener-attributes" - IngressLBSuffixMultiClusterTargetGroup = "multi-cluster-target-group" - IngressSuffixLoadBalancerCapacityReservation = "minimum-load-balancer-capacity" - IngressSuffixIPAMIPv4PoolId = "ipam-ipv4-pool-id" + IngressSuffixLoadBalancerName = "load-balancer-name" + IngressSuffixGroupName = "group.name" + IngressSuffixGroupOrder = "group.order" + IngressSuffixTags = "tags" + IngressSuffixIPAddressType = "ip-address-type" + IngressSuffixScheme = "scheme" + IngressSuffixSubnets = "subnets" + IngressSuffixCustomerOwnedIPv4Pool = "customer-owned-ipv4-pool" + IngressSuffixLoadBalancerAttributes = "load-balancer-attributes" + IngressSuffixWAFv2ACLARN = "wafv2-acl-arn" + IngressSuffixWAFACLID = "waf-acl-id" + IngressSuffixWebACLID = "web-acl-id" // deprecated, use "waf-acl-id" instead. + IngressSuffixShieldAdvancedProtection = "shield-advanced-protection" + IngressSuffixSecurityGroups = "security-groups" + IngressSuffixListenPorts = "listen-ports" + IngressSuffixSSLRedirect = "ssl-redirect" + IngressSuffixInboundCIDRs = "inbound-cidrs" + IngressSuffixCertificateARN = "certificate-arn" + IngressSuffixSSLPolicy = "ssl-policy" + IngressSuffixTargetType = "target-type" + IngressSuffixBackendProtocol = "backend-protocol" + IngressSuffixBackendProtocolVersion = "backend-protocol-version" + IngressSuffixTargetGroupAttributes = "target-group-attributes" + IngressSuffixHealthCheckPort = "healthcheck-port" + IngressSuffixHealthCheckProtocol = "healthcheck-protocol" + IngressSuffixHealthCheckPath = "healthcheck-path" + IngressSuffixHealthCheckIntervalSeconds = "healthcheck-interval-seconds" + IngressSuffixHealthCheckTimeoutSeconds = "healthcheck-timeout-seconds" + IngressSuffixHealthyThresholdCount = "healthy-threshold-count" + IngressSuffixUnhealthyThresholdCount = "unhealthy-threshold-count" + IngressSuffixSuccessCodes = "success-codes" + IngressSuffixAuthType = "auth-type" + IngressSuffixAuthIDPCognito = "auth-idp-cognito" + IngressSuffixAuthIDPOIDC = "auth-idp-oidc" + IngressSuffixAuthOnUnauthenticatedRequest = "auth-on-unauthenticated-request" + IngressSuffixAuthScope = "auth-scope" + IngressSuffixAuthSessionCookie = "auth-session-cookie" + IngressSuffixAuthSessionTimeout = "auth-session-timeout" + IngressSuffixTargetNodeLabels = "target-node-labels" + IngressSuffixManageSecurityGroupRules = "manage-backend-security-group-rules" + IngressSuffixMutualAuthentication = "mutual-authentication" + IngressSuffixSecurityGroupPrefixLists = "security-group-prefix-lists" + IngressSuffixlsAttsAnnotationPrefix = "listener-attributes" + IngressLBSuffixMultiClusterTargetGroup = "multi-cluster-target-group" + IngressSuffixLoadBalancerCapacityReservation = "minimum-load-balancer-capacity" + IngressSuffixIPAMIPv4PoolId = "ipam-ipv4-pool-id" + IngressSuffixEnableFrontendNlb = "enable-frontend-nlb" + IngressSuffixFrontendNlbScheme = "frontend-nlb-scheme" + IngressSuffixFrontendNlbSubnets = "frontend-nlb-subnets" + IngressSuffixFrontendNlbSecurityGroups = "frontend-nlb-security-groups" + IngressSuffixFrontendNlbListenerPortMapping = "frontend-nlb-listener-port-mapping" + IngressSuffixFrontendNlbHealthCheckPort = "frontend-nlb-healthcheck-port" + IngressSuffixFrontendNlbHealthCheckProtocol = "frontend-nlb-healthcheck-protocol" + IngressSuffixFrontendNlbHealthCheckPath = "frontend-nlb-healthcheck-path" + IngressSuffixFrontendNlbHealthCheckIntervalSeconds = "frontend-nlb-healthcheck-interval-seconds" + IngressSuffixFrontendNlbHealthCheckTimeoutSeconds = "frontend-nlb-healthcheck-timeout-seconds" + IngressSuffixFrontendNlbHealthCheckHealthyThresholdCount = "frontend-nlb-healthcheck-healthy-threshold-count" + IngressSuffixFrontendNlHealthCheckbUnhealthyThresholdCount = "frontend-nlb-healthcheck-unhealthy-threshold-count" + IngressSuffixFrontendNlbHealthCheckSuccessCodes = "frontend-nlb-healthcheck-success-codes" // NLB annotation suffixes // prefixes service.beta.kubernetes.io, service.kubernetes.io diff --git a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go new file mode 100644 index 0000000000..a21a6897a3 --- /dev/null +++ b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go @@ -0,0 +1,166 @@ +package elbv2 + +import ( + "context" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/go-logr/logr" + "github.com/pkg/errors" + "sigs.k8s.io/aws-load-balancer-controller/pkg/config" + "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func NewFrontendNlbTargetSynthesizer(k8sClient client.Client, trackingProvider tracking.Provider, taggingManager TaggingManager, frontendNlbTargetsManager FrontendNlbTargetsManager, logger logr.Logger, featureGates config.FeatureGates, stack core.Stack, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState) *frontendNlbTargetSynthesizer { + return &frontendNlbTargetSynthesizer{ + k8sClient: k8sClient, + trackingProvider: trackingProvider, + taggingManager: taggingManager, + frontendNlbTargetsManager: frontendNlbTargetsManager, + featureGates: featureGates, + logger: logger, + stack: stack, + frontendNlbTargetGroupDesiredState: frontendNlbTargetGroupDesiredState, + } +} + +type frontendNlbTargetSynthesizer struct { + k8sClient client.Client + trackingProvider tracking.Provider + taggingManager TaggingManager + frontendNlbTargetsManager FrontendNlbTargetsManager + featureGates config.FeatureGates + logger logr.Logger + stack core.Stack + frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState +} + +// Synthesize processes AWS target groups and deregisters ALB targets based on the desired state. +func (s *frontendNlbTargetSynthesizer) Synthesize(ctx context.Context) error { + var resTGs []*elbv2model.TargetGroup + s.stack.ListResources(&resTGs) + sdkTGs, err := s.findSDKTargetGroups(ctx) + if err != nil { + return err + } + _, _, unmatchedSDKTGs, err := matchResAndSDKTargetGroups(resTGs, sdkTGs, + s.trackingProvider.ResourceIDTagKey(), s.featureGates) + if err != nil { + return err + } + + for _, sdkTG := range unmatchedSDKTGs { + if sdkTG.TargetGroup.TargetType != elbv2types.TargetTypeEnumAlb { + continue + } + + err := s.deregisterCurrentTarget(ctx, sdkTG) + if err != nil { + return errors.Wrapf(err, "failed to deregister target for the target group: %s", *sdkTG.TargetGroup.TargetGroupArn) + } + } + + return nil + +} + +func (s *frontendNlbTargetSynthesizer) deregisterCurrentTarget(ctx context.Context, sdkTG TargetGroupWithTags) error { + // Retrieve the current targets for the target group + currentTargets, err := s.frontendNlbTargetsManager.ListTargets(ctx, *sdkTG.TargetGroup.TargetGroupArn) + if err != nil { + return errors.Wrapf(err, "failed to list current target for target group: %s", *sdkTG.TargetGroup.TargetGroupArn) + } + + // If there is no target, nothing to deregister + if len(currentTargets) == 0 { + return nil + } + + // Deregister current target + s.logger.Info("Deregistering current target", + "targetGroupARN", *sdkTG.TargetGroup.TargetGroupArn, + "target", currentTargets[0].Target.Id, + "port", currentTargets[0].Target.Port, + ) + + err = s.frontendNlbTargetsManager.DeregisterTargets(ctx, *sdkTG.TargetGroup.TargetGroupArn, elbv2types.TargetDescription{ + Id: awssdk.String(*currentTargets[0].Target.Id), + Port: awssdk.Int32(*currentTargets[0].Target.Port), + }) + + if err != nil { + return errors.Wrapf(err, "failed to deregister targets for target group: %s", *sdkTG.TargetGroup.TargetGroupArn) + } + + return nil +} + +func (s *frontendNlbTargetSynthesizer) PostSynthesize(ctx context.Context) error { + var resTGs []*elbv2model.TargetGroup + s.stack.ListResources(&resTGs) + + // Filter desired target group to include only ALB type target group + albResTGs := filterALBTargetGroups(resTGs) + + for _, resTG := range albResTGs { + + // Skip target group that are not yet created + if resTG.Status.TargetGroupARN == "" { + continue + } + + // List current targets + currentTargets, err := s.frontendNlbTargetsManager.ListTargets(ctx, resTG.Status.TargetGroupARN) + + if err != nil { + return err + } + + desiredTarget, err := s.frontendNlbTargetGroupDesiredState.TargetGroups[resTG.Spec.Name].TargetARN.Resolve(ctx) + desiredTargetPort := s.frontendNlbTargetGroupDesiredState.TargetGroups[resTG.Spec.Name].TargetPort + + if err != nil { + return errors.Wrapf(err, "failed to resolve the desiredTarget for target group: %s", desiredTarget) + } + + if len(currentTargets) == 0 || + currentTargets[0].Target == nil || + currentTargets[0].Target.Id == nil || + *currentTargets[0].Target.Id != desiredTarget { + err = s.frontendNlbTargetsManager.RegisterTargets(ctx, resTG.Status.TargetGroupARN, elbv2types.TargetDescription{ + Id: awssdk.String(desiredTarget), + Port: awssdk.Int32(desiredTargetPort), + }) + + if err != nil { + requeueMsg := "Failed to register target, retrying after deplay for target group: " + resTG.Status.TargetGroupARN + return runtime.NewRequeueNeededAfter(requeueMsg, 15*time.Second) + } + + } + + } + + return nil +} + +func filterALBTargetGroups(targetGroups []*elbv2model.TargetGroup) []*elbv2model.TargetGroup { + var filteredTargetGroups []*elbv2model.TargetGroup + for _, tg := range targetGroups { + if elbv2types.TargetTypeEnum(tg.Spec.TargetType) == elbv2types.TargetTypeEnumAlb { + filteredTargetGroups = append(filteredTargetGroups, tg) + } + } + return filteredTargetGroups +} + +func (s *frontendNlbTargetSynthesizer) findSDKTargetGroups(ctx context.Context) ([]TargetGroupWithTags, error) { + stackTags := s.trackingProvider.StackTags(s.stack) + return s.taggingManager.ListTargetGroups(ctx, + tracking.TagsAsTagFilter(stackTags)) +} diff --git a/pkg/deploy/elbv2/frontend_nlb_targets_manager.go b/pkg/deploy/elbv2/frontend_nlb_targets_manager.go new file mode 100644 index 0000000000..acb8c33469 --- /dev/null +++ b/pkg/deploy/elbv2/frontend_nlb_targets_manager.go @@ -0,0 +1,99 @@ +package elbv2 + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/go-logr/logr" + "github.com/pkg/errors" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" +) + +// FrontendNlbTargetsManager is an abstraction around ELBV2's targets API. +type FrontendNlbTargetsManager interface { + // Register Targets into TargetGroup. + RegisterTargets(ctx context.Context, tgArn string, albTarget elbv2types.TargetDescription) error + + // Deregister Targets from TargetGroup. + DeregisterTargets(ctx context.Context, tgArn string, albTarget elbv2types.TargetDescription) error + + // List Targets from TargetGroup. + ListTargets(ctx context.Context, tgArn string) ([]elbv2types.TargetHealthDescription, error) +} + +// NewFrontendNlbTargetsManager constructs new frontendNlbTargetsManager +func NewFrontendNlbTargetsManager(elbv2Client services.ELBV2, logger logr.Logger) *frontendNlbTargetsManager { + return &frontendNlbTargetsManager{ + elbv2Client: elbv2Client, + logger: logger, + } +} + +var _ FrontendNlbTargetsManager = &frontendNlbTargetsManager{} + +type frontendNlbTargetsManager struct { + elbv2Client services.ELBV2 + logger logr.Logger +} + +func (m *frontendNlbTargetsManager) RegisterTargets(ctx context.Context, tgARN string, albTarget elbv2types.TargetDescription) error { + targets := []elbv2types.TargetDescription{albTarget} + req := &elbv2sdk.RegisterTargetsInput{ + TargetGroupArn: aws.String(tgARN), + Targets: targets, + } + m.logger.Info("registering targets", + "arn", tgARN, + "targets", albTarget) + + _, err := m.elbv2Client.RegisterTargetsWithContext(ctx, req) + + if err != nil { + return errors.Wrap(err, "failed to register targets") + } + + m.logger.Info("registered targets", + "arn", tgARN) + return nil +} + +func (m *frontendNlbTargetsManager) DeregisterTargets(ctx context.Context, tgARN string, albTarget elbv2types.TargetDescription) error { + targets := []elbv2types.TargetDescription{albTarget} + m.logger.Info("deRegistering targets", + "arn", tgARN, + "targets", targets) + req := &elbv2sdk.DeregisterTargetsInput{ + TargetGroupArn: aws.String(tgARN), + Targets: targets, + } + _, err := m.elbv2Client.DeregisterTargetsWithContext(ctx, req) + if err != nil { + return errors.Wrap(err, "failed to deregister targets") + } + m.logger.Info("deregistered targets", + "arn", tgARN) + + return nil +} + +func (m *frontendNlbTargetsManager) ListTargets(ctx context.Context, tgARN string) ([]elbv2types.TargetHealthDescription, error) { + m.logger.Info("Listing targets", + "arn", tgARN) + + resp, err := m.elbv2Client.DescribeTargetHealthWithContext(ctx, &elbv2sdk.DescribeTargetHealthInput{ + TargetGroupArn: awssdk.String(tgARN), + }) + + if err != nil { + return make([]elbv2types.TargetHealthDescription, 0), err + } + + if len(resp.TargetHealthDescriptions) != 0 { + return resp.TargetHealthDescriptions, nil + } + + return make([]elbv2types.TargetHealthDescription, 0), nil +} diff --git a/pkg/deploy/elbv2/target_group_synthesizer.go b/pkg/deploy/elbv2/target_group_synthesizer.go index 690dae5a29..11d7e54221 100644 --- a/pkg/deploy/elbv2/target_group_synthesizer.go +++ b/pkg/deploy/elbv2/target_group_synthesizer.go @@ -2,6 +2,7 @@ package elbv2 import ( "context" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/go-logr/logr" "github.com/pkg/errors" @@ -166,6 +167,7 @@ func isSDKTargetGroupRequiresReplacement(sdkTG TargetGroupWithTags, resTG *elbv2 if string(resTG.Spec.TargetType) != string(sdkTG.TargetGroup.TargetType) { return true } + if string(resTG.Spec.Protocol) != string(sdkTG.TargetGroup.Protocol) { return true } diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index 7ca4aa3af8..d0ddc214bb 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -20,10 +20,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + ingressController = "ingress" +) + // StackDeployer will deploy a resource stack into AWS and K8S. type StackDeployer interface { // Deploy a resource stack. - Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string) error + Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState) error } // NewDefaultStackDeployer constructs new defaultStackDeployer. @@ -49,6 +53,7 @@ func NewDefaultStackDeployer(cloud services.Cloud, k8sClient client.Client, elbv2LRManager: elbv2.NewDefaultListenerRuleManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, config.ExternalManagedTags, config.FeatureGates, logger), elbv2TGManager: elbv2.NewDefaultTargetGroupManager(cloud.ELBV2(), trackingProvider, elbv2TaggingManager, cloud.VpcID(), config.ExternalManagedTags, logger), elbv2TGBManager: elbv2.NewDefaultTargetGroupBindingManager(k8sClient, trackingProvider, logger), + elbv2FrontendNlbTargetsManager: elbv2.NewFrontendNlbTargetsManager(cloud.ELBV2(), logger), wafv2WebACLAssociationManager: wafv2.NewDefaultWebACLAssociationManager(cloud.WAFv2(), logger), wafRegionalWebACLAssociationManager: wafregional.NewDefaultWebACLAssociationManager(cloud.WAFRegional(), logger), shieldProtectionManager: shield.NewDefaultProtectionManager(cloud.Shield(), logger), @@ -77,6 +82,7 @@ type defaultStackDeployer struct { elbv2LRManager elbv2.ListenerRuleManager elbv2TGManager elbv2.TargetGroupManager elbv2TGBManager elbv2.TargetGroupBindingManager + elbv2FrontendNlbTargetsManager elbv2.FrontendNlbTargetsManager wafv2WebACLAssociationManager wafv2.WebACLAssociationManager wafRegionalWebACLAssociationManager wafregional.WebACLAssociationManager shieldProtectionManager shield.ProtectionManager @@ -94,15 +100,21 @@ type ResourceSynthesizer interface { } // Deploy a resource stack. -func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string) error { +func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState) error { synthesizers := []ResourceSynthesizer{ ec2.NewSecurityGroupSynthesizer(d.cloud.EC2(), d.trackingProvider, d.ec2TaggingManager, d.ec2SGManager, d.vpcID, d.logger, stack), + } + + if controllerName == ingressController { + synthesizers = append(synthesizers, elbv2.NewFrontendNlbTargetSynthesizer(d.k8sClient, d.trackingProvider, d.elbv2TaggingManager, d.elbv2FrontendNlbTargetsManager, d.logger, d.featureGates, stack, frontendNlbTargetGroupDesiredState)) + } + + synthesizers = append(synthesizers, elbv2.NewTargetGroupSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2TGManager, d.logger, d.featureGates, stack), elbv2.NewLoadBalancerSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2LBManager, d.logger, d.featureGates, d.controllerConfig, stack), elbv2.NewListenerSynthesizer(d.cloud.ELBV2(), d.elbv2TaggingManager, d.elbv2LSManager, d.logger, stack), elbv2.NewListenerRuleSynthesizer(d.cloud.ELBV2(), d.elbv2TaggingManager, d.elbv2LRManager, d.logger, d.featureGates, stack), - elbv2.NewTargetGroupBindingSynthesizer(d.k8sClient, d.trackingProvider, d.elbv2TGBManager, d.logger, stack), - } + elbv2.NewTargetGroupBindingSynthesizer(d.k8sClient, d.trackingProvider, d.elbv2TGBManager, d.logger, stack)) if d.addonsConfig.WAFV2Enabled { synthesizers = append(synthesizers, wafv2.NewWebACLAssociationSynthesizer(d.wafv2WebACLAssociationManager, d.logger, stack)) diff --git a/pkg/ingress/model_build_frontend_nlb.go b/pkg/ingress/model_build_frontend_nlb.go new file mode 100644 index 0000000000..58c4824835 --- /dev/null +++ b/pkg/ingress/model_build_frontend_nlb.go @@ -0,0 +1,768 @@ +package ingress + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/google/go-cmp/cmp" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" + "sigs.k8s.io/aws-load-balancer-controller/pkg/equality" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" +) + +// FrontendNlbListenerConfig defines the configuration for an NLB listener +type FrontendNlbListenerConfig struct { + Protocol elbv2model.Protocol + Port int32 + TargetPort int32 + HealthCheckConfig elbv2model.TargetGroupHealthCheckConfig + HealthCheckConfigExplicit map[string]bool +} + +// FrontendNlbListenConfigWithIngress associates a listener config with its ingress resource +type FrontendNlbListenConfigWithIngress struct { + ingKey types.NamespacedName + FrontendNlbListenerConfig FrontendNlbListenerConfig +} + +// buildFrontendNlbModel constructs the frontend NLB model for the ingress +// It creates the load balancer, listeners, and target groups based on ingress configurations +func (t *defaultModelBuildTask) buildFrontendNlbModel(ctx context.Context, alb *elbv2model.LoadBalancer, listenerPortConfigByIngress map[types.NamespacedName]map[int32]listenPortConfig) error { + enableFrontendNlb, err := t.buildEnableFrontendNlbViaAnnotation(ctx) + if err != nil { + return err + } + + // If the annotation is not present or explicitly set to false, do not build the NLB model + if !enableFrontendNlb { + return nil + } + + scheme, err := t.buildFrontendNlbScheme(ctx, alb) + if err != nil { + return err + } + err = t.buildFrontendNlb(ctx, scheme, alb) + if err != nil { + return err + } + err = t.buildFrontendNlbListeners(ctx, listenerPortConfigByIngress) + if err != nil { + return err + } + return nil +} + +func (t *defaultModelBuildTask) buildEnableFrontendNlbViaAnnotation(ctx context.Context) (bool, error) { + var enableFrontendNlb *bool + for _, member := range t.ingGroup.Members { + rawEnableFrontendNlb := false + exists, err := t.annotationParser.ParseBoolAnnotation(annotations.IngressSuffixEnableFrontendNlb, &rawEnableFrontendNlb, member.Ing.Annotations) + if err != nil { + return false, err + } + + if exists { + if enableFrontendNlb == nil { + enableFrontendNlb = &rawEnableFrontendNlb + } else if *enableFrontendNlb != rawEnableFrontendNlb { + return false, errors.New("conflicting enable frontend NLB values") + } + + } + } + + if enableFrontendNlb == nil { + return false, nil + } + + return *enableFrontendNlb, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbScheme(ctx context.Context, alb *elbv2model.LoadBalancer) (elbv2model.LoadBalancerScheme, error) { + scheme, explicitSchemeSpecified, err := t.buildFrontendNlbSchemeViaAnnotation(ctx, alb) + if err != nil { + return alb.Spec.Scheme, err + } + if explicitSchemeSpecified { + return scheme, nil + } + + return t.defaultScheme, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbSubnetMappings(ctx context.Context, scheme elbv2model.LoadBalancerScheme) ([]elbv2model.SubnetMapping, error) { + var explicitSubnetNameOrIDsList [][]string + for _, member := range t.ingGroup.Members { + var rawSubnetNameOrIDs []string + if exists := t.annotationParser.ParseStringSliceAnnotation(annotations.IngressSuffixFrontendNlbSubnets, &rawSubnetNameOrIDs, member.Ing.Annotations); !exists { + continue + } + explicitSubnetNameOrIDsList = append(explicitSubnetNameOrIDsList, rawSubnetNameOrIDs) + } + + if len(explicitSubnetNameOrIDsList) != 0 { + chosenSubnetNameOrIDs := explicitSubnetNameOrIDsList[0] + for _, subnetNameOrIDs := range explicitSubnetNameOrIDsList[1:] { + if !cmp.Equal(chosenSubnetNameOrIDs, subnetNameOrIDs, equality.IgnoreStringSliceOrder()) { + return nil, errors.Errorf("conflicting subnets: %v | %v", chosenSubnetNameOrIDs, subnetNameOrIDs) + } + } + chosenSubnets, err := t.subnetsResolver.ResolveViaNameOrIDSlice(ctx, chosenSubnetNameOrIDs, + networking.WithSubnetsResolveLBType(elbv2model.LoadBalancerTypeNetwork), + networking.WithSubnetsResolveLBScheme(scheme), + ) + if err != nil { + return nil, err + } + + return buildFrontendNlbSubnetMappingsWithSubnets(chosenSubnets), nil + } + + return nil, nil + +} + +func buildFrontendNlbSubnetMappingsWithSubnets(subnets []ec2types.Subnet) []elbv2model.SubnetMapping { + subnetMappings := make([]elbv2model.SubnetMapping, 0, len(subnets)) + for _, subnet := range subnets { + subnetMappings = append(subnetMappings, elbv2model.SubnetMapping{ + SubnetID: awssdk.ToString(subnet.SubnetId), + }) + } + return subnetMappings +} + +func (t *defaultModelBuildTask) buildFrontendNlb(ctx context.Context, scheme elbv2model.LoadBalancerScheme, alb *elbv2model.LoadBalancer) error { + spec, err := t.buildFrontendNlbSpec(ctx, scheme, alb) + if err != nil { + return err + } + t.frontendNlb = elbv2model.NewLoadBalancer(t.stack, "FrontendNlb", spec) + + return nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbSpec(ctx context.Context, scheme elbv2model.LoadBalancerScheme, + alb *elbv2model.LoadBalancer) (elbv2model.LoadBalancerSpec, error) { + securityGroups, err := t.buildFrontendNlbSecurityGroups(ctx) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } + + // use alb security group if it is not explicitly specified + if securityGroups == nil { + securityGroups = alb.Spec.SecurityGroups + } + + subnetMappings, err := t.buildFrontendNlbSubnetMappings(ctx, scheme) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } + + // use alb subnetMappings if it is not explicitly specified + if subnetMappings == nil { + subnetMappings = alb.Spec.SubnetMappings + } + + name, err := t.buildFrontendNlbName(ctx, scheme, alb) + if err != nil { + return elbv2model.LoadBalancerSpec{}, err + } + + spec := elbv2model.LoadBalancerSpec{ + Name: name, + Type: elbv2model.LoadBalancerTypeNetwork, + Scheme: scheme, + IPAddressType: alb.Spec.IPAddressType, + SecurityGroups: securityGroups, + SubnetMappings: subnetMappings, + } + + return spec, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbName(_ context.Context, scheme elbv2model.LoadBalancerScheme, alb *elbv2model.LoadBalancer) (string, error) { + // build NLB name based upon ALB name + // keeping as much of the original name as possible while ensuring the "-nlb" suffix is always present and the total length never exceeds 32 characters. + if alb.Spec.Name != "" { + baseName := alb.Spec.Name + maxBaseLength := 28 + if len(baseName) > maxBaseLength { + baseName = baseName[:maxBaseLength] + } + nlbName := baseName + "-nlb" + + return nlbName, nil + } + + // Should not fall to this case, but keep it just in case + uuidHash := sha256.New() + _, _ = uuidHash.Write([]byte(t.clusterName)) + _, _ = uuidHash.Write([]byte(t.ingGroup.ID.String())) + _, _ = uuidHash.Write([]byte(scheme)) + uuid := hex.EncodeToString(uuidHash.Sum(nil)) + + if t.ingGroup.ID.IsExplicit() { + payload := invalidLoadBalancerNamePattern.ReplaceAllString(t.ingGroup.ID.Name, "") + return fmt.Sprintf("k8s-%.16s-%.8s-nlb", payload, uuid), nil + } + + sanitizedNamespace := invalidLoadBalancerNamePattern.ReplaceAllString(t.ingGroup.ID.Namespace, "") + sanitizedName := invalidLoadBalancerNamePattern.ReplaceAllString(t.ingGroup.ID.Name, "") + return fmt.Sprintf("k8s-%.8s-%.8s-%.6s-nlb", sanitizedNamespace, sanitizedName, uuid), nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbSecurityGroups(ctx context.Context) ([]core.StringToken, error) { + sgNameOrIDsViaAnnotation, err := t.buildNLBFrontendSGNameOrIDsFromAnnotation(ctx) + if err != nil { + return nil, err + } + + var lbSGTokens []core.StringToken + if len(sgNameOrIDsViaAnnotation) != 0 { + frontendSGIDs, err := t.sgResolver.ResolveViaNameOrID(ctx, sgNameOrIDsViaAnnotation) + + if err != nil { + return nil, err + } + for _, sgID := range frontendSGIDs { + lbSGTokens = append(lbSGTokens, core.LiteralStringToken(sgID)) + return lbSGTokens, nil + } + } + return nil, nil +} + +func (t *defaultModelBuildTask) buildNLBFrontendSGNameOrIDsFromAnnotation(ctx context.Context) ([]string, error) { + var explicitSGNameOrIDsList [][]string + for _, member := range t.ingGroup.Members { + var rawSGNameOrIDs []string + if exists := t.annotationParser.ParseStringSliceAnnotation(annotations.IngressSuffixFrontendNlbSecurityGroups, &rawSGNameOrIDs, member.Ing.Annotations); !exists { + continue + } + explicitSGNameOrIDsList = append(explicitSGNameOrIDsList, rawSGNameOrIDs) + } + if len(explicitSGNameOrIDsList) == 0 { + return nil, nil + } + chosenSGNameOrIDs := explicitSGNameOrIDsList[0] + for _, sgNameOrIDs := range explicitSGNameOrIDsList[1:] { + if !cmp.Equal(chosenSGNameOrIDs, sgNameOrIDs) { + return nil, errors.Errorf("conflicting securityGroups: %v | %v", chosenSGNameOrIDs, sgNameOrIDs) + } + } + return chosenSGNameOrIDs, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbListeners(ctx context.Context, listenerPortConfigByIngress map[types.NamespacedName]map[int32]listenPortConfig) error { + frontendNlbListenerConfigsByPort := make(map[int32][]FrontendNlbListenConfigWithIngress) + + // build frontend nlb config by port for ingress + for _, member := range t.ingGroup.Members { + ingKey := k8s.NamespacedName(member.Ing) + frontendNlbListenerConfigByPortForIngress, err := t.buildFrontendNlbListenerConfigByPortForIngress(ctx, &member, listenerPortConfigByIngress) + if err != nil { + return errors.Wrapf(err, "failed to compute listenPort config for ingress: %s", ingKey.String()) + } + for port, config := range frontendNlbListenerConfigByPortForIngress { + configWithIngress := FrontendNlbListenConfigWithIngress{ + ingKey: ingKey, + FrontendNlbListenerConfig: config, + } + frontendNlbListenerConfigsByPort[port] = append( + frontendNlbListenerConfigsByPort[port], + configWithIngress, + ) + } + } + + // merge frontend nlb listener configs + frontendNlbListenerConfigByPort := make(map[int32]FrontendNlbListenerConfig) + for port, cfgs := range frontendNlbListenerConfigsByPort { + mergedCfg, err := t.mergeFrontendNlbListenPortConfigs(ctx, cfgs) + if err != nil { + return errors.Wrapf(err, "failed to merge NLB listenPort config for port: %v", port) + } + frontendNlbListenerConfigByPort[port] = mergedCfg + } + + // build listener using the config + for port, cfg := range frontendNlbListenerConfigByPort { + _, err := t.buildFrontendNlbListener(ctx, port, cfg) + if err != nil { + return err + } + } + + return nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbListenerConfigByPortForIngress(ctx context.Context, ing *ClassifiedIngress, listenerPortConfigByIngress map[types.NamespacedName]map[int32]listenPortConfig) (map[int32]FrontendNlbListenerConfig, error) { + ingKey := k8s.NamespacedName(ing.Ing) + + frontendNlbListenerConfigByPort := make(map[int32]FrontendNlbListenerConfig) + + portMapping, err := t.parseFrontendNlbListenerPortMapping(ctx, ing.Ing.Annotations) + if err != nil { + return nil, err + } + + // Check if frontend-nlb-listener-port-mapping exists + if len(portMapping) > 0 { + //if exists: only create NLB listeners for explicitly mapped ALB listener ports + for nlbListenerPort, mappedAlbListenerPort := range portMapping { + + // check if the ALB listener port exists in the listener port set + if _, exists := listenerPortConfigByIngress[ingKey][mappedAlbListenerPort]; !exists { + t.logger.Info("Skipping NLB listener creation for unmapped ALB listener port", "mappedAlbListenerPort", mappedAlbListenerPort) + continue + } + + healthCheckConfig, isExplicit, err := t.buildFrontendNlbTargetGroupHealthCheckConfig(ctx, ing.Ing.Annotations, "TCP") + if err != nil { + return nil, err + } + + frontendNlbListenerConfigByPort[nlbListenerPort] = FrontendNlbListenerConfig{ + Protocol: elbv2model.ProtocolTCP, + Port: nlbListenerPort, + TargetPort: mappedAlbListenerPort, + HealthCheckConfig: healthCheckConfig, + HealthCheckConfigExplicit: isExplicit, + } + } + + } else { + // if not: Map ALB listener ports directly to NLB listener ports + for albListenerPort := range listenerPortConfigByIngress[ingKey] { + + healthCheckConfig, isExplicit, err := t.buildFrontendNlbTargetGroupHealthCheckConfig(ctx, ing.Ing.Annotations, "TCP") + if err != nil { + return nil, err + } + + // Add the listener configuration to the map + frontendNlbListenerConfigByPort[albListenerPort] = FrontendNlbListenerConfig{ + Protocol: elbv2model.ProtocolTCP, + Port: albListenerPort, + TargetPort: albListenerPort, + HealthCheckConfig: healthCheckConfig, + HealthCheckConfigExplicit: isExplicit, + } + } + } + + return frontendNlbListenerConfigByPort, nil +} + +func (t *defaultModelBuildTask) parseFrontendNlbListenerPortMapping(ctx context.Context, ingAnnotation map[string]string) (map[int32]int32, error) { + var rawPortMapping map[string]string + _, err := t.annotationParser.ParseStringMapAnnotation(annotations.IngressSuffixFrontendNlbListenerPortMapping, &rawPortMapping, ingAnnotation) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse frontend-nlb-listener-port-mapping for ingress %v", rawPortMapping) + } + + portMappping := make(map[int32]int32) + + for rawNlbPort, rawAlbPort := range rawPortMapping { + nlbPort, err := strconv.ParseInt(rawNlbPort, 10, 32) + if err != nil { + return nil, errors.Errorf("invalid NLB listener port: %s", rawNlbPort) + } + + albPort, err := strconv.ParseInt(rawAlbPort, 10, 32) + if err != nil { + return nil, errors.Errorf("invalid ALB listener port: %s", rawAlbPort) + } + + portMappping[int32(nlbPort)] = int32(albPort) + } + + return portMappping, nil + +} + +func (t *defaultModelBuildTask) mergeFrontendNlbListenPortConfigs(ctx context.Context, configs []FrontendNlbListenConfigWithIngress) (FrontendNlbListenerConfig, error) { + if len(configs) == 0 { + return FrontendNlbListenerConfig{}, errors.New("no NLB listener port configurations provided") + } + + // Initialize the final configuration + finalConfig := FrontendNlbListenerConfig{} + explicitFields := make(map[string]bool) + + // Port and Protocol are the same + finalConfig.Port = configs[0].FrontendNlbListenerConfig.Port + finalConfig.Protocol = configs[0].FrontendNlbListenerConfig.Protocol + + // Initialize the first Target port + finalConfig.TargetPort = configs[0].FrontendNlbListenerConfig.TargetPort + + // Iterate over all configurations to build the final configuration + for i, config := range configs { + healthCheckConfig := config.FrontendNlbListenerConfig.HealthCheckConfig + explicit := config.FrontendNlbListenerConfig.HealthCheckConfigExplicit + + // Merge intervalSeconds + err := mergeHealthCheckField("IntervalSeconds", &finalConfig.HealthCheckConfig.IntervalSeconds, healthCheckConfig.IntervalSeconds, explicit, explicitFields, i) + if err != nil { + return FrontendNlbListenerConfig{}, err + } + + // Merge timeoutSeconds + err = mergeHealthCheckField("TimeoutSeconds", &finalConfig.HealthCheckConfig.TimeoutSeconds, healthCheckConfig.TimeoutSeconds, explicit, explicitFields, i) + if err != nil { + return FrontendNlbListenerConfig{}, err + } + + // Merge healthyThresholdCount + err = mergeHealthCheckField("HealthyThresholdCount", &finalConfig.HealthCheckConfig.HealthyThresholdCount, healthCheckConfig.HealthyThresholdCount, explicit, explicitFields, i) + if err != nil { + return FrontendNlbListenerConfig{}, err + } + + // Merge unhealthyThresholdCount + err = mergeHealthCheckField("UnhealthyThresholdCount", &finalConfig.HealthCheckConfig.UnhealthyThresholdCount, healthCheckConfig.UnhealthyThresholdCount, explicit, explicitFields, i) + if err != nil { + return FrontendNlbListenerConfig{}, err + } + + // Merge protocol + if explicit["Protocol"] { + if explicitFields["Protocol"] { + if finalConfig.HealthCheckConfig.Protocol != healthCheckConfig.Protocol { + return FrontendNlbListenerConfig{}, errors.Errorf("conflicting Protocol, config %d: %s, previous: %s", + i+1, healthCheckConfig.Protocol, finalConfig.HealthCheckConfig.Protocol) + } + } else { + finalConfig.HealthCheckConfig.Protocol = healthCheckConfig.Protocol + explicitFields["Protocol"] = true + } + } else if !explicitFields["Protocol"] { + finalConfig.HealthCheckConfig.Protocol = healthCheckConfig.Protocol + } + + // Merge path + err = mergeHealthCheckField("Path", &finalConfig.HealthCheckConfig.Path, healthCheckConfig.Path, explicit, explicitFields, i) + if err != nil { + return FrontendNlbListenerConfig{}, err + } + + // Merge matcher + err = mergeHealthCheckField("Matcher", &finalConfig.HealthCheckConfig.Matcher, healthCheckConfig.Matcher, explicit, explicitFields, i) + if err != nil { + return FrontendNlbListenerConfig{}, err + } + + // Merge port + err = mergeHealthCheckField("Port", &finalConfig.HealthCheckConfig.Port, healthCheckConfig.Port, explicit, explicitFields, i) + if err != nil { + return FrontendNlbListenerConfig{}, err + } + + // Validate NLB-to-ALB port mappings to ensure each NLB listener port maps to exactly one ALB port, preventing connection collisions + if finalConfig.TargetPort != config.FrontendNlbListenerConfig.TargetPort { + return FrontendNlbListenerConfig{}, errors.Errorf("conflicting Target Port, config %d: %v, previous: %v", + i+1, config.FrontendNlbListenerConfig.TargetPort, finalConfig.TargetPort) + } else { + finalConfig.TargetPort = config.FrontendNlbListenerConfig.TargetPort + } + } + + return finalConfig, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbListener(ctx context.Context, port int32, config FrontendNlbListenerConfig) (*elbv2model.Listener, error) { + lsSpec, err := t.buildFrontendNlbListenerSpec(ctx, port, config) + if err != nil { + return nil, err + } + frontendNlbListenerResID := buildFrontendNlbResourceID("ls", config.Protocol, &port) + ls := elbv2model.NewListener(t.stack, frontendNlbListenerResID, lsSpec) + return ls, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbListenerSpec(ctx context.Context, port int32, config FrontendNlbListenerConfig) (elbv2model.ListenerSpec, error) { + listenerProtocol := elbv2model.Protocol(config.Protocol) + + targetGroup, err := t.buildFrontendNlbTargetGroup(ctx, port, config) + if err != nil { + return elbv2model.ListenerSpec{}, err + } + + defaultActions := t.buildFrontendNlbListenerDefaultActions(ctx, targetGroup) + + t.frontendNlbTargetGroupDesiredState.AddTargetGroup(targetGroup.Spec.Name, targetGroup.TargetGroupARN(), t.loadBalancer.LoadBalancerARN(), *targetGroup.Spec.Port, config.TargetPort) + + return elbv2model.ListenerSpec{ + LoadBalancerARN: t.frontendNlb.LoadBalancerARN(), + Port: port, + Protocol: listenerProtocol, + DefaultActions: defaultActions, + }, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbListenerDefaultActions(_ context.Context, targetGroup *elbv2model.TargetGroup) []elbv2model.Action { + return []elbv2model.Action{ + { + Type: elbv2model.ActionTypeForward, + ForwardConfig: &elbv2model.ForwardActionConfig{ + TargetGroups: []elbv2model.TargetGroupTuple{ + { + TargetGroupARN: targetGroup.TargetGroupARN(), + }, + }, + }, + }, + } +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroup(ctx context.Context, port int32, config FrontendNlbListenerConfig) (*elbv2model.TargetGroup, error) { + frontendNlbTgResID := buildFrontendNlbResourceID("tg", config.Protocol, &port) + tgSpec, err := t.buildFrontendNlbTargetGroupSpec(ctx, config.Protocol, port, &config.HealthCheckConfig) + if err != nil { + return nil, err + } + targetGroup := elbv2model.NewTargetGroup(t.stack, frontendNlbTgResID, tgSpec) + return targetGroup, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckConfig(ctx context.Context, svcAndIngAnnotations map[string]string, tgProtocol elbv2model.Protocol) (elbv2model.TargetGroupHealthCheckConfig, map[string]bool, error) { + isExplicit := make(map[string]bool) + + healthCheckPort, portExplicit, err := t.buildFrontendNlbTargetGroupHealthCheckPort(ctx, svcAndIngAnnotations) + if err != nil { + return elbv2model.TargetGroupHealthCheckConfig{}, nil, err + } + isExplicit["Port"] = portExplicit + + healthCheckProtocol, protocolExplicit, err := t.buildFrontendNlbTargetGroupHealthCheckProtocol(ctx, svcAndIngAnnotations, "HTTP") + if err != nil { + return elbv2model.TargetGroupHealthCheckConfig{}, nil, err + } + isExplicit["Protocol"] = protocolExplicit + + healthCheckPath, pathExplicit := t.buildFrontendNlbTargetGroupHealthCheckPath(ctx, svcAndIngAnnotations) + isExplicit["Path"] = pathExplicit + + healthCheckMatcher, matcherExplicit := t.buildFrontendNlbTargetGroupHealthCheckMatcher(ctx, svcAndIngAnnotations, elbv2model.ProtocolVersionHTTP1) + isExplicit["Matcher"] = matcherExplicit + + healthCheckIntervalSeconds, intervalExplicit, err := t.buildFrontendNlbTargetGroupHealthCheckIntervalSeconds(ctx, svcAndIngAnnotations) + if err != nil { + return elbv2model.TargetGroupHealthCheckConfig{}, nil, err + } + isExplicit["IntervalSeconds"] = intervalExplicit + + healthCheckTimeoutSeconds, timeoutExplicit, err := t.buildFrontendNlbTargetGroupHealthCheckTimeoutSeconds(ctx, svcAndIngAnnotations) + if err != nil { + return elbv2model.TargetGroupHealthCheckConfig{}, nil, err + } + isExplicit["TimeoutSeconds"] = timeoutExplicit + + healthCheckHealthyThresholdCount, healthyThresholdExplicit, err := t.buildFrontendNlbTargetGroupHealthCheckHealthyThresholdCount(ctx, svcAndIngAnnotations) + if err != nil { + return elbv2model.TargetGroupHealthCheckConfig{}, nil, err + } + isExplicit["HealthyThresholdCount"] = healthyThresholdExplicit + + healthCheckUnhealthyThresholdCount, unhealthyThresholdExplicit, err := t.buildFrontendNlbTargetGroupHealthCheckUnhealthyThresholdCount(ctx, svcAndIngAnnotations) + if err != nil { + return elbv2model.TargetGroupHealthCheckConfig{}, nil, err + } + isExplicit["UnhealthyThresholdCount"] = unhealthyThresholdExplicit + + return elbv2model.TargetGroupHealthCheckConfig{ + Port: &healthCheckPort, + Protocol: healthCheckProtocol, + Path: &healthCheckPath, + Matcher: &healthCheckMatcher, + IntervalSeconds: awssdk.Int32(healthCheckIntervalSeconds), + TimeoutSeconds: awssdk.Int32(healthCheckTimeoutSeconds), + HealthyThresholdCount: awssdk.Int32(healthCheckHealthyThresholdCount), + UnhealthyThresholdCount: awssdk.Int32(healthCheckUnhealthyThresholdCount), + }, isExplicit, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckPort(_ context.Context, svcAndIngAnnotations map[string]string) (intstr.IntOrString, bool, error) { + rawHealthCheckPort := "" + exists := t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixFrontendNlbHealthCheckPort, &rawHealthCheckPort, svcAndIngAnnotations) + if !exists { + return intstr.FromString(healthCheckPortTrafficPort), false, nil + } + if rawHealthCheckPort == healthCheckPortTrafficPort { + return intstr.FromString(healthCheckPortTrafficPort), true, nil + } + healthCheckPort := intstr.Parse(rawHealthCheckPort) + if healthCheckPort.Type == intstr.Int { + return healthCheckPort, true, nil + } + + return intstr.IntOrString{}, true, errors.New("failed to resolve healthCheckPort") +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckProtocol(_ context.Context, svcAndIngAnnotations map[string]string, tgProtocol elbv2model.Protocol) (elbv2model.Protocol, bool, error) { + rawHealthCheckProtocol := string(tgProtocol) + exists := t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixFrontendNlbHealthCheckProtocol, &rawHealthCheckProtocol, svcAndIngAnnotations) + switch rawHealthCheckProtocol { + case string(elbv2model.ProtocolHTTP): + return elbv2model.ProtocolHTTP, exists, nil + case string(elbv2model.ProtocolHTTPS): + return elbv2model.ProtocolHTTPS, exists, nil + default: + return "", exists, errors.Errorf("healthCheckProtocol must be within [%v, %v]", elbv2model.ProtocolHTTP, elbv2model.ProtocolHTTPS) + } +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckPath(_ context.Context, svcAndIngAnnotations map[string]string) (string, bool) { + rawHealthCheckPath := t.defaultHealthCheckPathHTTP + exists := t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixFrontendNlbHealthCheckPath, &rawHealthCheckPath, svcAndIngAnnotations) + return rawHealthCheckPath, exists +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckMatcher(_ context.Context, svcAndIngAnnotations map[string]string, tgProtocolVersion elbv2model.ProtocolVersion) (elbv2model.HealthCheckMatcher, bool) { + rawHealthCheckMatcherHTTPCode := t.defaultHealthCheckMatcherHTTPCode + exists := t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixFrontendNlbHealthCheckSuccessCodes, &rawHealthCheckMatcherHTTPCode, svcAndIngAnnotations) + return elbv2model.HealthCheckMatcher{ + HTTPCode: &rawHealthCheckMatcherHTTPCode, + }, exists +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckIntervalSeconds(_ context.Context, svcAndIngAnnotations map[string]string) (int32, bool, error) { + rawHealthCheckIntervalSeconds := t.defaultHealthCheckIntervalSeconds + exists, err := t.annotationParser.ParseInt32Annotation(annotations.IngressSuffixFrontendNlbHealthCheckIntervalSeconds, &rawHealthCheckIntervalSeconds, svcAndIngAnnotations) + if err != nil { + return 0, false, err + } + return rawHealthCheckIntervalSeconds, exists, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckTimeoutSeconds(_ context.Context, svcAndIngAnnotations map[string]string) (int32, bool, error) { + rawHealthCheckTimeoutSeconds := t.defaultHealthCheckTimeoutSeconds + exists, err := t.annotationParser.ParseInt32Annotation(annotations.IngressSuffixFrontendNlbHealthCheckTimeoutSeconds, &rawHealthCheckTimeoutSeconds, svcAndIngAnnotations) + if err != nil { + return 0, false, err + } + return rawHealthCheckTimeoutSeconds, exists, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckHealthyThresholdCount(_ context.Context, svcAndIngAnnotations map[string]string) (int32, bool, error) { + rawHealthCheckHealthyThresholdCount := t.defaultHealthCheckHealthyThresholdCount + exists, err := t.annotationParser.ParseInt32Annotation(annotations.IngressSuffixFrontendNlbHealthCheckHealthyThresholdCount, + &rawHealthCheckHealthyThresholdCount, svcAndIngAnnotations) + if err != nil { + return 0, false, err + } + return rawHealthCheckHealthyThresholdCount, exists, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckUnhealthyThresholdCount(_ context.Context, svcAndIngAnnotations map[string]string) (int32, bool, error) { + rawHealthCheckUnhealthyThresholdCount := t.defaultHealthCheckUnhealthyThresholdCount + exists, err := t.annotationParser.ParseInt32Annotation(annotations.IngressSuffixFrontendNlHealthCheckbUnhealthyThresholdCount, + &rawHealthCheckUnhealthyThresholdCount, svcAndIngAnnotations) + if err != nil { + return 0, false, err + } + return rawHealthCheckUnhealthyThresholdCount, exists, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupSpec(ctx context.Context, tgProtocol elbv2model.Protocol, + port int32, healthCheckConfig *elbv2model.TargetGroupHealthCheckConfig) (elbv2model.TargetGroupSpec, error) { + + tgName := t.buildFrontendNlbTargetGroupName(ctx, port, elbv2model.TargetTypeALB, tgProtocol, healthCheckConfig) + + return elbv2model.TargetGroupSpec{ + Name: tgName, + TargetType: elbv2model.TargetTypeALB, + Port: awssdk.Int32(port), + Protocol: tgProtocol, + IPAddressType: elbv2model.TargetGroupIPAddressType(t.loadBalancer.Spec.IPAddressType), + HealthCheckConfig: healthCheckConfig, + }, nil +} + +func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupName(_ context.Context, tgPort int32, + targetType elbv2model.TargetType, tgProtocol elbv2model.Protocol, hc *elbv2model.TargetGroupHealthCheckConfig) string { + uuidHash := sha256.New() + _, _ = uuidHash.Write([]byte(t.clusterName)) + _, _ = uuidHash.Write([]byte(t.ingGroup.ID.String())) + _, _ = uuidHash.Write([]byte(strconv.Itoa(int(tgPort)))) + _, _ = uuidHash.Write([]byte(targetType)) + _, _ = uuidHash.Write([]byte(tgProtocol)) + _, _ = uuidHash.Write([]byte(hc.Port.String())) + _, _ = uuidHash.Write([]byte(hc.Protocol)) + uuid := hex.EncodeToString(uuidHash.Sum(nil)) + + if t.ingGroup.ID.IsExplicit() { + sanitizedName := invalidLoadBalancerNamePattern.ReplaceAllString(t.ingGroup.ID.Name, "") + return fmt.Sprintf("k8s-%.17s-%.10s", sanitizedName, uuid) + } + + sanitizedNamespace := invalidTargetGroupNamePattern.ReplaceAllString(t.ingGroup.ID.Namespace, "") + sanitizedName := invalidTargetGroupNamePattern.ReplaceAllString(t.ingGroup.ID.Name, "") + return fmt.Sprintf("k8s-%.8s-%.8s-%.10s", sanitizedNamespace, sanitizedName, uuid) +} + +func (t *defaultModelBuildTask) buildFrontendNlbSchemeViaAnnotation(ctx context.Context, alb *elbv2model.LoadBalancer) (elbv2model.LoadBalancerScheme, bool, error) { + explicitSchemes := sets.Set[string]{} + for _, member := range t.ingGroup.Members { + rawSchema := "" + if exists := t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixFrontendNlbScheme, &rawSchema, member.Ing.Annotations); !exists { + continue + } + explicitSchemes.Insert(rawSchema) + } + if len(explicitSchemes) == 0 { + return alb.Spec.Scheme, false, nil + } + if len(explicitSchemes) > 1 { + return "", true, errors.Errorf("conflicting scheme: %v", explicitSchemes) + } + rawScheme, _ := explicitSchemes.PopAny() + switch rawScheme { + case string(elbv2model.LoadBalancerSchemeInternetFacing): + return elbv2model.LoadBalancerSchemeInternetFacing, true, nil + case string(elbv2model.LoadBalancerSchemeInternal): + return elbv2model.LoadBalancerSchemeInternal, true, nil + default: + return "", false, errors.Errorf("unknown scheme: %v", rawScheme) + } +} + +func buildFrontendNlbResourceID(resourceType string, protocol elbv2model.Protocol, port *int32) string { + if port != nil && protocol != "" { + return fmt.Sprintf("FrontendNlb-%s-%v-%v", resourceType, protocol, *port) + } + return fmt.Sprintf("FrontendNlb") +} + +func mergeHealthCheckField[T comparable](fieldName string, finalValue **T, currentValue *T, explicit map[string]bool, explicitFields map[string]bool, configIndex int) error { + if explicit[fieldName] { + if explicitFields[fieldName] { + fmt.Printf("*finalValue (%T): %v\n", **finalValue, **finalValue) + fmt.Printf("newValue (%T): %v\n", *currentValue, *currentValue) + if **finalValue != *currentValue { + return errors.Errorf("conflicting %s, config %d: %v, previous: %v", fieldName, configIndex+1, *currentValue, **finalValue) + } + } else { + *finalValue = currentValue + explicitFields[fieldName] = true + } + } else if !explicitFields[fieldName] { + *finalValue = currentValue + } + return nil +} diff --git a/pkg/ingress/model_build_frontend_nlb_test.go b/pkg/ingress/model_build_frontend_nlb_test.go new file mode 100644 index 0000000000..ef7c740957 --- /dev/null +++ b/pkg/ingress/model_build_frontend_nlb_test.go @@ -0,0 +1,966 @@ +package ingress + +import ( + "context" + "testing" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/go-logr/logr" + gomock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + networking "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + networking2 "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + networkingpkg "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +func Test_defaultModelBuildTask_buildFrontendNlbSecurityGroups(t *testing.T) { + type describeSecurityGroupsResult struct { + securityGroups []ec2types.SecurityGroup + err error + } + + type fields struct { + ingGroup Group + scheme elbv2.LoadBalancerScheme + describeSecurityGroupsResult []describeSecurityGroupsResult + } + + tests := []struct { + name string + fields fields + wantSGTokens []core.StringToken + wantErr string + }{ + { + name: "with no annotations", + fields: fields{ + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{}, + }, + }, + }, + }, + }, + scheme: elbv2.LoadBalancerSchemeInternal, + }, + }, + { + name: "with annotations", + fields: fields{ + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-security-groups": "sg-manual", + }, + }, + }, + }, + }, + }, + scheme: elbv2.LoadBalancerSchemeInternal, + describeSecurityGroupsResult: []describeSecurityGroupsResult{ + { + securityGroups: []ec2types.SecurityGroup{ + { + GroupId: awssdk.String("sg-manual"), + }, + }, + }, + }, + }, + wantSGTokens: []core.StringToken{core.LiteralStringToken("sg-manual")}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockEC2 := services.NewMockEC2(ctrl) + vpcID := "vpc-dummy" + for _, res := range tt.fields.describeSecurityGroupsResult { + mockEC2.EXPECT().DescribeSecurityGroupsAsList(gomock.Any(), gomock.Any()).Return(res.securityGroups, res.err) + } + sgResolver := networkingpkg.NewDefaultSecurityGroupResolver(mockEC2, vpcID) + + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + task := &defaultModelBuildTask{ + sgResolver: sgResolver, + ingGroup: tt.fields.ingGroup, + annotationParser: annotationParser, + } + + got, err := task.buildFrontendNlbSecurityGroups(context.Background()) + + if err != nil { + assert.EqualError(t, err, tt.wantErr) + } else { + + var gotSGTokens []core.StringToken + for _, sgToken := range got { + gotSGTokens = append(gotSGTokens, sgToken) + } + assert.Equal(t, tt.wantSGTokens, gotSGTokens) + } + }) + } +} + +func Test_buildFrontendNlbSubnetMappings(t *testing.T) { + + type fields struct { + ingGroup Group + scheme elbv2.LoadBalancerScheme + } + + tests := []struct { + name string + fields fields + want []string + wantErr string + }{ + { + name: "no annotation implicit subnets", + fields: fields{ + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{}, + }, + }, + }, + }, + }, + scheme: elbv2.LoadBalancerSchemeInternal, + }, + }, + { + name: "with subnets annoattion", + fields: fields{ + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-2", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-subnets": "subnet-1,subnet-2", + }, + }, + }, + }, + }, + }, + scheme: elbv2.LoadBalancerSchemeInternal, + }, + want: []string{"subnet-1", "subnet-2"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockEC2 := services.NewMockEC2(ctrl) + mockEC2.EXPECT().DescribeSubnetsAsList(gomock.Any(), gomock.Any()). + DoAndReturn(stubDescribeSubnetsAsList). + AnyTimes() + + azInfoProvider := networking2.NewMockAZInfoProvider(ctrl) + azInfoProvider.EXPECT().FetchAZInfos(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, availabilityZoneIDs []string) (map[string]ec2types.AvailabilityZone, error) { + ret := make(map[string]ec2types.AvailabilityZone, len(availabilityZoneIDs)) + for _, id := range availabilityZoneIDs { + ret[id] = ec2types.AvailabilityZone{ZoneType: awssdk.String("availability-zone")} + } + return ret, nil + }).AnyTimes() + + subnetsResolver := networking2.NewDefaultSubnetsResolver( + azInfoProvider, + mockEC2, + "vpc-1", + "test-cluster", + true, + true, + true, + logr.New(&log.NullLogSink{}), + ) + + annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io") + task := &defaultModelBuildTask{ + ingGroup: tt.fields.ingGroup, + annotationParser: annotationParser, + subnetsResolver: subnetsResolver, + } + got, err := task.buildFrontendNlbSubnetMappings(context.Background(), tt.fields.scheme) + + if err != nil { + assert.EqualError(t, err, tt.wantErr) + } else { + + var gotSubnets []string + for _, mapping := range got { + gotSubnets = append(gotSubnets, mapping.SubnetID) + } + assert.Equal(t, tt.want, gotSubnets) + } + }) + } +} + +func Test_buildFrontendNlbName(t *testing.T) { + tests := []struct { + name string + clusterName string + namespace string + ingName string + scheme elbv2model.LoadBalancerScheme + wantPrefix string + alb *elbv2model.LoadBalancer + }{ + { + name: "standard case", + clusterName: "test-cluster", + namespace: "awesome-ns", + ingName: "my-ingress", + scheme: elbv2model.LoadBalancerSchemeInternal, + wantPrefix: "test-alb", + alb: &elbv2model.LoadBalancer{ + Spec: elbv2model.LoadBalancerSpec{ + Name: "test-alb", + }, + }, + }, + { + name: "with special characters", + clusterName: "test-cluster", + namespace: "awesome-ns", + ingName: "my-ingress-1", + scheme: elbv2model.LoadBalancerSchemeInternal, + wantPrefix: "k8s-awesomen-myingr", + alb: &elbv2model.LoadBalancer{ + Spec: elbv2model.LoadBalancerSpec{}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &defaultModelBuildTask{ + clusterName: tt.clusterName, + ingGroup: Group{ + ID: GroupID{ + Namespace: tt.namespace, + Name: tt.ingName, + }, + }, + } + + got, err := task.buildFrontendNlbName(context.Background(), tt.scheme, tt.alb) + assert.NoError(t, err) + assert.Contains(t, got, tt.wantPrefix) + + }) + } +} + +func Test_buildFrontendNLBTargetGroupName(t *testing.T) { + tests := []struct { + name string + clusterName string + namespace string + ingName string + port int32 + svcPort intstr.IntOrString + targetType elbv2model.TargetType + protocol elbv2model.Protocol + wantPrefix string + }{ + { + name: "standard case", + clusterName: "test-cluster", + namespace: "default", + ingName: "my-ingress", + port: 80, + svcPort: intstr.FromInt(80), + targetType: "alb", + protocol: elbv2model.ProtocolTCP, + wantPrefix: "k8s-default-mying", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &defaultModelBuildTask{ + clusterName: tt.clusterName, + loadBalancer: &elbv2model.LoadBalancer{ + Spec: elbv2model.LoadBalancerSpec{ + Scheme: elbv2.LoadBalancerSchemeInternetFacing, + }, + }, + ingGroup: Group{ + ID: GroupID{ + Namespace: tt.namespace, + Name: tt.ingName, + }, + }, + } + + port80 := intstr.FromInt(80) + + healthCheckConfig := &elbv2model.TargetGroupHealthCheckConfig{ + Protocol: elbv2model.ProtocolTCP, + Port: &port80, + } + + got := task.buildFrontendNlbTargetGroupName( + context.Background(), + tt.port, + tt.targetType, + tt.protocol, + healthCheckConfig, + ) + + assert.Contains(t, got, tt.wantPrefix) + + }) + } +} + +func Test_buildFrontendNlbSchemeViaAnnotation(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + defaultScheme elbv2model.LoadBalancerScheme + wantScheme elbv2model.LoadBalancerScheme + wantExplicit bool + wantErr bool + }{ + { + name: "explicit internal scheme", + annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-scheme": "internal", + }, + defaultScheme: elbv2model.LoadBalancerSchemeInternetFacing, + wantScheme: elbv2model.LoadBalancerSchemeInternal, + wantExplicit: true, + wantErr: false, + }, + { + name: "explicit internet-facing scheme", + annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-scheme": "internet-facing", + }, + defaultScheme: elbv2model.LoadBalancerSchemeInternal, + wantScheme: elbv2model.LoadBalancerSchemeInternetFacing, + wantExplicit: true, + wantErr: false, + }, + { + name: "no annotation - use default", + annotations: map[string]string{}, + defaultScheme: elbv2model.LoadBalancerSchemeInternal, + wantScheme: elbv2model.LoadBalancerSchemeInternal, + wantExplicit: false, + wantErr: false, + }, + { + name: "invalid scheme", + annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-scheme": "invalid", + }, + defaultScheme: elbv2model.LoadBalancerSchemeInternal, + wantScheme: "", + wantExplicit: false, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ing := &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: tt.annotations, + }, + } + + task := &defaultModelBuildTask{ + ingGroup: Group{ + Members: []ClassifiedIngress{ + { + Ing: ing, + }, + }, + }, + annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), + } + + alb := &elbv2model.LoadBalancer{ + Spec: elbv2model.LoadBalancerSpec{ + Scheme: tt.defaultScheme, + }, + } + + gotScheme, gotExplicit, err := task.buildFrontendNlbSchemeViaAnnotation(context.Background(), alb) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.wantScheme, gotScheme) + assert.Equal(t, tt.wantExplicit, gotExplicit) + } + }) + } +} + +func Test_buildEnableFrontendNlbViaAnnotation(t *testing.T) { + + type fields struct { + ingGroup Group + } + + tests := []struct { + name string + fields fields + wantEnabled bool + wantErr bool + expectedErrMsg string + }{ + { + name: "single ingress with enable annotation true", + fields: fields{ + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/enable-frontend-nlb": "true", + }, + }, + }, + }, + }, + }, + }, + wantEnabled: true, + wantErr: false, + }, + { + name: "single ingress without annotation", + fields: fields{ + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{}, + }, + }, + }, + }, + }, + }, + wantEnabled: false, + wantErr: false, + }, + { + name: "multiple ingresses with same enable value", + fields: fields{ + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/enable-frontend-nlb": "true", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-2", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/enable-frontend-nlb": "true", + }, + }, + }, + }, + }, + }, + }, + wantEnabled: true, + wantErr: false, + }, + { + name: "multiple ingresses with conflicting enable values", + fields: fields{ + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/enable-frontend-nlb": "true", + }, + }, + }, + }, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-2", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/enable-frontend-nlb": "false", + }, + }, + }, + }, + }, + }, + }, + wantEnabled: false, + wantErr: true, + expectedErrMsg: "conflicting enable frontend NLB", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + task := &defaultModelBuildTask{ + ingGroup: tt.fields.ingGroup, + annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), + } + + got, err := task.buildEnableFrontendNlbViaAnnotation(context.Background()) + + if tt.wantErr { + assert.Error(t, err) + if tt.expectedErrMsg != "" { + assert.Contains(t, err.Error(), tt.expectedErrMsg) + } + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.wantEnabled, got) + }) + } +} + +func Test_mergeFrontendNlbListenPortConfigs(t *testing.T) { + tests := []struct { + name string + configs []FrontendNlbListenConfigWithIngress + expectedConfig FrontendNlbListenerConfig + wantErr bool + expectedErrMsg string + }{ + { + name: "valid config with no conflicts", + configs: []FrontendNlbListenConfigWithIngress{ + { + ingKey: types.NamespacedName{Namespace: "awesome-ns", Name: "ingress-1"}, + FrontendNlbListenerConfig: FrontendNlbListenerConfig{ + Port: 80, + Protocol: elbv2model.ProtocolTCP, + TargetPort: 80, + HealthCheckConfig: elbv2model.TargetGroupHealthCheckConfig{ + IntervalSeconds: awssdk.Int32(10), + TimeoutSeconds: awssdk.Int32(5), + }, + }, + }, + }, + expectedConfig: FrontendNlbListenerConfig{ + Port: 80, + Protocol: elbv2model.ProtocolTCP, + TargetPort: 80, + HealthCheckConfig: elbv2model.TargetGroupHealthCheckConfig{ + IntervalSeconds: awssdk.Int32(10), + TimeoutSeconds: awssdk.Int32(5), + }, + }, + wantErr: false, + }, + { + name: "conflicting health check interval", + configs: []FrontendNlbListenConfigWithIngress{ + { + ingKey: types.NamespacedName{Namespace: "awesome-ns", Name: "ingress-1"}, + FrontendNlbListenerConfig: FrontendNlbListenerConfig{ + Port: 80, + Protocol: elbv2model.ProtocolTCP, + HealthCheckConfig: elbv2model.TargetGroupHealthCheckConfig{ + IntervalSeconds: awssdk.Int32(10), + }, + HealthCheckConfigExplicit: map[string]bool{ + "IntervalSeconds": true, + }, + }, + }, + { + ingKey: types.NamespacedName{Namespace: "awesome-ns", Name: "ingress-2"}, + FrontendNlbListenerConfig: FrontendNlbListenerConfig{ + Port: 80, + Protocol: elbv2model.ProtocolTCP, + HealthCheckConfig: elbv2model.TargetGroupHealthCheckConfig{ + IntervalSeconds: awssdk.Int32(15), + }, + HealthCheckConfigExplicit: map[string]bool{ + "IntervalSeconds": true, + }, + }, + }, + }, + wantErr: true, + expectedErrMsg: "conflicting IntervalSeconds", + }, + { + name: "valid multiple configs with same values", + configs: []FrontendNlbListenConfigWithIngress{ + { + ingKey: types.NamespacedName{Namespace: "awesome-ns", Name: "ingress-1"}, + FrontendNlbListenerConfig: FrontendNlbListenerConfig{ + Port: 80, + Protocol: elbv2model.ProtocolTCP, + HealthCheckConfig: elbv2model.TargetGroupHealthCheckConfig{ + IntervalSeconds: awssdk.Int32(10), + TimeoutSeconds: awssdk.Int32(5), + }, + TargetPort: 80, + HealthCheckConfigExplicit: map[string]bool{ + "IntervalSeconds": true, + "TimeoutSeconds": true, + }, + }, + }, + { + ingKey: types.NamespacedName{Namespace: "awesome-ns", Name: "ingress-2"}, + FrontendNlbListenerConfig: FrontendNlbListenerConfig{ + Port: 80, + Protocol: elbv2model.ProtocolTCP, + HealthCheckConfig: elbv2model.TargetGroupHealthCheckConfig{ + IntervalSeconds: awssdk.Int32(10), + TimeoutSeconds: awssdk.Int32(5), + }, + TargetPort: 80, + HealthCheckConfigExplicit: map[string]bool{ + "IntervalSeconds": true, + "TimeoutSeconds": true, + }, + }, + }, + }, + expectedConfig: FrontendNlbListenerConfig{ + Port: 80, + Protocol: elbv2model.ProtocolTCP, + HealthCheckConfig: elbv2model.TargetGroupHealthCheckConfig{ + IntervalSeconds: awssdk.Int32(10), + TimeoutSeconds: awssdk.Int32(5), + }, + TargetPort: 80, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + task := &defaultModelBuildTask{} + got, err := task.mergeFrontendNlbListenPortConfigs(context.Background(), tt.configs) + + if tt.wantErr { + assert.Error(t, err) + if tt.expectedErrMsg != "" { + assert.Contains(t, err.Error(), tt.expectedErrMsg) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedConfig, got) + } + }) + } +} + +func Test_defaultModelBuildTask_buildFrontendNlbListeners(t *testing.T) { + tests := []struct { + name string + ingGroup Group + albListenerPorts []int32 + listenerPortConfigByIngress map[types.NamespacedName]map[int32]listenPortConfig + loadBalancer *elbv2model.LoadBalancer + wantErr bool + expectedErrMsg string + }{ + { + name: "valid listener configurations", + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{"alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping": "80=443"}, + }, + }, + }, + }, + }, + listenerPortConfigByIngress: map[types.NamespacedName]map[int32]listenPortConfig{ + {Namespace: "awesome-ns", Name: "ing-1"}: { + 443: listenPortConfig{}, + }, + }, + loadBalancer: &elbv2model.LoadBalancer{ + Spec: elbv2model.LoadBalancerSpec{ + IPAddressType: elbv2model.IPAddressTypeIPV4, + }, + }, + wantErr: false, + }, + { + name: "conflicting listener configurations", + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-2", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping": "80=443", + }, + }, + }}, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-3", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping": "80=8443", + }}, + }, + }, + }, + }, + listenerPortConfigByIngress: map[types.NamespacedName]map[int32]listenPortConfig{ + {Namespace: "awesome-ns", Name: "ing-2"}: { + 443: listenPortConfig{}, + }, + {Namespace: "awesome-ns", Name: "ing-3"}: { + 8443: listenPortConfig{}, + }, + }, + wantErr: true, + expectedErrMsg: "failed to merge NLB listenPort config for port: 80", + }, + { + name: "valid listener configurations for multiple ingress", + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-4", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping": "80=443", + }, + }, + }}, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-5", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping": "8080=80", + }}, + }, + }, + }, + }, + listenerPortConfigByIngress: map[types.NamespacedName]map[int32]listenPortConfig{ + {Namespace: "awesome-ns", Name: "ing-4"}: { + 443: listenPortConfig{}, + }, + {Namespace: "awesome-ns", Name: "ing-5"}: { + 80: listenPortConfig{}, + }, + }, + loadBalancer: &elbv2model.LoadBalancer{ + Spec: elbv2model.LoadBalancerSpec{ + IPAddressType: elbv2model.IPAddressTypeIPV4, + }, + }, + wantErr: false, + }, + { + name: "valid listener configurations for multiple ingress, 8443 is ingnored", + ingGroup: Group{ + ID: GroupID{ + Namespace: "awesome-ns", + Name: "my-ingress", + }, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-6", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping": "80=443", + }, + }, + }}, + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-7", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/frontend-nlb-listener-port-mapping": "80=8443", + }}, + }, + }, + }, + }, + listenerPortConfigByIngress: map[types.NamespacedName]map[int32]listenPortConfig{ + {Namespace: "awesome-ns", Name: "ing-6"}: { + 443: listenPortConfig{}, + }, + {Namespace: "awesome-ns", Name: "ing-7"}: {}, + }, + loadBalancer: &elbv2model.LoadBalancer{ + Spec: elbv2model.LoadBalancerSpec{ + IPAddressType: elbv2model.IPAddressTypeIPV4, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + stack := core.NewDefaultStack(core.StackID{Name: "awesome-stack"}) + desiredState := &core.FrontendNlbTargetGroupDesiredState{ + TargetGroups: make(map[string]*core.FrontendNlbTargetGroupState), + } + mockLoadBalancer := elbv2model.NewLoadBalancer(stack, "FrontendNlb", elbv2model.LoadBalancerSpec{ + IPAddressType: elbv2model.IPAddressTypeIPV4, + }) + + task := &defaultModelBuildTask{ + ingGroup: tt.ingGroup, + annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), + loadBalancer: tt.loadBalancer, + frontendNlb: mockLoadBalancer, + stack: stack, + frontendNlbTargetGroupDesiredState: desiredState, + } + + err := task.buildFrontendNlbListeners(context.Background(), tt.listenerPortConfigByIngress) + + if tt.wantErr { + assert.Error(t, err) + if tt.expectedErrMsg != "" { + assert.Contains(t, err.Error(), tt.expectedErrMsg) + } + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index 1e341f24c1..7273c3884a 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -37,7 +37,7 @@ const ( // ModelBuilder is responsible for build mode stack for a IngressGroup. type ModelBuilder interface { // build mode stack for a IngressGroup. - Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, error) + Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *core.FrontendNlbTargetGroupDesiredState, *elbv2model.LoadBalancer, error) } // NewDefaultModelBuilder constructs new defaultModelBuilder. @@ -121,8 +121,10 @@ type defaultModelBuilder struct { } // build mode stack for a IngressGroup. -func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, error) { +func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *core.FrontendNlbTargetGroupDesiredState, *elbv2model.LoadBalancer, error) { stack := core.NewDefaultStack(core.StackID(ingGroup.ID)) + frontendNlbTargetGroupDesiredState := core.NewFrontendNlbTargetGroupDesiredState() + task := &defaultModelBuildTask{ k8sClient: b.k8sClient, eventRecorder: b.eventRecorder, @@ -148,8 +150,9 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metrics enableIPTargetType: b.enableIPTargetType, metricsCollector: b.metricsCollector, - ingGroup: ingGroup, - stack: stack, + ingGroup: ingGroup, + stack: stack, + frontendNlbTargetGroupDesiredState: frontendNlbTargetGroupDesiredState, defaultTags: b.defaultTags, externalManagedTags: b.externalManagedTags, @@ -169,13 +172,14 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metrics defaultHealthCheckMatcherGRPCCode: "12", loadBalancer: nil, + frontendNlb: nil, tgByResID: make(map[string]*elbv2model.TargetGroup), backendServices: make(map[types.NamespacedName]*corev1.Service), } if err := task.run(ctx); err != nil { - return nil, nil, nil, false, err + return nil, nil, nil, false, nil, nil, err } - return task.stack, task.loadBalancer, task.secretKeys, task.backendSGAllocated, nil + return task.stack, task.loadBalancer, task.secretKeys, task.backendSGAllocated, frontendNlbTargetGroupDesiredState, task.frontendNlb, nil } // the default model build task @@ -226,10 +230,12 @@ type defaultModelBuildTask struct { defaultHealthCheckMatcherHTTPCode string defaultHealthCheckMatcherGRPCCode string - loadBalancer *elbv2model.LoadBalancer - tgByResID map[string]*elbv2model.TargetGroup - backendServices map[types.NamespacedName]*corev1.Service - secretKeys []types.NamespacedName + loadBalancer *elbv2model.LoadBalancer + tgByResID map[string]*elbv2model.TargetGroup + backendServices map[types.NamespacedName]*corev1.Service + secretKeys []types.NamespacedName + frontendNlb *elbv2model.LoadBalancer + frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState metricsCollector lbcmetrics.MetricCollector } @@ -250,6 +256,7 @@ func (t *defaultModelBuildTask) run(ctx context.Context) error { return nil } + listenerPortConfigByIngress := make(map[types.NamespacedName]map[int32]listenPortConfig) ingListByPort := make(map[int32][]ClassifiedIngress) listenPortConfigsByPort := make(map[int32][]listenPortConfigWithIngress) for _, member := range t.ingGroup.Members { @@ -258,6 +265,9 @@ func (t *defaultModelBuildTask) run(ctx context.Context) error { if err != nil { return errors.Wrapf(err, "ingress: %v", ingKey.String()) } + + listenerPortConfigByIngress[ingKey] = listenPortConfigByPortForIngress + for port, cfg := range listenPortConfigByPortForIngress { ingListByPort[port] = append(ingListByPort[port], member) listenPortConfigsByPort[port] = append(listenPortConfigsByPort[port], listenPortConfigWithIngress{ @@ -299,6 +309,11 @@ func (t *defaultModelBuildTask) run(ctx context.Context) error { if err := t.buildLoadBalancerAddOns(ctx, lb.LoadBalancerARN()); err != nil { return errmetrics.NewErrorWithMetrics(controllerName, "build_load_balancer_addons", err, t.metricsCollector) } + + if err := t.buildFrontendNlbModel(ctx, lb, listenerPortConfigByIngress); err != nil { + return errmetrics.NewErrorWithMetrics(controllerName, "build_frontend_nlb", err, t.metricsCollector) + } + return nil } diff --git a/pkg/ingress/model_builder_test.go b/pkg/ingress/model_builder_test.go index 0c27e4743c..dfcd3a683d 100644 --- a/pkg/ingress/model_builder_test.go +++ b/pkg/ingress/model_builder_test.go @@ -3955,7 +3955,7 @@ func Test_defaultModelBuilder_Build(t *testing.T) { b.enableIPTargetType = *tt.enableIPTargetType } - gotStack, _, _, _, err := b.Build(context.Background(), tt.args.ingGroup, b.metricsCollector) + gotStack, _, _, _, _, _, err := b.Build(context.Background(), tt.args.ingGroup, b.metricsCollector) if tt.wantErr != "" { assert.EqualError(t, err, tt.wantErr) } else { diff --git a/pkg/model/core/frontend_nlb_target_group.go b/pkg/model/core/frontend_nlb_target_group.go new file mode 100644 index 0000000000..f83ef844b0 --- /dev/null +++ b/pkg/model/core/frontend_nlb_target_group.go @@ -0,0 +1,31 @@ +package core + +// FrontendNlbTargetGroupState represents the state of a single ALB Target Type target group with its ALB target +type FrontendNlbTargetGroupState struct { + Name string + ARN StringToken + Port int32 + TargetARN StringToken + TargetPort int32 +} + +// FrontendNlbTargetGroupDesiredState maintains a mapping of target groups targeting ALB +type FrontendNlbTargetGroupDesiredState struct { + TargetGroups map[string]*FrontendNlbTargetGroupState +} + +func NewFrontendNlbTargetGroupDesiredState() *FrontendNlbTargetGroupDesiredState { + return &FrontendNlbTargetGroupDesiredState{ + TargetGroups: make(map[string]*FrontendNlbTargetGroupState), + } +} + +func (m *FrontendNlbTargetGroupDesiredState) AddTargetGroup(targetGroupName string, targetGroupARN StringToken, targetARN StringToken, port int32, targetPort int32) { + m.TargetGroups[targetGroupName] = &FrontendNlbTargetGroupState{ + Name: targetGroupName, + ARN: targetGroupARN, + Port: port, + TargetARN: targetARN, + TargetPort: targetPort, + } +} diff --git a/pkg/model/elbv2/target_group.go b/pkg/model/elbv2/target_group.go index ffd6a3e12a..2d57fbd35e 100644 --- a/pkg/model/elbv2/target_group.go +++ b/pkg/model/elbv2/target_group.go @@ -2,6 +2,7 @@ package elbv2 import ( "context" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" @@ -55,6 +56,7 @@ type TargetType string const ( TargetTypeInstance TargetType = "instance" TargetTypeIP TargetType = "ip" + TargetTypeALB TargetType = "alb" ) type TargetGroupIPAddressType string diff --git a/test/e2e/ingress/utils.go b/test/e2e/ingress/utils.go index c536b8c156..609143e708 100644 --- a/test/e2e/ingress/utils.go +++ b/test/e2e/ingress/utils.go @@ -1,6 +1,11 @@ package ingress -import networking "k8s.io/api/networking/v1" +import ( + "fmt" + "strings" + + networking "k8s.io/api/networking/v1" +) func FindIngressDNSName(ing *networking.Ingress) string { for _, ingSTS := range ing.Status.LoadBalancer.Ingress { @@ -10,3 +15,17 @@ func FindIngressDNSName(ing *networking.Ingress) string { } return "" } + +func FindIngressTwoDNSName(ing *networking.Ingress) (albDNS string, nlbDNS string) { + for _, ingSTS := range ing.Status.LoadBalancer.Ingress { + if ingSTS.Hostname != "" { + fmt.Println("ingSTS.Hostname", ingSTS.Hostname) + if strings.Contains(ingSTS.Hostname, "elb.amazonaws.com") { + albDNS = ingSTS.Hostname + } else { + nlbDNS = ingSTS.Hostname + } + } + } + return albDNS, nlbDNS +} diff --git a/test/e2e/ingress/vanilla_ingress_test.go b/test/e2e/ingress/vanilla_ingress_test.go index a79a3e673d..327242bca5 100644 --- a/test/e2e/ingress/vanilla_ingress_test.go +++ b/test/e2e/ingress/vanilla_ingress_test.go @@ -828,6 +828,60 @@ var _ = Describe("vanilla ingress tests", func() { Body().Equal("Hello World!") }) }) + + Context("with frontend NLB enabled", func() { + It("should create a frontend NLB and route traffic correctly", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/enable-frontend-nlb": "true", + "alb.ingress.kubernetes.io/frontend-nlb-healthcheck-path": "/path", + "alb.ingress.kubernetes.io/frontend-nlb-scheme": "internet-facing", + } + + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) + + defer resStack.TearDown(ctx) + + time.Sleep(6 * time.Minute) // Waiting 6 minutes for target registration and DNS propagation, and health check + + albARN, albDNS, nlbARN, nlbDNS := ExpectTwoLBProvisionedForIngress(ctx, tf, ing) + + // test alb traffic + ExpectLBDNSBeAvailable(ctx, tf, albARN, albDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", albDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + + // test nlb traffic + ExpectLBDNSBeAvailable(ctx, tf, nlbARN, nlbDNS) + nlbHttpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", nlbDNS)) + nlbHttpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + + }) + + }) + }) // ExpectOneLBProvisionedForIngress expects one LoadBalancer provisioned for Ingress. @@ -847,6 +901,30 @@ func ExpectOneLBProvisionedForIngress(ctx context.Context, tf *framework.Framewo return lbARN, lbDNS } +// ExpectTwoLBProvisionedForIngress expects one ALB and one frontend NLB provisioned for the Ingress. +func ExpectTwoLBProvisionedForIngress(ctx context.Context, tf *framework.Framework, ing *networking.Ingress) (albARN string, albDNS string, nlbARN string, nlbDNS string) { + // Verify ALB is provisioned + Eventually(func(g Gomega) { + err := tf.K8sClient.Get(ctx, k8s.NamespacedName(ing), ing) + g.Expect(err).NotTo(HaveOccurred()) + albDNS, nlbDNS = FindIngressTwoDNSName(ing) + g.Expect(albDNS).ShouldNot(BeEmpty()) + g.Expect(nlbDNS).ShouldNot(BeEmpty()) + }, utils.IngressReconcileTimeout, utils.PollIntervalShort).Should(Succeed()) + tf.Logger.Info("ingress DNS populated", "dnsName", albDNS) + tf.Logger.Info("ingress DNS populated", "dnsName", nlbDNS) + + var err error + albARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, albDNS) + Expect(err).ShouldNot(HaveOccurred()) + tf.Logger.Info("ALB provisioned", "arn", albARN) + + nlbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, nlbDNS) + Expect(err).ShouldNot(HaveOccurred()) + tf.Logger.Info("NLB provisioned", "arn", nlbARN) + return albARN, albDNS, nlbARN, nlbDNS +} + // ExpectNoLBProvisionedForIngress expects no LoadBalancer provisioned for Ingress. func ExpectNoLBProvisionedForIngress(ctx context.Context, tf *framework.Framework, ing *networking.Ingress) { Consistently(func(g Gomega) { From d94d4c415261f333d994e74e071bfda3956cef60 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 10 Apr 2025 17:53:42 -0700 Subject: [PATCH 15/40] lb creation --- .../v1beta1/loadbalancerconfig_types.go | 28 +- apis/gateway/v1beta1/zz_generated.deepcopy.go | 37 +- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 298 ++++++++++++ ...way.k8s.aws_targetgroupconfigurations.yaml | 456 ++++++++++++++++++ controllers/gateway/gateway_controller.go | 2 +- pkg/gateway/model/base_model_builder.go | 86 +++- pkg/gateway/model/gateway_tag_helper.go | 56 +++ pkg/gateway/model/loadbalancer/mutator.go | 7 + pkg/gateway/model/model_build_loadbalancer.go | 138 ++++-- .../model/model_build_security_group.go | 313 ++++++++++++ pkg/gateway/model/model_build_subnet.go | 20 +- pkg/gateway/model/model_build_subnet_test.go | 6 +- 12 files changed, 1360 insertions(+), 87 deletions(-) create mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml create mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml create mode 100644 pkg/gateway/model/gateway_tag_helper.go create mode 100644 pkg/gateway/model/loadbalancer/mutator.go create mode 100644 pkg/gateway/model/model_build_security_group.go diff --git a/apis/gateway/v1beta1/loadbalancerconfig_types.go b/apis/gateway/v1beta1/loadbalancerconfig_types.go index 076561bc59..799e3e9208 100644 --- a/apis/gateway/v1beta1/loadbalancerconfig_types.go +++ b/apis/gateway/v1beta1/loadbalancerconfig_types.go @@ -44,8 +44,8 @@ type ListenerAttribute struct { Value string `json:"value"` } -// Tag defines a AWS Tag on resources. -type LoadBalancerTag struct { +// AWSTag defines a AWS Tag on resources. +type AWSTag struct { // The key of the tag. Key string `json:"key"` @@ -73,7 +73,7 @@ type SubnetConfiguration struct { // SourceNatIPv6Prefix [Network LoadBalancer] The IPv6 prefix to use for source NAT. Specify an IPv6 prefix (/80 netmask) from the subnet CIDR block or auto_assigned to use an IPv6 prefix selected at random from the subnet CIDR block. // +optional - SourceNatIPv6Prefix *string `json:"sourceNatIPv6Prefix,omitempty"` + SourceNatIPv6Prefix *string `json:"sourceNAT,omitempty"` } // +kubebuilder:validation:Enum=HTTP1Only;HTTP2Only;HTTP2Optional;HTTP2Preferred;None @@ -183,10 +183,16 @@ type LoadBalancerConfigurationSpec struct { // +optional EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic *string `json:"enforceSecurityGroupInboundRulesOnPrivateLinkTraffic,omitempty"` - // customerOwnedIpv4Pool is the ID of the customer-owned address for Application Load Balancers on Outposts pool. + // customerOwnedIpv4Pool [Application LoadBalancer] + // is the ID of the customer-owned address for Application Load Balancers on Outposts pool. // +optional CustomerOwnedIpv4Pool *string `json:"customerOwnedIpv4Pool,omitempty"` + // IPv4IPAMPoolId [Application LoadBalancer] + // defines the IPAM pool ID used for IPv4 Addresses on the ALB. + // +optional + IPv4IPAMPoolId *string `json:"ipv4IPAMPoolId,omitempty"` + // loadBalancerSubnets is an optional list of subnet configurations to be used in the LB // This value takes precedence over loadBalancerSubnetsSelector if both are selected. // +optional @@ -224,7 +230,19 @@ type LoadBalancerConfigurationSpec struct { // Tags defines list of Tags on LB. // +optional - Tags []LoadBalancerTag `json:"tags,omitempty"` + Tags []AWSTag `json:"tags,omitempty"` + + // EnableICMP [Network LoadBalancer] + // enables the creation of security group rules to the managed security group + // to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs + // +optional + EnableICMP bool `json:"enableICMP,omitempty"` + + // ManageBackendSecurityGroupRules [Application / Network LoadBalancer] + // specifies whether you want the controller to configure security group rules on Node/Pod for traffic access + // when you specify securityGroups + // +optional + ManageBackendSecurityGroupRules bool `json:"manageBackendSecurityGroupRules,omitempty"` } // TODO -- these can be used to set what generation the gateway is currently on to track progress on reconcile. diff --git a/apis/gateway/v1beta1/zz_generated.deepcopy.go b/apis/gateway/v1beta1/zz_generated.deepcopy.go index 41bb66a02f..d4c9a325ab 100644 --- a/apis/gateway/v1beta1/zz_generated.deepcopy.go +++ b/apis/gateway/v1beta1/zz_generated.deepcopy.go @@ -25,6 +25,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSTag) DeepCopyInto(out *AWSTag) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSTag. +func (in *AWSTag) DeepCopy() *AWSTag { + if in == nil { + return nil + } + out := new(AWSTag) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HealthCheckConfiguration) DeepCopyInto(out *HealthCheckConfiguration) { *out = *in @@ -273,6 +288,11 @@ func (in *LoadBalancerConfigurationSpec) DeepCopyInto(out *LoadBalancerConfigura *out = new(string) **out = **in } + if in.IPv4IPAMPoolId != nil { + in, out := &in.IPv4IPAMPoolId, &out.IPv4IPAMPoolId + *out = new(string) + **out = **in + } if in.LoadBalancerSubnets != nil { in, out := &in.LoadBalancerSubnets, &out.LoadBalancerSubnets *out = new([]SubnetConfiguration) @@ -354,7 +374,7 @@ func (in *LoadBalancerConfigurationSpec) DeepCopyInto(out *LoadBalancerConfigura } if in.Tags != nil { in, out := &in.Tags, &out.Tags - *out = make([]LoadBalancerTag, len(*in)) + *out = make([]AWSTag, len(*in)) copy(*out, *in) } } @@ -394,21 +414,6 @@ func (in *LoadBalancerConfigurationStatus) DeepCopy() *LoadBalancerConfiguration return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LoadBalancerTag) DeepCopyInto(out *LoadBalancerTag) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerTag. -func (in *LoadBalancerTag) DeepCopy() *LoadBalancerTag { - if in == nil { - return nil - } - out := new(LoadBalancerTag) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MutualAuthenticationAttributes) DeepCopyInto(out *MutualAuthenticationAttributes) { *out = *in diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml new file mode 100644 index 0000000000..98ed0757eb --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -0,0 +1,298 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: loadbalancerconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: LoadBalancerConfiguration + listKind: LoadBalancerConfigurationList + plural: loadbalancerconfigurations + singular: loadbalancerconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LoadBalancerConfigurationSpec defines the desired state of + LoadBalancerConfiguration + properties: + customerOwnedIpv4Pool: + description: |- + customerOwnedIpv4Pool [Application LoadBalancer] + is the ID of the customer-owned address for Application Load Balancers on Outposts pool. + type: string + enableICMP: + description: |- + EnableICMP [Network LoadBalancer] + enables the creation of security group rules to the managed security group + to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs + type: boolean + enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: + description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic + Indicates whether to evaluate inbound security group rules for traffic + sent to a Network Load Balancer through Amazon Web Services PrivateLink. + type: string + ipAddressType: + description: loadBalancerIPType defines what kind of load balancer + to provision (ipv4, dual stack) + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string + ipv4IPAMPoolId: + description: |- + IPv4IPAMPoolId [Application LoadBalancer] + defines the IPAM pool ID used for IPv4 Addresses on the ALB. + type: string + listenerConfigurations: + description: listenerConfigurations is an optional list of configurations + for each listener on LB + items: + properties: + alpnPolicy: + description: alpnPolicy an optional string that allows you to + configure ALPN policies on your Load Balancer + enum: + - HTTP1Only + - HTTP2Only + - HTTP2Optional + - HTTP2Preferred + - None + type: string + certificates: + description: certificates is the list of other certificates + to add to the listener. + items: + type: string + type: array + defaultCertificate: + description: |- + TODO: Add validation in admission webhook to make it required for secure protocols + defaultCertificate the cert arn to be used by default. + type: string + listenerAttributes: + description: listenerAttributes defines the attributes for the + listener + items: + description: ListenerAttribute defines listener attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + mutualAuthentication: + description: mutualAuthentication defines the mutual authentication + configuration information. + properties: + advertiseTrustStoreCaNames: + description: Indicates whether trust store CA certificate + names are advertised. + enum: + - "on" + - "off" + type: string + ignoreClientCertificateExpiry: + description: Indicates whether expired client certificates + are ignored. + type: boolean + mode: + description: The client certificate handling method. Options + are off , passthrough or verify + enum: + - "off" + - passthrough + - verify + type: string + trustStore: + description: The Name or ARN of the trust store. + type: string + required: + - mode + type: object + protocolPort: + description: protocolPort is identifier for the listener on + load balancer. It should be of the form PROTOCOL:PORT + pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ + type: string + sslPolicy: + description: sslPolicy is the security policy that defines which + protocols and ciphers are supported for secure listeners [HTTPS + or TLS listener]. + type: string + required: + - protocolPort + type: object + type: array + loadBalancerAttributes: + description: LoadBalancerAttributes defines the attribute of LB + items: + description: LoadBalancerAttribute defines LB attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + loadBalancerName: + description: loadBalancerName defines the name of the LB to provision. + If unspecified, it will be automatically generated. + maxLength: 32 + minLength: 1 + type: string + loadBalancerSubnets: + description: |- + loadBalancerSubnets is an optional list of subnet configurations to be used in the LB + This value takes precedence over loadBalancerSubnetsSelector if both are selected. + items: + description: SubnetConfiguration defines the subnet settings for + a Load Balancer. + properties: + eipAllocation: + description: eipAllocation [Network LoadBalancer] the EIP name + for this subnet. + type: string + identifier: + description: identifier [Application LoadBalancer / Network + LoadBalancer] name or id for the subnet + type: string + ipv6Allocation: + description: IPv6Allocation [Network LoadBalancer] the ipv6 + address to assign to this subnet. + type: string + privateIPv4Allocation: + description: privateIPv4Allocation [Network LoadBalancer] the + private ipv4 address to assign to this subnet. + type: string + sourceNAT: + description: SourceNatIPv6Prefix [Network LoadBalancer] The + IPv6 prefix to use for source NAT. Specify an IPv6 prefix + (/80 netmask) from the subnet CIDR block or auto_assigned + to use an IPv6 prefix selected at random from the subnet CIDR + block. + type: string + type: object + type: array + loadBalancerSubnetsSelector: + additionalProperties: + items: + type: string + type: array + description: |- + LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each + tag specified in the map key contains one of the values in the corresponding + value list. + type: object + manageBackendSecurityGroupRules: + description: |- + ManageBackendSecurityGroupRules [Application / Network LoadBalancer] + specifies whether you want the controller to configure security group rules on Node/Pod for traffic access + when you specify securityGroups + type: boolean + scheme: + description: scheme defines the type of LB to provision. If unspecified, + it will be automatically inferred. + enum: + - internal + - internet-facing + type: string + securityGroupPrefixes: + description: securityGroupPrefixes an optional list of prefixes that + are allowed to access the LB. + items: + type: string + type: array + securityGroups: + description: securityGroups an optional list of security group ids + or names to apply to the LB + items: + type: string + type: array + sourceRanges: + description: sourceRanges an optional list of CIDRs that are allowed + to access the LB. + items: + type: string + type: array + tags: + description: Tags defines list of Tags on LB. + items: + description: AWSTag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + vpcId: + description: vpcId is the ID of the VPC for the load balancer. + type: string + type: object + status: + description: LoadBalancerConfigurationStatus defines the observed state + of TargetGroupBinding + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml new file mode 100644 index 0000000000..c34856f29b --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -0,0 +1,456 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: targetgroupconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: TargetGroupConfiguration + listKind: TargetGroupConfigurationList + plural: targetgroupconfigurations + singular: targetgroupconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The Kubernetes Service's name + jsonPath: .spec.targetReference.name + name: SERVICE-NAME + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TargetGroupConfiguration is the Schema for defining TargetGroups + with an AWS ELB Gateway + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TargetGroupConfigurationSpec defines the TargetGroup properties + for a route. + properties: + defaultConfiguration: + description: defaultRouteConfiguration fallback configuration applied + to all routes, unless overridden by route-specific configurations. + properties: + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount of + time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health checks + on the targets. + type: string + healthCheckPort: + description: healthCheckPort The port to use to connect with + the target. + format: int32 + type: integer + healthCheckProtocol: + description: healthCheckProtocol The protocol to use to connect + with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols + are not supported for health checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, in seconds, + during which no response means a failed health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering an unhealthy + target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes to use + when checking for a successful response from a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering the target + unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target group + is of type IPv4 or IPv6. If unspecified, it will be automatically + inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups to + only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute of target + group + items: + description: TargetGroupAttribute defines target group attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetType: + description: targetType is the TargetType of TargetGroup. If unspecified, + it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + routeConfigurations: + description: routeConfigurations the route configuration for specific + routes + items: + description: RouteConfiguration defines the per route configuration + properties: + name: + description: name the name of the route, it should be in the + form of ROUTE:NAMESPACE:NAME + pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ + type: string + targetGroupProps: + description: targetGroupProps the target group specific properties + properties: + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount + of time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health + checks on the targets. + type: string + healthCheckPort: + description: healthCheckPort The port to use to connect + with the target. + format: int32 + type: integer + healthCheckProtocol: + description: healthCheckProtocol The protocol to use + to connect with the target. The GENEVE, TLS, UDP, + and TCP_UDP protocols are not supported for health + checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, + in seconds, during which no response means a failed + health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering + an unhealthy target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes + to use when checking for a successful response from + a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering + the target unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target + group is of type IPv4 or IPv6. If unspecified, it will + be automatically inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups + to only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute + of target group + items: + description: TargetGroupAttribute defines target group + attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetType: + description: targetType is the TargetType of TargetGroup. + If unspecified, it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + required: + - name + - targetGroupProps + type: object + type: array + targetReference: + description: targetReference the kubernetes object to attach the Target + Group settings to. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + + Defaults to "Service" when not specified. + type: string + name: + description: Name is the name of the referent. + type: string + required: + - name + type: object + required: + - targetReference + type: object + status: + description: TargetGroupConfigurationStatus defines the observed state + of TargetGroupConfiguration + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 76b80a0db2..e8da41531a 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -59,7 +59,7 @@ func NewALBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud func newGatewayReconciler(controllerName string, lbType elbv2model.LoadBalancerType, maxConcurrentReconciles int, gatewayTagPrefix string, finalizer string, routeLoader routeutils.Loader, routeFilter routeutils.LoadRouteFilter, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileTracker func(namespaceName types.NamespacedName)) Reconciler { trackingProvider := tracking.NewDefaultProvider(gatewayTagPrefix, controllerConfig.ClusterName) - modelBuilder := gatewaymodel.NewModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), lbType, trackingProvider, elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger) + modelBuilder := gatewaymodel.NewModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), lbType, trackingProvider, elbv2TaggingManager, controllerConfig, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName) diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go index 83f22b266e..513de2b489 100644 --- a/pkg/gateway/model/base_model_builder.go +++ b/pkg/gateway/model/base_model_builder.go @@ -28,15 +28,23 @@ type Builder interface { // NewModelBuilder construct a new baseModelBuilder func NewModelBuilder(subnetsResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, vpcID string, loadBalancerType elbv2model.LoadBalancerType, trackingProvider tracking.Provider, - elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string, + elbv2TaggingManager elbv2deploy.TaggingManager, lbcConfig config.ControllerConfig, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string, externalManagedTags sets.Set[string], defaultSSLPolicy string, defaultTargetType string, defaultLoadBalancerScheme string, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool, disableRestrictedSGRules bool, logger logr.Logger) Builder { + gwTagHelper := newTagHelper(sets.New(lbcConfig.ExternalManagedTags...), lbcConfig.DefaultTags) subnetBuilder := newSubnetModelBuilder(loadBalancerType, trackingProvider, subnetsResolver, elbv2TaggingManager) + sgBuilder := newSecurityGroupBuilder(gwTagHelper, clusterName, enableBackendSG, sgResolver, backendSGProvider, logger) return &baseModelBuilder{ - lbBuilder: newLoadBalancerBuilder(subnetBuilder, defaultLoadBalancerScheme), + subnetBuilder: subnetBuilder, + securityGroupBuilder: sgBuilder, + lbBuilder: newLoadBalancerBuilder(loadBalancerType, gwTagHelper, clusterName), + logger: logger, + + defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), + defaultIPType: elbv2model.IPAddressTypeIPV4, } } @@ -45,6 +53,12 @@ var _ Builder = &baseModelBuilder{} type baseModelBuilder struct { lbBuilder loadBalancerBuilder logger logr.Logger + + subnetBuilder subnetModelBuilder + securityGroupBuilder securityGroupBuilder + + defaultLoadBalancerScheme elbv2model.LoadBalancerScheme + defaultIPType elbv2model.IPAddressType } func (baseBuilder *baseModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) { @@ -56,8 +70,39 @@ func (baseBuilder *baseModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(gw))) + /* Basic LB stuff (Scheme, IP Address Type) */ + scheme, err := baseBuilder.buildLoadBalancerScheme(lbConf) + + if err != nil { + return nil, nil, false, err + } + + ipAddressType, err := baseBuilder.buildLoadBalancerIPAddressType(lbConf) + + if err != nil { + return nil, nil, false, err + } + + /* Subnets */ + + subnets, err := baseBuilder.subnetBuilder.buildLoadBalancerSubnets(ctx, lbConf.Spec.LoadBalancerSubnets, lbConf.Spec.LoadBalancerSubnetsSelector, scheme, ipAddressType, stack) + + if err != nil { + return nil, nil, false, err + } + + /* Security Groups */ + + securityGroups, err := baseBuilder.securityGroupBuilder.buildSecurityGroups(ctx, stack, lbConf, gw, routes, ipAddressType) + + if err != nil { + return nil, nil, false, err + } + + /* Combine everything to form a LoadBalancer */ + // TODO - Fix - _, err := baseBuilder.lbBuilder.buildLoadBalancerSpec(ctx, gw, stack, lbConf, routes) + _, err = baseBuilder.lbBuilder.buildLoadBalancerSpec(ctx, scheme, ipAddressType, gw, lbConf, subnets, securityGroups.securityGroupTokens) if err != nil { return nil, nil, false, err @@ -86,3 +131,38 @@ func (baseBuilder *baseModelBuilder) isDeleteProtected(lbConf *elbv2gw.LoadBalan return false } + +func (baseBuilder *baseModelBuilder) buildLoadBalancerScheme(lbConf *elbv2gw.LoadBalancerConfiguration) (elbv2model.LoadBalancerScheme, error) { + scheme := lbConf.Spec.Scheme + + if scheme == nil { + return baseBuilder.defaultLoadBalancerScheme, nil + } + switch *scheme { + case elbv2gw.LoadBalancerScheme(elbv2model.LoadBalancerSchemeInternetFacing): + return elbv2model.LoadBalancerSchemeInternetFacing, nil + case elbv2gw.LoadBalancerScheme(elbv2model.LoadBalancerSchemeInternal): + return elbv2model.LoadBalancerSchemeInternal, nil + default: + return "", errors.Errorf("unknown scheme: %v", *scheme) + } +} + +// buildLoadBalancerIPAddressType builds the LoadBalancer IPAddressType. +func (baseBuilder *baseModelBuilder) buildLoadBalancerIPAddressType(lbConf *elbv2gw.LoadBalancerConfiguration) (elbv2model.IPAddressType, error) { + + if lbConf.Spec.IpAddressType == nil { + return baseBuilder.defaultIPType, nil + } + + switch *lbConf.Spec.IpAddressType { + case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeIPV4): + return elbv2model.IPAddressTypeIPV4, nil + case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeDualStack): + return elbv2model.IPAddressTypeDualStack, nil + case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeDualStackWithoutPublicIPV4): + return elbv2model.IPAddressTypeDualStackWithoutPublicIPV4, nil + default: + return "", errors.Errorf("unknown IPAddressType: %v", *lbConf.Spec.IpAddressType) + } +} diff --git a/pkg/gateway/model/gateway_tag_helper.go b/pkg/gateway/model/gateway_tag_helper.go new file mode 100644 index 0000000000..892abdd899 --- /dev/null +++ b/pkg/gateway/model/gateway_tag_helper.go @@ -0,0 +1,56 @@ +package model + +import ( + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/sets" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm" +) + +type tagHelper interface { + getGatewayTags(lbConf *elbv2gw.LoadBalancerConfiguration) (map[string]string, error) +} + +type tagHelperImpl struct { + externalManagedTags sets.Set[string] + defaultTags map[string]string +} + +func newTagHelper(externalManagedTags sets.Set[string], defaultTags map[string]string) tagHelper { + return &tagHelperImpl{ + externalManagedTags: externalManagedTags, + defaultTags: defaultTags, + } +} + +func (t *tagHelperImpl) getGatewayTags(lbConf *elbv2gw.LoadBalancerConfiguration) (map[string]string, error) { + var annotationTags map[string]string + + if lbConf != nil { + annotationTags = t.convertTagsToMap(lbConf.Spec.Tags) + } + + if err := t.validateTagCollisionWithExternalManagedTags(annotationTags); err != nil { + return nil, err + } + + return algorithm.MergeStringMap(t.defaultTags, annotationTags), nil +} + +func (t *tagHelperImpl) validateTagCollisionWithExternalManagedTags(tags map[string]string) error { + for tagKey := range tags { + if t.externalManagedTags.Has(tagKey) { + return errors.Errorf("external managed tag key %v cannot be specified", tagKey) + } + } + return nil +} + +func (t *tagHelperImpl) convertTagsToMap(tags []elbv2gw.AWSTag) map[string]string { + m := make(map[string]string) + + for _, tag := range tags { + m[tag.Key] = tag.Value + } + return m +} diff --git a/pkg/gateway/model/loadbalancer/mutator.go b/pkg/gateway/model/loadbalancer/mutator.go new file mode 100644 index 0000000000..373fd6949a --- /dev/null +++ b/pkg/gateway/model/loadbalancer/mutator.go @@ -0,0 +1,7 @@ +package loadbalancer + +import elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + +type Mutator interface { + Mutate(spec *elbv2model.LoadBalancerSpec) error +} diff --git a/pkg/gateway/model/model_build_loadbalancer.go b/pkg/gateway/model/model_build_loadbalancer.go index 98945e4d8d..a86e643540 100644 --- a/pkg/gateway/model/model_build_loadbalancer.go +++ b/pkg/gateway/model/model_build_loadbalancer.go @@ -2,55 +2,87 @@ package model import ( "context" + "crypto/sha256" + "encoding/hex" + "fmt" "github.com/pkg/errors" + "regexp" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" - "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) +var invalidLoadBalancerNamePattern = regexp.MustCompile("[[:^alnum:]]") + +const ( + lbAttrsAccessLogsS3Enabled = "access_logs.s3.enabled" + lbAttrsAccessLogsS3Bucket = "access_logs.s3.bucket" + lbAttrsAccessLogsS3Prefix = "access_logs.s3.prefix" + lbAttrsLoadBalancingCrossZoneEnabled = "load_balancing.cross_zone.enabled" + lbAttrsLoadBalancingDnsClientRoutingPolicy = "dns_record.client_routing_policy" + availabilityZoneAffinity = "availability_zone_affinity" + partialAvailabilityZoneAffinity = "partial_availability_zone_affinity" + anyAvailabilityZone = "any_availability_zone" + resourceIDLoadBalancer = "LoadBalancer" +) + type loadBalancerBuilder interface { - buildLoadBalancerSpec(ctx context.Context, gw *gwv1.Gateway, stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor) (elbv2model.LoadBalancerSpec, error) + buildLoadBalancerSpec(ctx context.Context, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, subnets buildLoadBalancerSubnetsOutput, securityGroupTokens []core.StringToken) (elbv2model.LoadBalancerSpec, error) } type loadBalancerBuilderImpl struct { - subnetBuilder subnetModelBuilder - - defaultLoadBalancerScheme elbv2model.LoadBalancerScheme - defaultIPType elbv2model.IPAddressType + loadBalancerType elbv2model.LoadBalancerType + clusterName string + tagHelper tagHelper } -func newLoadBalancerBuilder(subnetBuilder subnetModelBuilder, defaultLoadBalancerScheme string) loadBalancerBuilder { - +func newLoadBalancerBuilder(loadBalancerType elbv2model.LoadBalancerType, tagHelper tagHelper, clusterName string) loadBalancerBuilder { return &loadBalancerBuilderImpl{ - subnetBuilder: subnetBuilder, - defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), - defaultIPType: elbv2model.IPAddressTypeIPV4, + loadBalancerType: loadBalancerType, + clusterName: clusterName, + tagHelper: tagHelper, } } -func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerSpec(ctx context.Context, gw *gwv1.Gateway, stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor) (elbv2model.LoadBalancerSpec, error) { - scheme, err := lbModelBuilder.buildLoadBalancerScheme(lbConf) +func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerSpec(ctx context.Context, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, subnets buildLoadBalancerSubnetsOutput, securityGroupTokens []core.StringToken) (elbv2model.LoadBalancerSpec, error) { + + name, err := lbModelBuilder.buildLoadBalancerName(lbConf, gw, scheme) if err != nil { return elbv2model.LoadBalancerSpec{}, err } - ipAddressType, err := lbModelBuilder.buildLoadBalancerIPAddressType(lbConf) + + tags, err := lbModelBuilder.tagHelper.getGatewayTags(lbConf) if err != nil { return elbv2model.LoadBalancerSpec{}, err } - configuredSubnets, sourcePrefixEnabled, err := lbModelBuilder.subnetBuilder.buildLoadBalancerSubnets(ctx, lbConf.Spec.LoadBalancerSubnets, lbConf.Spec.LoadBalancerSubnetsSelector, scheme, ipAddressType, stack) - if err != nil { - return elbv2model.LoadBalancerSpec{}, err + + spec := elbv2model.LoadBalancerSpec{ + Name: name, + Type: lbModelBuilder.loadBalancerType, + Scheme: scheme, + IPAddressType: ipAddressType, + SubnetMappings: subnets.subnets, + SecurityGroups: securityGroupTokens, + LoadBalancerAttributes: lbModelBuilder.buildLoadBalancerAttributes(lbConf), + MinimumLoadBalancerCapacity: lbModelBuilder.buildLoadBalancerMinimumCapacity(lbConf), + Tags: tags, + } + + if lbModelBuilder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { + spec.EnablePrefixForIpv6SourceNat = lbModelBuilder.translateSourcePrefixEnabled(subnets.sourceIPv6NatEnabled) + + if lbConf.Spec.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic != nil { + spec.SecurityGroupsInboundRulesOnPrivateLink = (*elbv2model.SecurityGroupsInboundRulesOnPrivateLinkStatus)(lbConf.Spec.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic) + } + } + + if lbModelBuilder.loadBalancerType == elbv2model.LoadBalancerTypeApplication { + spec.CustomerOwnedIPv4Pool = lbConf.Spec.CustomerOwnedIpv4Pool + spec.IPv4IPAMPool = lbConf.Spec.IPv4IPAMPoolId } - return elbv2model.LoadBalancerSpec{ - Type: elbv2model.LoadBalancerTypeApplication, - Scheme: scheme, - IPAddressType: ipAddressType, - SubnetMappings: configuredSubnets, - EnablePrefixForIpv6SourceNat: lbModelBuilder.translateSourcePrefixEnabled(sourcePrefixEnabled), - }, nil + return spec, nil } func (lbModelBuilder *loadBalancerBuilderImpl) translateSourcePrefixEnabled(b bool) elbv2model.EnablePrefixForIpv6SourceNat { @@ -58,40 +90,40 @@ func (lbModelBuilder *loadBalancerBuilderImpl) translateSourcePrefixEnabled(b bo return elbv2model.EnablePrefixForIpv6SourceNatOn } return elbv2model.EnablePrefixForIpv6SourceNatOff - } -func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerScheme(lbConf *elbv2gw.LoadBalancerConfiguration) (elbv2model.LoadBalancerScheme, error) { - scheme := lbConf.Spec.Scheme - - if scheme == nil { - return lbModelBuilder.defaultLoadBalancerScheme, nil +func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerName(lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, scheme elbv2model.LoadBalancerScheme) (string, error) { + if lbConf.Spec.LoadBalancerName != nil { + name := *lbConf.Spec.LoadBalancerName + // The name of the loadbalancer can only have up to 32 characters + if len(name) > 32 { + return "", errors.New("load balancer name cannot be longer than 32 characters") + } + return name, nil } - switch *scheme { - case elbv2gw.LoadBalancerScheme(elbv2model.LoadBalancerSchemeInternetFacing): - return elbv2model.LoadBalancerSchemeInternetFacing, nil - case elbv2gw.LoadBalancerScheme(elbv2model.LoadBalancerSchemeInternal): - return elbv2model.LoadBalancerSchemeInternal, nil - default: - return "", errors.Errorf("unknown scheme: %v", *scheme) - } -} + uuidHash := sha256.New() + _, _ = uuidHash.Write([]byte(lbModelBuilder.clusterName)) + _, _ = uuidHash.Write([]byte(gw.UID)) + _, _ = uuidHash.Write([]byte(scheme)) + uuid := hex.EncodeToString(uuidHash.Sum(nil)) -// buildLoadBalancerIPAddressType builds the LoadBalancer IPAddressType. -func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerIPAddressType(lbConf *elbv2gw.LoadBalancerConfiguration) (elbv2model.IPAddressType, error) { + sanitizedNamespace := invalidLoadBalancerNamePattern.ReplaceAllString(gw.Namespace, "") + sanitizedName := invalidLoadBalancerNamePattern.ReplaceAllString(gw.Name, "") + return fmt.Sprintf("k8s-%.8s-%.8s-%.10s", sanitizedNamespace, sanitizedName, uuid), nil +} - if lbConf.Spec.IpAddressType == nil { - return lbModelBuilder.defaultIPType, nil +func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerAttributes(lbConf *elbv2gw.LoadBalancerConfiguration) []elbv2model.LoadBalancerAttribute { + var attributes []elbv2model.LoadBalancerAttribute + for _, attr := range lbConf.Spec.LoadBalancerAttributes { + attributes = append(attributes, elbv2model.LoadBalancerAttribute{ + Key: attr.Key, + Value: attr.Value, + }) } + return attributes +} - switch *lbConf.Spec.IpAddressType { - case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeIPV4): - return elbv2model.IPAddressTypeIPV4, nil - case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeDualStack): - return elbv2model.IPAddressTypeDualStack, nil - case elbv2gw.LoadBalancerIpAddressType(elbv2model.IPAddressTypeDualStackWithoutPublicIPV4): - return elbv2model.IPAddressTypeDualStackWithoutPublicIPV4, nil - default: - return "", errors.Errorf("unknown IPAddressType: %v", *lbConf.Spec.IpAddressType) - } +// TODO -- Fill this in at a later time. +func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerMinimumCapacity(lbConf *elbv2gw.LoadBalancerConfiguration) *elbv2model.MinimumLoadBalancerCapacity { + return nil } diff --git a/pkg/gateway/model/model_build_security_group.go b/pkg/gateway/model/model_build_security_group.go new file mode 100644 index 0000000000..ed6090a530 --- /dev/null +++ b/pkg/gateway/model/model_build_security_group.go @@ -0,0 +1,313 @@ +package model + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/go-logr/logr" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "regexp" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + ec2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/ec2" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strings" +) + +var ( + invalidSecurityGroupNamePtn = regexp.MustCompile("[[:^alnum:]]") +) + +const ( + icmpv4Protocol = "icmp" + icmpv6Protocol = "icmpv6" + + icmpv4TypeForPathMtu = 3 // https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml#icmp-parameters-codes-3 + icmpv4CodeForPathMtu = 4 + + icmpv6TypeForPathMtu = 2 // https://www.iana.org/assignments/icmpv6-parameters/icmpv6-parameters.xhtml#icmpv6-parameters-codes-2 + icmpv6CodeForPathMtu = 0 + + resourceIDManagedSecurityGroup = "ManagedLBSecurityGroup" +) + +type securityGroupOutput struct { + securityGroupTokens []core.StringToken + backendSecurityGroupToken core.StringToken + backendSecurityGroupAllocated bool +} + +type securityGroupBuilder interface { + buildSecurityGroups(ctx context.Context, stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, routes map[int][]routeutils.RouteDescriptor, ipAddressType elbv2model.IPAddressType) (securityGroupOutput, error) +} + +type securityGroupBuilderImpl struct { + tagHelper tagHelper + clusterName string + + sgResolver networking.SecurityGroupResolver + backendSGProvider networking.BackendSGProvider + + enableBackendSG bool + logger logr.Logger +} + +func newSecurityGroupBuilder(tagHelper tagHelper, clusterName string, enableBackendSG bool, sgResolver networking.SecurityGroupResolver, backendSGProvider networking.BackendSGProvider, logger logr.Logger) securityGroupBuilder { + return &securityGroupBuilderImpl{ + tagHelper: tagHelper, + clusterName: clusterName, + logger: logger, + enableBackendSG: enableBackendSG, + sgResolver: sgResolver, + backendSGProvider: backendSGProvider, + } +} + +func (builder *securityGroupBuilderImpl) buildSecurityGroups(ctx context.Context, stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, routes map[int][]routeutils.RouteDescriptor, ipAddressType elbv2model.IPAddressType) (securityGroupOutput, error) { + var sgNameOrIds []string + if lbConf != nil && lbConf.Spec.SecurityGroups != nil { + sgNameOrIds = *lbConf.Spec.SecurityGroups + } + + if len(sgNameOrIds) == 0 { + return builder.handleManagedSecurityGroup(ctx, stack, lbConf, gw, routes, ipAddressType) + } + + return builder.handleCustomerSpecifiedSecurityGroups(ctx, lbConf, gw, sgNameOrIds) +} + +func (builder *securityGroupBuilderImpl) handleManagedSecurityGroup(ctx context.Context, stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, routes map[int][]routeutils.RouteDescriptor, ipAddressType elbv2model.IPAddressType) (securityGroupOutput, error) { + var lbSGTokens []core.StringToken + managedSG, err := builder.buildManagedSecurityGroup(stack, lbConf, gw, routes, ipAddressType) + if err != nil { + return securityGroupOutput{}, err + } + lbSGTokens = append(lbSGTokens, managedSG.GroupID()) + var backendSecurityGroupToken core.StringToken + var backendSGAllocated bool + if !builder.enableBackendSG { + backendSecurityGroupToken = managedSG.GroupID() + } else { + backendSecurityGroupToken, err = builder.getBackendSecurityGroup(ctx, gw) + if err != nil { + return securityGroupOutput{}, err + } + backendSGAllocated = true + } + builder.logger.Info("Auto Create SG", "LB SGs", lbSGTokens, "backend SG", backendSecurityGroupToken) + return securityGroupOutput{ + securityGroupTokens: lbSGTokens, + backendSecurityGroupToken: backendSecurityGroupToken, + backendSecurityGroupAllocated: backendSGAllocated, + }, nil +} + +func (builder *securityGroupBuilderImpl) handleCustomerSpecifiedSecurityGroups(ctx context.Context, lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, sgNameOrIds []string) (securityGroupOutput, error) { + var lbSGTokens []core.StringToken + manageBackendSGRules := lbConf.Spec.ManageBackendSecurityGroupRules + frontendSGIDs, err := builder.sgResolver.ResolveViaNameOrID(ctx, sgNameOrIds) + if err != nil { + return securityGroupOutput{}, err + } + for _, sgID := range frontendSGIDs { + lbSGTokens = append(lbSGTokens, core.LiteralStringToken(sgID)) + } + + var backendSecurityGroupToken core.StringToken + var backendSGAllocated bool + if manageBackendSGRules { + if !builder.enableBackendSG { + return securityGroupOutput{}, errors.New("backendSG feature is required to manage worker node SG rules when frontendSG manually specified") + } + backendSecurityGroupToken, err = builder.getBackendSecurityGroup(ctx, gw) + if err != nil { + return securityGroupOutput{}, err + } + backendSGAllocated = true + } + builder.logger.Info("SG configured via annotation", "LB SGs", lbSGTokens, "backend SG", backendSecurityGroupToken) + return securityGroupOutput{ + securityGroupTokens: lbSGTokens, + backendSecurityGroupToken: backendSecurityGroupToken, + backendSecurityGroupAllocated: backendSGAllocated, + }, nil +} + +func (builder *securityGroupBuilderImpl) getBackendSecurityGroup(ctx context.Context, gw *gwv1.Gateway) (core.StringToken, error) { + backendSGID, err := builder.backendSGProvider.Get(ctx, networking.ResourceTypeIngress, []types.NamespacedName{k8s.NamespacedName(gw)}) + if err != nil { + return nil, err + } + return core.LiteralStringToken(backendSGID), nil +} + +func (builder *securityGroupBuilderImpl) buildManagedSecurityGroup(stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, routes map[int][]routeutils.RouteDescriptor, ipAddressType elbv2model.IPAddressType) (*ec2model.SecurityGroup, error) { + name := builder.buildManagedSecurityGroupName(gw) + tags, err := builder.tagHelper.getGatewayTags(lbConf) + if err != nil { + return nil, err + } + + ingressPermissions := builder.buildManagedSecurityGroupIngressPermissions(lbConf, routes, ipAddressType) + return ec2model.NewSecurityGroup(stack, resourceIDManagedSecurityGroup, ec2model.SecurityGroupSpec{ + GroupName: name, + Description: "[k8s] Managed SecurityGroup for LoadBalancer", + Tags: tags, + Ingress: ingressPermissions, + }), nil +} + +func (builder *securityGroupBuilderImpl) buildManagedSecurityGroupName(gw *gwv1.Gateway) string { + uuidHash := sha256.New() + _, _ = uuidHash.Write([]byte(builder.clusterName)) + _, _ = uuidHash.Write([]byte(gw.Name)) + _, _ = uuidHash.Write([]byte(gw.Namespace)) + _, _ = uuidHash.Write([]byte(gw.UID)) + uuid := hex.EncodeToString(uuidHash.Sum(nil)) + + sanitizedNamespace := invalidSecurityGroupNamePtn.ReplaceAllString(gw.Namespace, "") + sanitizedName := invalidSecurityGroupNamePtn.ReplaceAllString(gw.Name, "") + return fmt.Sprintf("k8s-%.8s-%.8s-%.10s", sanitizedNamespace, sanitizedName, uuid) +} + +func (builder *securityGroupBuilderImpl) buildManagedSecurityGroupIngressPermissions(lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor, ipAddressType elbv2model.IPAddressType) []ec2model.IPPermission { + var permissions []ec2model.IPPermission + + var sourceRanges []string + var prefixes []string + var enableICMP bool + + if lbConf.Spec.SourceRanges != nil { + sourceRanges = *lbConf.Spec.SourceRanges + } + + if lbConf.Spec.SecurityGroupPrefixes != nil { + prefixes = *lbConf.Spec.SecurityGroupPrefixes + } + + if lbConf.Spec.EnableICMP { + enableICMP = true + } + + includeIPv6 := isIPv6Supported(ipAddressType) + + // Port Loop + for port, cfg := range routes { + // Protocol Loop + for _, protocol := range generateProtocolListFromRoutes(cfg) { + // CIDR Loop + for _, cidr := range sourceRanges { + isIPv6 := isIPv6CIDR(cidr) + + if !isIPv6 { + permissions = append(permissions, ec2model.IPPermission{ + IPProtocol: protocol, + FromPort: awssdk.Int32(int32(port)), + ToPort: awssdk.Int32(int32(port)), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: cidr, + }, + }, + }) + + if enableICMP { + permissions = append(permissions, ec2model.IPPermission{ + IPProtocol: icmpv4Protocol, + FromPort: awssdk.Int32(icmpv4TypeForPathMtu), + ToPort: awssdk.Int32(icmpv4CodeForPathMtu), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: cidr, + }, + }, + }) + } + + } else if includeIPv6 { + permissions = append(permissions, ec2model.IPPermission{ + IPProtocol: protocol, + FromPort: awssdk.Int32(int32(port)), + ToPort: awssdk.Int32(int32(port)), + IPv6Range: []ec2model.IPv6Range{ + { + CIDRIPv6: cidr, + }, + }, + }) + + if enableICMP { + permissions = append(permissions, ec2model.IPPermission{ + IPProtocol: icmpv6Protocol, + FromPort: awssdk.Int32(icmpv6TypeForPathMtu), + ToPort: awssdk.Int32(icmpv6CodeForPathMtu), + IPv6Range: []ec2model.IPv6Range{ + { + CIDRIPv6: cidr, + }, + }, + }) + } + } + } // CIDR Loop + // PL loop + for _, prefixID := range prefixes { + permissions = append(permissions, ec2model.IPPermission{ + IPProtocol: protocol, + FromPort: awssdk.Int32(int32(port)), + ToPort: awssdk.Int32(int32(port)), + PrefixLists: []ec2model.PrefixList{ + { + ListID: prefixID, + }, + }, + }) + } // PL Loop + } // Protocol Loop + } // Port Loop + return permissions +} + +func generateProtocolListFromRoutes(routes []routeutils.RouteDescriptor) []string { + protocolSet := sets.New[string]() + + for _, route := range routes { + switch route.GetRouteKind() { + case routeutils.HTTPRouteKind: + case routeutils.GRPCRouteKind: + case routeutils.TCPRouteKind: + case routeutils.TLSRouteKind: + protocolSet.Insert(string(ec2types.ProtocolTcp)) + break + case routeutils.UDPRouteKind: + protocolSet.Insert(string(ec2types.ProtocolUdp)) + break + default: + // Ignore? Throw error? + } + } + return protocolSet.UnsortedList() +} + +func isIPv6Supported(ipAddressType elbv2model.IPAddressType) bool { + switch ipAddressType { + case elbv2model.IPAddressTypeDualStack, elbv2model.IPAddressTypeDualStackWithoutPublicIPV4: + return true + default: + return false + } +} + +// TODO - Refactor? +func isIPv6CIDR(cidr string) bool { + return strings.Contains(cidr, ":") +} diff --git a/pkg/gateway/model/model_build_subnet.go b/pkg/gateway/model/model_build_subnet.go index 9b2c13301d..adfce1aa93 100644 --- a/pkg/gateway/model/model_build_subnet.go +++ b/pkg/gateway/model/model_build_subnet.go @@ -15,8 +15,13 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" ) +type buildLoadBalancerSubnetsOutput struct { + subnets []elbv2model.SubnetMapping + sourceIPv6NatEnabled bool +} + type subnetModelBuilder interface { - buildLoadBalancerSubnets(ctx context.Context, gwSubnetConfig *[]elbv2gw.SubnetConfiguration, gwSubnetTagSelectors *map[string][]string, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, stack core.Stack) ([]elbv2model.SubnetMapping, bool, error) + buildLoadBalancerSubnets(ctx context.Context, gwSubnetConfig *[]elbv2gw.SubnetConfiguration, gwSubnetTagSelectors *map[string][]string, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, stack core.Stack) (buildLoadBalancerSubnetsOutput, error) } type subnetModelBuilderImpl struct { @@ -50,17 +55,17 @@ func newSubnetModelBuilder(loadBalancerType elbv2model.LoadBalancerType, trackin } } -func (subnetBuilder *subnetModelBuilderImpl) buildLoadBalancerSubnets(ctx context.Context, gwSubnetConfig *[]elbv2gw.SubnetConfiguration, gwSubnetTagSelectors *map[string][]string, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, stack core.Stack) ([]elbv2model.SubnetMapping, bool, error) { +func (subnetBuilder *subnetModelBuilderImpl) buildLoadBalancerSubnets(ctx context.Context, gwSubnetConfig *[]elbv2gw.SubnetConfiguration, gwSubnetTagSelectors *map[string][]string, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, stack core.Stack) (buildLoadBalancerSubnetsOutput, error) { sourceNATEnabled, err := subnetBuilder.validateSubnetsInput(gwSubnetConfig, scheme, ipAddressType) if err != nil { - return nil, false, err + return buildLoadBalancerSubnetsOutput{}, err } resolvedEC2Subnets, err := subnetBuilder.resolveEC2Subnets(ctx, stack, gwSubnetConfig, gwSubnetTagSelectors, scheme) if err != nil { - return nil, false, err + return buildLoadBalancerSubnetsOutput{}, err } resultPtrs := make([]*elbv2model.SubnetMapping, 0) @@ -80,7 +85,7 @@ func (subnetBuilder *subnetModelBuilderImpl) buildLoadBalancerSubnets(ctx contex for _, mutator := range subnetBuilder.subnetMutatorChain { err := mutator.Mutate(resultPtrs, resolvedEC2Subnets, subnetConfig) if err != nil { - return nil, false, err + return buildLoadBalancerSubnetsOutput{}, err } } @@ -90,7 +95,10 @@ func (subnetBuilder *subnetModelBuilderImpl) buildLoadBalancerSubnets(ctx contex result = append(result, *v) } - return result, sourceNATEnabled, nil + return buildLoadBalancerSubnetsOutput{ + subnets: result, + sourceIPv6NatEnabled: sourceNATEnabled, + }, nil } func (subnetBuilder *subnetModelBuilderImpl) validateSubnetsInput(subnetConfigsPtr *[]elbv2gw.SubnetConfiguration, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType) (bool, error) { diff --git a/pkg/gateway/model/model_build_subnet_test.go b/pkg/gateway/model/model_build_subnet_test.go index d59fce9513..b7fdeba931 100644 --- a/pkg/gateway/model/model_build_subnet_test.go +++ b/pkg/gateway/model/model_build_subnet_test.go @@ -97,11 +97,11 @@ func Test_BuildLoadBalancerSubnets(t *testing.T) { }, } - subnetMappings, ipv6SourceNatEnabled, err := builder.buildLoadBalancerSubnets(context.Background(), &gwSubnetConfig, nil, elbv2model.LoadBalancerSchemeInternal, elbv2model.IPAddressTypeIPV4, nil) + output, err := builder.buildLoadBalancerSubnets(context.Background(), &gwSubnetConfig, nil, elbv2model.LoadBalancerSchemeInternal, elbv2model.IPAddressTypeIPV4, nil) assert.NoError(t, err) - assert.Equal(t, expectedMappings, subnetMappings) - assert.False(t, ipv6SourceNatEnabled) + assert.Equal(t, expectedMappings, output.subnets) + assert.False(t, output.sourceIPv6NatEnabled) assert.Equal(t, 1, mm.called) } From 2b8d4f56d933b0aee267cb9c09a45f31b3af1856 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Fri, 11 Apr 2025 16:07:28 -0700 Subject: [PATCH 16/40] sg logic and refactoring --- controllers/gateway/gateway_controller.go | 5 +- controllers/service/service_controller.go | 10 ++-- main.go | 8 ++- pkg/config/controller_config.go | 9 ++- pkg/config/controller_config_test.go | 5 +- pkg/deploy/tracking/provider.go | 10 ++-- pkg/deploy/tracking/provider_test.go | 19 +++--- pkg/gateway/constants/controller_constants.go | 6 -- pkg/gateway/model/base_model_builder.go | 15 ++--- pkg/gateway/model/model_build_loadbalancer.go | 5 +- .../model/model_build_security_group.go | 9 +-- pkg/ingress/finalizer.go | 10 +--- pkg/ingress/group_loader.go | 7 ++- pkg/ingress/model_builder_test.go | 5 +- pkg/metrics/util/reconcile_counter.go | 2 +- pkg/networking/backend_sg_provider.go | 60 +++++++++++++++---- pkg/networking/backend_sg_provider_test.go | 4 +- pkg/service/model_build_load_balancer_test.go | 5 +- pkg/shared_constants/finalizers.go | 18 ++++++ pkg/shared_constants/tag_keys.go | 9 +++ 20 files changed, 143 insertions(+), 78 deletions(-) create mode 100644 pkg/shared_constants/finalizers.go create mode 100644 pkg/shared_constants/tag_keys.go diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index e8da41531a..66f223be1d 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -27,6 +27,7 @@ import ( elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -47,12 +48,12 @@ var _ Reconciler = &gatewayReconciler{} // NewNLBGatewayReconciler constructs a gateway reconciler to handle specifically for NLB gateways func NewNLBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { - return newGatewayReconciler(constants.NLBGatewayController, elbv2model.LoadBalancerTypeNetwork, controllerConfig.NLBGatewayMaxConcurrentReconciles, constants.NLBGatewayTagPrefix, constants.NLBGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementNLBGateway) + return newGatewayReconciler(constants.NLBGatewayController, elbv2model.LoadBalancerTypeNetwork, controllerConfig.NLBGatewayMaxConcurrentReconciles, constants.NLBGatewayTagPrefix, shared_constants.NLBGatewayFinalizer, routeLoader, routeutils.L4RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementNLBGateway) } // NewALBGatewayReconciler constructs a gateway reconciler to handle specifically for ALB gateways func NewALBGatewayReconciler(routeLoader routeutils.Loader, cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, controllerConfig config.ControllerConfig, finalizerManager k8s.FinalizerManager, networkingSGReconciler networking.SecurityGroupReconciler, networkingSGManager networking.SecurityGroupManager, elbv2TaggingManager elbv2deploy.TaggingManager, subnetResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters) Reconciler { - return newGatewayReconciler(constants.ALBGatewayController, elbv2model.LoadBalancerTypeApplication, controllerConfig.ALBGatewayMaxConcurrentReconciles, constants.ALBGatewayTagPrefix, constants.ALBGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementALBGateway) + return newGatewayReconciler(constants.ALBGatewayController, elbv2model.LoadBalancerTypeApplication, controllerConfig.ALBGatewayMaxConcurrentReconciles, constants.ALBGatewayTagPrefix, shared_constants.ALBGatewayFinalizer, routeLoader, routeutils.L7RouteFilter, cloud, k8sClient, eventRecorder, controllerConfig, finalizerManager, networkingSGReconciler, networkingSGManager, elbv2TaggingManager, subnetResolver, vpcInfoProvider, backendSGProvider, sgResolver, logger, metricsCollector, reconcileCounters.IncrementALBGateway) } // newGatewayReconciler constructs a reconciler that responds to gateway object changes diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index e89bc546c5..dbadd0e1d6 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -3,6 +3,7 @@ package service import ( "context" "fmt" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -33,7 +34,6 @@ import ( ) const ( - serviceFinalizer = "service.k8s.aws/resources" serviceTagPrefix = "service.k8s.aws" serviceAnnotationPrefix = "service.beta.kubernetes.io" controllerName = "service" @@ -47,7 +47,7 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe annotationParser := annotations.NewSuffixAnnotationParser(serviceAnnotationPrefix) trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName) - serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates) + serviceUtils := service.NewServiceUtils(annotationParser, shared_constants.ServiceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates) modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider, elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils, @@ -170,7 +170,7 @@ func (r *serviceReconciler) reconcileLoadBalancerResources(ctx context.Context, var err error addFinalizersFn := func() { - err = r.finalizerManager.AddFinalizers(ctx, svc, serviceFinalizer) + err = r.finalizerManager.AddFinalizers(ctx, svc, shared_constants.ServiceFinalizer) } r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "add_finalizers", addFinalizersFn) if err != nil { @@ -214,7 +214,7 @@ func (r *serviceReconciler) reconcileLoadBalancerResources(ctx context.Context, } func (r *serviceReconciler) cleanupLoadBalancerResources(ctx context.Context, svc *corev1.Service, stack core.Stack) error { - if k8s.HasFinalizer(svc, serviceFinalizer) { + if k8s.HasFinalizer(svc, shared_constants.ServiceFinalizer) { err := r.deployModel(ctx, svc, stack) if err != nil { return err @@ -226,7 +226,7 @@ func (r *serviceReconciler) cleanupLoadBalancerResources(ctx context.Context, sv r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedCleanupStatus, fmt.Sprintf("Failed update status due to %v", err)) return err } - if err := r.finalizerManager.RemoveFinalizers(ctx, svc, serviceFinalizer); err != nil { + if err := r.finalizerManager.RemoveFinalizers(ctx, svc, shared_constants.ServiceFinalizer); err != nil { r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed remove finalizer due to %v", err)) return err } diff --git a/main.go b/main.go index f0f1250dfc..68c37fda06 100644 --- a/main.go +++ b/main.go @@ -148,6 +148,8 @@ func main() { os.Exit(1) } + nlbGatewayEnabled := controllerCFG.FeatureGates.Enabled(config.NLBGatewayAPI) + albGatewayEnabled := controllerCFG.FeatureGates.Enabled(config.ALBGatewayAPI) podInfoRepo := k8s.NewDefaultPodInfoRepo(clientSet.CoreV1().RESTClient(), controllerCFG.RuntimeConfig.WatchNamespace, ctrl.Log) finalizerManager := k8s.NewDefaultFinalizerManager(mgr.GetClient(), ctrl.Log) sgManager := networking.NewDefaultSecurityGroupManager(cloud.EC2(), ctrl.Log) @@ -165,7 +167,7 @@ func main() { cloud.VpcID(), controllerCFG.ClusterName, controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules, controllerCFG.ServiceTargetENISGTags, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log) backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup, - cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, ctrl.Log.WithName("backend-sg-provider")) + cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, nlbGatewayEnabled, albGatewayEnabled, ctrl.Log.WithName("backend-sg-provider")) sgResolver := networking.NewDefaultSecurityGroupResolver(cloud.EC2(), cloud.VpcID()) elbv2TaggingManager := elbv2deploy.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerCFG.FeatureGates, cloud.RGT(), ctrl.Log) ingGroupReconciler := ingress.NewGroupReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("ingress"), @@ -221,7 +223,7 @@ func main() { } // Setup NLB Gateway controller if enabled - if controllerCFG.FeatureGates.Enabled(config.NLBGatewayAPI) { + if nlbGatewayEnabled { gwControllerConfig.routeLoader = routeutils.NewLoader(mgr.GetClient()) if err := setupGatewayController(ctx, mgr, gwControllerConfig, constants.NLBGatewayController); err != nil { setupLog.Error(err, "failed to setup NLB Gateway controller") @@ -230,7 +232,7 @@ func main() { } // Setup ALB Gateway controller if enabled - if controllerCFG.FeatureGates.Enabled(config.ALBGatewayAPI) { + if albGatewayEnabled { if gwControllerConfig.routeLoader == nil { gwControllerConfig.routeLoader = routeutils.NewLoader(mgr.GetClient()) } diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 1f0903af8a..3aa091c4fc 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -1,6 +1,7 @@ package config import ( + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strings" "time" @@ -45,12 +46,16 @@ const ( var ( trackingTagKeys = sets.NewString( - "elbv2.k8s.aws/cluster", - "elbv2.k8s.aws/resource", + shared_constants.TagKeyK8sCluster, + shared_constants.TagKeyResource, "ingress.k8s.aws/stack", "ingress.k8s.aws/resource", "service.k8s.aws/stack", "service.k8s.aws/resource", + "gateway.k8s.aws.nlb/resource", + "gateway.k8s.aws.alb/resource", + "gateway.k8s.aws.nlb/stack", + "gateway.k8s.aws.alb/stack", ) ) diff --git a/pkg/config/controller_config_test.go b/pkg/config/controller_config_test.go index 0351ded53f..74ec7ae382 100644 --- a/pkg/config/controller_config_test.go +++ b/pkg/config/controller_config_test.go @@ -3,6 +3,7 @@ package config import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "testing" ) @@ -28,7 +29,7 @@ func TestControllerConfig_validateDefaultTagsCollisionWithTrackingTags(t *testin name: "default tags and tracking tags have collision", fields: fields{ DefaultTags: map[string]string{ - "elbv2.k8s.aws/cluster": "value-a", + shared_constants.TagKeyK8sCluster: "value-a", }, }, wantErr: errors.New("tag key elbv2.k8s.aws/cluster cannot be specified in default-tags flag"), @@ -75,7 +76,7 @@ func TestControllerConfig_validateExternalManagedTagsCollisionWithTrackingTags(t { name: "external managed tags and tracking tags have collision", fields: fields{ - ExternalManagedTags: []string{"elbv2.k8s.aws/cluster"}, + ExternalManagedTags: []string{shared_constants.TagKeyK8sCluster}, }, wantErr: errors.New("tag key elbv2.k8s.aws/cluster cannot be specified in external-managed-tags flag"), }, diff --git a/pkg/deploy/tracking/provider.go b/pkg/deploy/tracking/provider.go index 32194e5b3d..7545fabfc4 100644 --- a/pkg/deploy/tracking/provider.go +++ b/pkg/deploy/tracking/provider.go @@ -4,8 +4,11 @@ import ( "fmt" "sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" ) +// TODO(ztn) - Add Gateway documentation here? + //we use AWS tags and K8s labels to track resources we have created. // //For AWS resources created by this controller, the tagging strategy is as follows: @@ -32,9 +35,6 @@ import ( // * `service.k8s.aws/stack-namespace: namespace` // * `service.k8s.aws/stack-name: serviceName` -// AWS TagKey for cluster resources. -const clusterNameTagKey = "elbv2.k8s.aws/cluster" - // Legacy AWS TagKey for cluster resources, which is used by AWSALBIngressController(v1.1.3+) const clusterNameTagKeyLegacy = "ingress.k8s.aws/cluster" @@ -85,8 +85,8 @@ func (p *defaultProvider) ResourceIDTagKey() string { func (p *defaultProvider) StackTags(stack core.Stack) map[string]string { stackID := stack.StackID() return map[string]string{ - clusterNameTagKey: p.clusterName, - p.prefixedTrackingKey("stack"): stackID.String(), + shared_constants.TagKeyK8sCluster: p.clusterName, + p.prefixedTrackingKey("stack"): stackID.String(), } } diff --git a/pkg/deploy/tracking/provider_test.go b/pkg/deploy/tracking/provider_test.go index 2d1e926682..4ce75e6d11 100644 --- a/pkg/deploy/tracking/provider_test.go +++ b/pkg/deploy/tracking/provider_test.go @@ -3,6 +3,7 @@ package tracking import ( "github.com/stretchr/testify/assert" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "testing" ) @@ -46,8 +47,8 @@ func Test_defaultProvider_StackTags(t *testing.T) { provider: NewDefaultProvider("ingress.k8s.aws", "cluster-name"), args: args{stack: core.NewDefaultStack(core.StackID{Namespace: "", Name: "awesome-group"})}, want: map[string]string{ - "elbv2.k8s.aws/cluster": "cluster-name", - "ingress.k8s.aws/stack": "awesome-group", + shared_constants.TagKeyK8sCluster: "cluster-name", + "ingress.k8s.aws/stack": "awesome-group", }, }, { @@ -55,8 +56,8 @@ func Test_defaultProvider_StackTags(t *testing.T) { provider: NewDefaultProvider("ingress.k8s.aws", "cluster-name"), args: args{stack: core.NewDefaultStack(core.StackID{Namespace: "namespace", Name: "ingressName"})}, want: map[string]string{ - "elbv2.k8s.aws/cluster": "cluster-name", - "ingress.k8s.aws/stack": "namespace/ingressName", + shared_constants.TagKeyK8sCluster: "cluster-name", + "ingress.k8s.aws/stack": "namespace/ingressName", }, }, { @@ -64,8 +65,8 @@ func Test_defaultProvider_StackTags(t *testing.T) { provider: NewDefaultProvider("service.k8s.aws", "cluster-name"), args: args{stack: core.NewDefaultStack(core.StackID{Namespace: "namespace", Name: "serviceName"})}, want: map[string]string{ - "elbv2.k8s.aws/cluster": "cluster-name", - "service.k8s.aws/stack": "namespace/serviceName", + shared_constants.TagKeyK8sCluster: "cluster-name", + "service.k8s.aws/stack": "namespace/serviceName", }, }, } @@ -100,9 +101,9 @@ func Test_defaultProvider_ResourceTags(t *testing.T) { res: fakeRes, }, want: map[string]string{ - "elbv2.k8s.aws/cluster": "cluster-name", - "ingress.k8s.aws/stack": "namespace/ingressName", - "ingress.k8s.aws/resource": "fake-id", + shared_constants.TagKeyK8sCluster: "cluster-name", + "ingress.k8s.aws/stack": "namespace/ingressName", + "ingress.k8s.aws/resource": "fake-id", }, }, } diff --git a/pkg/gateway/constants/controller_constants.go b/pkg/gateway/constants/controller_constants.go index 194a0373fa..cac44acb3b 100644 --- a/pkg/gateway/constants/controller_constants.go +++ b/pkg/gateway/constants/controller_constants.go @@ -25,9 +25,6 @@ const ( // NLBRouteResourceGroupVersion the groupVersion used by TCPRoute and UDPRoute NLBRouteResourceGroupVersion = "gateway.networking.k8s.io/v1alpha2" - - // NLBGatewayFinalizer the finalizer we attach the NLB Gateway object - NLBGatewayFinalizer = "gateway.k8s.aws/nlb-finalizer" ) /* @@ -43,7 +40,4 @@ const ( // ALBRouteResourceGroupVersion the groupVersion used by HTTPRoute and GRPCRoute ALBRouteResourceGroupVersion = "gateway.networking.k8s.io/v1" - - // ALBGatewayFinalizer the finalizer we attach the ALB Gateway object - ALBGatewayFinalizer = "gateway.k8s.aws/alb-finalizer" ) diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go index 513de2b489..905cea7d3c 100644 --- a/pkg/gateway/model/base_model_builder.go +++ b/pkg/gateway/model/base_model_builder.go @@ -36,11 +36,12 @@ func NewModelBuilder(subnetsResolver networking.SubnetsResolver, gwTagHelper := newTagHelper(sets.New(lbcConfig.ExternalManagedTags...), lbcConfig.DefaultTags) subnetBuilder := newSubnetModelBuilder(loadBalancerType, trackingProvider, subnetsResolver, elbv2TaggingManager) sgBuilder := newSecurityGroupBuilder(gwTagHelper, clusterName, enableBackendSG, sgResolver, backendSGProvider, logger) + lbBuilder := newLoadBalancerBuilder(loadBalancerType, gwTagHelper, clusterName) return &baseModelBuilder{ subnetBuilder: subnetBuilder, securityGroupBuilder: sgBuilder, - lbBuilder: newLoadBalancerBuilder(loadBalancerType, gwTagHelper, clusterName), + lbBuilder: lbBuilder, logger: logger, defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), @@ -62,14 +63,14 @@ type baseModelBuilder struct { } func (baseBuilder *baseModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor) (core.Stack, *elbv2model.LoadBalancer, bool, error) { + stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(gw))) if gw.DeletionTimestamp != nil && !gw.DeletionTimestamp.IsZero() { if baseBuilder.isDeleteProtected(lbConf) { return nil, nil, false, errors.Errorf("Unable to delete gateway %+v because deletion protection is enabled.", k8s.NamespacedName(gw)) } + return stack, nil, false, nil } - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(gw))) - /* Basic LB stuff (Scheme, IP Address Type) */ scheme, err := baseBuilder.buildLoadBalancerScheme(lbConf) @@ -100,15 +101,15 @@ func (baseBuilder *baseModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway } /* Combine everything to form a LoadBalancer */ - - // TODO - Fix - _, err = baseBuilder.lbBuilder.buildLoadBalancerSpec(ctx, scheme, ipAddressType, gw, lbConf, subnets, securityGroups.securityGroupTokens) + spec, err := baseBuilder.lbBuilder.buildLoadBalancerSpec(scheme, ipAddressType, gw, lbConf, subnets, securityGroups.securityGroupTokens) if err != nil { return nil, nil, false, err } - return stack, nil, false, nil + lb := elbv2model.NewLoadBalancer(stack, resourceIDLoadBalancer, spec) + + return stack, lb, securityGroups.backendSecurityGroupAllocated, nil } func (baseBuilder *baseModelBuilder) isDeleteProtected(lbConf *elbv2gw.LoadBalancerConfiguration) bool { diff --git a/pkg/gateway/model/model_build_loadbalancer.go b/pkg/gateway/model/model_build_loadbalancer.go index a86e643540..b4ea69b85c 100644 --- a/pkg/gateway/model/model_build_loadbalancer.go +++ b/pkg/gateway/model/model_build_loadbalancer.go @@ -1,7 +1,6 @@ package model import ( - "context" "crypto/sha256" "encoding/hex" "fmt" @@ -28,7 +27,7 @@ const ( ) type loadBalancerBuilder interface { - buildLoadBalancerSpec(ctx context.Context, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, subnets buildLoadBalancerSubnetsOutput, securityGroupTokens []core.StringToken) (elbv2model.LoadBalancerSpec, error) + buildLoadBalancerSpec(scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, subnets buildLoadBalancerSubnetsOutput, securityGroupTokens []core.StringToken) (elbv2model.LoadBalancerSpec, error) } type loadBalancerBuilderImpl struct { @@ -45,7 +44,7 @@ func newLoadBalancerBuilder(loadBalancerType elbv2model.LoadBalancerType, tagHel } } -func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerSpec(ctx context.Context, scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, subnets buildLoadBalancerSubnetsOutput, securityGroupTokens []core.StringToken) (elbv2model.LoadBalancerSpec, error) { +func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerSpec(scheme elbv2model.LoadBalancerScheme, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, lbConf *elbv2gw.LoadBalancerConfiguration, subnets buildLoadBalancerSubnetsOutput, securityGroupTokens []core.StringToken) (elbv2model.LoadBalancerSpec, error) { name, err := lbModelBuilder.buildLoadBalancerName(lbConf, gw, scheme) if err != nil { diff --git a/pkg/gateway/model/model_build_security_group.go b/pkg/gateway/model/model_build_security_group.go index ed6090a530..25286da52d 100644 --- a/pkg/gateway/model/model_build_security_group.go +++ b/pkg/gateway/model/model_build_security_group.go @@ -51,9 +51,8 @@ type securityGroupBuilder interface { } type securityGroupBuilderImpl struct { - tagHelper tagHelper - clusterName string - + tagHelper tagHelper + clusterName string sgResolver networking.SecurityGroupResolver backendSGProvider networking.BackendSGProvider @@ -102,6 +101,7 @@ func (builder *securityGroupBuilderImpl) handleManagedSecurityGroup(ctx context. return securityGroupOutput{}, err } backendSGAllocated = true + lbSGTokens = append(lbSGTokens, backendSecurityGroupToken) } builder.logger.Info("Auto Create SG", "LB SGs", lbSGTokens, "backend SG", backendSecurityGroupToken) return securityGroupOutput{ @@ -133,6 +133,7 @@ func (builder *securityGroupBuilderImpl) handleCustomerSpecifiedSecurityGroups(c return securityGroupOutput{}, err } backendSGAllocated = true + lbSGTokens = append(lbSGTokens, backendSecurityGroupToken) } builder.logger.Info("SG configured via annotation", "LB SGs", lbSGTokens, "backend SG", backendSecurityGroupToken) return securityGroupOutput{ @@ -143,7 +144,7 @@ func (builder *securityGroupBuilderImpl) handleCustomerSpecifiedSecurityGroups(c } func (builder *securityGroupBuilderImpl) getBackendSecurityGroup(ctx context.Context, gw *gwv1.Gateway) (core.StringToken, error) { - backendSGID, err := builder.backendSGProvider.Get(ctx, networking.ResourceTypeIngress, []types.NamespacedName{k8s.NamespacedName(gw)}) + backendSGID, err := builder.backendSGProvider.Get(ctx, networking.ResourceTypeGateway, []types.NamespacedName{k8s.NamespacedName(gw)}) if err != nil { return nil, err } diff --git a/pkg/ingress/finalizer.go b/pkg/ingress/finalizer.go index c1c0cac67c..4b4d73e06b 100644 --- a/pkg/ingress/finalizer.go +++ b/pkg/ingress/finalizer.go @@ -5,11 +5,7 @@ import ( "fmt" networking "k8s.io/api/networking/v1" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" -) - -const ( - explicitGroupFinalizerPrefix = "group.ingress.k8s.aws/" - implicitGroupFinalizer = "ingress.k8s.aws/resources" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" ) // FinalizerManager manages finalizer for ingresses. @@ -62,7 +58,7 @@ func (m *defaultFinalizerManager) RemoveGroupFinalizer(ctx context.Context, grou // for implicit group, the format is "ingress.k8s.aws/resources" func buildGroupFinalizer(groupID GroupID) string { if groupID.IsExplicit() { - return fmt.Sprintf("%s%s", explicitGroupFinalizerPrefix, groupID.Name) + return fmt.Sprintf("%s%s", shared_constants.ExplicitGroupFinalizerPrefix, groupID.Name) } - return implicitGroupFinalizer + return shared_constants.ImplicitGroupFinalizer } diff --git a/pkg/ingress/group_loader.go b/pkg/ingress/group_loader.go index 2bd98ab61a..0aad20eeea 100644 --- a/pkg/ingress/group_loader.go +++ b/pkg/ingress/group_loader.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sort" "strings" @@ -117,10 +118,10 @@ func (m *defaultGroupLoader) LoadGroupIDIfAny(ctx context.Context, ing *networki func (m *defaultGroupLoader) LoadGroupIDsPendingFinalization(_ context.Context, ing *networking.Ingress) []GroupID { var groupIDs []GroupID for _, finalizer := range ing.GetFinalizers() { - if finalizer == implicitGroupFinalizer { + if finalizer == shared_constants.ImplicitGroupFinalizer { groupIDs = append(groupIDs, NewGroupIDForImplicitGroup(k8s.NamespacedName(ing))) - } else if strings.HasPrefix(finalizer, explicitGroupFinalizerPrefix) { - groupName := finalizer[len(explicitGroupFinalizerPrefix):] + } else if strings.HasPrefix(finalizer, shared_constants.ExplicitGroupFinalizerPrefix) { + groupName := finalizer[len(shared_constants.ExplicitGroupFinalizerPrefix):] groupIDs = append(groupIDs, NewGroupIDForExplicitGroup(groupName)) } } diff --git a/pkg/ingress/model_builder_test.go b/pkg/ingress/model_builder_test.go index dfcd3a683d..82679efc65 100644 --- a/pkg/ingress/model_builder_test.go +++ b/pkg/ingress/model_builder_test.go @@ -3,6 +3,7 @@ package ingress import ( "context" "encoding/json" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "testing" "time" @@ -2169,8 +2170,8 @@ func Test_defaultModelBuilder_Build(t *testing.T) { Scheme: elbv2types.LoadBalancerSchemeEnumInternal, }, Tags: map[string]string{ - "elbv2.k8s.aws/cluster": "cluster-name", - "ingress.k8s.aws/stack": "ns-1/ing-1", + shared_constants.TagKeyK8sCluster: "cluster-name", + "ingress.k8s.aws/stack": "ns-1/ing-1", }, }, { diff --git a/pkg/metrics/util/reconcile_counter.go b/pkg/metrics/util/reconcile_counter.go index 25910aa654..9b6c62c6d1 100644 --- a/pkg/metrics/util/reconcile_counter.go +++ b/pkg/metrics/util/reconcile_counter.go @@ -26,8 +26,8 @@ func NewReconcileCounters() *ReconcileCounters { serviceReconciles: make(map[types.NamespacedName]int), ingressReconciles: make(map[types.NamespacedName]int), tgbReconciles: make(map[types.NamespacedName]int), - nlbGatewayReconciles: make(map[types.NamespacedName]int), albGatewayReconciles: make(map[types.NamespacedName]int), + nlbGatewayReconciles: make(map[types.NamespacedName]int), mutex: sync.Mutex{}, } } diff --git a/pkg/networking/backend_sg_provider.go b/pkg/networking/backend_sg_provider.go index 8b3900e523..a34d3a5414 100644 --- a/pkg/networking/backend_sg_provider.go +++ b/pkg/networking/backend_sg_provider.go @@ -8,6 +8,7 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/smithy-go" "regexp" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sort" "strings" "sync" @@ -24,6 +25,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) const ( @@ -31,14 +33,8 @@ const ( defaultSGDeletionTimeout = 2 * time.Minute resourceTypeSecurityGroup = "security-group" - tagKeyK8sCluster = "elbv2.k8s.aws/cluster" - tagKeyResource = "elbv2.k8s.aws/resource" tagValueBackend = "backend-sg" - explicitGroupFinalizerPrefix = "group.ingress.k8s.aws/" - implicitGroupFinalizer = "ingress.k8s.aws/resources" - serviceFinalizer = "service.k8s.aws/resources" - sgDescription = "[k8s] Shared Backend SecurityGroup for LoadBalancer" ) @@ -47,6 +43,7 @@ type ResourceType string const ( ResourceTypeIngress = "ingress" ResourceTypeService = "service" + ResourceTypeGateway = "gateway" ) // BackendSGProvider is responsible for providing backend security groups @@ -59,7 +56,7 @@ type BackendSGProvider interface { // NewBackendSGProvider constructs a new defaultBackendSGProvider func NewBackendSGProvider(clusterName string, backendSG string, vpcID string, - ec2Client services.EC2, k8sClient client.Client, defaultTags map[string]string, logger logr.Logger) *defaultBackendSGProvider { + ec2Client services.EC2, k8sClient client.Client, defaultTags map[string]string, enableNLBGateway bool, enableALBGateway bool, logger logr.Logger) *defaultBackendSGProvider { return &defaultBackendSGProvider{ vpcID: vpcID, clusterName: clusterName, @@ -70,9 +67,11 @@ func NewBackendSGProvider(clusterName string, backendSG string, vpcID string, logger: logger, mutex: sync.Mutex{}, + enableGatewayCheck: enableALBGateway || enableNLBGateway, + checkIngressFinalizersFunc: func(finalizers []string) bool { for _, fin := range finalizers { - if fin == implicitGroupFinalizer || strings.HasPrefix(fin, explicitGroupFinalizerPrefix) { + if fin == shared_constants.ImplicitGroupFinalizer || strings.HasPrefix(fin, shared_constants.ExplicitGroupFinalizerPrefix) { return true } } @@ -81,7 +80,16 @@ func NewBackendSGProvider(clusterName string, backendSG string, vpcID string, checkServiceFinalizersFunc: func(finalizers []string) bool { for _, fin := range finalizers { - if fin == serviceFinalizer { + if fin == shared_constants.ServiceFinalizer { + return true + } + } + return false + }, + + checkGatewayFinalizersFunc: func(finalizers []string) bool { + for _, fin := range finalizers { + if fin == shared_constants.ALBGatewayFinalizer || fin == shared_constants.NLBGatewayFinalizer { return true } } @@ -113,8 +121,11 @@ type defaultBackendSGProvider struct { // controller deletes the backend SG. objectsMap sync.Map + enableGatewayCheck bool + checkServiceFinalizersFunc func([]string) bool checkIngressFinalizersFunc func([]string) bool + checkGatewayFinalizersFunc func([]string) bool defaultDeletionPollInterval time.Duration defaultDeletionTimeout time.Duration @@ -175,6 +186,9 @@ func (p *defaultBackendSGProvider) isBackendSGRequired(ctx context.Context) (boo if required, err := p.checkServiceListForUnmapped(ctx); required || err != nil { return required, err } + if required, err := p.checkServiceListForUnmapped(ctx); required || err != nil { + return required, err + } return false, nil } @@ -210,6 +224,26 @@ func (p *defaultBackendSGProvider) checkServiceListForUnmapped(ctx context.Conte return false, nil } +func (p *defaultBackendSGProvider) checkGatewayListForUnmapped(ctx context.Context) (bool, error) { + if !p.enableGatewayCheck { + return false, nil + } + + gwList := &gwv1.GatewayList{} + if err := p.k8sClient.List(ctx, gwList); err != nil { + return true, errors.Wrapf(err, "unable to list gateways") + } + for _, gw := range gwList.Items { + if !p.checkGatewayFinalizersFunc(gw.GetFinalizers()) { + continue + } + if !p.existsInObjectMap(ResourceTypeGateway, k8s.NamespacedName(&gw)) { + return true, nil + } + } + return false, nil +} + func (p *defaultBackendSGProvider) existsInObjectMap(resourceType ResourceType, resource types.NamespacedName) bool { if _, exists := p.objectsMap.Load(getObjectKey(resourceType, resource)); exists { return true @@ -269,11 +303,11 @@ func (p *defaultBackendSGProvider) buildBackendSGTags(_ context.Context) []ec2ty ResourceType: resourceTypeSecurityGroup, Tags: append(defaultTags, []ec2types.Tag{ { - Key: awssdk.String(tagKeyK8sCluster), + Key: awssdk.String(shared_constants.TagKeyK8sCluster), Value: awssdk.String(p.clusterName), }, { - Key: awssdk.String(tagKeyResource), + Key: awssdk.String(shared_constants.TagKeyResource), Value: awssdk.String(tagValueBackend), }, }...), @@ -289,11 +323,11 @@ func (p *defaultBackendSGProvider) getBackendSGFromEC2(ctx context.Context, sgNa Values: []string{vpcID}, }, { - Name: awssdk.String(fmt.Sprintf("tag:%v", tagKeyK8sCluster)), + Name: awssdk.String(fmt.Sprintf("tag:%v", shared_constants.TagKeyK8sCluster)), Values: []string{p.clusterName}, }, { - Name: awssdk.String(fmt.Sprintf("tag:%v", tagKeyResource)), + Name: awssdk.String(fmt.Sprintf("tag:%v", shared_constants.TagKeyResource)), Values: []string{tagValueBackend}, }, }, diff --git a/pkg/networking/backend_sg_provider_test.go b/pkg/networking/backend_sg_provider_test.go index 4850d7d30d..c751d4b374 100644 --- a/pkg/networking/backend_sg_provider_test.go +++ b/pkg/networking/backend_sg_provider_test.go @@ -285,7 +285,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { } k8sClient := mock_client.NewMockClient(ctrl) sgProvider := NewBackendSGProvider(defaultClusterName, tt.fields.backendSG, - defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, logr.New(&log.NullLogSink{})) + defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, false, false, logr.New(&log.NullLogSink{})) resourceType := ResourceTypeIngress var activeResources []types.NamespacedName @@ -732,7 +732,7 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) { ec2Client := services.NewMockEC2(ctrl) k8sClient := mock_client.NewMockClient(ctrl) sgProvider := NewBackendSGProvider(defaultClusterName, tt.fields.backendSG, - defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, logr.New(&log.NullLogSink{})) + defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, false, false, logr.New(&log.NullLogSink{})) if len(tt.fields.autogenSG) > 0 { sgProvider.backendSG = "" sgProvider.autoGeneratedSG = tt.fields.autogenSG diff --git a/pkg/service/model_build_load_balancer_test.go b/pkg/service/model_build_load_balancer_test.go index 02a6cdfbb0..7883d31ba9 100644 --- a/pkg/service/model_build_load_balancer_test.go +++ b/pkg/service/model_build_load_balancer_test.go @@ -6,6 +6,7 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/aws/aws-sdk-go/service/ec2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "testing" "k8s.io/apimachinery/pkg/util/sets" @@ -1265,8 +1266,8 @@ func Test_defaultModelBuilderTask_buildLoadBalancerSubnets(t *testing.T) { Scheme: elbv2types.LoadBalancerSchemeEnumInternal, }, Tags: map[string]string{ - "elbv2.k8s.aws/cluster": "cluster-name", - "service.k8s.aws/stack": "namespace/serviceName", + shared_constants.TagKeyK8sCluster: "cluster-name", + "service.k8s.aws/stack": "namespace/serviceName", }, }, }, diff --git a/pkg/shared_constants/finalizers.go b/pkg/shared_constants/finalizers.go new file mode 100644 index 0000000000..7f90abf231 --- /dev/null +++ b/pkg/shared_constants/finalizers.go @@ -0,0 +1,18 @@ +package shared_constants + +const ( + // ExplicitGroupFinalizerPrefix the prefix for finalizers applied to an ingress group + ExplicitGroupFinalizerPrefix = "group.ingress.k8s.aws/" + + // ImplicitGroupFinalizer the finalizer used on an ingress resource + ImplicitGroupFinalizer = "ingress.k8s.aws/resources" + + // ServiceFinalizer the finalizer used on service resources + ServiceFinalizer = "service.k8s.aws/resources" + + // NLBGatewayFinalizer the finalizer we attach to an NLB Gateway resource + NLBGatewayFinalizer = "gateway.k8s.aws/nlb" + + // ALBGatewayFinalizer the finalizer we attach to an ALB Gateway resource + ALBGatewayFinalizer = "gateway.k8s.aws/alb" +) diff --git a/pkg/shared_constants/tag_keys.go b/pkg/shared_constants/tag_keys.go new file mode 100644 index 0000000000..cc6a370112 --- /dev/null +++ b/pkg/shared_constants/tag_keys.go @@ -0,0 +1,9 @@ +package shared_constants + +const ( + // TagKeyK8sCluster AWS TagKey for cluster resources. + TagKeyK8sCluster = "elbv2.k8s.aws/cluster" + + // TagKeyResource AWS TagKey to denote what resource is being represented. + TagKeyResource = "elbv2.k8s.aws/resource" +) From b8aa4b6966f154a608429079416219ec007d96c0 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 14 Apr 2025 11:08:37 -0700 Subject: [PATCH 17/40] refactor duplication of deletion protection attribute --- pkg/deploy/elbv2/load_balancer_synthesizer.go | 7 ++----- pkg/gateway/model/base_model_builder.go | 3 ++- pkg/gateway/model/constants.go | 5 ----- pkg/ingress/model_builder.go | 8 ++++---- pkg/service/model_build_load_balancer_test.go | 6 ++---- pkg/service/model_builder.go | 6 +++--- pkg/shared_constants/attributes.go | 6 ++++++ 7 files changed, 19 insertions(+), 22 deletions(-) delete mode 100644 pkg/gateway/model/constants.go create mode 100644 pkg/shared_constants/attributes.go diff --git a/pkg/deploy/elbv2/load_balancer_synthesizer.go b/pkg/deploy/elbv2/load_balancer_synthesizer.go index 629071cce8..aee7c696fb 100644 --- a/pkg/deploy/elbv2/load_balancer_synthesizer.go +++ b/pkg/deploy/elbv2/load_balancer_synthesizer.go @@ -14,13 +14,10 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strings" ) -const ( - lbAttrsDeletionProtectionEnabled = "deletion_protection.enabled" -) - // NewLoadBalancerSynthesizer constructs loadBalancerSynthesizer func NewLoadBalancerSynthesizer(elbv2Client services.ELBV2, trackingProvider tracking.Provider, taggingManager TaggingManager, lbManager LoadBalancerManager, logger logr.Logger, featureGates config.FeatureGates, controllerConfig config.ControllerConfig, stack core.Stack) *loadBalancerSynthesizer { @@ -116,7 +113,7 @@ func (s *loadBalancerSynthesizer) disableDeletionProtection(ctx context.Context, input := &elbv2sdk.ModifyLoadBalancerAttributesInput{ Attributes: []elbv2types.LoadBalancerAttribute{ { - Key: awssdk.String(lbAttrsDeletionProtectionEnabled), + Key: awssdk.String(shared_constants.LBAttributeDeletionProtection), Value: awssdk.String("false"), }, }, diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go index 905cea7d3c..78245db66d 100644 --- a/pkg/gateway/model/base_model_builder.go +++ b/pkg/gateway/model/base_model_builder.go @@ -15,6 +15,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "strconv" ) @@ -118,7 +119,7 @@ func (baseBuilder *baseModelBuilder) isDeleteProtected(lbConf *elbv2gw.LoadBalan } for _, attr := range lbConf.Spec.LoadBalancerAttributes { - if attr.Key == deletionProtectionAttributeKey { + if attr.Key == shared_constants.LBAttributeDeletionProtection { deletionProtectionEnabled, err := strconv.ParseBool(attr.Value) if err != nil { diff --git a/pkg/gateway/model/constants.go b/pkg/gateway/model/constants.go deleted file mode 100644 index 327cd6f28a..0000000000 --- a/pkg/gateway/model/constants.go +++ /dev/null @@ -1,5 +0,0 @@ -package model - -const ( - deletionProtectionAttributeKey = "deletion_protection.enabled" -) diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index 7273c3884a..ba519044fa 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -3,6 +3,7 @@ package ingress import ( "context" "reflect" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strconv" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" @@ -30,8 +31,7 @@ import ( ) const ( - lbAttrsDeletionProtectionEnabled = "deletion_protection.enabled" - controllerName = "ingress" + controllerName = "ingress" ) // ModelBuilder is responsible for build mode stack for a IngressGroup. @@ -458,8 +458,8 @@ func (t *defaultModelBuildTask) getDeletionProtectionViaAnnotation(ing *networki if err != nil { return false, err } - if _, deletionProtectionSpecified := lbAttributes[lbAttrsDeletionProtectionEnabled]; deletionProtectionSpecified { - deletionProtectionEnabled, err := strconv.ParseBool(lbAttributes[lbAttrsDeletionProtectionEnabled]) + if _, deletionProtectionSpecified := lbAttributes[shared_constants.LBAttributeDeletionProtection]; deletionProtectionSpecified { + deletionProtectionEnabled, err := strconv.ParseBool(lbAttributes[shared_constants.LBAttributeDeletionProtection]) if err != nil { return false, err } diff --git a/pkg/service/model_build_load_balancer_test.go b/pkg/service/model_build_load_balancer_test.go index 7883d31ba9..edc838f6c3 100644 --- a/pkg/service/model_build_load_balancer_test.go +++ b/pkg/service/model_build_load_balancer_test.go @@ -25,8 +25,6 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" ) -const lbAttrsDeletionProtectionEnabled = "deletion_protection.enabled" - func Test_defaultModelBuilderTask_buildLBAttributes(t *testing.T) { tests := []struct { testName string @@ -79,7 +77,7 @@ func Test_defaultModelBuilderTask_buildLBAttributes(t *testing.T) { Value: "true", }, { - Key: lbAttrsDeletionProtectionEnabled, + Key: shared_constants.LBAttributeDeletionProtection, Value: "true", }, }, @@ -113,7 +111,7 @@ func Test_defaultModelBuilderTask_buildLBAttributes(t *testing.T) { Value: "true", }, { - Key: lbAttrsDeletionProtectionEnabled, + Key: shared_constants.LBAttributeDeletionProtection, Value: "true", }, { diff --git a/pkg/service/model_builder.go b/pkg/service/model_builder.go index 759a957554..4b12004aeb 100644 --- a/pkg/service/model_builder.go +++ b/pkg/service/model_builder.go @@ -2,6 +2,7 @@ package service import ( "context" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strconv" "sync" @@ -29,7 +30,6 @@ const ( LoadBalancerTypeExternal = "external" LoadBalancerTargetTypeIP = "ip" LoadBalancerTargetTypeInstance = "instance" - lbAttrsDeletionProtection = "deletion_protection.enabled" controllerName = "service" ) @@ -274,8 +274,8 @@ func (t *defaultModelBuildTask) getDeletionProtectionViaAnnotation(svc corev1.Se if err != nil { return false, err } - if _, deletionProtectionSpecified := lbAttributes[lbAttrsDeletionProtection]; deletionProtectionSpecified { - deletionProtectionEnabled, err := strconv.ParseBool(lbAttributes[lbAttrsDeletionProtection]) + if _, deletionProtectionSpecified := lbAttributes[shared_constants.LBAttributeDeletionProtection]; deletionProtectionSpecified { + deletionProtectionEnabled, err := strconv.ParseBool(lbAttributes[shared_constants.LBAttributeDeletionProtection]) if err != nil { return false, err } diff --git a/pkg/shared_constants/attributes.go b/pkg/shared_constants/attributes.go new file mode 100644 index 0000000000..73092d7746 --- /dev/null +++ b/pkg/shared_constants/attributes.go @@ -0,0 +1,6 @@ +package shared_constants + +const ( + // LBAttributeDeletionProtection deletion protection attribute name + LBAttributeDeletionProtection = "deletion_protection.enabled" +) From d9f54062a44898b9845c66ee8ffe7b8a23963540 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 14 Apr 2025 11:24:31 -0700 Subject: [PATCH 18/40] refactor duplicated icmp values --- pkg/gateway/model/model_build_loadbalancer.go | 10 +------ .../model/model_build_security_group.go | 26 +++++++------------ pkg/service/model_build_managed_sg.go | 22 +++++----------- pkg/shared_constants/icmp.go | 12 +++++++++ 4 files changed, 30 insertions(+), 40 deletions(-) create mode 100644 pkg/shared_constants/icmp.go diff --git a/pkg/gateway/model/model_build_loadbalancer.go b/pkg/gateway/model/model_build_loadbalancer.go index b4ea69b85c..45a5e1495c 100644 --- a/pkg/gateway/model/model_build_loadbalancer.go +++ b/pkg/gateway/model/model_build_loadbalancer.go @@ -15,15 +15,7 @@ import ( var invalidLoadBalancerNamePattern = regexp.MustCompile("[[:^alnum:]]") const ( - lbAttrsAccessLogsS3Enabled = "access_logs.s3.enabled" - lbAttrsAccessLogsS3Bucket = "access_logs.s3.bucket" - lbAttrsAccessLogsS3Prefix = "access_logs.s3.prefix" - lbAttrsLoadBalancingCrossZoneEnabled = "load_balancing.cross_zone.enabled" - lbAttrsLoadBalancingDnsClientRoutingPolicy = "dns_record.client_routing_policy" - availabilityZoneAffinity = "availability_zone_affinity" - partialAvailabilityZoneAffinity = "partial_availability_zone_affinity" - anyAvailabilityZone = "any_availability_zone" - resourceIDLoadBalancer = "LoadBalancer" + resourceIDLoadBalancer = "LoadBalancer" ) type loadBalancerBuilder interface { diff --git a/pkg/gateway/model/model_build_security_group.go b/pkg/gateway/model/model_build_security_group.go index 25286da52d..7b39307a3f 100644 --- a/pkg/gateway/model/model_build_security_group.go +++ b/pkg/gateway/model/model_build_security_group.go @@ -19,6 +19,7 @@ import ( ec2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/ec2" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "strings" ) @@ -28,16 +29,9 @@ var ( ) const ( - icmpv4Protocol = "icmp" - icmpv6Protocol = "icmpv6" - - icmpv4TypeForPathMtu = 3 // https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml#icmp-parameters-codes-3 - icmpv4CodeForPathMtu = 4 - - icmpv6TypeForPathMtu = 2 // https://www.iana.org/assignments/icmpv6-parameters/icmpv6-parameters.xhtml#icmpv6-parameters-codes-2 - icmpv6CodeForPathMtu = 0 - resourceIDManagedSecurityGroup = "ManagedLBSecurityGroup" + + managedSGDescription = "[k8s] Managed SecurityGroup for LoadBalancer" ) type securityGroupOutput struct { @@ -161,7 +155,7 @@ func (builder *securityGroupBuilderImpl) buildManagedSecurityGroup(stack core.St ingressPermissions := builder.buildManagedSecurityGroupIngressPermissions(lbConf, routes, ipAddressType) return ec2model.NewSecurityGroup(stack, resourceIDManagedSecurityGroup, ec2model.SecurityGroupSpec{ GroupName: name, - Description: "[k8s] Managed SecurityGroup for LoadBalancer", + Description: managedSGDescription, Tags: tags, Ingress: ingressPermissions, }), nil @@ -223,9 +217,9 @@ func (builder *securityGroupBuilderImpl) buildManagedSecurityGroupIngressPermiss if enableICMP { permissions = append(permissions, ec2model.IPPermission{ - IPProtocol: icmpv4Protocol, - FromPort: awssdk.Int32(icmpv4TypeForPathMtu), - ToPort: awssdk.Int32(icmpv4CodeForPathMtu), + IPProtocol: shared_constants.ICMPV4Protocol, + FromPort: awssdk.Int32(shared_constants.ICMPV4TypeForPathMtu), + ToPort: awssdk.Int32(shared_constants.ICMPV4CodeForPathMtu), IPRanges: []ec2model.IPRange{ { CIDRIP: cidr, @@ -248,9 +242,9 @@ func (builder *securityGroupBuilderImpl) buildManagedSecurityGroupIngressPermiss if enableICMP { permissions = append(permissions, ec2model.IPPermission{ - IPProtocol: icmpv6Protocol, - FromPort: awssdk.Int32(icmpv6TypeForPathMtu), - ToPort: awssdk.Int32(icmpv6CodeForPathMtu), + IPProtocol: shared_constants.ICMPV6Protocol, + FromPort: awssdk.Int32(shared_constants.ICMPV6TypeForPathMtu), + ToPort: awssdk.Int32(shared_constants.ICMPV6CodeForPathMtu), IPv6Range: []ec2model.IPv6Range{ { CIDRIPv6: cidr, diff --git a/pkg/service/model_build_managed_sg.go b/pkg/service/model_build_managed_sg.go index fd66a21304..0416468cfe 100644 --- a/pkg/service/model_build_managed_sg.go +++ b/pkg/service/model_build_managed_sg.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "regexp" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strings" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -17,15 +18,6 @@ import ( ) const ( - icmpv4Protocol = "icmp" - icmpv6Protocol = "icmpv6" - - icmpv4TypeForPathMtu = 3 // https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml#icmp-parameters-codes-3 - icmpv4CodeForPathMtu = 4 - - icmpv6TypeForPathMtu = 2 // https://www.iana.org/assignments/icmpv6-parameters/icmpv6-parameters.xhtml#icmpv6-parameters-codes-2 - icmpv6CodeForPathMtu = 0 - resourceIDManagedSecurityGroup = "ManagedLBSecurityGroup" ) @@ -99,9 +91,9 @@ func (t *defaultModelBuildTask) buildManagedSecurityGroupIngressPermissions(ctx }) if icmpForPathMtuConfigured && icmpForPathMtuConfiguredFlag == "on" { permissions = append(permissions, ec2model.IPPermission{ - IPProtocol: string(icmpv4Protocol), - FromPort: awssdk.Int32(icmpv4TypeForPathMtu), - ToPort: awssdk.Int32(icmpv4CodeForPathMtu), + IPProtocol: shared_constants.ICMPV4Protocol, + FromPort: awssdk.Int32(shared_constants.ICMPV4TypeForPathMtu), + ToPort: awssdk.Int32(shared_constants.ICMPV4CodeForPathMtu), IPRanges: []ec2model.IPRange{ { CIDRIP: cidr, @@ -122,9 +114,9 @@ func (t *defaultModelBuildTask) buildManagedSecurityGroupIngressPermissions(ctx }) if icmpForPathMtuConfigured && icmpForPathMtuConfiguredFlag == "on" { permissions = append(permissions, ec2model.IPPermission{ - IPProtocol: string(icmpv6Protocol), - FromPort: awssdk.Int32(icmpv6TypeForPathMtu), - ToPort: awssdk.Int32(icmpv6CodeForPathMtu), + IPProtocol: shared_constants.ICMPV6Protocol, + FromPort: awssdk.Int32(shared_constants.ICMPV6TypeForPathMtu), + ToPort: awssdk.Int32(shared_constants.ICMPV6CodeForPathMtu), IPv6Range: []ec2model.IPv6Range{ { CIDRIPv6: cidr, diff --git a/pkg/shared_constants/icmp.go b/pkg/shared_constants/icmp.go new file mode 100644 index 0000000000..f156a552b1 --- /dev/null +++ b/pkg/shared_constants/icmp.go @@ -0,0 +1,12 @@ +package shared_constants + +const ( + ICMPV4Protocol = "icmp" + ICMPV6Protocol = "icmpv6" + + ICMPV4CodeForPathMtu = 3 // https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml#icmp-parameters-codes-3 + ICMPV6CodeForPathMtu = 4 + + ICMPV4TypeForPathMtu = 2 // https://www.iana.org/assignments/icmpv6-parameters/icmpv6-parameters.xhtml#icmpv6-parameters-codes-2 + ICMPV6TypeForPathMtu = 0 +) From eaf77edae0e9bda6a1432f04f90ff6a7450f48f5 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 14 Apr 2025 15:18:25 -0700 Subject: [PATCH 19/40] add sg tests --- main.go | 2 +- pkg/gateway/model/loadbalancer/mutator.go | 7 - pkg/gateway/model/mock_gateway_tag_helper.go | 14 + .../model/model_build_security_group.go | 4 +- .../model/model_build_security_group_test.go | 311 ++++++++++++++++++ pkg/networking/backend_sg_provider.go | 8 +- pkg/networking/backend_sg_provider_test.go | 128 ++++++- 7 files changed, 452 insertions(+), 22 deletions(-) delete mode 100644 pkg/gateway/model/loadbalancer/mutator.go create mode 100644 pkg/gateway/model/mock_gateway_tag_helper.go create mode 100644 pkg/gateway/model/model_build_security_group_test.go diff --git a/main.go b/main.go index 68c37fda06..e57c640037 100644 --- a/main.go +++ b/main.go @@ -167,7 +167,7 @@ func main() { cloud.VpcID(), controllerCFG.ClusterName, controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules, controllerCFG.ServiceTargetENISGTags, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log) backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup, - cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, nlbGatewayEnabled, albGatewayEnabled, ctrl.Log.WithName("backend-sg-provider")) + cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, nlbGatewayEnabled || albGatewayEnabled, ctrl.Log.WithName("backend-sg-provider")) sgResolver := networking.NewDefaultSecurityGroupResolver(cloud.EC2(), cloud.VpcID()) elbv2TaggingManager := elbv2deploy.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerCFG.FeatureGates, cloud.RGT(), ctrl.Log) ingGroupReconciler := ingress.NewGroupReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("ingress"), diff --git a/pkg/gateway/model/loadbalancer/mutator.go b/pkg/gateway/model/loadbalancer/mutator.go deleted file mode 100644 index 373fd6949a..0000000000 --- a/pkg/gateway/model/loadbalancer/mutator.go +++ /dev/null @@ -1,7 +0,0 @@ -package loadbalancer - -import elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" - -type Mutator interface { - Mutate(spec *elbv2model.LoadBalancerSpec) error -} diff --git a/pkg/gateway/model/mock_gateway_tag_helper.go b/pkg/gateway/model/mock_gateway_tag_helper.go new file mode 100644 index 0000000000..fe063ff75a --- /dev/null +++ b/pkg/gateway/model/mock_gateway_tag_helper.go @@ -0,0 +1,14 @@ +package model + +import elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + +type mockTagHelper struct { + tags map[string]string + err error +} + +func (m *mockTagHelper) getGatewayTags(lbConf *elbv2gw.LoadBalancerConfiguration) (map[string]string, error) { + return m.tags, m.err +} + +var _ tagHelper = &mockTagHelper{} diff --git a/pkg/gateway/model/model_build_security_group.go b/pkg/gateway/model/model_build_security_group.go index 7b39307a3f..ef495a01a2 100644 --- a/pkg/gateway/model/model_build_security_group.go +++ b/pkg/gateway/model/model_build_security_group.go @@ -75,7 +75,7 @@ func (builder *securityGroupBuilderImpl) buildSecurityGroups(ctx context.Context return builder.handleManagedSecurityGroup(ctx, stack, lbConf, gw, routes, ipAddressType) } - return builder.handleCustomerSpecifiedSecurityGroups(ctx, lbConf, gw, sgNameOrIds) + return builder.handleExplicitSecurityGroups(ctx, lbConf, gw, sgNameOrIds) } func (builder *securityGroupBuilderImpl) handleManagedSecurityGroup(ctx context.Context, stack core.Stack, lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, routes map[int][]routeutils.RouteDescriptor, ipAddressType elbv2model.IPAddressType) (securityGroupOutput, error) { @@ -105,7 +105,7 @@ func (builder *securityGroupBuilderImpl) handleManagedSecurityGroup(ctx context. }, nil } -func (builder *securityGroupBuilderImpl) handleCustomerSpecifiedSecurityGroups(ctx context.Context, lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, sgNameOrIds []string) (securityGroupOutput, error) { +func (builder *securityGroupBuilderImpl) handleExplicitSecurityGroups(ctx context.Context, lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, sgNameOrIds []string) (securityGroupOutput, error) { var lbSGTokens []core.StringToken manageBackendSGRules := lbConf.Spec.ManageBackendSecurityGroupRules frontendSGIDs, err := builder.sgResolver.ResolveViaNameOrID(ctx, sgNameOrIds) diff --git a/pkg/gateway/model/model_build_security_group_test.go b/pkg/gateway/model/model_build_security_group_test.go new file mode 100644 index 0000000000..f162b3a397 --- /dev/null +++ b/pkg/gateway/model/model_build_security_group_test.go @@ -0,0 +1,311 @@ +package model + +import ( + "context" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/types" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + coremodel "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + ec2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/ec2" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "testing" +) + +func Test_BuildSecurityGroups_Specified(t *testing.T) { + const clusterName = "my-cluster" + + type resolveSgCall struct { + securityGroups []string + err error + } + + type backendSgProviderCall struct { + sgId string + err error + } + + testCases := []struct { + name string + lbConf *elbv2gw.LoadBalancerConfiguration + ipAddressType elbv2model.IPAddressType + expectedTags map[string]string + tagErr error + enableBackendSg bool + + resolveSg *resolveSgCall + providerCall *backendSgProviderCall + + expectErr bool + expectedBackendSgToken coremodel.StringToken + expectedSgTokens []coremodel.StringToken + backendSgAllocated bool + }{ + { + name: "sg specified - no backend sg", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SecurityGroups: &[]string{ + "sg1", + "sg2", + }, + }, + }, + resolveSg: &resolveSgCall{ + securityGroups: []string{ + "sg1", + "sg2", + }, + }, + expectedSgTokens: []coremodel.StringToken{ + coremodel.LiteralStringToken("sg1"), + coremodel.LiteralStringToken("sg2"), + }, + }, + { + name: "sg specified - with backend sg", + enableBackendSg: true, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + ManageBackendSecurityGroupRules: true, + SecurityGroups: &[]string{ + "sg1", + "sg2", + }, + }, + }, + resolveSg: &resolveSgCall{ + securityGroups: []string{ + "sg1", + "sg2", + }, + }, + providerCall: &backendSgProviderCall{ + sgId: "auto-allocated", + }, + expectedSgTokens: []coremodel.StringToken{ + coremodel.LiteralStringToken("sg1"), + coremodel.LiteralStringToken("sg2"), + coremodel.LiteralStringToken("auto-allocated"), + }, + expectedBackendSgToken: coremodel.LiteralStringToken("auto-allocated"), + backendSgAllocated: true, + }, + { + name: "sg specified - with backend sg - error - backendsg not enabled", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + ManageBackendSecurityGroupRules: true, + SecurityGroups: &[]string{ + "sg1", + "sg2", + }, + }, + }, + resolveSg: &resolveSgCall{ + securityGroups: []string{ + "sg1", + "sg2", + }, + }, + expectErr: true, + }, + { + name: "sg specified - with backend sg - error - resolve sg error", + enableBackendSg: true, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + ManageBackendSecurityGroupRules: true, + SecurityGroups: &[]string{ + "sg1", + "sg2", + }, + }, + }, + resolveSg: &resolveSgCall{ + err: errors.New("bad thing"), + }, + expectErr: true, + }, + { + name: "sg specified - with backend sg - error - resolve sg error", + enableBackendSg: true, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + ManageBackendSecurityGroupRules: true, + SecurityGroups: &[]string{ + "sg1", + "sg2", + }, + }, + }, + resolveSg: &resolveSgCall{ + securityGroups: []string{ + "sg1", + "sg2", + }, + }, + providerCall: &backendSgProviderCall{ + err: errors.New("bad thing"), + }, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockTagger := &mockTagHelper{ + tags: tc.expectedTags, + err: tc.tagErr, + } + + gw := &gwv1.Gateway{} + gw.Name = "my-gw" + gw.Namespace = "my-namespace" + + mockSgProvider := networking.NewMockBackendSGProvider(ctrl) + mockSgResolver := networking.NewMockSecurityGroupResolver(ctrl) + + if tc.resolveSg != nil { + mockSgResolver.EXPECT().ResolveViaNameOrID(gomock.Any(), gomock.Eq(*tc.lbConf.Spec.SecurityGroups)).Return(tc.resolveSg.securityGroups, tc.resolveSg.err).Times(1) + } + + if tc.providerCall != nil { + mockSgProvider.EXPECT().Get(gomock.Any(), gomock.Eq(networking.ResourceType(networking.ResourceTypeGateway)), gomock.Eq([]types.NamespacedName{k8s.NamespacedName(gw)})).Return(tc.providerCall.sgId, tc.providerCall.err).Times(1) + } + + stack := coremodel.NewDefaultStack(coremodel.StackID{Namespace: "namespace", Name: "name"}) + builder := newSecurityGroupBuilder(mockTagger, clusterName, tc.enableBackendSg, mockSgResolver, mockSgProvider, logr.Discard()) + + out, err := builder.buildSecurityGroups(context.Background(), stack, tc.lbConf, gw, make(map[int][]routeutils.RouteDescriptor), tc.ipAddressType) + + if tc.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expectedBackendSgToken, out.backendSecurityGroupToken) + assert.Equal(t, tc.expectedSgTokens, out.securityGroupTokens) + assert.Equal(t, tc.backendSgAllocated, out.backendSecurityGroupAllocated) + }) + } +} + +func Test_BuildSecurityGroups_Allocate(t *testing.T) { + const clusterName = "my-cluster" + + type backendSgProviderCall struct { + sgId string + err error + } + + testCases := []struct { + name string + lbConf *elbv2gw.LoadBalancerConfiguration + ipAddressType elbv2model.IPAddressType + expectedTags map[string]string + tagErr error + enableBackendSg bool + + providerCall *backendSgProviderCall + + expectErr bool + hasBackendSg bool + backendSgAllocated bool + expectedStackResources int + }{ + { + name: "sg allocate - no backend sg", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{}, + }, + expectedStackResources: 1, + }, + { + name: "sg allocate - with backend sg", + enableBackendSg: true, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + ManageBackendSecurityGroupRules: true, + }, + }, + providerCall: &backendSgProviderCall{ + sgId: "auto-allocated", + }, + backendSgAllocated: true, + expectedStackResources: 1, + }, + { + name: "sg allocate - provider error", + enableBackendSg: true, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + ManageBackendSecurityGroupRules: true, + }, + }, + providerCall: &backendSgProviderCall{ + err: errors.New("bad thing"), + }, + expectErr: true, + }, + { + name: "sg allocate - tag error", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{}, + }, + expectErr: true, + tagErr: errors.New("bad thing"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockTagger := &mockTagHelper{ + tags: tc.expectedTags, + err: tc.tagErr, + } + + gw := &gwv1.Gateway{} + gw.Name = "my-gw" + gw.Namespace = "my-namespace" + + mockSgProvider := networking.NewMockBackendSGProvider(ctrl) + mockSgResolver := networking.NewMockSecurityGroupResolver(ctrl) + + if tc.providerCall != nil { + mockSgProvider.EXPECT().Get(gomock.Any(), gomock.Eq(networking.ResourceType(networking.ResourceTypeGateway)), gomock.Eq([]types.NamespacedName{k8s.NamespacedName(gw)})).Return(tc.providerCall.sgId, tc.providerCall.err).Times(1) + } + + stack := coremodel.NewDefaultStack(coremodel.StackID{Namespace: "namespace", Name: "name"}) + builder := newSecurityGroupBuilder(mockTagger, clusterName, tc.enableBackendSg, mockSgResolver, mockSgProvider, logr.Discard()) + + out, err := builder.buildSecurityGroups(context.Background(), stack, tc.lbConf, gw, make(map[int][]routeutils.RouteDescriptor), tc.ipAddressType) + + if tc.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.backendSgAllocated, out.backendSecurityGroupAllocated) + var resSGs []*ec2model.SecurityGroup + listErr := stack.ListResources(&resSGs) + assert.NoError(t, listErr) + assert.Equal(t, tc.expectedStackResources, len(resSGs)) + if tc.hasBackendSg { + assert.NotNil(t, out.backendSecurityGroupToken) + } + }) + } +} diff --git a/pkg/networking/backend_sg_provider.go b/pkg/networking/backend_sg_provider.go index a34d3a5414..d48332c321 100644 --- a/pkg/networking/backend_sg_provider.go +++ b/pkg/networking/backend_sg_provider.go @@ -56,7 +56,7 @@ type BackendSGProvider interface { // NewBackendSGProvider constructs a new defaultBackendSGProvider func NewBackendSGProvider(clusterName string, backendSG string, vpcID string, - ec2Client services.EC2, k8sClient client.Client, defaultTags map[string]string, enableNLBGateway bool, enableALBGateway bool, logger logr.Logger) *defaultBackendSGProvider { + ec2Client services.EC2, k8sClient client.Client, defaultTags map[string]string, enableGatewayCheck bool, logger logr.Logger) *defaultBackendSGProvider { return &defaultBackendSGProvider{ vpcID: vpcID, clusterName: clusterName, @@ -67,7 +67,7 @@ func NewBackendSGProvider(clusterName string, backendSG string, vpcID string, logger: logger, mutex: sync.Mutex{}, - enableGatewayCheck: enableALBGateway || enableNLBGateway, + enableGatewayCheck: enableGatewayCheck, checkIngressFinalizersFunc: func(finalizers []string) bool { for _, fin := range finalizers { @@ -186,7 +186,7 @@ func (p *defaultBackendSGProvider) isBackendSGRequired(ctx context.Context) (boo if required, err := p.checkServiceListForUnmapped(ctx); required || err != nil { return required, err } - if required, err := p.checkServiceListForUnmapped(ctx); required || err != nil { + if required, err := p.checkGatewayListForUnmapped(ctx); required || err != nil { return required, err } return false, nil @@ -332,7 +332,7 @@ func (p *defaultBackendSGProvider) getBackendSGFromEC2(ctx context.Context, sgNa }, }, } - p.logger.V(1).Info("Queriying existing SG", "vpc-id", vpcID, "name", sgName) + p.logger.V(1).Info("Querying existing SG", "vpc-id", vpcID, "name", sgName) sgs, err := p.ec2Client.DescribeSecurityGroupsAsList(ctx, req) if err != nil && !isEC2SecurityGroupNotFoundError(err) { return "", err diff --git a/pkg/networking/backend_sg_provider_test.go b/pkg/networking/backend_sg_provider_test.go index c751d4b374..839b0d2ee7 100644 --- a/pkg/networking/backend_sg_provider_test.go +++ b/pkg/networking/backend_sg_provider_test.go @@ -7,6 +7,7 @@ import ( "k8s.io/apimachinery/pkg/types" "reflect" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" "testing" "github.com/go-logr/logr" @@ -42,12 +43,13 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { err error } type fields struct { - backendSG string - ingResources []*networking.Ingress - svcResource *corev1.Service - defaultTags map[string]string - describeSGCalls []describeSecurityGroupsAsListCall - createSGCalls []createSecurityGroupWithContexCall + backendSG string + ingResources []*networking.Ingress + svcResource *corev1.Service + enableGatewayCheck bool + defaultTags map[string]string + describeSGCalls []describeSecurityGroupsAsListCall + createSGCalls []createSecurityGroupWithContexCall } defaultEC2Filters := []ec2types.Filter{ { @@ -285,7 +287,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { } k8sClient := mock_client.NewMockClient(ctrl) sgProvider := NewBackendSGProvider(defaultClusterName, tt.fields.backendSG, - defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, false, false, logr.New(&log.NullLogSink{})) + defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, tt.fields.enableGatewayCheck, logr.New(&log.NullLogSink{})) resourceType := ResourceTypeIngress var activeResources []types.NamespacedName @@ -317,6 +319,11 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) { services []*corev1.Service err error } + type listGatewaysCall struct { + gateways []*gwv1.Gateway + err error + } + type deleteSecurityGroupWithContextCall struct { req *ec2sdk.DeleteSecurityGroupInput resp *ec2sdk.DeleteSecurityGroupOutput @@ -333,11 +340,13 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) { listIngressCalls []listIngressCall deleteSGCalls []deleteSecurityGroupWithContextCall listServicesCalls []listServicesCall + listGatewaysCall []listGatewaysCall activeIngresses []*networking.Ingress inactiveIngresses []*networking.Ingress svcResource *corev1.Service resourceMapItems []mapItem backendSGRequiredForActive bool + enableGatewayCheck bool } ing := &networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -545,6 +554,94 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) { inactiveIngresses: []*networking.Ingress{ing}, }, }, + { + name: "backend sg required for gw - nlb", + fields: fields{ + autogenSG: "sg-autogen", + listIngressCalls: []listIngressCall{ + {}, + }, + listServicesCalls: []listServicesCall{ + {}, + }, + listGatewaysCall: []listGatewaysCall{ + { + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "gw-1", + Finalizers: []string{"gateway.k8s.aws/nlb"}, + }, + }, + }, + }, + }, + enableGatewayCheck: true, + inactiveIngresses: []*networking.Ingress{ing}, + }, + }, + { + name: "backend sg required for gw - alb", + fields: fields{ + autogenSG: "sg-autogen", + listIngressCalls: []listIngressCall{ + {}, + }, + listServicesCalls: []listServicesCall{ + {}, + }, + listGatewaysCall: []listGatewaysCall{ + { + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "gw-1", + Finalizers: []string{"gateway.k8s.aws/alb"}, + }, + }, + }, + }, + }, + enableGatewayCheck: true, + inactiveIngresses: []*networking.Ingress{ing}, + }, + }, + { + name: "backend sg required for gw - alb but gw not enabled.", + fields: fields{ + autogenSG: "sg-autogen", + listIngressCalls: []listIngressCall{ + {}, + }, + listServicesCalls: []listServicesCall{ + {}, + }, + listGatewaysCall: []listGatewaysCall{ + { + gateways: []*gwv1.Gateway{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "gw-1", + Finalizers: []string{"gateway.k8s.aws/alb"}, + }, + }, + }, + }, + }, + inactiveIngresses: []*networking.Ingress{ing}, + deleteSGCalls: []deleteSecurityGroupWithContextCall{ + { + req: &ec2sdk.DeleteSecurityGroupInput{ + GroupId: awssdk.String("sg-autogen"), + }, + resp: &ec2sdk.DeleteSecurityGroupOutput{}, + }, + }, + }, + }, { name: "backend sg requirement for service already known", fields: fields{ @@ -732,7 +829,7 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) { ec2Client := services.NewMockEC2(ctrl) k8sClient := mock_client.NewMockClient(ctrl) sgProvider := NewBackendSGProvider(defaultClusterName, tt.fields.backendSG, - defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, false, false, logr.New(&log.NullLogSink{})) + defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, tt.fields.enableGatewayCheck, logr.New(&log.NullLogSink{})) if len(tt.fields.autogenSG) > 0 { sgProvider.backendSG = "" sgProvider.autoGeneratedSG = tt.fields.autogenSG @@ -771,6 +868,21 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) { }, ).AnyTimes() } + + for _, call := range tt.fields.listGatewaysCall { + if !tt.fields.enableGatewayCheck { + break + } + k8sClient.EXPECT().List(gomock.Any(), &gwv1.GatewayList{}, gomock.Any()).DoAndReturn( + func(ctx context.Context, gatewayList *gwv1.GatewayList, opts ...client.ListOption) error { + for _, gw := range call.gateways { + gatewayList.Items = append(gatewayList.Items, *(gw.DeepCopy())) + } + return call.err + }, + ).AnyTimes() + } + for _, ing := range tt.env.ingresses { assert.NoError(t, k8sClient.Create(context.Background(), ing.DeepCopy())) } From 1a5690044766a9d97ada2f4731838858f451ef24 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 14 Apr 2025 17:10:18 -0700 Subject: [PATCH 20/40] add tests for permissions genertaion --- .../model/model_build_security_group.go | 7 +- .../model/model_build_security_group_test.go | 543 ++++++++++++++++++ pkg/gateway/routeutils/mock_route.go | 41 ++ 3 files changed, 586 insertions(+), 5 deletions(-) create mode 100644 pkg/gateway/routeutils/mock_route.go diff --git a/pkg/gateway/model/model_build_security_group.go b/pkg/gateway/model/model_build_security_group.go index ef495a01a2..10e576340f 100644 --- a/pkg/gateway/model/model_build_security_group.go +++ b/pkg/gateway/model/model_build_security_group.go @@ -277,14 +277,11 @@ func generateProtocolListFromRoutes(routes []routeutils.RouteDescriptor) []strin for _, route := range routes { switch route.GetRouteKind() { - case routeutils.HTTPRouteKind: - case routeutils.GRPCRouteKind: - case routeutils.TCPRouteKind: - case routeutils.TLSRouteKind: + case routeutils.HTTPRouteKind, routeutils.GRPCRouteKind, routeutils.TCPRouteKind, routeutils.TLSRouteKind: protocolSet.Insert(string(ec2types.ProtocolTcp)) break case routeutils.UDPRouteKind: - protocolSet.Insert(string(ec2types.ProtocolUdp)) + protocolSet = protocolSet.Insert(string(ec2types.ProtocolUdp)) break default: // Ignore? Throw error? diff --git a/pkg/gateway/model/model_build_security_group_test.go b/pkg/gateway/model/model_build_security_group_test.go index f162b3a397..a3f15ebb22 100644 --- a/pkg/gateway/model/model_build_security_group_test.go +++ b/pkg/gateway/model/model_build_security_group_test.go @@ -2,6 +2,8 @@ package model import ( "context" + "fmt" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/go-logr/logr" "github.com/golang/mock/gomock" "github.com/pkg/errors" @@ -309,3 +311,544 @@ func Test_BuildSecurityGroups_Allocate(t *testing.T) { }) } } + +func Test_BuildSecurityGroups_BuildManagedSecurityGroupIngressPermissions(t *testing.T) { + testCases := []struct { + name string + lbConf *elbv2gw.LoadBalancerConfiguration + ipAddressType elbv2model.IPAddressType + routes map[int][]routeutils.RouteDescriptor + expected []ec2model.IPPermission + }{ + { + name: "no routes", + lbConf: &elbv2gw.LoadBalancerConfiguration{}, + expected: make([]ec2model.IPPermission, 0), + }, + { + name: "ipv4 - tcp - with source range", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "127.0.0.1/24", + "127.100.0.1/24", + "127.200.0.1/24", + }, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.100.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.200.0.1/24", + }, + }, + }, + }, + }, + { + name: "ipv4 - udp - with source range", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "127.0.0.1/24", + "127.100.0.1/24", + "127.200.0.1/24", + }, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.UDPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "udp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "udp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.100.0.1/24", + }, + }, + }, + { + IPProtocol: "udp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.200.0.1/24", + }, + }, + }, + }, + }, + { + name: "ipv4 - udp - with source range - icmp enabled", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "127.0.0.1/24", + }, + EnableICMP: true, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.UDPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "udp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "icmp", + FromPort: awssdk.Int32(2), + ToPort: awssdk.Int32(3), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + }, + }, + { + name: "ipv4 - with duplicated route type - with source range", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "127.0.0.1/24", + "127.100.0.1/24", + "127.200.0.1/24", + }, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.100.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.200.0.1/24", + }, + }, + }, + }, + }, + { + name: "ipv4 - with duplicated route type - with source range - multiple ports", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "127.0.0.1/24", + "127.100.0.1/24", + "127.200.0.1/24", + }, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + }, + }, + 85: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + }, + }, + 90: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.100.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.200.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(85), + ToPort: awssdk.Int32(85), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(85), + ToPort: awssdk.Int32(85), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.100.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(85), + ToPort: awssdk.Int32(85), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.200.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(90), + ToPort: awssdk.Int32(90), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(90), + ToPort: awssdk.Int32(90), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.100.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(90), + ToPort: awssdk.Int32(90), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.200.0.1/24", + }, + }, + }, + }, + }, + { + name: "ipv6 - with source range", + ipAddressType: elbv2model.IPAddressTypeDualStack, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "2001:db8::/32", + }, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + }, + }, + 85: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + }, + }, + 90: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPv6Range: []ec2model.IPv6Range{ + { + CIDRIPv6: "2001:db8::/32", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(85), + ToPort: awssdk.Int32(85), + IPv6Range: []ec2model.IPv6Range{ + { + CIDRIPv6: "2001:db8::/32", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(90), + ToPort: awssdk.Int32(90), + IPv6Range: []ec2model.IPv6Range{ + { + CIDRIPv6: "2001:db8::/32", + }, + }, + }, + }, + }, + { + name: "ipv6 + ipv4 - with source range", + ipAddressType: elbv2model.IPAddressTypeDualStack, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "2001:db8::/32", + "127.0.0.1/24", + }, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPv6Range: []ec2model.IPv6Range{ + { + CIDRIPv6: "2001:db8::/32", + }, + }, + }, + }, + }, + { + name: "ipv6 + ipv4 - with source range - but lb type doesnt support ipv6", + ipAddressType: elbv2model.IPAddressTypeIPV4, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "2001:db8::/32", + "127.0.0.1/24", + }, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + }, + }, + { + name: "prefix list", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + SourceRanges: &[]string{ + "127.0.0.1/24", + }, + SecurityGroupPrefixes: &[]string{"pl1", "pl2"}, + }, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "127.0.0.1/24", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + PrefixLists: []ec2model.PrefixList{ + { + ListID: "pl1", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + PrefixLists: []ec2model.PrefixList{ + { + ListID: "pl2", + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := &securityGroupBuilderImpl{} + permissions := builder.buildManagedSecurityGroupIngressPermissions(tc.lbConf, tc.routes, tc.ipAddressType) + assert.ElementsMatch(t, tc.expected, permissions, fmt.Sprintf("%+v", permissions)) + }) + } +} diff --git a/pkg/gateway/routeutils/mock_route.go b/pkg/gateway/routeutils/mock_route.go new file mode 100644 index 0000000000..898e4d8d50 --- /dev/null +++ b/pkg/gateway/routeutils/mock_route.go @@ -0,0 +1,41 @@ +package routeutils + +import ( + "k8s.io/apimachinery/pkg/types" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +type MockRoute struct { + Kind string +} + +func (m *MockRoute) GetRouteNamespacedName() types.NamespacedName { + //TODO implement me + panic("implement me") +} + +func (m *MockRoute) GetRouteKind() string { + return m.Kind +} + +func (m *MockRoute) GetHostnames() []gwv1.Hostname { + //TODO implement me + panic("implement me") +} + +func (m *MockRoute) GetParentRefs() []gwv1.ParentReference { + //TODO implement me + panic("implement me") +} + +func (m *MockRoute) GetRawRoute() interface{} { + //TODO implement me + panic("implement me") +} + +func (m *MockRoute) GetAttachedRules() []RouteRule { + //TODO implement me + panic("implement me") +} + +var _ RouteDescriptor = &MockRoute{} From 0a93d9e79ab664fe9199a8b27aad97ead17746d8 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Tue, 15 Apr 2025 14:43:36 -0700 Subject: [PATCH 21/40] fix quick-ci --- config/crd/gateway/gateway-crds.yaml | 26 ++++++++++++++++--- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 26 ++++++++++++++++--- config/rbac/role.yaml | 24 +++++++++++++++++ 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/config/crd/gateway/gateway-crds.yaml b/config/crd/gateway/gateway-crds.yaml index 735b106b5e..27186e6969 100644 --- a/config/crd/gateway/gateway-crds.yaml +++ b/config/crd/gateway/gateway-crds.yaml @@ -45,9 +45,16 @@ spec: LoadBalancerConfiguration properties: customerOwnedIpv4Pool: - description: customerOwnedIpv4Pool is the ID of the customer-owned - address for Application Load Balancers on Outposts pool. + description: |- + customerOwnedIpv4Pool [Application LoadBalancer] + is the ID of the customer-owned address for Application Load Balancers on Outposts pool. type: string + enableICMP: + description: |- + EnableICMP [Network LoadBalancer] + enables the creation of security group rules to the managed security group + to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs + type: boolean enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic Indicates whether to evaluate inbound security group rules for traffic @@ -61,6 +68,11 @@ spec: - dualstack - dualstack-without-public-ipv4 type: string + ipv4IPAMPoolId: + description: |- + IPv4IPAMPoolId [Application LoadBalancer] + defines the IPAM pool ID used for IPv4 Addresses on the ALB. + type: string listenerConfigurations: description: listenerConfigurations is an optional list of configurations for each listener on LB @@ -193,7 +205,7 @@ spec: description: privateIPv4Allocation [Network LoadBalancer] the private ipv4 address to assign to this subnet. type: string - sourceNatIPv6Prefix: + sourceNAT: description: SourceNatIPv6Prefix [Network LoadBalancer] The IPv6 prefix to use for source NAT. Specify an IPv6 prefix (/80 netmask) from the subnet CIDR block or auto_assigned @@ -212,6 +224,12 @@ spec: tag specified in the map key contains one of the values in the corresponding value list. type: object + manageBackendSecurityGroupRules: + description: |- + ManageBackendSecurityGroupRules [Application / Network LoadBalancer] + specifies whether you want the controller to configure security group rules on Node/Pod for traffic access + when you specify securityGroups + type: boolean scheme: description: scheme defines the type of LB to provision. If unspecified, it will be automatically inferred. @@ -240,7 +258,7 @@ spec: tags: description: Tags defines list of Tags on LB. items: - description: Tag defines a AWS Tag on resources. + description: AWSTag defines a AWS Tag on resources. properties: key: description: The key of the tag. diff --git a/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml index c858e905ce..98ed0757eb 100644 --- a/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ b/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -46,9 +46,16 @@ spec: LoadBalancerConfiguration properties: customerOwnedIpv4Pool: - description: customerOwnedIpv4Pool is the ID of the customer-owned - address for Application Load Balancers on Outposts pool. + description: |- + customerOwnedIpv4Pool [Application LoadBalancer] + is the ID of the customer-owned address for Application Load Balancers on Outposts pool. type: string + enableICMP: + description: |- + EnableICMP [Network LoadBalancer] + enables the creation of security group rules to the managed security group + to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs + type: boolean enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic Indicates whether to evaluate inbound security group rules for traffic @@ -62,6 +69,11 @@ spec: - dualstack - dualstack-without-public-ipv4 type: string + ipv4IPAMPoolId: + description: |- + IPv4IPAMPoolId [Application LoadBalancer] + defines the IPAM pool ID used for IPv4 Addresses on the ALB. + type: string listenerConfigurations: description: listenerConfigurations is an optional list of configurations for each listener on LB @@ -194,7 +206,7 @@ spec: description: privateIPv4Allocation [Network LoadBalancer] the private ipv4 address to assign to this subnet. type: string - sourceNatIPv6Prefix: + sourceNAT: description: SourceNatIPv6Prefix [Network LoadBalancer] The IPv6 prefix to use for source NAT. Specify an IPv6 prefix (/80 netmask) from the subnet CIDR block or auto_assigned @@ -213,6 +225,12 @@ spec: tag specified in the map key contains one of the values in the corresponding value list. type: object + manageBackendSecurityGroupRules: + description: |- + ManageBackendSecurityGroupRules [Application / Network LoadBalancer] + specifies whether you want the controller to configure security group rules on Node/Pod for traffic access + when you specify securityGroups + type: boolean scheme: description: scheme defines the type of LB to provision. If unspecified, it will be automatically inferred. @@ -241,7 +259,7 @@ spec: tags: description: Tags defines list of Tags on LB. items: - description: Tag defines a AWS Tag on resources. + description: AWSTag defines a AWS Tag on resources. properties: key: description: The key of the tag. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 313f45d488..0d26c1555f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -194,6 +194,28 @@ rules: - get - patch - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/finalizers + verbs: + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + verbs: + - get + - patch + - update - apiGroups: - gateway.networking.k8s.io resources: @@ -201,12 +223,14 @@ rules: verbs: - get - list + - patch - watch - apiGroups: - gateway.networking.k8s.io resources: - gateways/finalizers verbs: + - patch - update - apiGroups: - gateway.networking.k8s.io From 46febcf50f55e90e999c6480f487151152b5d243 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 17 Apr 2025 15:45:45 -0700 Subject: [PATCH 22/40] delete crd --- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 298 ------------ ...way.k8s.aws_targetgroupconfigurations.yaml | 456 ------------------ 2 files changed, 754 deletions(-) delete mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml delete mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml deleted file mode 100644 index 98ed0757eb..0000000000 --- a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ /dev/null @@ -1,298 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: loadbalancerconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: LoadBalancerConfiguration - listKind: LoadBalancerConfigurationList - plural: loadbalancerconfigurations - singular: loadbalancerconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LoadBalancerConfigurationSpec defines the desired state of - LoadBalancerConfiguration - properties: - customerOwnedIpv4Pool: - description: |- - customerOwnedIpv4Pool [Application LoadBalancer] - is the ID of the customer-owned address for Application Load Balancers on Outposts pool. - type: string - enableICMP: - description: |- - EnableICMP [Network LoadBalancer] - enables the creation of security group rules to the managed security group - to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs - type: boolean - enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: - description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic - Indicates whether to evaluate inbound security group rules for traffic - sent to a Network Load Balancer through Amazon Web Services PrivateLink. - type: string - ipAddressType: - description: loadBalancerIPType defines what kind of load balancer - to provision (ipv4, dual stack) - enum: - - ipv4 - - dualstack - - dualstack-without-public-ipv4 - type: string - ipv4IPAMPoolId: - description: |- - IPv4IPAMPoolId [Application LoadBalancer] - defines the IPAM pool ID used for IPv4 Addresses on the ALB. - type: string - listenerConfigurations: - description: listenerConfigurations is an optional list of configurations - for each listener on LB - items: - properties: - alpnPolicy: - description: alpnPolicy an optional string that allows you to - configure ALPN policies on your Load Balancer - enum: - - HTTP1Only - - HTTP2Only - - HTTP2Optional - - HTTP2Preferred - - None - type: string - certificates: - description: certificates is the list of other certificates - to add to the listener. - items: - type: string - type: array - defaultCertificate: - description: |- - TODO: Add validation in admission webhook to make it required for secure protocols - defaultCertificate the cert arn to be used by default. - type: string - listenerAttributes: - description: listenerAttributes defines the attributes for the - listener - items: - description: ListenerAttribute defines listener attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - mutualAuthentication: - description: mutualAuthentication defines the mutual authentication - configuration information. - properties: - advertiseTrustStoreCaNames: - description: Indicates whether trust store CA certificate - names are advertised. - enum: - - "on" - - "off" - type: string - ignoreClientCertificateExpiry: - description: Indicates whether expired client certificates - are ignored. - type: boolean - mode: - description: The client certificate handling method. Options - are off , passthrough or verify - enum: - - "off" - - passthrough - - verify - type: string - trustStore: - description: The Name or ARN of the trust store. - type: string - required: - - mode - type: object - protocolPort: - description: protocolPort is identifier for the listener on - load balancer. It should be of the form PROTOCOL:PORT - pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ - type: string - sslPolicy: - description: sslPolicy is the security policy that defines which - protocols and ciphers are supported for secure listeners [HTTPS - or TLS listener]. - type: string - required: - - protocolPort - type: object - type: array - loadBalancerAttributes: - description: LoadBalancerAttributes defines the attribute of LB - items: - description: LoadBalancerAttribute defines LB attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - loadBalancerName: - description: loadBalancerName defines the name of the LB to provision. - If unspecified, it will be automatically generated. - maxLength: 32 - minLength: 1 - type: string - loadBalancerSubnets: - description: |- - loadBalancerSubnets is an optional list of subnet configurations to be used in the LB - This value takes precedence over loadBalancerSubnetsSelector if both are selected. - items: - description: SubnetConfiguration defines the subnet settings for - a Load Balancer. - properties: - eipAllocation: - description: eipAllocation [Network LoadBalancer] the EIP name - for this subnet. - type: string - identifier: - description: identifier [Application LoadBalancer / Network - LoadBalancer] name or id for the subnet - type: string - ipv6Allocation: - description: IPv6Allocation [Network LoadBalancer] the ipv6 - address to assign to this subnet. - type: string - privateIPv4Allocation: - description: privateIPv4Allocation [Network LoadBalancer] the - private ipv4 address to assign to this subnet. - type: string - sourceNAT: - description: SourceNatIPv6Prefix [Network LoadBalancer] The - IPv6 prefix to use for source NAT. Specify an IPv6 prefix - (/80 netmask) from the subnet CIDR block or auto_assigned - to use an IPv6 prefix selected at random from the subnet CIDR - block. - type: string - type: object - type: array - loadBalancerSubnetsSelector: - additionalProperties: - items: - type: string - type: array - description: |- - LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each - tag specified in the map key contains one of the values in the corresponding - value list. - type: object - manageBackendSecurityGroupRules: - description: |- - ManageBackendSecurityGroupRules [Application / Network LoadBalancer] - specifies whether you want the controller to configure security group rules on Node/Pod for traffic access - when you specify securityGroups - type: boolean - scheme: - description: scheme defines the type of LB to provision. If unspecified, - it will be automatically inferred. - enum: - - internal - - internet-facing - type: string - securityGroupPrefixes: - description: securityGroupPrefixes an optional list of prefixes that - are allowed to access the LB. - items: - type: string - type: array - securityGroups: - description: securityGroups an optional list of security group ids - or names to apply to the LB - items: - type: string - type: array - sourceRanges: - description: sourceRanges an optional list of CIDRs that are allowed - to access the LB. - items: - type: string - type: array - tags: - description: Tags defines list of Tags on LB. - items: - description: AWSTag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - vpcId: - description: vpcId is the ID of the VPC for the load balancer. - type: string - type: object - status: - description: LoadBalancerConfigurationStatus defines the observed state - of TargetGroupBinding - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml deleted file mode 100644 index c34856f29b..0000000000 --- a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml +++ /dev/null @@ -1,456 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: targetgroupconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: TargetGroupConfiguration - listKind: TargetGroupConfigurationList - plural: targetgroupconfigurations - singular: targetgroupconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: The Kubernetes Service's name - jsonPath: .spec.targetReference.name - name: SERVICE-NAME - type: string - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: TargetGroupConfiguration is the Schema for defining TargetGroups - with an AWS ELB Gateway - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TargetGroupConfigurationSpec defines the TargetGroup properties - for a route. - properties: - defaultConfiguration: - description: defaultRouteConfiguration fallback configuration applied - to all routes, unless overridden by route-specific configurations. - properties: - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount of - time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health checks - on the targets. - type: string - healthCheckPort: - description: healthCheckPort The port to use to connect with - the target. - format: int32 - type: integer - healthCheckProtocol: - description: healthCheckProtocol The protocol to use to connect - with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols - are not supported for health checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, in seconds, - during which no response means a failed health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering an unhealthy - target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes to use - when checking for a successful response from a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering the target - unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target group - is of type IPv4 or IPv6. If unspecified, it will be automatically - inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups to - only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute of target - group - items: - description: TargetGroupAttribute defines target group attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetType: - description: targetType is the TargetType of TargetGroup. If unspecified, - it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - routeConfigurations: - description: routeConfigurations the route configuration for specific - routes - items: - description: RouteConfiguration defines the per route configuration - properties: - name: - description: name the name of the route, it should be in the - form of ROUTE:NAMESPACE:NAME - pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ - type: string - targetGroupProps: - description: targetGroupProps the target group specific properties - properties: - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount - of time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health - checks on the targets. - type: string - healthCheckPort: - description: healthCheckPort The port to use to connect - with the target. - format: int32 - type: integer - healthCheckProtocol: - description: healthCheckProtocol The protocol to use - to connect with the target. The GENEVE, TLS, UDP, - and TCP_UDP protocols are not supported for health - checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, - in seconds, during which no response means a failed - health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering - an unhealthy target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes - to use when checking for a successful response from - a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering - the target unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target - group is of type IPv4 or IPv6. If unspecified, it will - be automatically inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups - to only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute - of target group - items: - description: TargetGroupAttribute defines target group - attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetType: - description: targetType is the TargetType of TargetGroup. - If unspecified, it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - required: - - name - - targetGroupProps - type: object - type: array - targetReference: - description: targetReference the kubernetes object to attach the Target - Group settings to. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - - Defaults to "Service" when not specified. - type: string - name: - description: Name is the name of the referent. - type: string - required: - - name - type: object - required: - - targetReference - type: object - status: - description: TargetGroupConfigurationStatus defines the observed state - of TargetGroupConfiguration - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} From a7671b3beed2378b80c7b2fb29365514def1e2eb Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 17 Apr 2025 16:01:23 -0700 Subject: [PATCH 23/40] address pr comments --- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 298 ++++++++++++ ...way.k8s.aws_targetgroupconfigurations.yaml | 456 ++++++++++++++++++ pkg/gateway/model/model_build_loadbalancer.go | 31 +- 3 files changed, 771 insertions(+), 14 deletions(-) create mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml create mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml new file mode 100644 index 0000000000..98ed0757eb --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -0,0 +1,298 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: loadbalancerconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: LoadBalancerConfiguration + listKind: LoadBalancerConfigurationList + plural: loadbalancerconfigurations + singular: loadbalancerconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LoadBalancerConfigurationSpec defines the desired state of + LoadBalancerConfiguration + properties: + customerOwnedIpv4Pool: + description: |- + customerOwnedIpv4Pool [Application LoadBalancer] + is the ID of the customer-owned address for Application Load Balancers on Outposts pool. + type: string + enableICMP: + description: |- + EnableICMP [Network LoadBalancer] + enables the creation of security group rules to the managed security group + to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs + type: boolean + enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: + description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic + Indicates whether to evaluate inbound security group rules for traffic + sent to a Network Load Balancer through Amazon Web Services PrivateLink. + type: string + ipAddressType: + description: loadBalancerIPType defines what kind of load balancer + to provision (ipv4, dual stack) + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string + ipv4IPAMPoolId: + description: |- + IPv4IPAMPoolId [Application LoadBalancer] + defines the IPAM pool ID used for IPv4 Addresses on the ALB. + type: string + listenerConfigurations: + description: listenerConfigurations is an optional list of configurations + for each listener on LB + items: + properties: + alpnPolicy: + description: alpnPolicy an optional string that allows you to + configure ALPN policies on your Load Balancer + enum: + - HTTP1Only + - HTTP2Only + - HTTP2Optional + - HTTP2Preferred + - None + type: string + certificates: + description: certificates is the list of other certificates + to add to the listener. + items: + type: string + type: array + defaultCertificate: + description: |- + TODO: Add validation in admission webhook to make it required for secure protocols + defaultCertificate the cert arn to be used by default. + type: string + listenerAttributes: + description: listenerAttributes defines the attributes for the + listener + items: + description: ListenerAttribute defines listener attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + mutualAuthentication: + description: mutualAuthentication defines the mutual authentication + configuration information. + properties: + advertiseTrustStoreCaNames: + description: Indicates whether trust store CA certificate + names are advertised. + enum: + - "on" + - "off" + type: string + ignoreClientCertificateExpiry: + description: Indicates whether expired client certificates + are ignored. + type: boolean + mode: + description: The client certificate handling method. Options + are off , passthrough or verify + enum: + - "off" + - passthrough + - verify + type: string + trustStore: + description: The Name or ARN of the trust store. + type: string + required: + - mode + type: object + protocolPort: + description: protocolPort is identifier for the listener on + load balancer. It should be of the form PROTOCOL:PORT + pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ + type: string + sslPolicy: + description: sslPolicy is the security policy that defines which + protocols and ciphers are supported for secure listeners [HTTPS + or TLS listener]. + type: string + required: + - protocolPort + type: object + type: array + loadBalancerAttributes: + description: LoadBalancerAttributes defines the attribute of LB + items: + description: LoadBalancerAttribute defines LB attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + loadBalancerName: + description: loadBalancerName defines the name of the LB to provision. + If unspecified, it will be automatically generated. + maxLength: 32 + minLength: 1 + type: string + loadBalancerSubnets: + description: |- + loadBalancerSubnets is an optional list of subnet configurations to be used in the LB + This value takes precedence over loadBalancerSubnetsSelector if both are selected. + items: + description: SubnetConfiguration defines the subnet settings for + a Load Balancer. + properties: + eipAllocation: + description: eipAllocation [Network LoadBalancer] the EIP name + for this subnet. + type: string + identifier: + description: identifier [Application LoadBalancer / Network + LoadBalancer] name or id for the subnet + type: string + ipv6Allocation: + description: IPv6Allocation [Network LoadBalancer] the ipv6 + address to assign to this subnet. + type: string + privateIPv4Allocation: + description: privateIPv4Allocation [Network LoadBalancer] the + private ipv4 address to assign to this subnet. + type: string + sourceNAT: + description: SourceNatIPv6Prefix [Network LoadBalancer] The + IPv6 prefix to use for source NAT. Specify an IPv6 prefix + (/80 netmask) from the subnet CIDR block or auto_assigned + to use an IPv6 prefix selected at random from the subnet CIDR + block. + type: string + type: object + type: array + loadBalancerSubnetsSelector: + additionalProperties: + items: + type: string + type: array + description: |- + LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each + tag specified in the map key contains one of the values in the corresponding + value list. + type: object + manageBackendSecurityGroupRules: + description: |- + ManageBackendSecurityGroupRules [Application / Network LoadBalancer] + specifies whether you want the controller to configure security group rules on Node/Pod for traffic access + when you specify securityGroups + type: boolean + scheme: + description: scheme defines the type of LB to provision. If unspecified, + it will be automatically inferred. + enum: + - internal + - internet-facing + type: string + securityGroupPrefixes: + description: securityGroupPrefixes an optional list of prefixes that + are allowed to access the LB. + items: + type: string + type: array + securityGroups: + description: securityGroups an optional list of security group ids + or names to apply to the LB + items: + type: string + type: array + sourceRanges: + description: sourceRanges an optional list of CIDRs that are allowed + to access the LB. + items: + type: string + type: array + tags: + description: Tags defines list of Tags on LB. + items: + description: AWSTag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + vpcId: + description: vpcId is the ID of the VPC for the load balancer. + type: string + type: object + status: + description: LoadBalancerConfigurationStatus defines the observed state + of TargetGroupBinding + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml new file mode 100644 index 0000000000..c34856f29b --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -0,0 +1,456 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: targetgroupconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: TargetGroupConfiguration + listKind: TargetGroupConfigurationList + plural: targetgroupconfigurations + singular: targetgroupconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The Kubernetes Service's name + jsonPath: .spec.targetReference.name + name: SERVICE-NAME + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TargetGroupConfiguration is the Schema for defining TargetGroups + with an AWS ELB Gateway + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TargetGroupConfigurationSpec defines the TargetGroup properties + for a route. + properties: + defaultConfiguration: + description: defaultRouteConfiguration fallback configuration applied + to all routes, unless overridden by route-specific configurations. + properties: + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount of + time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health checks + on the targets. + type: string + healthCheckPort: + description: healthCheckPort The port to use to connect with + the target. + format: int32 + type: integer + healthCheckProtocol: + description: healthCheckProtocol The protocol to use to connect + with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols + are not supported for health checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, in seconds, + during which no response means a failed health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering an unhealthy + target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes to use + when checking for a successful response from a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering the target + unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target group + is of type IPv4 or IPv6. If unspecified, it will be automatically + inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups to + only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute of target + group + items: + description: TargetGroupAttribute defines target group attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetType: + description: targetType is the TargetType of TargetGroup. If unspecified, + it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + routeConfigurations: + description: routeConfigurations the route configuration for specific + routes + items: + description: RouteConfiguration defines the per route configuration + properties: + name: + description: name the name of the route, it should be in the + form of ROUTE:NAMESPACE:NAME + pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ + type: string + targetGroupProps: + description: targetGroupProps the target group specific properties + properties: + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount + of time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health + checks on the targets. + type: string + healthCheckPort: + description: healthCheckPort The port to use to connect + with the target. + format: int32 + type: integer + healthCheckProtocol: + description: healthCheckProtocol The protocol to use + to connect with the target. The GENEVE, TLS, UDP, + and TCP_UDP protocols are not supported for health + checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, + in seconds, during which no response means a failed + health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering + an unhealthy target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes + to use when checking for a successful response from + a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering + the target unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target + group is of type IPv4 or IPv6. If unspecified, it will + be automatically inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups + to only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute + of target group + items: + description: TargetGroupAttribute defines target group + attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetType: + description: targetType is the TargetType of TargetGroup. + If unspecified, it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + required: + - name + - targetGroupProps + type: object + type: array + targetReference: + description: targetReference the kubernetes object to attach the Target + Group settings to. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + + Defaults to "Service" when not specified. + type: string + name: + description: Name is the name of the referent. + type: string + required: + - name + type: object + required: + - targetReference + type: object + status: + description: TargetGroupConfigurationStatus defines the observed state + of TargetGroupConfiguration + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/gateway/model/model_build_loadbalancer.go b/pkg/gateway/model/model_build_loadbalancer.go index 45a5e1495c..3eddc0016b 100644 --- a/pkg/gateway/model/model_build_loadbalancer.go +++ b/pkg/gateway/model/model_build_loadbalancer.go @@ -4,7 +4,6 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "github.com/pkg/errors" "regexp" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" @@ -61,23 +60,31 @@ func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerSpec(scheme elbv } if lbModelBuilder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { - spec.EnablePrefixForIpv6SourceNat = lbModelBuilder.translateSourcePrefixEnabled(subnets.sourceIPv6NatEnabled) - - if lbConf.Spec.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic != nil { - spec.SecurityGroupsInboundRulesOnPrivateLink = (*elbv2model.SecurityGroupsInboundRulesOnPrivateLinkStatus)(lbConf.Spec.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic) - } + lbModelBuilder.addL4Fields(&spec, lbConf, subnets) } if lbModelBuilder.loadBalancerType == elbv2model.LoadBalancerTypeApplication { - spec.CustomerOwnedIPv4Pool = lbConf.Spec.CustomerOwnedIpv4Pool - spec.IPv4IPAMPool = lbConf.Spec.IPv4IPAMPoolId + lbModelBuilder.addL7Fields(&spec, lbConf) } return spec, nil } -func (lbModelBuilder *loadBalancerBuilderImpl) translateSourcePrefixEnabled(b bool) elbv2model.EnablePrefixForIpv6SourceNat { - if b { +func (lbModelBuilder *loadBalancerBuilderImpl) addL4Fields(spec *elbv2model.LoadBalancerSpec, lbConf *elbv2gw.LoadBalancerConfiguration, subnets buildLoadBalancerSubnetsOutput) { + spec.EnablePrefixForIpv6SourceNat = lbModelBuilder.translateSourcePrefixEnabled(subnets.sourceIPv6NatEnabled) + + if lbConf.Spec.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic != nil { + spec.SecurityGroupsInboundRulesOnPrivateLink = (*elbv2model.SecurityGroupsInboundRulesOnPrivateLinkStatus)(lbConf.Spec.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic) + } +} + +func (lbModelBuilder *loadBalancerBuilderImpl) addL7Fields(spec *elbv2model.LoadBalancerSpec, lbConf *elbv2gw.LoadBalancerConfiguration) { + spec.CustomerOwnedIPv4Pool = lbConf.Spec.CustomerOwnedIpv4Pool + spec.IPv4IPAMPool = lbConf.Spec.IPv4IPAMPoolId +} + +func (lbModelBuilder *loadBalancerBuilderImpl) translateSourcePrefixEnabled(sourceNATEnabled bool) elbv2model.EnablePrefixForIpv6SourceNat { + if sourceNATEnabled { return elbv2model.EnablePrefixForIpv6SourceNatOn } return elbv2model.EnablePrefixForIpv6SourceNatOff @@ -86,10 +93,6 @@ func (lbModelBuilder *loadBalancerBuilderImpl) translateSourcePrefixEnabled(b bo func (lbModelBuilder *loadBalancerBuilderImpl) buildLoadBalancerName(lbConf *elbv2gw.LoadBalancerConfiguration, gw *gwv1.Gateway, scheme elbv2model.LoadBalancerScheme) (string, error) { if lbConf.Spec.LoadBalancerName != nil { name := *lbConf.Spec.LoadBalancerName - // The name of the loadbalancer can only have up to 32 characters - if len(name) > 32 { - return "", errors.New("load balancer name cannot be longer than 32 characters") - } return name, nil } uuidHash := sha256.New() From 98a5a04a26e25c7a55344bf4e8719423316ddded Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 17 Apr 2025 16:08:54 -0700 Subject: [PATCH 24/40] . --- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 298 ------------ ...way.k8s.aws_targetgroupconfigurations.yaml | 456 ------------------ 2 files changed, 754 deletions(-) delete mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml delete mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml deleted file mode 100644 index 98ed0757eb..0000000000 --- a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ /dev/null @@ -1,298 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: loadbalancerconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: LoadBalancerConfiguration - listKind: LoadBalancerConfigurationList - plural: loadbalancerconfigurations - singular: loadbalancerconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LoadBalancerConfigurationSpec defines the desired state of - LoadBalancerConfiguration - properties: - customerOwnedIpv4Pool: - description: |- - customerOwnedIpv4Pool [Application LoadBalancer] - is the ID of the customer-owned address for Application Load Balancers on Outposts pool. - type: string - enableICMP: - description: |- - EnableICMP [Network LoadBalancer] - enables the creation of security group rules to the managed security group - to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs - type: boolean - enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: - description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic - Indicates whether to evaluate inbound security group rules for traffic - sent to a Network Load Balancer through Amazon Web Services PrivateLink. - type: string - ipAddressType: - description: loadBalancerIPType defines what kind of load balancer - to provision (ipv4, dual stack) - enum: - - ipv4 - - dualstack - - dualstack-without-public-ipv4 - type: string - ipv4IPAMPoolId: - description: |- - IPv4IPAMPoolId [Application LoadBalancer] - defines the IPAM pool ID used for IPv4 Addresses on the ALB. - type: string - listenerConfigurations: - description: listenerConfigurations is an optional list of configurations - for each listener on LB - items: - properties: - alpnPolicy: - description: alpnPolicy an optional string that allows you to - configure ALPN policies on your Load Balancer - enum: - - HTTP1Only - - HTTP2Only - - HTTP2Optional - - HTTP2Preferred - - None - type: string - certificates: - description: certificates is the list of other certificates - to add to the listener. - items: - type: string - type: array - defaultCertificate: - description: |- - TODO: Add validation in admission webhook to make it required for secure protocols - defaultCertificate the cert arn to be used by default. - type: string - listenerAttributes: - description: listenerAttributes defines the attributes for the - listener - items: - description: ListenerAttribute defines listener attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - mutualAuthentication: - description: mutualAuthentication defines the mutual authentication - configuration information. - properties: - advertiseTrustStoreCaNames: - description: Indicates whether trust store CA certificate - names are advertised. - enum: - - "on" - - "off" - type: string - ignoreClientCertificateExpiry: - description: Indicates whether expired client certificates - are ignored. - type: boolean - mode: - description: The client certificate handling method. Options - are off , passthrough or verify - enum: - - "off" - - passthrough - - verify - type: string - trustStore: - description: The Name or ARN of the trust store. - type: string - required: - - mode - type: object - protocolPort: - description: protocolPort is identifier for the listener on - load balancer. It should be of the form PROTOCOL:PORT - pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ - type: string - sslPolicy: - description: sslPolicy is the security policy that defines which - protocols and ciphers are supported for secure listeners [HTTPS - or TLS listener]. - type: string - required: - - protocolPort - type: object - type: array - loadBalancerAttributes: - description: LoadBalancerAttributes defines the attribute of LB - items: - description: LoadBalancerAttribute defines LB attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - loadBalancerName: - description: loadBalancerName defines the name of the LB to provision. - If unspecified, it will be automatically generated. - maxLength: 32 - minLength: 1 - type: string - loadBalancerSubnets: - description: |- - loadBalancerSubnets is an optional list of subnet configurations to be used in the LB - This value takes precedence over loadBalancerSubnetsSelector if both are selected. - items: - description: SubnetConfiguration defines the subnet settings for - a Load Balancer. - properties: - eipAllocation: - description: eipAllocation [Network LoadBalancer] the EIP name - for this subnet. - type: string - identifier: - description: identifier [Application LoadBalancer / Network - LoadBalancer] name or id for the subnet - type: string - ipv6Allocation: - description: IPv6Allocation [Network LoadBalancer] the ipv6 - address to assign to this subnet. - type: string - privateIPv4Allocation: - description: privateIPv4Allocation [Network LoadBalancer] the - private ipv4 address to assign to this subnet. - type: string - sourceNAT: - description: SourceNatIPv6Prefix [Network LoadBalancer] The - IPv6 prefix to use for source NAT. Specify an IPv6 prefix - (/80 netmask) from the subnet CIDR block or auto_assigned - to use an IPv6 prefix selected at random from the subnet CIDR - block. - type: string - type: object - type: array - loadBalancerSubnetsSelector: - additionalProperties: - items: - type: string - type: array - description: |- - LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each - tag specified in the map key contains one of the values in the corresponding - value list. - type: object - manageBackendSecurityGroupRules: - description: |- - ManageBackendSecurityGroupRules [Application / Network LoadBalancer] - specifies whether you want the controller to configure security group rules on Node/Pod for traffic access - when you specify securityGroups - type: boolean - scheme: - description: scheme defines the type of LB to provision. If unspecified, - it will be automatically inferred. - enum: - - internal - - internet-facing - type: string - securityGroupPrefixes: - description: securityGroupPrefixes an optional list of prefixes that - are allowed to access the LB. - items: - type: string - type: array - securityGroups: - description: securityGroups an optional list of security group ids - or names to apply to the LB - items: - type: string - type: array - sourceRanges: - description: sourceRanges an optional list of CIDRs that are allowed - to access the LB. - items: - type: string - type: array - tags: - description: Tags defines list of Tags on LB. - items: - description: AWSTag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - vpcId: - description: vpcId is the ID of the VPC for the load balancer. - type: string - type: object - status: - description: LoadBalancerConfigurationStatus defines the observed state - of TargetGroupBinding - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml deleted file mode 100644 index c34856f29b..0000000000 --- a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml +++ /dev/null @@ -1,456 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: targetgroupconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: TargetGroupConfiguration - listKind: TargetGroupConfigurationList - plural: targetgroupconfigurations - singular: targetgroupconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: The Kubernetes Service's name - jsonPath: .spec.targetReference.name - name: SERVICE-NAME - type: string - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: TargetGroupConfiguration is the Schema for defining TargetGroups - with an AWS ELB Gateway - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TargetGroupConfigurationSpec defines the TargetGroup properties - for a route. - properties: - defaultConfiguration: - description: defaultRouteConfiguration fallback configuration applied - to all routes, unless overridden by route-specific configurations. - properties: - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount of - time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health checks - on the targets. - type: string - healthCheckPort: - description: healthCheckPort The port to use to connect with - the target. - format: int32 - type: integer - healthCheckProtocol: - description: healthCheckProtocol The protocol to use to connect - with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols - are not supported for health checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, in seconds, - during which no response means a failed health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering an unhealthy - target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes to use - when checking for a successful response from a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering the target - unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target group - is of type IPv4 or IPv6. If unspecified, it will be automatically - inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups to - only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute of target - group - items: - description: TargetGroupAttribute defines target group attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetType: - description: targetType is the TargetType of TargetGroup. If unspecified, - it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - routeConfigurations: - description: routeConfigurations the route configuration for specific - routes - items: - description: RouteConfiguration defines the per route configuration - properties: - name: - description: name the name of the route, it should be in the - form of ROUTE:NAMESPACE:NAME - pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ - type: string - targetGroupProps: - description: targetGroupProps the target group specific properties - properties: - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount - of time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health - checks on the targets. - type: string - healthCheckPort: - description: healthCheckPort The port to use to connect - with the target. - format: int32 - type: integer - healthCheckProtocol: - description: healthCheckProtocol The protocol to use - to connect with the target. The GENEVE, TLS, UDP, - and TCP_UDP protocols are not supported for health - checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, - in seconds, during which no response means a failed - health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering - an unhealthy target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes - to use when checking for a successful response from - a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering - the target unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target - group is of type IPv4 or IPv6. If unspecified, it will - be automatically inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups - to only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute - of target group - items: - description: TargetGroupAttribute defines target group - attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetType: - description: targetType is the TargetType of TargetGroup. - If unspecified, it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - required: - - name - - targetGroupProps - type: object - type: array - targetReference: - description: targetReference the kubernetes object to attach the Target - Group settings to. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - - Defaults to "Service" when not specified. - type: string - name: - description: Name is the name of the referent. - type: string - required: - - name - type: object - required: - - targetReference - type: object - status: - description: TargetGroupConfigurationStatus defines the observed state - of TargetGroupConfiguration - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} From b7b38a7d14130df766b7dc37ac47e34c459f3a51 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Fri, 18 Apr 2025 11:40:44 -0700 Subject: [PATCH 25/40] refactor to use default source ranges --- .../v1beta1/loadbalancerconfig_types.go | 2 +- config/crd/gateway/gateway-crds.yaml | 2 +- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 2 +- config/rbac/role.yaml | 24 ------- .../model/model_build_security_group.go | 14 ++-- .../model/model_build_security_group_test.go | 64 +++++++++++++++++++ pkg/gateway/routeutils/mock_route.go | 5 ++ 7 files changed, 82 insertions(+), 31 deletions(-) diff --git a/apis/gateway/v1beta1/loadbalancerconfig_types.go b/apis/gateway/v1beta1/loadbalancerconfig_types.go index 799e3e9208..c9a3e164a5 100644 --- a/apis/gateway/v1beta1/loadbalancerconfig_types.go +++ b/apis/gateway/v1beta1/loadbalancerconfig_types.go @@ -73,7 +73,7 @@ type SubnetConfiguration struct { // SourceNatIPv6Prefix [Network LoadBalancer] The IPv6 prefix to use for source NAT. Specify an IPv6 prefix (/80 netmask) from the subnet CIDR block or auto_assigned to use an IPv6 prefix selected at random from the subnet CIDR block. // +optional - SourceNatIPv6Prefix *string `json:"sourceNAT,omitempty"` + SourceNatIPv6Prefix *string `json:"sourceNatIPv6Prefix,omitempty"` } // +kubebuilder:validation:Enum=HTTP1Only;HTTP2Only;HTTP2Optional;HTTP2Preferred;None diff --git a/config/crd/gateway/gateway-crds.yaml b/config/crd/gateway/gateway-crds.yaml index 27186e6969..846f287ce9 100644 --- a/config/crd/gateway/gateway-crds.yaml +++ b/config/crd/gateway/gateway-crds.yaml @@ -205,7 +205,7 @@ spec: description: privateIPv4Allocation [Network LoadBalancer] the private ipv4 address to assign to this subnet. type: string - sourceNAT: + sourceNatIPv6Prefix: description: SourceNatIPv6Prefix [Network LoadBalancer] The IPv6 prefix to use for source NAT. Specify an IPv6 prefix (/80 netmask) from the subnet CIDR block or auto_assigned diff --git a/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml index 98ed0757eb..f146a9eb99 100644 --- a/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ b/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -206,7 +206,7 @@ spec: description: privateIPv4Allocation [Network LoadBalancer] the private ipv4 address to assign to this subnet. type: string - sourceNAT: + sourceNatIPv6Prefix: description: SourceNatIPv6Prefix [Network LoadBalancer] The IPv6 prefix to use for source NAT. Specify an IPv6 prefix (/80 netmask) from the subnet CIDR block or auto_assigned diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0d26c1555f..313f45d488 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -194,28 +194,6 @@ rules: - get - patch - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses - verbs: - - get - - list - - watch -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses/finalizers - verbs: - - update -- apiGroups: - - gateway.networking.k8s.io - resources: - - gatewayclasses/status - verbs: - - get - - patch - - update - apiGroups: - gateway.networking.k8s.io resources: @@ -223,14 +201,12 @@ rules: verbs: - get - list - - patch - watch - apiGroups: - gateway.networking.k8s.io resources: - gateways/finalizers verbs: - - patch - update - apiGroups: - gateway.networking.k8s.io diff --git a/pkg/gateway/model/model_build_security_group.go b/pkg/gateway/model/model_build_security_group.go index 10e576340f..d657398383 100644 --- a/pkg/gateway/model/model_build_security_group.go +++ b/pkg/gateway/model/model_build_security_group.go @@ -177,19 +177,25 @@ func (builder *securityGroupBuilderImpl) buildManagedSecurityGroupName(gw *gwv1. func (builder *securityGroupBuilderImpl) buildManagedSecurityGroupIngressPermissions(lbConf *elbv2gw.LoadBalancerConfiguration, routes map[int][]routeutils.RouteDescriptor, ipAddressType elbv2model.IPAddressType) []ec2model.IPPermission { var permissions []ec2model.IPPermission - var sourceRanges []string + // Default to 0.0.0.0/0 and ::/0 + // If user specified actual ranges, then these values will be overridden. + // TODO - Document this + sourceRanges := []string{ + "0.0.0.0/0", + "::/0", + } var prefixes []string var enableICMP bool - if lbConf.Spec.SourceRanges != nil { + if lbConf != nil && lbConf.Spec.SourceRanges != nil { sourceRanges = *lbConf.Spec.SourceRanges } - if lbConf.Spec.SecurityGroupPrefixes != nil { + if lbConf != nil && lbConf.Spec.SecurityGroupPrefixes != nil { prefixes = *lbConf.Spec.SecurityGroupPrefixes } - if lbConf.Spec.EnableICMP { + if lbConf != nil && lbConf.Spec.EnableICMP { enableICMP = true } diff --git a/pkg/gateway/model/model_build_security_group_test.go b/pkg/gateway/model/model_build_security_group_test.go index a3f15ebb22..a7bfacbed7 100644 --- a/pkg/gateway/model/model_build_security_group_test.go +++ b/pkg/gateway/model/model_build_security_group_test.go @@ -325,6 +325,31 @@ func Test_BuildSecurityGroups_BuildManagedSecurityGroupIngressPermissions(t *tes lbConf: &elbv2gw.LoadBalancerConfiguration{}, expected: make([]ec2model.IPPermission, 0), }, + { + name: "ipv4 - tcp - with default source ranges", + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{}, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "0.0.0.0/0", + }, + }, + }, + }, + }, { name: "ipv4 - tcp - with source range", lbConf: &elbv2gw.LoadBalancerConfiguration{ @@ -651,6 +676,45 @@ func Test_BuildSecurityGroups_BuildManagedSecurityGroupIngressPermissions(t *tes }, }, }, + { + name: "ipv6 - with default source ranges", + ipAddressType: elbv2model.IPAddressTypeDualStack, + lbConf: &elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{}, + }, + routes: map[int][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + }, + &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + }, + }, + }, + expected: []ec2model.IPPermission{ + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPRanges: []ec2model.IPRange{ + { + CIDRIP: "0.0.0.0/0", + }, + }, + }, + { + IPProtocol: "tcp", + FromPort: awssdk.Int32(80), + ToPort: awssdk.Int32(80), + IPv6Range: []ec2model.IPv6Range{ + { + CIDRIPv6: "::/0", + }, + }, + }, + }, + }, { name: "ipv6 - with source range", ipAddressType: elbv2model.IPAddressTypeDualStack, diff --git a/pkg/gateway/routeutils/mock_route.go b/pkg/gateway/routeutils/mock_route.go index 898e4d8d50..13b438c58e 100644 --- a/pkg/gateway/routeutils/mock_route.go +++ b/pkg/gateway/routeutils/mock_route.go @@ -9,6 +9,11 @@ type MockRoute struct { Kind string } +func (m *MockRoute) GetBackendRefs() []gwv1.BackendRef { + //TODO implement me + panic("implement me") +} + func (m *MockRoute) GetRouteNamespacedName() types.NamespacedName { //TODO implement me panic("implement me") From 0b7788082da7d467bd58d87d1d31ec9459e8a2e4 Mon Sep 17 00:00:00 2001 From: shraddha bang Date: Fri, 18 Apr 2025 14:15:50 -0700 Subject: [PATCH 26/40] [feat: gw api] Add eventhandler for the gateway resource --- .../gateway/eventhandlers/gateway_events.go | 70 +++++++++++++++++++ controllers/gateway/gateway_controller.go | 5 ++ 2 files changed, 75 insertions(+) create mode 100644 controllers/gateway/eventhandlers/gateway_events.go diff --git a/controllers/gateway/eventhandlers/gateway_events.go b/controllers/gateway/eventhandlers/gateway_events.go new file mode 100644 index 0000000000..364b2af0e8 --- /dev/null +++ b/controllers/gateway/eventhandlers/gateway_events.go @@ -0,0 +1,70 @@ +package eventhandlers + +import ( + "context" + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// NewEnqueueRequestsForGatewayEvent creates handler for Gateway resources +func NewEnqueueRequestsForGatewayEventHandler( + k8sClient client.Client, eventRecorder record.EventRecorder, gwController string, logger logr.Logger) handler.TypedEventHandler[*gwv1.Gateway, reconcile.Request] { + return &enqueueRequestsForGatewayEvent{ + k8sClient: k8sClient, + eventRecorder: eventRecorder, + gwController: gwController, + logger: logger, + } +} + +var _ handler.TypedEventHandler[*gwv1.Gateway, reconcile.Request] = (*enqueueRequestsForGatewayEvent)(nil) + +// enqueueRequestsForGatewayEvent handles GatewayClass events +type enqueueRequestsForGatewayEvent struct { + k8sClient client.Client + eventRecorder record.EventRecorder + gwController string + logger logr.Logger +} + +func (h *enqueueRequestsForGatewayEvent) Create(ctx context.Context, e event.TypedCreateEvent[*gwv1.Gateway], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gw := e.Object + h.logger.V(1).Info("enqueue gateway create event", "gateway", k8s.NamespacedName(gw)) + h.enqueueImpactedGateway(ctx, gw, queue) +} + +func (h *enqueueRequestsForGatewayEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*gwv1.Gateway], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gw := e.ObjectNew + h.logger.V(1).Info("enqueue gateway update event", "gateway", k8s.NamespacedName(gw)) + h.enqueueImpactedGateway(ctx, gw, queue) +} + +func (h *enqueueRequestsForGatewayEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*gwv1.Gateway], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gw := e.Object + h.logger.V(1).Info("enqueue gateway delete event", "gateway", k8s.NamespacedName(gw)) + h.enqueueImpactedGateway(ctx, gw, queue) +} + +func (h *enqueueRequestsForGatewayEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gwv1.Gateway], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + gw := e.Object + h.logger.V(1).Info("enqueue gateway delete event", "gateway", k8s.NamespacedName(gw)) + h.enqueueImpactedGateway(ctx, gw, queue) +} + +func (h *enqueueRequestsForGatewayEvent) enqueueImpactedGateway(ctx context.Context, gw *gwv1.Gateway, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if gw == nil { + return + } + if IsGatewayManagedByLBController(ctx, h.k8sClient, gw, h.gwController) { + h.logger.V(1).Info("enqueue gateway", + "gateway", k8s.NamespacedName(gw)) + queue.Add(reconcile.Request{NamespacedName: k8s.NamespacedName(gw)}) + } +} diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 66f223be1d..d6565122f6 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -331,6 +331,11 @@ func (r *gatewayReconciler) SetupWatches(ctx context.Context, c controller.Contr func (r *gatewayReconciler) setupCommonGatewayControllerWatches(ctrl controller.Controller, mgr ctrl.Manager) error { loggerPrefix := r.logger.WithName("eventHandlers") + + gwEventHandler := eventhandlers.NewEnqueueRequestsForGatewayEventHandler(r.k8sClient, r.eventRecorder, r.controllerName, + loggerPrefix.WithName("Gateway")) + ctrl.Watch(source.Kind(mgr.GetCache(), &gwv1.Gateway{}, gwEventHandler)) + gwClassEventChan := make(chan event.TypedGenericEvent[*gwv1.GatewayClass]) lbConfigEventChan := make(chan event.TypedGenericEvent[*elbv2gw.LoadBalancerConfiguration]) From 080700985d5dcf01c0b9e6c6183e8fb51670216a Mon Sep 17 00:00:00 2001 From: u-kai <76635578+u-kai@users.noreply.github.com> Date: Sat, 29 Mar 2025 19:44:09 +0900 Subject: [PATCH 27/40] Allow the same certificate to be specified for both the default and SNI certificate --- pkg/ingress/model_builder.go | 19 +- pkg/ingress/model_builder_test.go | 289 ++++++++++++++++++++++++++++++ 2 files changed, 306 insertions(+), 2 deletions(-) diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index ba519044fa..09907b0da8 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -332,12 +332,22 @@ func (t *defaultModelBuildTask) mergeListenPortConfigs(_ context.Context, listen var mergedSSLPolicy *string var mergedTLSCerts []string + + // Set the default cert as the first cert + // This process allows the same certificate to be specified for both the default certificate and the SNI certificate. + for _, cfg := range listenPortConfigs { + if len(cfg.listenPortConfig.tlsCerts) > 0 { + mergedTLSCerts = append(mergedTLSCerts, cfg.listenPortConfig.tlsCerts[0]) + break + } + } + mergedTLSCertsSet := sets.NewString() var mergedMtlsAttributesProvider *types.NamespacedName var mergedMtlsAttributes *elbv2model.MutualAuthenticationAttributes - for _, cfg := range listenPortConfigs { + for i, cfg := range listenPortConfigs { if mergedProtocolProvider == nil { mergedProtocolProvider = &cfg.ingKey mergedProtocol = cfg.listenPortConfig.protocol @@ -380,7 +390,12 @@ func (t *defaultModelBuildTask) mergeListenPortConfigs(_ context.Context, listen } } - for _, cert := range cfg.listenPortConfig.tlsCerts { + for j, cert := range cfg.listenPortConfig.tlsCerts { + // Ignore the first cert as it is the default cert + // Default cert is already added to the mergedTLSCerts + if i == 0 && j == 0 { + continue + } if mergedTLSCertsSet.Has(cert) { continue } diff --git a/pkg/ingress/model_builder_test.go b/pkg/ingress/model_builder_test.go index 82679efc65..f33f5dad69 100644 --- a/pkg/ingress/model_builder_test.go +++ b/pkg/ingress/model_builder_test.go @@ -1004,6 +1004,295 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } } +}`, + }, + { + name: "Ingress - using acm and internet-facing case with the same acm certificate for default and sni listener", + env: env{ + svcs: []*corev1.Service{ns_1_svc_1, ns_1_svc_2, ns_1_svc_3}, + }, + fields: fields{ + resolveViaDiscoveryCalls: []resolveViaDiscoveryCall{resolveViaDiscoveryCallForInternetFacingLB}, + listLoadBalancersCalls: []listLoadBalancersCall{listLoadBalancerCallForEmptyLB}, + enableBackendSG: true, + }, + args: args{ + ingGroup: Group{ + ID: GroupID{Namespace: "ns-1", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns-1", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/certificate-arn": "arn:aws:acm:us-east-1:9999999:certificate/11111111,arn:aws:acm:us-east-1:9999999:certificate/33333333,arn:aws:acm:us-east-1:9999999:certificate/22222222,,arn:aws:acm:us-east-1:9999999:certificate/11111111", + "alb.ingress.kubernetes.io/mutual-authentication": `[{"port":443,"mode":"off"}]`, + }, + }, + Spec: networking.IngressSpec{ + Rules: []networking.IngressRule{ + { + Host: "app-1.example.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/svc-1", + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: ns_1_svc_1.Name, + Port: networking.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }, + { + Path: "/svc-2", + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: ns_1_svc_2.Name, + Port: networking.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }, + }, + }, + }, + }, + { + Host: "app-2.example.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/svc-3", + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: ns_1_svc_3.Name, + Port: networking.ServiceBackendPort{ + Name: "https", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantStackPatch: ` +{ + "resources": { + "AWS::EC2::SecurityGroup": { + "ManagedLBSecurityGroup": { + "spec": { + "ingress": [ + { + "fromPort": 443, + "ipProtocol": "tcp", + "ipRanges": [ + { + "cidrIP": "0.0.0.0/0" + } + ], + "toPort": 443 + } + ] + } + } + }, + "AWS::ElasticLoadBalancingV2::Listener": { + "443": { + "spec": { + "certificates": [ + { + "certificateARN": "arn:aws:acm:us-east-1:9999999:certificate/11111111" + }, + { + "certificateARN": "arn:aws:acm:us-east-1:9999999:certificate/33333333" + }, + { + "certificateARN": "arn:aws:acm:us-east-1:9999999:certificate/22222222" + }, + { + "certificateARN": "arn:aws:acm:us-east-1:9999999:certificate/11111111" + } + ], + "defaultActions": [ + { + "fixedResponseConfig": { + "contentType": "text/plain", + "statusCode": "404" + }, + "type": "fixed-response" + } + ], + "loadBalancerARN": { + "$ref": "#/resources/AWS::ElasticLoadBalancingV2::LoadBalancer/LoadBalancer/status/loadBalancerARN" + }, + "port": 443, + "protocol": "HTTPS", + "sslPolicy": "ELBSecurityPolicy-2016-08", + "mutualAuthentication" : { + "mode" : "off", + "trustStoreArn": "" + } + } + }, + "80": null + }, + "AWS::ElasticLoadBalancingV2::ListenerRule": { + "443:1": { + "spec": { + "actions": [ + { + "forwardConfig": { + "targetGroups": [ + { + "targetGroupARN": { + "$ref": "#/resources/AWS::ElasticLoadBalancingV2::TargetGroup/ns-1/ing-1-svc-1:http/status/targetGroupARN" + } + } + ] + }, + "type": "forward" + } + ], + "conditions": [ + { + "field": "host-header", + "hostHeaderConfig": { + "values": [ + "app-1.example.com" + ] + } + }, + { + "field": "path-pattern", + "pathPatternConfig": { + "values": [ + "/svc-1" + ] + } + } + ], + "listenerARN": { + "$ref": "#/resources/AWS::ElasticLoadBalancingV2::Listener/443/status/listenerARN" + }, + "priority": 1 + } + }, + "443:2": { + "spec": { + "actions": [ + { + "forwardConfig": { + "targetGroups": [ + { + "targetGroupARN": { + "$ref": "#/resources/AWS::ElasticLoadBalancingV2::TargetGroup/ns-1/ing-1-svc-2:http/status/targetGroupARN" + } + } + ] + }, + "type": "forward" + } + ], + "conditions": [ + { + "field": "host-header", + "hostHeaderConfig": { + "values": [ + "app-1.example.com" + ] + } + }, + { + "field": "path-pattern", + "pathPatternConfig": { + "values": [ + "/svc-2" + ] + } + } + ], + "listenerARN": { + "$ref": "#/resources/AWS::ElasticLoadBalancingV2::Listener/443/status/listenerARN" + }, + "priority": 2 + } + }, + "443:3": { + "spec": { + "actions": [ + { + "forwardConfig": { + "targetGroups": [ + { + "targetGroupARN": { + "$ref": "#/resources/AWS::ElasticLoadBalancingV2::TargetGroup/ns-1/ing-1-svc-3:https/status/targetGroupARN" + } + } + ] + }, + "type": "forward" + } + ], + "conditions": [ + { + "field": "host-header", + "hostHeaderConfig": { + "values": [ + "app-2.example.com" + ] + } + }, + { + "field": "path-pattern", + "pathPatternConfig": { + "values": [ + "/svc-3" + ] + } + } + ], + "listenerARN": { + "$ref": "#/resources/AWS::ElasticLoadBalancingV2::Listener/443/status/listenerARN" + }, + "priority": 3 + } + }, + "80:1": null, + "80:2": null, + "80:3": null + }, + "AWS::ElasticLoadBalancingV2::LoadBalancer": { + "LoadBalancer": { + "spec": { + "name": "k8s-ns1-ing1-159dd7a143", + "scheme": "internet-facing", + "subnetMapping": [ + { + "subnetID": "subnet-c" + }, + { + "subnetID": "subnet-d" + } + ] + } + } + } + } }`, }, { From 6c8b246da7c81de28bfd848b6648c01913ba2bd0 Mon Sep 17 00:00:00 2001 From: u-kai <76635578+u-kai@users.noreply.github.com> Date: Sat, 29 Mar 2025 20:03:41 +0900 Subject: [PATCH 28/40] update docs --- docs/guide/ingress/annotations.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/guide/ingress/annotations.md b/docs/guide/ingress/annotations.md index af72f5a599..25e99a782c 100644 --- a/docs/guide/ingress/annotations.md +++ b/docs/guide/ingress/annotations.md @@ -800,7 +800,9 @@ TLS support can be controlled with the following annotations: - `alb.ingress.kubernetes.io/certificate-arn` specifies the ARN of one or more certificate managed by [AWS Certificate Manager](https://aws.amazon.com/certificate-manager) !!!tip "" - The first certificate in the list will be added as default certificate. And remaining certificate will be added to the optional certificate list. + The first certificate in the list will be added as the default certificate. + The remaining certificates will be added to the optional SNI certificate list. + If the same certificate as the default certificate is also listed again (either explicitly in the list or via annotations from other IngressGroup members), it will still be added to the SNI list as well. See [SSL Certificates](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#https-listener-certificates) for more details. !!!tip "Certificate Discovery" From 87161a56f5e03923bf5d1494241f1686c7e8133d Mon Sep 17 00:00:00 2001 From: u-kai <76635578+u-kai@users.noreply.github.com> Date: Sat, 29 Mar 2025 20:08:47 +0900 Subject: [PATCH 29/40] fix default cert member index --- pkg/ingress/model_builder.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index 09907b0da8..50cd258771 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -335,9 +335,11 @@ func (t *defaultModelBuildTask) mergeListenPortConfigs(_ context.Context, listen // Set the default cert as the first cert // This process allows the same certificate to be specified for both the default certificate and the SNI certificate. - for _, cfg := range listenPortConfigs { + var defaultCertMemberIndex int + for i, cfg := range listenPortConfigs { if len(cfg.listenPortConfig.tlsCerts) > 0 { mergedTLSCerts = append(mergedTLSCerts, cfg.listenPortConfig.tlsCerts[0]) + defaultCertMemberIndex = i break } } @@ -391,9 +393,8 @@ func (t *defaultModelBuildTask) mergeListenPortConfigs(_ context.Context, listen } for j, cert := range cfg.listenPortConfig.tlsCerts { - // Ignore the first cert as it is the default cert - // Default cert is already added to the mergedTLSCerts - if i == 0 && j == 0 { + // The first certificate is ignored as it is the default certificate, which has already been added to the mergedTLSCerts. + if i == defaultCertMemberIndex && j == 0 { continue } if mergedTLSCertsSet.Has(cert) { From 1ffc3e715da14f753a9eb1874d9912b70e07e2b5 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 21 Apr 2025 17:37:40 -0700 Subject: [PATCH 30/40] [feat: gw-api] Creating Target Group + TGB from Gateway spec (#4150) * [gw api] tg creation * fixes to get tg + tgb working * make logging less noisy * refactor duplicated consts * tg tests * fix interval and timeout to appease alb * use shared constant for health check port * refactor multicluster to target group props * refactor to use route kind enum * infer target group type from route --- .../v1beta1/targetgroupconfig_types.go | 52 +- apis/gateway/v1beta1/zz_generated.deepcopy.go | 7 +- config/crd/gateway/gateway-crds.yaml | 66 +- ...way.k8s.aws_targetgroupconfigurations.yaml | 66 +- controllers/gateway/gateway_controller.go | 3 + main.go | 4 +- pkg/gateway/model/base_model_builder.go | 28 + .../model/model_build_security_group.go | 15 - pkg/gateway/model/model_build_target_group.go | 617 ++++++++++++++++++ .../model/model_build_target_group_test.go | 362 ++++++++++ pkg/gateway/model/utilities.go | 20 + pkg/gateway/routeutils/backend.go | 6 +- pkg/gateway/routeutils/constants.go | 16 +- pkg/gateway/routeutils/descriptor.go | 2 +- pkg/gateway/routeutils/grpc.go | 4 +- pkg/gateway/routeutils/grpc_test.go | 2 +- pkg/gateway/routeutils/http.go | 4 +- pkg/gateway/routeutils/http_test.go | 2 +- .../routeutils/listener_attachment_helper.go | 20 +- .../listener_attachment_helper_test.go | 12 +- pkg/gateway/routeutils/loader.go | 20 +- pkg/gateway/routeutils/loader_test.go | 20 +- pkg/gateway/routeutils/mock_route.go | 12 +- .../routeutils/route_attachment_helper.go | 13 +- .../route_attachment_helper_test.go | 9 +- .../routeutils/route_listener_mapper.go | 16 +- .../routeutils/route_listener_mapper_test.go | 2 + pkg/gateway/routeutils/tcp.go | 5 +- pkg/gateway/routeutils/tcp_test.go | 2 +- pkg/gateway/routeutils/tls.go | 4 +- pkg/gateway/routeutils/tls_test.go | 2 +- pkg/gateway/routeutils/udp.go | 4 +- pkg/gateway/routeutils/udp_test.go | 2 +- pkg/gateway/routeutils/utils.go | 4 +- pkg/gateway/routeutils/utils_test.go | 2 +- pkg/ingress/model_build_frontend_nlb.go | 7 +- pkg/ingress/model_build_target_group.go | 15 +- pkg/service/model_build_load_balancer_test.go | 4 +- pkg/service/model_build_target_group.go | 35 +- pkg/service/model_build_target_group_test.go | 31 +- pkg/service/model_builder.go | 2 +- pkg/shared_constants/attributes.go | 5 + pkg/shared_constants/healthcheck.go | 6 + 43 files changed, 1359 insertions(+), 171 deletions(-) create mode 100644 pkg/gateway/model/model_build_target_group.go create mode 100644 pkg/gateway/model/model_build_target_group_test.go create mode 100644 pkg/gateway/model/utilities.go create mode 100644 pkg/shared_constants/healthcheck.go diff --git a/apis/gateway/v1beta1/targetgroupconfig_types.go b/apis/gateway/v1beta1/targetgroupconfig_types.go index af9def9608..c5fe018f16 100644 --- a/apis/gateway/v1beta1/targetgroupconfig_types.go +++ b/apis/gateway/v1beta1/targetgroupconfig_types.go @@ -65,9 +65,10 @@ type HealthCheckConfiguration struct { // +optional HealthCheckPath *string `json:"healthCheckPath,omitempty"` - // healthCheckPort The port to use to connect with the target. + // healthCheckPort The port the load balancer uses when performing health checks on targets. + // The default is to use the port on which each target receives traffic from the load balancer. // +optional - HealthCheckPort *int32 `json:"healthCheckPort,omitempty"` + HealthCheckPort *string `json:"healthCheckPort,omitempty"` // healthCheckProtocol The protocol to use to connect with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols are not supported for health checks. // +optional @@ -116,6 +117,18 @@ const ( TargetGroupHealthCheckProtocolTCP TargetGroupHealthCheckProtocol = "tcp" ) +// +kubebuilder:validation:Enum=HTTP;HTTPS;TCP;TLS;UDP;TCP_UDP +type Protocol string + +const ( + ProtocolHTTP Protocol = "HTTP" + ProtocolHTTPS Protocol = "HTTPS" + ProtocolTCP Protocol = "TCP" + ProtocolTLS Protocol = "TLS" + ProtocolUDP Protocol = "UDP" + ProtocolTCP_UDP Protocol = "TCP_UDP" +) + // +kubebuilder:validation:Enum=http1;http2;grpc type ProtocolVersion string @@ -125,12 +138,23 @@ const ( ProtocolVersionGRPC ProtocolVersion = "grpc" ) +// +kubebuilder:validation:Enum=none;prefer-route-specific;prefer-default +type MergeMode string + +const ( + MergeModeNone MergeMode = "none" + MergeModePreferRouteSpecific MergeMode = "prefer-route-specific" + MergeModePreferDefault MergeMode = "prefer-default" +) + // TargetGroupConfigurationSpec defines the TargetGroup properties for a route. type TargetGroupConfigurationSpec struct { // targetReference the kubernetes object to attach the Target Group settings to. TargetReference Reference `json:"targetReference"` + // mergeMode the mode to use for merging the identified per-route configuration and default configuration. + // defaultRouteConfiguration fallback configuration applied to all routes, unless overridden by route-specific configurations. // +optional DefaultConfiguration TargetGroupProps `json:"defaultConfiguration,omitempty"` @@ -141,12 +165,12 @@ type TargetGroupConfigurationSpec struct { } // +kubebuilder:validation:Pattern="^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$" -type RouteName string +type RouteIdentifier string // RouteConfiguration defines the per route configuration type RouteConfiguration struct { - // name the name of the route, it should be in the form of ROUTE:NAMESPACE:NAME - Name RouteName `json:"name"` + // name the identifier of the route, it should be in the form of ROUTE:NAMESPACE:NAME + Identifier RouteIdentifier `json:"identifier"` // targetGroupProps the target group specific properties TargetGroupProps TargetGroupProps `json:"targetGroupProps"` @@ -154,6 +178,10 @@ type RouteConfiguration struct { // TargetGroupProps defines the target group properties type TargetGroupProps struct { + // targetGroupName specifies the name to assign to the Target Group. If not defined, then one is generated. + // +optional + TargetGroupName string `json:"targetGroupName,omitempty"` + // ipAddressType specifies whether the target group is of type IPv4 or IPv6. If unspecified, it will be automatically inferred. // +optional IPAddressType *TargetGroupIPAddressType `json:"ipAddressType,omitempty"` @@ -174,10 +202,20 @@ type TargetGroupProps struct { // +optional TargetType *TargetType `json:"targetType,omitempty"` + // Protocol [Application / Network Load Balancer] the protocol for the target group. + // If unspecified, it will be automatically inferred. + // +optional + Protocol *Protocol `json:"protocol,omitempty"` + // protocolVersion [HTTP/HTTPS protocol] The protocol version. The possible values are GRPC , HTTP1 and HTTP2 // +optional ProtocolVersion *ProtocolVersion `json:"protocolVersion,omitempty"` + // EnableMultiCluster [Application / Network LoadBalancer] + // Allows for multiple Clusters / Services to use the generated TargetGroup ARN + // +optional + EnableMultiCluster bool `json:"enableMultiCluster,omitempty"` + // vpcID is the VPC of the TargetGroup. If unspecified, it will be automatically inferred. // +optional VpcID *string `json:"vpcID,omitempty"` @@ -231,6 +269,10 @@ type TargetGroupConfiguration struct { Status TargetGroupConfigurationStatus `json:"status,omitempty"` } +func (tgConfig *TargetGroupConfiguration) GetTargetGroupConfigForRoute(name, namespace, kind string) *TargetGroupProps { + return &tgConfig.Spec.DefaultConfiguration +} + // +kubebuilder:object:root=true // TargetGroupConfigurationList contains a list of TargetGroupConfiguration diff --git a/apis/gateway/v1beta1/zz_generated.deepcopy.go b/apis/gateway/v1beta1/zz_generated.deepcopy.go index d4c9a325ab..fe1edd2f62 100644 --- a/apis/gateway/v1beta1/zz_generated.deepcopy.go +++ b/apis/gateway/v1beta1/zz_generated.deepcopy.go @@ -60,7 +60,7 @@ func (in *HealthCheckConfiguration) DeepCopyInto(out *HealthCheckConfiguration) } if in.HealthCheckPort != nil { in, out := &in.HealthCheckPort, &out.HealthCheckPort - *out = new(int32) + *out = new(string) **out = **in } if in.HealthCheckProtocol != nil { @@ -686,6 +686,11 @@ func (in *TargetGroupProps) DeepCopyInto(out *TargetGroupProps) { *out = new(TargetType) **out = **in } + if in.Protocol != nil { + in, out := &in.Protocol, &out.Protocol + *out = new(Protocol) + **out = **in + } if in.ProtocolVersion != nil { in, out := &in.ProtocolVersion, &out.ProtocolVersion *out = new(ProtocolVersion) diff --git a/config/crd/gateway/gateway-crds.yaml b/config/crd/gateway/gateway-crds.yaml index 846f287ce9..1d4c8ea66c 100644 --- a/config/crd/gateway/gateway-crds.yaml +++ b/config/crd/gateway/gateway-crds.yaml @@ -350,6 +350,11 @@ spec: description: defaultRouteConfiguration fallback configuration applied to all routes, unless overridden by route-specific configurations. properties: + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + Allows for multiple Clusters / Services to use the generated TargetGroup ARN + type: boolean healthCheckConfig: description: healthCheckConfig The Health Check configuration for this backend. @@ -365,10 +370,10 @@ spec: on the targets. type: string healthCheckPort: - description: healthCheckPort The port to use to connect with - the target. - format: int32 - type: integer + description: |- + healthCheckPort The port the load balancer uses when performing health checks on targets. + The default is to use the port on which each target receives traffic from the load balancer. + type: string healthCheckProtocol: description: healthCheckProtocol The protocol to use to connect with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols @@ -462,6 +467,18 @@ spec: type: object type: object x-kubernetes-map-type: atomic + protocol: + description: |- + Protocol [Application / Network Load Balancer] the protocol for the target group. + If unspecified, it will be automatically inferred. + enum: + - HTTP + - HTTPS + - TCP + - TLS + - UDP + - TCP_UDP + type: string protocolVersion: description: protocolVersion [HTTP/HTTPS protocol] The protocol version. The possible values are GRPC , HTTP1 and HTTP2 @@ -503,6 +520,10 @@ spec: - value type: object type: array + targetGroupName: + description: targetGroupName specifies the name to assign to the + Target Group. If not defined, then one is generated. + type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred as instance. @@ -521,14 +542,19 @@ spec: items: description: RouteConfiguration defines the per route configuration properties: - name: - description: name the name of the route, it should be in the - form of ROUTE:NAMESPACE:NAME + identifier: + description: name the identifier of the route, it should be + in the form of ROUTE:NAMESPACE:NAME pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ type: string targetGroupProps: description: targetGroupProps the target group specific properties properties: + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + Allows for multiple Clusters / Services to use the generated TargetGroup ARN + type: boolean healthCheckConfig: description: healthCheckConfig The Health Check configuration for this backend. @@ -544,10 +570,10 @@ spec: checks on the targets. type: string healthCheckPort: - description: healthCheckPort The port to use to connect - with the target. - format: int32 - type: integer + description: |- + healthCheckPort The port the load balancer uses when performing health checks on targets. + The default is to use the port on which each target receives traffic from the load balancer. + type: string healthCheckProtocol: description: healthCheckProtocol The protocol to use to connect with the target. The GENEVE, TLS, UDP, @@ -644,6 +670,18 @@ spec: type: object type: object x-kubernetes-map-type: atomic + protocol: + description: |- + Protocol [Application / Network Load Balancer] the protocol for the target group. + If unspecified, it will be automatically inferred. + enum: + - HTTP + - HTTPS + - TCP + - TLS + - UDP + - TCP_UDP + type: string protocolVersion: description: protocolVersion [HTTP/HTTPS protocol] The protocol version. The possible values are GRPC , HTTP1 and HTTP2 @@ -686,6 +724,10 @@ spec: - value type: object type: array + targetGroupName: + description: targetGroupName specifies the name to assign + to the Target Group. If not defined, then one is generated. + type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred as instance. @@ -699,7 +741,7 @@ spec: type: string type: object required: - - name + - identifier - targetGroupProps type: object type: array diff --git a/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml index c34856f29b..b2b21ca7b2 100644 --- a/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml +++ b/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -53,6 +53,11 @@ spec: description: defaultRouteConfiguration fallback configuration applied to all routes, unless overridden by route-specific configurations. properties: + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + Allows for multiple Clusters / Services to use the generated TargetGroup ARN + type: boolean healthCheckConfig: description: healthCheckConfig The Health Check configuration for this backend. @@ -68,10 +73,10 @@ spec: on the targets. type: string healthCheckPort: - description: healthCheckPort The port to use to connect with - the target. - format: int32 - type: integer + description: |- + healthCheckPort The port the load balancer uses when performing health checks on targets. + The default is to use the port on which each target receives traffic from the load balancer. + type: string healthCheckProtocol: description: healthCheckProtocol The protocol to use to connect with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols @@ -165,6 +170,18 @@ spec: type: object type: object x-kubernetes-map-type: atomic + protocol: + description: |- + Protocol [Application / Network Load Balancer] the protocol for the target group. + If unspecified, it will be automatically inferred. + enum: + - HTTP + - HTTPS + - TCP + - TLS + - UDP + - TCP_UDP + type: string protocolVersion: description: protocolVersion [HTTP/HTTPS protocol] The protocol version. The possible values are GRPC , HTTP1 and HTTP2 @@ -206,6 +223,10 @@ spec: - value type: object type: array + targetGroupName: + description: targetGroupName specifies the name to assign to the + Target Group. If not defined, then one is generated. + type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred as instance. @@ -224,14 +245,19 @@ spec: items: description: RouteConfiguration defines the per route configuration properties: - name: - description: name the name of the route, it should be in the - form of ROUTE:NAMESPACE:NAME + identifier: + description: name the identifier of the route, it should be + in the form of ROUTE:NAMESPACE:NAME pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ type: string targetGroupProps: description: targetGroupProps the target group specific properties properties: + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + Allows for multiple Clusters / Services to use the generated TargetGroup ARN + type: boolean healthCheckConfig: description: healthCheckConfig The Health Check configuration for this backend. @@ -247,10 +273,10 @@ spec: checks on the targets. type: string healthCheckPort: - description: healthCheckPort The port to use to connect - with the target. - format: int32 - type: integer + description: |- + healthCheckPort The port the load balancer uses when performing health checks on targets. + The default is to use the port on which each target receives traffic from the load balancer. + type: string healthCheckProtocol: description: healthCheckProtocol The protocol to use to connect with the target. The GENEVE, TLS, UDP, @@ -347,6 +373,18 @@ spec: type: object type: object x-kubernetes-map-type: atomic + protocol: + description: |- + Protocol [Application / Network Load Balancer] the protocol for the target group. + If unspecified, it will be automatically inferred. + enum: + - HTTP + - HTTPS + - TCP + - TLS + - UDP + - TCP_UDP + type: string protocolVersion: description: protocolVersion [HTTP/HTTPS protocol] The protocol version. The possible values are GRPC , HTTP1 and HTTP2 @@ -389,6 +427,10 @@ spec: - value type: object type: array + targetGroupName: + description: targetGroupName specifies the name to assign + to the Target Group. If not defined, then one is generated. + type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred as instance. @@ -402,7 +444,7 @@ spec: type: string type: object required: - - name + - identifier - targetGroupProps type: object type: array diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index d6565122f6..76e124e4ab 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -144,6 +144,9 @@ type gatewayReconciler struct { // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes/status,verbs=get;update;patch // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes/finalizers,verbs=update +//+kubebuilder:rbac:groups=gateway.k8s.aws,resources=loadbalancerconfigurations,verbs=get;list;watch +//+kubebuilder:rbac:groups=gateway.k8s.aws,resources=targetgroupconfigurations,verbs=get;list;watch + func (r *gatewayReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { r.reconcileTracker(req.NamespacedName) err := r.reconcileHelper(ctx, req) diff --git a/main.go b/main.go index e57c640037..c02654b4e2 100644 --- a/main.go +++ b/main.go @@ -224,7 +224,7 @@ func main() { // Setup NLB Gateway controller if enabled if nlbGatewayEnabled { - gwControllerConfig.routeLoader = routeutils.NewLoader(mgr.GetClient()) + gwControllerConfig.routeLoader = routeutils.NewLoader(mgr.GetClient(), mgr.GetLogger().WithName("gateway-route-loader")) if err := setupGatewayController(ctx, mgr, gwControllerConfig, constants.NLBGatewayController); err != nil { setupLog.Error(err, "failed to setup NLB Gateway controller") os.Exit(1) @@ -234,7 +234,7 @@ func main() { // Setup ALB Gateway controller if enabled if albGatewayEnabled { if gwControllerConfig.routeLoader == nil { - gwControllerConfig.routeLoader = routeutils.NewLoader(mgr.GetClient()) + gwControllerConfig.routeLoader = routeutils.NewLoader(mgr.GetClient(), mgr.GetLogger().WithName("gateway-route-loader")) } if err := setupGatewayController(ctx, mgr, gwControllerConfig, constants.ALBGatewayController); err != nil { setupLog.Error(err, "failed to setup ALB Gateway controller") diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go index 78245db66d..b99d9888ee 100644 --- a/pkg/gateway/model/base_model_builder.go +++ b/pkg/gateway/model/base_model_builder.go @@ -38,11 +38,13 @@ func NewModelBuilder(subnetsResolver networking.SubnetsResolver, subnetBuilder := newSubnetModelBuilder(loadBalancerType, trackingProvider, subnetsResolver, elbv2TaggingManager) sgBuilder := newSecurityGroupBuilder(gwTagHelper, clusterName, enableBackendSG, sgResolver, backendSGProvider, logger) lbBuilder := newLoadBalancerBuilder(loadBalancerType, gwTagHelper, clusterName) + tgBuilder := newTargetGroupBuilder(clusterName, vpcID, gwTagHelper, loadBalancerType, disableRestrictedSGRules, defaultTargetType) return &baseModelBuilder{ subnetBuilder: subnetBuilder, securityGroupBuilder: sgBuilder, lbBuilder: lbBuilder, + tgBuilder: tgBuilder, logger: logger, defaultLoadBalancerScheme: elbv2model.LoadBalancerScheme(defaultLoadBalancerScheme), @@ -58,6 +60,7 @@ type baseModelBuilder struct { subnetBuilder subnetModelBuilder securityGroupBuilder securityGroupBuilder + tgBuilder targetGroupBuilder defaultLoadBalancerScheme elbv2model.LoadBalancerScheme defaultIPType elbv2model.IPAddressType @@ -110,6 +113,31 @@ func (baseBuilder *baseModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway lb := elbv2model.NewLoadBalancer(stack, resourceIDLoadBalancer, spec) + baseBuilder.logger.Info("Got this route details", "routes", routes) + /* Target Groups */ + // TODO - Figure out how to map this back to a listener? + tgByResID := make(map[string]buildTargetGroupOutput) + for _, descriptors := range routes { + for _, descriptor := range descriptors { + for _, rule := range descriptor.GetAttachedRules() { + for _, backend := range rule.GetBackends() { + // TODO -- Figure out what to do with the return value (it's also inserted into the tgByResID map) + // TODO -- I'm not in love with this API. + _, tgErr := baseBuilder.tgBuilder.buildTargetGroup(&tgByResID, gw, lbConf, lb.Spec.IPAddressType, descriptor, backend, securityGroups.backendSecurityGroupToken) + if tgErr != nil { + return nil, nil, false, err + } + } + } + } + } + + for tgResID, tgOut := range tgByResID { + tg := elbv2model.NewTargetGroup(stack, tgResID, tgOut.targetGroupSpec) + tgOut.bindingSpec.Template.Spec.TargetGroupARN = tg.TargetGroupARN() + elbv2model.NewTargetGroupBindingResource(stack, tg.ID(), tgOut.bindingSpec) + } + return stack, lb, securityGroups.backendSecurityGroupAllocated, nil } diff --git a/pkg/gateway/model/model_build_security_group.go b/pkg/gateway/model/model_build_security_group.go index d657398383..7f45c2c2aa 100644 --- a/pkg/gateway/model/model_build_security_group.go +++ b/pkg/gateway/model/model_build_security_group.go @@ -21,7 +21,6 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" gwv1 "sigs.k8s.io/gateway-api/apis/v1" - "strings" ) var ( @@ -295,17 +294,3 @@ func generateProtocolListFromRoutes(routes []routeutils.RouteDescriptor) []strin } return protocolSet.UnsortedList() } - -func isIPv6Supported(ipAddressType elbv2model.IPAddressType) bool { - switch ipAddressType { - case elbv2model.IPAddressTypeDualStack, elbv2model.IPAddressTypeDualStackWithoutPublicIPV4: - return true - default: - return false - } -} - -// TODO - Refactor? -func isIPv6CIDR(cidr string) bool { - return strings.Contains(cidr, ":") -} diff --git a/pkg/gateway/model/model_build_target_group.go b/pkg/gateway/model/model_build_target_group.go new file mode 100644 index 0000000000..0f5baafa4c --- /dev/null +++ b/pkg/gateway/model/model_build_target_group.go @@ -0,0 +1,617 @@ +package model + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "regexp" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strconv" +) + +type buildTargetGroupOutput struct { + targetGroupSpec elbv2model.TargetGroupSpec + bindingSpec elbv2model.TargetGroupBindingResourceSpec +} + +type targetGroupBuilder interface { + buildTargetGroup(tgByResID *map[string]buildTargetGroupOutput, + gw *gwv1.Gateway, lbConfig *elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend, backendSGIDToken core.StringToken) (buildTargetGroupOutput, error) +} + +type targetGroupBuilderImpl struct { + loadBalancerType elbv2model.LoadBalancerType + + clusterName string + vpcID string + + tagHelper tagHelper + disableRestrictedSGRules bool + + defaultTargetType elbv2model.TargetType + + defaultHealthCheckMatcherHTTPCode string + defaultHealthCheckMatcherGRPCCode string + + defaultHealthCheckPathHTTP string + defaultHealthCheckPathGRPC string + + defaultHealthCheckUnhealthyThresholdCount int32 + defaultHealthyThresholdCount int32 + defaultHealthCheckTimeout int32 + defaultHealthCheckInterval int32 +} + +func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper, loadBalancerType elbv2model.LoadBalancerType, disableRestrictedSGRules bool, defaultTargetType string) targetGroupBuilder { + return &targetGroupBuilderImpl{ + loadBalancerType: loadBalancerType, + clusterName: clusterName, + vpcID: vpcId, + tagHelper: tagHelper, + disableRestrictedSGRules: disableRestrictedSGRules, + defaultTargetType: elbv2model.TargetType(defaultTargetType), + defaultHealthCheckMatcherHTTPCode: "200-399", + defaultHealthCheckMatcherGRPCCode: "12", + defaultHealthCheckPathHTTP: "/", + defaultHealthCheckPathGRPC: "/AWS.ALB/healthcheck", + defaultHealthCheckUnhealthyThresholdCount: 3, + defaultHealthyThresholdCount: 3, + defaultHealthCheckTimeout: 5, + defaultHealthCheckInterval: 15, + } +} + +func (builder *targetGroupBuilderImpl) buildTargetGroup(tgByResID *map[string]buildTargetGroupOutput, + gw *gwv1.Gateway, lbConfig *elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend, backendSGIDToken core.StringToken) (buildTargetGroupOutput, error) { + + targetGroupProps := builder.getTargetGroupProps(routeDescriptor, backend) + + tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), k8s.NamespacedName(backend.Service), routeDescriptor.GetRouteNamespacedName(), backend.ServicePort.TargetPort) + if tg, exists := (*tgByResID)[tgResID]; exists { + return tg, nil + } + + tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbConfig, lbIPType, backend, targetGroupProps) + if err != nil { + return buildTargetGroupOutput{}, err + } + nodeSelector := builder.buildTargetGroupBindingNodeSelector(targetGroupProps, tgSpec.TargetType) + bindingSpec := builder.buildTargetGroupBindingSpec(lbConfig, targetGroupProps, tgSpec, nodeSelector, backend, backendSGIDToken) + + output := buildTargetGroupOutput{ + targetGroupSpec: tgSpec, + bindingSpec: bindingSpec, + } + + (*tgByResID)[tgResID] = output + return output, nil +} + +func (builder *targetGroupBuilderImpl) getTargetGroupProps(routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) *elbv2gw.TargetGroupProps { + var targetGroupProps *elbv2gw.TargetGroupProps + if backend.ELBv2TargetGroupConfig != nil { + routeNamespacedName := routeDescriptor.GetRouteNamespacedName() + targetGroupProps = backend.ELBv2TargetGroupConfig.GetTargetGroupConfigForRoute(routeNamespacedName.Name, routeNamespacedName.Namespace, string(routeDescriptor.GetRouteKind())) + } + return targetGroupProps +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(lbConfig *elbv2gw.LoadBalancerConfiguration, tgProps *elbv2gw.TargetGroupProps, tgSpec elbv2model.TargetGroupSpec, nodeSelector *metav1.LabelSelector, backend routeutils.Backend, backendSGIDToken core.StringToken) elbv2model.TargetGroupBindingResourceSpec { + targetType := elbv2api.TargetType(tgSpec.TargetType) + targetPort := backend.ServicePort.TargetPort + if targetType == elbv2api.TargetTypeInstance { + targetPort = intstr.FromInt32(backend.ServicePort.NodePort) + } + tgbNetworking := builder.buildTargetGroupBindingNetworking(targetPort, *tgSpec.HealthCheckConfig.Port, *backend.ServicePort, backendSGIDToken) + + multiClusterEnabled := builder.buildTargetGroupBindingMultiClusterFlag(tgProps) + + return elbv2model.TargetGroupBindingResourceSpec{ + Template: elbv2model.TargetGroupBindingTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backend.Service.Namespace, + Name: tgSpec.Name, + }, + Spec: elbv2model.TargetGroupBindingSpec{ + TargetGroupARN: nil, // This should get filled in later! + TargetType: &targetType, + ServiceRef: elbv2api.ServiceReference{ + Name: backend.Service.Name, + Port: intstr.FromInt32(backend.ServicePort.Port), + }, + Networking: tgbNetworking, + NodeSelector: nodeSelector, + IPAddressType: elbv2api.TargetGroupIPAddressType(tgSpec.IPAddressType), + VpcID: builder.vpcID, + MultiClusterTargetGroup: multiClusterEnabled, + }, + }, + } +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetPort intstr.IntOrString, healthCheckPort intstr.IntOrString, port corev1.ServicePort, backendSGIDToken core.StringToken) *elbv2model.TargetGroupBindingNetworking { + if backendSGIDToken == nil { + return nil + } + protocolTCP := elbv2api.NetworkingProtocolTCP + protocolUDP := elbv2api.NetworkingProtocolUDP + + udpSupported := port.Protocol == corev1.ProtocolUDP + + if builder.disableRestrictedSGRules { + ports := []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: nil, + }, + } + + if udpSupported { + ports = append(ports, elbv2api.NetworkingPort{ + Protocol: &protocolUDP, + Port: nil, + }) + } + + return &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: backendSGIDToken, + }, + }, + }, + Ports: ports, + }, + }, + } + } + + var networkingPorts []elbv2api.NetworkingPort + var networkingRules []elbv2model.NetworkingIngressRule + + protocolToUse := &protocolTCP + if udpSupported { + protocolToUse = &protocolUDP + } + + networkingPorts = append(networkingPorts, elbv2api.NetworkingPort{ + Protocol: protocolToUse, + Port: &targetPort, + }) + + if udpSupported || (healthCheckPort.Type == intstr.Int && healthCheckPort.IntValue() != targetPort.IntValue()) { + var hcPortToUse intstr.IntOrString + if healthCheckPort.Type == intstr.String { + hcPortToUse = targetPort + } else { + hcPortToUse = healthCheckPort + } + + networkingPorts = append(networkingPorts, elbv2api.NetworkingPort{ + Protocol: &protocolTCP, + Port: &hcPortToUse, + }) + } + + for _, port := range networkingPorts { + networkingRules = append(networkingRules, elbv2model.NetworkingIngressRule{ + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: backendSGIDToken, + }, + }, + }, + Ports: []elbv2api.NetworkingPort{port}, + }) + } + return &elbv2model.TargetGroupBindingNetworking{ + Ingress: networkingRules, + } +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbConfig *elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, backend routeutils.Backend, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { + targetType := builder.buildTargetGroupTargetType(targetGroupProps) + tgProtocol, err := builder.buildTargetGroupProtocol(targetGroupProps, route) + if err != nil { + return elbv2model.TargetGroupSpec{}, err + } + tgProtocolVersion := builder.buildTargetGroupProtocolVersion(targetGroupProps) + + healthCheckConfig, err := builder.buildTargetGroupHealthCheckConfig(targetGroupProps, tgProtocol, tgProtocolVersion, targetType, backend) + if err != nil { + return elbv2model.TargetGroupSpec{}, err + } + tgAttributesMap := builder.buildTargetGroupAttributes(targetGroupProps) + ipAddressType, err := builder.buildTargetGroupIPAddressType(backend.Service, lbIPType) + if err != nil { + return elbv2model.TargetGroupSpec{}, err + } + + tags, err := builder.tagHelper.getGatewayTags(lbConfig) + if err != nil { + return elbv2model.TargetGroupSpec{}, err + } + tgPort := builder.buildTargetGroupPort(targetType, *backend.ServicePort) + // TODO - backend.ServicePort.TargetPort might not be correct. + name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), k8s.NamespacedName(backend.Service), backend.ServicePort.TargetPort, tgPort, targetType, tgProtocol, tgProtocolVersion) + return elbv2model.TargetGroupSpec{ + Name: name, + TargetType: targetType, + Port: awssdk.Int32(tgPort), + Protocol: tgProtocol, + ProtocolVersion: tgProtocolVersion, + IPAddressType: ipAddressType, + HealthCheckConfig: &healthCheckConfig, + TargetGroupAttributes: builder.convertMapToAttributes(tgAttributesMap), + Tags: tags, + }, nil +} + +var invalidTargetGroupNamePattern = regexp.MustCompile("[[:^alnum:]]") + +// buildTargetGroupName will calculate the targetGroup's name. +func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *elbv2gw.TargetGroupProps, + gwKey types.NamespacedName, routeKey types.NamespacedName, svcKey types.NamespacedName, port intstr.IntOrString, tgPort int32, + targetType elbv2model.TargetType, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion) string { + + if targetGroupProps != nil && targetGroupProps.TargetGroupName != "" { + return targetGroupProps.TargetGroupName + } + + uuidHash := sha256.New() + _, _ = uuidHash.Write([]byte(builder.clusterName)) + _, _ = uuidHash.Write([]byte(gwKey.Namespace)) + _, _ = uuidHash.Write([]byte(gwKey.Name)) + _, _ = uuidHash.Write([]byte(routeKey.Namespace)) + _, _ = uuidHash.Write([]byte(routeKey.Name)) + _, _ = uuidHash.Write([]byte(svcKey.Namespace)) + _, _ = uuidHash.Write([]byte(svcKey.Name)) + _, _ = uuidHash.Write([]byte(port.String())) + _, _ = uuidHash.Write([]byte(strconv.Itoa(int(tgPort)))) + _, _ = uuidHash.Write([]byte(targetType)) + _, _ = uuidHash.Write([]byte(tgProtocol)) + if tgProtocolVersion != nil { + _, _ = uuidHash.Write([]byte(*tgProtocolVersion)) + } + uuid := hex.EncodeToString(uuidHash.Sum(nil)) + + sanitizedNamespace := invalidTargetGroupNamePattern.ReplaceAllString(routeKey.Namespace, "") + sanitizedName := invalidTargetGroupNamePattern.ReplaceAllString(routeKey.Name, "") + return fmt.Sprintf("k8s-%.8s-%.8s-%.10s", sanitizedNamespace, sanitizedName, uuid) +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupTargetType(targetGroupProps *elbv2gw.TargetGroupProps) elbv2model.TargetType { + if targetGroupProps == nil || targetGroupProps.TargetType == nil { + return builder.defaultTargetType + } + + return elbv2model.TargetType(*targetGroupProps.TargetType) +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupIPAddressType(svc *corev1.Service, loadBalancerIPAddressType elbv2model.IPAddressType) (elbv2model.TargetGroupIPAddressType, error) { + var ipv6Configured bool + for _, ipFamily := range svc.Spec.IPFamilies { + if ipFamily == corev1.IPv6Protocol { + ipv6Configured = true + break + } + } + if ipv6Configured { + if !isIPv6Supported(loadBalancerIPAddressType) { + return "", errors.New("unsupported IPv6 configuration, lb not dual-stack") + } + return elbv2model.TargetGroupIPAddressTypeIPv6, nil + } + return elbv2model.TargetGroupIPAddressTypeIPv4, nil +} + +// buildTargetGroupPort constructs the TargetGroup's port. +// Note: TargetGroup's port is not in the data path as we always register targets with port specified. +// so these settings don't really matter to our controller, +// and we do our best to use the most appropriate port as targetGroup's port to avoid UX confusing. +func (builder *targetGroupBuilderImpl) buildTargetGroupPort(targetType elbv2model.TargetType, svcPort corev1.ServicePort) int32 { + if targetType == elbv2model.TargetTypeInstance { + // Maybe an error? Because the service has no node port, instance type targets don't work. + if svcPort.NodePort == 0 { + return 1 + } + return svcPort.NodePort + } + if svcPort.TargetPort.Type == intstr.Int { + return int32(svcPort.TargetPort.IntValue()) + } + // If all else fails, just return 1 as alluded to above, this setting doesn't really matter. + return 1 +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { + // TODO - Not convinced that this is good, maybe auto detect certs == HTTPS / TLS. + if builder.loadBalancerType == elbv2model.LoadBalancerTypeApplication { + return builder.buildL7TargetGroupProtocol(targetGroupProps, route) + } + + return builder.buildL4TargetGroupProtocol(targetGroupProps, route) +} + +func (builder *targetGroupBuilderImpl) buildL7TargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { + if targetGroupProps == nil || targetGroupProps.Protocol == nil { + return builder.inferTargetGroupProtocolFromRoute(route), nil + } + switch string(*targetGroupProps.Protocol) { + case string(elbv2model.ProtocolHTTP): + return elbv2model.ProtocolHTTP, nil + case string(elbv2model.ProtocolHTTPS): + return elbv2model.ProtocolHTTPS, nil + default: + return "", errors.Errorf("backend protocol must be within [%v, %v]: %v", elbv2model.ProtocolHTTP, elbv2model.ProtocolHTTPS, *targetGroupProps.Protocol) + } +} + +func (builder *targetGroupBuilderImpl) buildL4TargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { + // TODO, auto infer? + if targetGroupProps == nil || targetGroupProps.Protocol == nil { + // infer this somehow!? + // use the backend config to get the protocol type. + return builder.inferTargetGroupProtocolFromRoute(route), nil + } + + switch string(*targetGroupProps.Protocol) { + case string(elbv2model.ProtocolTCP): + return elbv2model.ProtocolTCP, nil + case string(elbv2model.ProtocolTLS): + return elbv2model.ProtocolTLS, nil + case string(elbv2model.ProtocolUDP): + return elbv2model.ProtocolUDP, nil + case string(elbv2model.ProtocolTCP_UDP): + return elbv2model.ProtocolTCP_UDP, nil + default: + return "", errors.Errorf("backend protocol must be within [%v, %v, %v, %v]: %v", elbv2model.ProtocolTCP, elbv2model.ProtocolUDP, elbv2model.ProtocolTCP_UDP, elbv2model.ProtocolTLS, *targetGroupProps.Protocol) + } +} + +func (builder *targetGroupBuilderImpl) inferTargetGroupProtocolFromRoute(route routeutils.RouteDescriptor) elbv2model.Protocol { + switch route.GetRouteKind() { + case routeutils.TCPRouteKind: + return elbv2model.ProtocolTCP + case routeutils.UDPRouteKind: + return elbv2model.ProtocolUDP + case routeutils.HTTPRouteKind: + return elbv2model.ProtocolHTTP + case routeutils.GRPCRouteKind: + return elbv2model.ProtocolHTTP + case routeutils.TLSRouteKind: + if builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { + return elbv2model.ProtocolTLS + } + return elbv2model.ProtocolHTTPS + } + // This should never happen. + return elbv2model.ProtocolTCP +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupProtocolVersion(targetGroupProps *elbv2gw.TargetGroupProps) *elbv2model.ProtocolVersion { + // NLB doesn't support protocol version + if builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { + return nil + } + if targetGroupProps != nil && targetGroupProps.ProtocolVersion != nil { + // TODO - We can infer GRPC here from route + pv := elbv2model.ProtocolVersion(*targetGroupProps.ProtocolVersion) + return &pv + } + http1 := elbv2model.ProtocolVersionHTTP1 + return &http1 +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion, targetType elbv2model.TargetType, backend routeutils.Backend) (elbv2model.TargetGroupHealthCheckConfig, error) { + // For my notes, when translating from svc to gateway: + // https://github.com/kubernetes-sigs/gateway-api/issues/451 + // Gateway API doesn't have the same ServiceExternalTrafficPolicyLocal support. + // TODO - Maybe a TargetGroupConfig attribute to support the same behavior? + healthCheckPort, err := builder.buildTargetGroupHealthCheckPort(targetGroupProps, targetType, backend) + if err != nil { + return elbv2model.TargetGroupHealthCheckConfig{}, err + } + healthCheckProtocol := builder.buildTargetGroupHealthCheckProtocol(targetGroupProps, tgProtocol) + healthCheckPath := builder.buildTargetGroupHealthCheckPath(targetGroupProps, tgProtocolVersion, healthCheckProtocol) + healthCheckMatcher := builder.buildTargetGroupHealthCheckMatcher(targetGroupProps, healthCheckProtocol) + healthCheckIntervalSeconds := builder.buildTargetGroupHealthCheckIntervalSeconds(targetGroupProps) + healthCheckTimeoutSeconds := builder.buildTargetGroupHealthCheckTimeoutSeconds(targetGroupProps) + healthCheckHealthyThresholdCount := builder.buildTargetGroupHealthCheckHealthyThresholdCount(targetGroupProps) + healthCheckUnhealthyThresholdCount := builder.buildTargetGroupHealthCheckUnhealthyThresholdCount(targetGroupProps) + hcConfig := elbv2model.TargetGroupHealthCheckConfig{ + Port: &healthCheckPort, + Protocol: healthCheckProtocol, + Path: healthCheckPath, + Matcher: healthCheckMatcher, + IntervalSeconds: awssdk.Int32(healthCheckIntervalSeconds), + TimeoutSeconds: awssdk.Int32(healthCheckTimeoutSeconds), + HealthyThresholdCount: awssdk.Int32(healthCheckHealthyThresholdCount), + UnhealthyThresholdCount: awssdk.Int32(healthCheckUnhealthyThresholdCount), + } + + return hcConfig, nil +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckPort(targetGroupProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType, backend routeutils.Backend) (intstr.IntOrString, error) { + if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckPort == nil || *targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil + } + + healthCheckPort := intstr.Parse(*targetGroupProps.HealthCheckConfig.HealthCheckPort) + if healthCheckPort.Type == intstr.Int { + return healthCheckPort, nil + } + + /* TODO - Zac revisit this? */ + if targetType == elbv2model.TargetTypeInstance { + return intstr.FromInt(int(backend.ServicePort.NodePort)), nil + } + if backend.ServicePort.TargetPort.Type == intstr.Int { + return backend.ServicePort.TargetPort, nil + } + return intstr.IntOrString{}, errors.New("cannot use named healthCheckPort for IP TargetType when service's targetPort is a named port") +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckProtocol(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol) elbv2model.Protocol { + + if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckProtocol == nil { + if builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { + return elbv2model.ProtocolTCP + } + return tgProtocol + } + + switch *targetGroupProps.HealthCheckConfig.HealthCheckProtocol { + case elbv2gw.TargetGroupHealthCheckProtocolTCP: + return elbv2model.ProtocolTCP + case elbv2gw.TargetGroupHealthCheckProtocolHTTP: + return elbv2model.ProtocolHTTP + case elbv2gw.TargetGroupHealthCheckProtocolHTTPS: + return elbv2model.ProtocolHTTPS + default: + return tgProtocol + } +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckPath(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocolVersion *elbv2model.ProtocolVersion, hcProtocol elbv2model.Protocol) *string { + if hcProtocol == elbv2model.ProtocolTCP { + return nil + } + + if targetGroupProps != nil && targetGroupProps.HealthCheckConfig.HealthCheckPath != nil { + return targetGroupProps.HealthCheckConfig.HealthCheckPath + } + + if tgProtocolVersion != nil && *tgProtocolVersion == elbv2model.ProtocolVersionGRPC { + return &builder.defaultHealthCheckPathGRPC + } + + return &builder.defaultHealthCheckPathHTTP +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckMatcher(targetGroupProps *elbv2gw.TargetGroupProps, hcProtocol elbv2model.Protocol) *elbv2model.HealthCheckMatcher { + + if hcProtocol == elbv2model.ProtocolTCP { + return nil + } + + if targetGroupProps != nil && targetGroupProps.ProtocolVersion != nil && string(*targetGroupProps.ProtocolVersion) == string(elbv2model.ProtocolVersionGRPC) { + matcher := builder.defaultHealthCheckMatcherGRPCCode + if targetGroupProps.ProtocolVersion != nil && targetGroupProps.HealthCheckConfig != nil && targetGroupProps.HealthCheckConfig.Matcher != nil && targetGroupProps.HealthCheckConfig.Matcher.GRPCCode != nil { + matcher = *targetGroupProps.HealthCheckConfig.Matcher.GRPCCode + } + return &elbv2model.HealthCheckMatcher{ + GRPCCode: &matcher, + } + } + matcher := builder.defaultHealthCheckMatcherHTTPCode + if targetGroupProps != nil && targetGroupProps.ProtocolVersion != nil && targetGroupProps.HealthCheckConfig != nil && targetGroupProps.HealthCheckConfig.Matcher != nil && targetGroupProps.HealthCheckConfig.Matcher.HTTPCode != nil { + matcher = *targetGroupProps.HealthCheckConfig.Matcher.HTTPCode + } + return &elbv2model.HealthCheckMatcher{ + HTTPCode: &matcher, + } +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckIntervalSeconds(targetGroupProps *elbv2gw.TargetGroupProps) int32 { + if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckInterval == nil { + return builder.defaultHealthCheckInterval + } + return *targetGroupProps.HealthCheckConfig.HealthCheckInterval +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckTimeoutSeconds(targetGroupProps *elbv2gw.TargetGroupProps) int32 { + if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckTimeout == nil { + return builder.defaultHealthCheckTimeout + } + return *targetGroupProps.HealthCheckConfig.HealthCheckTimeout +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckHealthyThresholdCount(targetGroupProps *elbv2gw.TargetGroupProps) int32 { + if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthyThresholdCount == nil { + return builder.defaultHealthyThresholdCount + } + return *targetGroupProps.HealthCheckConfig.HealthyThresholdCount +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckUnhealthyThresholdCount(targetGroupProps *elbv2gw.TargetGroupProps) int32 { + if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.UnhealthyThresholdCount == nil { + return builder.defaultHealthCheckUnhealthyThresholdCount + } + return *targetGroupProps.HealthCheckConfig.UnhealthyThresholdCount +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupAttributes(targetGroupProps *elbv2gw.TargetGroupProps) map[string]string { + attributeMap := make(map[string]string) + + if targetGroupProps == nil { + return attributeMap + } + + for _, attr := range targetGroupProps.TargetGroupAttributes { + attributeMap[attr.Key] = attr.Value + } + + if builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { + builder.buildL4TargetGroupAttributes(&attributeMap, targetGroupProps) + } + + return attributeMap +} + +func (builder *targetGroupBuilderImpl) convertMapToAttributes(attributeMap map[string]string) []elbv2model.TargetGroupAttribute { + convertedAttributes := make([]elbv2model.TargetGroupAttribute, 0) + for key, value := range attributeMap { + convertedAttributes = append(convertedAttributes, elbv2model.TargetGroupAttribute{ + Key: key, + Value: value, + }) + } + return convertedAttributes +} + +func (builder *targetGroupBuilderImpl) buildL4TargetGroupAttributes(attributeMap *map[string]string, targetGroupProps *elbv2gw.TargetGroupProps) { + if targetGroupProps == nil { + return + } + // TODO -- buildPreserveClientIPFlag +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupResourceID(gwKey types.NamespacedName, svcKey types.NamespacedName, routeKey types.NamespacedName, port intstr.IntOrString) string { + return fmt.Sprintf("%s/%s:%s-%s:%s-%s:%s", gwKey.Namespace, gwKey.Name, routeKey.Namespace, routeKey.Name, svcKey.Namespace, svcKey.Name, port.String()) +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNodeSelector(tgProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType) *metav1.LabelSelector { + if targetType != elbv2model.TargetTypeInstance { + return nil + } + if tgProps == nil { + return nil + } + return tgProps.NodeSelector +} + +func (builder *targetGroupBuilderImpl) buildTargetGroupBindingMultiClusterFlag(tgProps *elbv2gw.TargetGroupProps) bool { + if tgProps == nil { + return false + } + return tgProps.EnableMultiCluster +} diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go new file mode 100644 index 0000000000..292263c022 --- /dev/null +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -0,0 +1,362 @@ +package model + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "testing" +) + +func Test_buildTargetGroup(t *testing.T) { + instanceType := elbv2api.TargetType(elbv2model.TargetTypeInstance) + ipType := elbv2api.TargetType(elbv2model.TargetTypeIP) + http1 := elbv2model.ProtocolVersionHTTP1 + testCases := []struct { + name string + tags map[string]string + lbType elbv2model.LoadBalancerType + disableRestrictedSGRules bool + defaultTargetType string + gateway *gwv1.Gateway + route *routeutils.MockRoute + backend routeutils.Backend + tagErr error + expectErr bool + expectedTgSpec elbv2model.TargetGroupSpec + expectedTgBindingSpec elbv2model.TargetGroupBindingResourceSpec + }{ + { + name: "no tg config - instance - nlb", + tags: make(map[string]string), + lbType: elbv2model.LoadBalancerTypeNetwork, + disableRestrictedSGRules: false, + defaultTargetType: string(elbv2model.TargetTypeInstance), + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-gw-ns", + Name: "my-gw", + }, + }, + route: &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + Name: "my-route", + Namespace: "my-route-ns", + }, + backend: routeutils.Backend{ + Service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-svc-ns", + Name: "my-svc", + }, + }, + ServicePort: &corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.IntOrString{ + IntVal: 80, + Type: intstr.Int, + }, + NodePort: 8080, + }, + }, + expectedTgSpec: elbv2model.TargetGroupSpec{ + Name: "k8s-myrouten-myroute-1949ae79d7", + TargetType: elbv2model.TargetTypeInstance, + Port: awssdk.Int32(8080), + Protocol: elbv2model.ProtocolTCP, + IPAddressType: elbv2model.TargetGroupIPAddressTypeIPv4, + HealthCheckConfig: &elbv2model.TargetGroupHealthCheckConfig{ + Port: &intstr.IntOrString{ + StrVal: shared_constants.HealthCheckPortTrafficPort, + Type: intstr.String, + }, + Protocol: elbv2model.ProtocolTCP, + IntervalSeconds: awssdk.Int32(15), + TimeoutSeconds: awssdk.Int32(5), + HealthyThresholdCount: awssdk.Int32(3), + UnhealthyThresholdCount: awssdk.Int32(3), + }, + TargetGroupAttributes: make([]elbv2model.TargetGroupAttribute, 0), + Tags: make(map[string]string), + }, + expectedTgBindingSpec: elbv2model.TargetGroupBindingResourceSpec{ + Template: elbv2model.TargetGroupBindingTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-svc-ns", + Name: "k8s-myrouten-myroute-1949ae79d7", + }, + Spec: elbv2model.TargetGroupBindingSpec{ + TargetType: &instanceType, + ServiceRef: elbv2api.ServiceReference{ + Name: "my-svc", + Port: intstr.FromInt32(80), // TODO - Figure out why this port is added and not the node port. + }, + IPAddressType: elbv2api.TargetGroupIPAddressType(elbv2model.IPAddressTypeIPV4), + VpcID: "vpc-xxx", + }, + }, + }, + }, + { + name: "no tg config - instance - alb", + tags: make(map[string]string), + lbType: elbv2model.LoadBalancerTypeApplication, + disableRestrictedSGRules: false, + defaultTargetType: string(elbv2model.TargetTypeInstance), + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-gw-ns", + Name: "my-gw", + }, + }, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "my-route", + Namespace: "my-route-ns", + }, + backend: routeutils.Backend{ + Service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-svc-ns", + Name: "my-svc", + }, + }, + ServicePort: &corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.IntOrString{ + IntVal: 80, + Type: intstr.Int, + }, + NodePort: 8080, + }, + }, + expectedTgSpec: elbv2model.TargetGroupSpec{ + Name: "k8s-myrouten-myroute-e99d898968", + TargetType: elbv2model.TargetTypeInstance, + Port: awssdk.Int32(8080), + Protocol: elbv2model.ProtocolHTTP, + ProtocolVersion: &http1, + IPAddressType: elbv2model.TargetGroupIPAddressTypeIPv4, + HealthCheckConfig: &elbv2model.TargetGroupHealthCheckConfig{ + Port: &intstr.IntOrString{ + StrVal: shared_constants.HealthCheckPortTrafficPort, + Type: intstr.String, + }, + Path: awssdk.String("/"), + Matcher: &elbv2model.HealthCheckMatcher{ + HTTPCode: awssdk.String("200-399"), + }, + Protocol: elbv2model.ProtocolHTTP, + IntervalSeconds: awssdk.Int32(15), + TimeoutSeconds: awssdk.Int32(5), + HealthyThresholdCount: awssdk.Int32(3), + UnhealthyThresholdCount: awssdk.Int32(3), + }, + TargetGroupAttributes: make([]elbv2model.TargetGroupAttribute, 0), + Tags: make(map[string]string), + }, + expectedTgBindingSpec: elbv2model.TargetGroupBindingResourceSpec{ + Template: elbv2model.TargetGroupBindingTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-svc-ns", + Name: "k8s-myrouten-myroute-e99d898968", + }, + Spec: elbv2model.TargetGroupBindingSpec{ + TargetType: &instanceType, + ServiceRef: elbv2api.ServiceReference{ + Name: "my-svc", + Port: intstr.FromInt32(80), // TODO - Figure out why this port is added and not the node port. + }, + IPAddressType: elbv2api.TargetGroupIPAddressType(elbv2model.IPAddressTypeIPV4), + VpcID: "vpc-xxx", + }, + }, + }, + }, + { + name: "no tg config - ip - nlb", + tags: make(map[string]string), + lbType: elbv2model.LoadBalancerTypeNetwork, + disableRestrictedSGRules: false, + defaultTargetType: string(elbv2model.TargetTypeIP), + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-gw-ns", + Name: "my-gw", + }, + }, + route: &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + Name: "my-route", + Namespace: "my-route-ns", + }, + backend: routeutils.Backend{ + Service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-svc-ns", + Name: "my-svc", + }, + }, + ServicePort: &corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.IntOrString{ + IntVal: 80, + Type: intstr.Int, + }, + NodePort: 8080, + }, + }, + expectedTgSpec: elbv2model.TargetGroupSpec{ + Name: "k8s-myrouten-myroute-7ac9e90fa0", + TargetType: elbv2model.TargetTypeIP, + Port: awssdk.Int32(80), + Protocol: elbv2model.ProtocolTCP, + IPAddressType: elbv2model.TargetGroupIPAddressTypeIPv4, + HealthCheckConfig: &elbv2model.TargetGroupHealthCheckConfig{ + Port: &intstr.IntOrString{ + StrVal: shared_constants.HealthCheckPortTrafficPort, + Type: intstr.String, + }, + Protocol: elbv2model.ProtocolTCP, + IntervalSeconds: awssdk.Int32(15), + TimeoutSeconds: awssdk.Int32(5), + HealthyThresholdCount: awssdk.Int32(3), + UnhealthyThresholdCount: awssdk.Int32(3), + }, + TargetGroupAttributes: make([]elbv2model.TargetGroupAttribute, 0), + Tags: make(map[string]string), + }, + expectedTgBindingSpec: elbv2model.TargetGroupBindingResourceSpec{ + Template: elbv2model.TargetGroupBindingTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-svc-ns", + Name: "k8s-myrouten-myroute-7ac9e90fa0", + }, + Spec: elbv2model.TargetGroupBindingSpec{ + TargetType: &ipType, + ServiceRef: elbv2api.ServiceReference{ + Name: "my-svc", + Port: intstr.FromInt32(80), + }, + IPAddressType: elbv2api.TargetGroupIPAddressType(elbv2model.IPAddressTypeIPV4), + VpcID: "vpc-xxx", + }, + }, + }, + }, + { + name: "no tg config - ip - alb", + tags: make(map[string]string), + lbType: elbv2model.LoadBalancerTypeApplication, + disableRestrictedSGRules: false, + defaultTargetType: string(elbv2model.TargetTypeIP), + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-gw-ns", + Name: "my-gw", + }, + }, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "my-route", + Namespace: "my-route-ns", + }, + backend: routeutils.Backend{ + Service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-svc-ns", + Name: "my-svc", + }, + }, + ServicePort: &corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.IntOrString{ + IntVal: 80, + Type: intstr.Int, + }, + NodePort: 8080, + }, + }, + expectedTgSpec: elbv2model.TargetGroupSpec{ + Name: "k8s-myrouten-myroute-8a97d3dcbe", + TargetType: elbv2model.TargetTypeIP, + Port: awssdk.Int32(80), + Protocol: elbv2model.ProtocolHTTP, + ProtocolVersion: &http1, + IPAddressType: elbv2model.TargetGroupIPAddressTypeIPv4, + HealthCheckConfig: &elbv2model.TargetGroupHealthCheckConfig{ + Port: &intstr.IntOrString{ + StrVal: shared_constants.HealthCheckPortTrafficPort, + Type: intstr.String, + }, + Path: awssdk.String("/"), + Matcher: &elbv2model.HealthCheckMatcher{ + HTTPCode: awssdk.String("200-399"), + }, + Protocol: elbv2model.ProtocolHTTP, + IntervalSeconds: awssdk.Int32(15), + TimeoutSeconds: awssdk.Int32(5), + HealthyThresholdCount: awssdk.Int32(3), + UnhealthyThresholdCount: awssdk.Int32(3), + }, + TargetGroupAttributes: make([]elbv2model.TargetGroupAttribute, 0), + Tags: make(map[string]string), + }, + expectedTgBindingSpec: elbv2model.TargetGroupBindingResourceSpec{ + Template: elbv2model.TargetGroupBindingTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-svc-ns", + Name: "k8s-myrouten-myroute-8a97d3dcbe", + }, + Spec: elbv2model.TargetGroupBindingSpec{ + TargetType: &ipType, + ServiceRef: elbv2api.ServiceReference{ + Name: "my-svc", + Port: intstr.FromInt32(80), // TODO - Figure out why this port is added and not the node port. + }, + IPAddressType: elbv2api.TargetGroupIPAddressType(elbv2model.IPAddressTypeIPV4), + VpcID: "vpc-xxx", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + tagger := &mockTagHelper{ + tags: tc.tags, + err: tc.tagErr, + } + + builder := newTargetGroupBuilder("my-cluster", "vpc-xxx", tagger, tc.lbType, tc.disableRestrictedSGRules, tc.defaultTargetType) + + result := make(map[string]buildTargetGroupOutput) + + out, err := builder.buildTargetGroup(&result, tc.gateway, nil, elbv2model.IPAddressTypeIPV4, tc.route, tc.backend, nil) + if tc.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expectedTgSpec, out.targetGroupSpec) + assert.Equal(t, tc.expectedTgBindingSpec, out.bindingSpec) + assert.Equal(t, 1, len(result)) + for _, v := range result { + assert.Equal(t, tc.expectedTgSpec, v.targetGroupSpec) + assert.Equal(t, tc.expectedTgBindingSpec, v.bindingSpec) + } + }) + } +} diff --git a/pkg/gateway/model/utilities.go b/pkg/gateway/model/utilities.go new file mode 100644 index 0000000000..ada833061e --- /dev/null +++ b/pkg/gateway/model/utilities.go @@ -0,0 +1,20 @@ +package model + +import ( + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "strings" +) + +func isIPv6Supported(ipAddressType elbv2model.IPAddressType) bool { + switch ipAddressType { + case elbv2model.IPAddressTypeDualStack, elbv2model.IPAddressTypeDualStackWithoutPublicIPV4: + return true + default: + return false + } +} + +// TODO - Refactor? +func isIPv6CIDR(cidr string) bool { + return strings.Contains(cidr, ":") +} diff --git a/pkg/gateway/routeutils/backend.go b/pkg/gateway/routeutils/backend.go index c34baa7aac..31816272e9 100644 --- a/pkg/gateway/routeutils/backend.go +++ b/pkg/gateway/routeutils/backend.go @@ -27,7 +27,7 @@ type Backend struct { } // commonBackendLoader this function will load the services and target group configurations associated with this gateway backend. -func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) { +func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) { // We only support references of type service. if backendRef.Kind != nil && *backendRef.Kind != "Service" { @@ -144,7 +144,7 @@ func lookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client // Implements the reference grant API // https://gateway-api.sigs.k8s.io/api-types/referencegrant/ -func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdentifier types.NamespacedName, routeIdentifier types.NamespacedName, routeKind string) (bool, error) { +func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdentifier types.NamespacedName, routeIdentifier types.NamespacedName, routeKind RouteKind) (bool, error) { referenceGrantList := &gwbeta1.ReferenceGrantList{} if err := k8sClient.List(ctx, referenceGrantList, client.InNamespace(svcIdentifier.Namespace)); err != nil { return false, err @@ -155,7 +155,7 @@ func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdenti for _, from := range grant.Spec.From { // Kind check maybe? - if string(from.Kind) == routeKind && string(from.Namespace) == routeIdentifier.Namespace { + if string(from.Kind) == string(routeKind) && string(from.Namespace) == routeIdentifier.Namespace { routeAllowed = true break } diff --git a/pkg/gateway/routeutils/constants.go b/pkg/gateway/routeutils/constants.go index 5e6478fa18..4c1a70e9f1 100644 --- a/pkg/gateway/routeutils/constants.go +++ b/pkg/gateway/routeutils/constants.go @@ -6,17 +6,19 @@ import ( gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) +type RouteKind string + // Route Kinds const ( - TCPRouteKind = "TCPRoute" - UDPRouteKind = "UDPRoute" - TLSRouteKind = "TLSRoute" - HTTPRouteKind = "HTTPRoute" - GRPCRouteKind = "GRPCRoute" + TCPRouteKind RouteKind = "TCPRoute" + UDPRouteKind RouteKind = "UDPRoute" + TLSRouteKind RouteKind = "TLSRoute" + HTTPRouteKind RouteKind = "HTTPRoute" + GRPCRouteKind RouteKind = "GRPCRoute" ) // RouteKind to Route Loader. These functions will pull data directly from the kube api or local cache. -var allRoutes = map[string]func(context context.Context, client client.Client) ([]preLoadRouteDescriptor, error){ +var allRoutes = map[RouteKind]func(context context.Context, client client.Client) ([]preLoadRouteDescriptor, error){ TCPRouteKind: ListTCPRoutes, UDPRouteKind: ListUDPRoutes, TLSRouteKind: ListTLSRoutes, @@ -25,7 +27,7 @@ var allRoutes = map[string]func(context context.Context, client client.Client) ( } // Default protocol map used to infer accepted route kinds when a listener doesn't specify the `allowedRoutes` field. -var defaultProtocolToRouteKindMap = map[gwv1.ProtocolType]string{ +var defaultProtocolToRouteKindMap = map[gwv1.ProtocolType]RouteKind{ gwv1.TCPProtocolType: TCPRouteKind, gwv1.UDPProtocolType: UDPRouteKind, gwv1.TLSProtocolType: TLSRouteKind, diff --git a/pkg/gateway/routeutils/descriptor.go b/pkg/gateway/routeutils/descriptor.go index 59e43db82d..9f776e79ed 100644 --- a/pkg/gateway/routeutils/descriptor.go +++ b/pkg/gateway/routeutils/descriptor.go @@ -12,7 +12,7 @@ import ( // however, consumers can use `GetRawRoute()` to inspect the actual route fields if needed. type routeMetadataDescriptor interface { GetRouteNamespacedName() types.NamespacedName - GetRouteKind() string + GetRouteKind() RouteKind GetHostnames() []gwv1.Hostname GetParentRefs() []gwv1.ParentReference GetRawRoute() interface{} diff --git a/pkg/gateway/routeutils/grpc.go b/pkg/gateway/routeutils/grpc.go index 27e693cde6..8a1586d76a 100644 --- a/pkg/gateway/routeutils/grpc.go +++ b/pkg/gateway/routeutils/grpc.go @@ -47,7 +47,7 @@ func (t *convertedGRPCRouteRule) GetBackends() []Backend { type grpcRouteDescription struct { route *gwv1.GRPCRoute rules []RouteRule - backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) + backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) } func (grpcRoute *grpcRouteDescription) loadAttachedRules(ctx context.Context, k8sClient client.Client) (RouteDescriptor, error) { @@ -83,7 +83,7 @@ func (grpcRoute *grpcRouteDescription) GetParentRefs() []gwv1.ParentReference { return grpcRoute.route.Spec.ParentRefs } -func (grpcRoute *grpcRouteDescription) GetRouteKind() string { +func (grpcRoute *grpcRouteDescription) GetRouteKind() RouteKind { return GRPCRouteKind } diff --git a/pkg/gateway/routeutils/grpc_test.go b/pkg/gateway/routeutils/grpc_test.go index a00bb6e3a3..fe6adc4d58 100644 --- a/pkg/gateway/routeutils/grpc_test.go +++ b/pkg/gateway/routeutils/grpc_test.go @@ -125,7 +125,7 @@ func Test_ListGRPCRoutes(t *testing.T) { func Test_GRPC_LoadAttachedRules(t *testing.T) { weight := 0 - mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) { + mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) { weight++ return &Backend{ Weight: weight, diff --git a/pkg/gateway/routeutils/http.go b/pkg/gateway/routeutils/http.go index 4dab813022..22e42934f4 100644 --- a/pkg/gateway/routeutils/http.go +++ b/pkg/gateway/routeutils/http.go @@ -47,7 +47,7 @@ func (t *convertedHTTPRouteRule) GetBackends() []Backend { type httpRouteDescription struct { route *gwv1.HTTPRoute rules []RouteRule - backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) + backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) } func (httpRoute *httpRouteDescription) GetAttachedRules() []RouteRule { @@ -84,7 +84,7 @@ func (httpRoute *httpRouteDescription) GetParentRefs() []gwv1.ParentReference { return httpRoute.route.Spec.ParentRefs } -func (httpRoute *httpRouteDescription) GetRouteKind() string { +func (httpRoute *httpRouteDescription) GetRouteKind() RouteKind { return HTTPRouteKind } diff --git a/pkg/gateway/routeutils/http_test.go b/pkg/gateway/routeutils/http_test.go index 9d0d98f868..87c0ce7736 100644 --- a/pkg/gateway/routeutils/http_test.go +++ b/pkg/gateway/routeutils/http_test.go @@ -125,7 +125,7 @@ func Test_ListHTTPRoutes(t *testing.T) { func Test_HTTP_LoadAttachedRules(t *testing.T) { weight := 0 - mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) { + mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) { weight++ return &Backend{ Weight: weight, diff --git a/pkg/gateway/routeutils/listener_attachment_helper.go b/pkg/gateway/routeutils/listener_attachment_helper.go index 1653e2e59a..a53d3fccb0 100644 --- a/pkg/gateway/routeutils/listener_attachment_helper.go +++ b/pkg/gateway/routeutils/listener_attachment_helper.go @@ -2,6 +2,7 @@ package routeutils import ( "context" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -18,11 +19,13 @@ var _ listenerAttachmentHelper = &listenerAttachmentHelperImpl{} // listenerAttachmentHelperImpl implements the listenerAttachmentHelper interface. type listenerAttachmentHelperImpl struct { namespaceSelector namespaceSelector + logger logr.Logger } -func newListenerAttachmentHelper(k8sClient client.Client) listenerAttachmentHelper { +func newListenerAttachmentHelper(k8sClient client.Client, logger logr.Logger) listenerAttachmentHelper { return &listenerAttachmentHelperImpl{ namespaceSelector: newNamespaceSelector(k8sClient), + logger: logger, } } @@ -33,15 +36,10 @@ func (attachmentHelper *listenerAttachmentHelperImpl) listenerAllowsAttachment(c if err != nil { return false, err } - if !namespaceOK { return false, nil } - - if !attachmentHelper.kindCheck(listener, route) { - return false, nil - } - return true, nil + return attachmentHelper.kindCheck(listener, route), nil } // namespaceCheck namespace check implements the Gateway API spec for namespace matching between listener @@ -86,7 +84,7 @@ func (attachmentHelper *listenerAttachmentHelperImpl) namespaceCheck(ctx context // and route to determine compatibility. func (attachmentHelper *listenerAttachmentHelperImpl) kindCheck(listener gwv1.Listener, route preLoadRouteDescriptor) bool { - var allowedRoutes sets.Set[string] + var allowedRoutes sets.Set[RouteKind] /* ... @@ -95,13 +93,13 @@ func (attachmentHelper *listenerAttachmentHelperImpl) kindCheck(listener gwv1.Li ... */ if listener.AllowedRoutes == nil || listener.AllowedRoutes.Kinds == nil || len(listener.AllowedRoutes.Kinds) == 0 { - allowedRoutes = sets.New[string](defaultProtocolToRouteKindMap[listener.Protocol]) + allowedRoutes = sets.New[RouteKind](defaultProtocolToRouteKindMap[listener.Protocol]) } else { // TODO - Not sure how to handle versioning (correctly) here. // So going to ignore the group checks for now :x - allowedRoutes = sets.New[string]() + allowedRoutes = sets.New[RouteKind]() for _, v := range listener.AllowedRoutes.Kinds { - allowedRoutes.Insert(string(v.Kind)) + allowedRoutes.Insert(RouteKind(v.Kind)) } } return allowedRoutes.Has(route.GetRouteKind()) diff --git a/pkg/gateway/routeutils/listener_attachment_helper_test.go b/pkg/gateway/routeutils/listener_attachment_helper_test.go index 3e3af83720..5272bc3134 100644 --- a/pkg/gateway/routeutils/listener_attachment_helper_test.go +++ b/pkg/gateway/routeutils/listener_attachment_helper_test.go @@ -2,6 +2,7 @@ package routeutils import ( "context" + "github.com/go-logr/logr" "github.com/pkg/errors" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -65,7 +66,9 @@ func Test_listenerAllowsAttachment(t *testing.T) { Namespace: tc.routeNamespace, }, }} - attachmentHelper := listenerAttachmentHelperImpl{} + attachmentHelper := listenerAttachmentHelperImpl{ + logger: logr.Discard(), + } result, err := attachmentHelper.listenerAllowsAttachment(context.Background(), gw, gwv1.Listener{ Protocol: tc.listenerProtocol, }, route) @@ -262,6 +265,7 @@ func Test_namespaceCheck(t *testing.T) { err: tc.namespaceSelectorError, nss: tc.namespaceSelectorResult, }, + logger: logr.Discard(), } gw := gwv1.Gateway{ @@ -337,7 +341,7 @@ func Test_kindCheck(t *testing.T) { listener: gwv1.Listener{ Protocol: gwv1.UDPProtocolType, AllowedRoutes: &gwv1.AllowedRoutes{Kinds: []gwv1.RouteGroupKind{ - {Kind: HTTPRouteKind}, + {Kind: gwv1.Kind(HTTPRouteKind)}, }}, }, expectedResult: true, @@ -346,7 +350,9 @@ func Test_kindCheck(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - attachmentHelper := listenerAttachmentHelperImpl{} + attachmentHelper := listenerAttachmentHelperImpl{ + logger: logr.Discard(), + } assert.Equal(t, tc.expectedResult, attachmentHelper.kindCheck(tc.listener, tc.route)) }) } diff --git a/pkg/gateway/routeutils/loader.go b/pkg/gateway/routeutils/loader.go index 1f5f64b48a..7e67247a5b 100644 --- a/pkg/gateway/routeutils/loader.go +++ b/pkg/gateway/routeutils/loader.go @@ -3,6 +3,7 @@ package routeutils import ( "context" "fmt" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -10,15 +11,15 @@ import ( // LoadRouteFilter is an interface that consumers can use to tell the loader which routes to load. type LoadRouteFilter interface { - IsApplicable(kind string) bool + IsApplicable(kind RouteKind) bool } // routeFilterImpl implements LoadRouteFilter type routeFilterImpl struct { - acceptedKinds sets.Set[string] + acceptedKinds sets.Set[RouteKind] } -func (r *routeFilterImpl) IsApplicable(kind string) bool { +func (r *routeFilterImpl) IsApplicable(kind RouteKind) bool { return r.acceptedKinds.Has(kind) } @@ -54,14 +55,16 @@ var _ Loader = &loaderImpl{} type loaderImpl struct { mapper listenerToRouteMapper k8sClient client.Client - allRouteLoaders map[string]func(context context.Context, client client.Client) ([]preLoadRouteDescriptor, error) + logger logr.Logger + allRouteLoaders map[RouteKind]func(context context.Context, client client.Client) ([]preLoadRouteDescriptor, error) } -func NewLoader(k8sClient client.Client) Loader { +func NewLoader(k8sClient client.Client, logger logr.Logger) Loader { return &loaderImpl{ - mapper: newListenerToRouteMapper(k8sClient), + mapper: newListenerToRouteMapper(k8sClient, logger.WithName("route-mapper")), k8sClient: k8sClient, allRouteLoaders: allRoutes, + logger: logger, } } @@ -70,7 +73,10 @@ func (l *loaderImpl) LoadRoutesForGateway(ctx context.Context, gw gwv1.Gateway, // 1. Load all relevant routes according to the filter loadedRoutes := make([]preLoadRouteDescriptor, 0) for route, loader := range l.allRouteLoaders { - if filter.IsApplicable(route) { + + applicable := filter.IsApplicable(route) + l.logger.V(1).Info("Processing route", "route", route, "is applicable", applicable) + if applicable { data, err := loader(ctx, l.k8sClient) if err != nil { return nil, err diff --git a/pkg/gateway/routeutils/loader_test.go b/pkg/gateway/routeutils/loader_test.go index 6a8a1145e5..1368f93f51 100644 --- a/pkg/gateway/routeutils/loader_test.go +++ b/pkg/gateway/routeutils/loader_test.go @@ -2,6 +2,7 @@ package routeutils import ( "context" + "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -25,7 +26,7 @@ var _ RouteDescriptor = &mockRoute{} type mockRoute struct { namespacedName types.NamespacedName - routeKind string + routeKind RouteKind } func (m *mockRoute) loadAttachedRules(context context.Context, k8sClient client.Client) (RouteDescriptor, error) { @@ -36,7 +37,7 @@ func (m *mockRoute) GetRouteNamespacedName() types.NamespacedName { return m.namespacedName } -func (m *mockRoute) GetRouteKind() string { +func (m *mockRoute) GetRouteKind() RouteKind { return m.routeKind } @@ -126,7 +127,7 @@ func TestLoadRoutesForGateway(t *testing.T) { loadedTCPRoutes = append(loadedTCPRoutes, r) } - allRouteLoaders := map[string]func(ctx context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error){ + allRouteLoaders := map[RouteKind]func(ctx context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error){ HTTPRouteKind: func(ctx context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error) { return preLoadHTTPRoutes, nil }, @@ -137,7 +138,7 @@ func TestLoadRoutesForGateway(t *testing.T) { testCases := []struct { name string - acceptedKinds sets.Set[string] + acceptedKinds sets.Set[RouteKind] expectedMap map[int][]RouteDescriptor expectedPreloadMap map[int][]preLoadRouteDescriptor expectedPreMappedRoutes []preLoadRouteDescriptor @@ -145,13 +146,13 @@ func TestLoadRoutesForGateway(t *testing.T) { }{ { name: "filter allows no routes", - acceptedKinds: make(sets.Set[string]), + acceptedKinds: make(sets.Set[RouteKind]), expectedPreMappedRoutes: make([]preLoadRouteDescriptor, 0), expectedMap: make(map[int][]RouteDescriptor), }, { name: "filter only allows http route", - acceptedKinds: sets.New[string](HTTPRouteKind), + acceptedKinds: sets.New[RouteKind](HTTPRouteKind), expectedPreMappedRoutes: preLoadHTTPRoutes, expectedPreloadMap: map[int][]preLoadRouteDescriptor{ 80: preLoadHTTPRoutes, @@ -162,7 +163,7 @@ func TestLoadRoutesForGateway(t *testing.T) { }, { name: "filter only allows http route, multiple ports", - acceptedKinds: sets.New[string](HTTPRouteKind), + acceptedKinds: sets.New[RouteKind](HTTPRouteKind), expectedPreMappedRoutes: preLoadHTTPRoutes, expectedPreloadMap: map[int][]preLoadRouteDescriptor{ 80: preLoadHTTPRoutes, @@ -175,7 +176,7 @@ func TestLoadRoutesForGateway(t *testing.T) { }, { name: "filter only allows tcp route", - acceptedKinds: sets.New[string](TCPRouteKind), + acceptedKinds: sets.New[RouteKind](TCPRouteKind), expectedPreMappedRoutes: preLoadTCPRoutes, expectedPreloadMap: map[int][]preLoadRouteDescriptor{ 80: preLoadTCPRoutes, @@ -186,7 +187,7 @@ func TestLoadRoutesForGateway(t *testing.T) { }, { name: "filter allows both route kinds", - acceptedKinds: sets.New[string](TCPRouteKind, HTTPRouteKind), + acceptedKinds: sets.New[RouteKind](TCPRouteKind, HTTPRouteKind), expectedPreMappedRoutes: append(preLoadHTTPRoutes, preLoadTCPRoutes...), expectedPreloadMap: map[int][]preLoadRouteDescriptor{ 80: preLoadTCPRoutes, @@ -208,6 +209,7 @@ func TestLoadRoutesForGateway(t *testing.T) { mapToReturn: tc.expectedPreloadMap, }, allRouteLoaders: allRouteLoaders, + logger: logr.Discard(), } filter := &routeFilterImpl{acceptedKinds: tc.acceptedKinds} diff --git a/pkg/gateway/routeutils/mock_route.go b/pkg/gateway/routeutils/mock_route.go index 13b438c58e..eae318cfae 100644 --- a/pkg/gateway/routeutils/mock_route.go +++ b/pkg/gateway/routeutils/mock_route.go @@ -6,7 +6,9 @@ import ( ) type MockRoute struct { - Kind string + Kind RouteKind + Name string + Namespace string } func (m *MockRoute) GetBackendRefs() []gwv1.BackendRef { @@ -15,11 +17,13 @@ func (m *MockRoute) GetBackendRefs() []gwv1.BackendRef { } func (m *MockRoute) GetRouteNamespacedName() types.NamespacedName { - //TODO implement me - panic("implement me") + return types.NamespacedName{ + Namespace: m.Namespace, + Name: m.Name, + } } -func (m *MockRoute) GetRouteKind() string { +func (m *MockRoute) GetRouteKind() RouteKind { return m.Kind } diff --git a/pkg/gateway/routeutils/route_attachment_helper.go b/pkg/gateway/routeutils/route_attachment_helper.go index 7644f45ffd..995d612387 100644 --- a/pkg/gateway/routeutils/route_attachment_helper.go +++ b/pkg/gateway/routeutils/route_attachment_helper.go @@ -1,6 +1,7 @@ package routeutils import ( + "github.com/go-logr/logr" gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -13,10 +14,13 @@ type routeAttachmentHelper interface { var _ routeAttachmentHelper = &routeAttachmentHelperImpl{} type routeAttachmentHelperImpl struct { + logger logr.Logger } -func newRouteAttachmentHelper() routeAttachmentHelper { - return &routeAttachmentHelperImpl{} +func newRouteAttachmentHelper(logger logr.Logger) routeAttachmentHelper { + return &routeAttachmentHelperImpl{ + logger: logger, + } } // doesRouteAttachToGateway is responsible for determining if a route and gateway should be connected. @@ -37,11 +41,12 @@ func (rah *routeAttachmentHelperImpl) doesRouteAttachToGateway(gw gwv1.Gateway, namespaceToCompare = gw.Namespace } - if string(parentRef.Name) == gw.Name && gw.Namespace == namespaceToCompare { + nameCheck := string(parentRef.Name) == gw.Name + nsCheck := gw.Namespace == namespaceToCompare + if nameCheck && nsCheck { return true } } - return false } diff --git a/pkg/gateway/routeutils/route_attachment_helper_test.go b/pkg/gateway/routeutils/route_attachment_helper_test.go index 8439d34587..c1abba2dc9 100644 --- a/pkg/gateway/routeutils/route_attachment_helper_test.go +++ b/pkg/gateway/routeutils/route_attachment_helper_test.go @@ -2,6 +2,7 @@ package routeutils import ( awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/go-logr/logr" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" gwv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -221,7 +222,9 @@ func Test_doesRouteAttachToGateway(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - helper := &routeAttachmentHelperImpl{} + helper := &routeAttachmentHelperImpl{ + logger: logr.Discard(), + } assert.Equal(t, tc.result, helper.doesRouteAttachToGateway(tc.gw, tc.route)) }) } @@ -359,7 +362,9 @@ func Test_routeAllowsAttachmentToListener(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - helper := &routeAttachmentHelperImpl{} + helper := &routeAttachmentHelperImpl{ + logger: logr.Discard(), + } assert.Equal(t, tc.result, helper.routeAllowsAttachmentToListener(tc.listener, tc.route)) }) } diff --git a/pkg/gateway/routeutils/route_listener_mapper.go b/pkg/gateway/routeutils/route_listener_mapper.go index 6e9d15de41..cdf33726b6 100644 --- a/pkg/gateway/routeutils/route_listener_mapper.go +++ b/pkg/gateway/routeutils/route_listener_mapper.go @@ -2,6 +2,7 @@ package routeutils import ( "context" + "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -17,12 +18,14 @@ var _ listenerToRouteMapper = &listenerToRouteMapperImpl{} type listenerToRouteMapperImpl struct { listenerAttachmentHelper listenerAttachmentHelper routeAttachmentHelper routeAttachmentHelper + logger logr.Logger } -func newListenerToRouteMapper(k8sClient client.Client) listenerToRouteMapper { +func newListenerToRouteMapper(k8sClient client.Client, logger logr.Logger) listenerToRouteMapper { return &listenerToRouteMapperImpl{ - listenerAttachmentHelper: newListenerAttachmentHelper(k8sClient), - routeAttachmentHelper: newRouteAttachmentHelper(), + listenerAttachmentHelper: newListenerAttachmentHelper(k8sClient, logger.WithName("listener-attachment-helper")), + routeAttachmentHelper: newRouteAttachmentHelper(logger.WithName("route-attachment-helper")), + logger: logger, } } @@ -33,7 +36,9 @@ func (ltr *listenerToRouteMapperImpl) mapGatewayAndRoutes(ctx context.Context, g // First filter out any routes that are not intended for this Gateway. routesForGateway := make([]preLoadRouteDescriptor, 0) for _, route := range routes { - if ltr.routeAttachmentHelper.doesRouteAttachToGateway(gw, route) { + allowsAttachment := ltr.routeAttachmentHelper.doesRouteAttachToGateway(gw, route) + ltr.logger.V(1).Info("Route is eligible for attachment", "route", route.GetRouteNamespacedName(), "allowed attachment", allowsAttachment) + if allowsAttachment { routesForGateway = append(routesForGateway, route) } } @@ -45,6 +50,7 @@ func (ltr *listenerToRouteMapperImpl) mapGatewayAndRoutes(ctx context.Context, g // We need to check both paths (route -> listener) and (listener -> route) // for connection viability. if !ltr.routeAttachmentHelper.routeAllowsAttachmentToListener(listener, route) { + ltr.logger.V(1).Info("Route doesnt allow attachment") continue } @@ -53,10 +59,12 @@ func (ltr *listenerToRouteMapperImpl) mapGatewayAndRoutes(ctx context.Context, g return nil, err } + ltr.logger.V(1).Info("lister allows attachment", "route", route.GetRouteNamespacedName(), "allowedAttachment", allowedAttachment) if allowedAttachment { result[int(listener.Port)] = append(result[int(listener.Port)], route) } } } + ltr.logger.Info("Final result is this", "result", result) return result, nil } diff --git a/pkg/gateway/routeutils/route_listener_mapper_test.go b/pkg/gateway/routeutils/route_listener_mapper_test.go index 3f44c3ecb5..3681ce3411 100644 --- a/pkg/gateway/routeutils/route_listener_mapper_test.go +++ b/pkg/gateway/routeutils/route_listener_mapper_test.go @@ -3,6 +3,7 @@ package routeutils import ( "context" "fmt" + "github.com/go-logr/logr" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" gwv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -309,6 +310,7 @@ func Test_mapGatewayAndRoutes(t *testing.T) { routeListenerMap: tc.routeListenerMap, routeGatewayMap: tc.routeGatewayMap, }, + logger: logr.Discard(), } result, err := mapper.mapGatewayAndRoutes(context.Background(), tc.gw, tc.routes) diff --git a/pkg/gateway/routeutils/tcp.go b/pkg/gateway/routeutils/tcp.go index 2425f7fae9..a34e79d468 100644 --- a/pkg/gateway/routeutils/tcp.go +++ b/pkg/gateway/routeutils/tcp.go @@ -48,7 +48,7 @@ func (t *convertedTCPRouteRule) GetBackends() []Backend { type tcpRouteDescription struct { route *gwalpha2.TCPRoute rules []RouteRule - backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) + backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) } func (tcpRoute *tcpRouteDescription) GetAttachedRules() []RouteRule { @@ -82,7 +82,7 @@ func (tcpRoute *tcpRouteDescription) GetHostnames() []gwv1.Hostname { return []gwv1.Hostname{} } -func (tcpRoute *tcpRouteDescription) GetRouteKind() string { +func (tcpRoute *tcpRouteDescription) GetRouteKind() RouteKind { return TCPRouteKind } @@ -128,6 +128,5 @@ func ListTCPRoutes(context context.Context, k8sClient client.Client) ([]preLoadR for _, item := range routeList.Items { result = append(result, convertTCPRoute(item)) } - return result, err } diff --git a/pkg/gateway/routeutils/tcp_test.go b/pkg/gateway/routeutils/tcp_test.go index 0ceef13f4b..051e063cd7 100644 --- a/pkg/gateway/routeutils/tcp_test.go +++ b/pkg/gateway/routeutils/tcp_test.go @@ -109,7 +109,7 @@ func Test_ListTCPRoutes(t *testing.T) { func Test_TCP_LoadAttachedRules(t *testing.T) { weight := 0 - mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) { + mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) { weight++ return &Backend{ Weight: weight, diff --git a/pkg/gateway/routeutils/tls.go b/pkg/gateway/routeutils/tls.go index f665c25b9e..911b18dfdf 100644 --- a/pkg/gateway/routeutils/tls.go +++ b/pkg/gateway/routeutils/tls.go @@ -48,7 +48,7 @@ func (t *convertedTLSRouteRule) GetBackends() []Backend { type tlsRouteDescription struct { route *gwalpha2.TLSRoute rules []RouteRule - backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) + backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) } func (tlsRoute *tlsRouteDescription) GetAttachedRules() []RouteRule { @@ -86,7 +86,7 @@ func (tlsRoute *tlsRouteDescription) GetParentRefs() []gwv1.ParentReference { return tlsRoute.route.Spec.ParentRefs } -func (tlsRoute *tlsRouteDescription) GetRouteKind() string { +func (tlsRoute *tlsRouteDescription) GetRouteKind() RouteKind { return TLSRouteKind } diff --git a/pkg/gateway/routeutils/tls_test.go b/pkg/gateway/routeutils/tls_test.go index 801480a5a6..6a1335c7dc 100644 --- a/pkg/gateway/routeutils/tls_test.go +++ b/pkg/gateway/routeutils/tls_test.go @@ -123,7 +123,7 @@ func Test_ListTLSRoutes(t *testing.T) { func Test_TLS_LoadAttachedRules(t *testing.T) { weight := 0 - mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) { + mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) { weight++ return &Backend{ Weight: weight, diff --git a/pkg/gateway/routeutils/udp.go b/pkg/gateway/routeutils/udp.go index 818dd770a1..fca6c9d124 100644 --- a/pkg/gateway/routeutils/udp.go +++ b/pkg/gateway/routeutils/udp.go @@ -48,7 +48,7 @@ func (t *convertedUDPRouteRule) GetBackends() []Backend { type udpRouteDescription struct { route *gwalpha2.UDPRoute rules []RouteRule - backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) + backendLoader func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) } func (udpRoute *udpRouteDescription) GetAttachedRules() []RouteRule { @@ -85,7 +85,7 @@ func (udpRoute *udpRouteDescription) GetParentRefs() []gwv1.ParentReference { return udpRoute.route.Spec.ParentRefs } -func (udpRoute *udpRouteDescription) GetRouteKind() string { +func (udpRoute *udpRouteDescription) GetRouteKind() RouteKind { return UDPRouteKind } diff --git a/pkg/gateway/routeutils/udp_test.go b/pkg/gateway/routeutils/udp_test.go index c3ae71dde8..58c57661dc 100644 --- a/pkg/gateway/routeutils/udp_test.go +++ b/pkg/gateway/routeutils/udp_test.go @@ -109,7 +109,7 @@ func Test_ListUDPRoutes(t *testing.T) { func Test_UDP_LoadAttachedRules(t *testing.T) { weight := 0 - mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind string) (*Backend, error) { + mockLoader := func(ctx context.Context, k8sClient client.Client, typeSpecificBackend interface{}, backendRef gwv1.BackendRef, routeIdentifier types.NamespacedName, routeKind RouteKind) (*Backend, error) { weight++ return &Backend{ Weight: weight, diff --git a/pkg/gateway/routeutils/utils.go b/pkg/gateway/routeutils/utils.go index 75e002b270..06293f078e 100644 --- a/pkg/gateway/routeutils/utils.go +++ b/pkg/gateway/routeutils/utils.go @@ -11,7 +11,7 @@ import ( // ListL4Routes retrieves all Layer 4 routes (TCP, UDP, TLS) from the cluster. func ListL4Routes(ctx context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error) { l4Routes := make([]preLoadRouteDescriptor, 0) - routekinds := []string{} + var routekinds []RouteKind tcpRoutes, err := ListTCPRoutes(ctx, k8sClient) if err != nil { routekinds = append(routekinds, TCPRouteKind) @@ -36,7 +36,7 @@ func ListL4Routes(ctx context.Context, k8sClient client.Client) ([]preLoadRouteD // ListL7Routes retrieves all Layer 7 routes (HTTP, gRPC) from the cluster. func ListL7Routes(ctx context.Context, k8sClient client.Client) ([]preLoadRouteDescriptor, error) { l7Routes := make([]preLoadRouteDescriptor, 0) - routekinds := []string{} + var routekinds []RouteKind httpRoutes, err := ListHTTPRoutes(ctx, k8sClient) if err != nil { routekinds = append(routekinds, HTTPRouteKind) diff --git a/pkg/gateway/routeutils/utils_test.go b/pkg/gateway/routeutils/utils_test.go index e82cea4ea7..67493188c6 100644 --- a/pkg/gateway/routeutils/utils_test.go +++ b/pkg/gateway/routeutils/utils_test.go @@ -34,7 +34,7 @@ func (m mockPreLoadRouteDescriptor) GetRouteNamespacedName() types.NamespacedNam return m.namespacedName } -func (m mockPreLoadRouteDescriptor) GetRouteKind() string { +func (m mockPreLoadRouteDescriptor) GetRouteKind() RouteKind { //TODO implement me panic("implement me") } diff --git a/pkg/ingress/model_build_frontend_nlb.go b/pkg/ingress/model_build_frontend_nlb.go index 58c4824835..ddb7f1242c 100644 --- a/pkg/ingress/model_build_frontend_nlb.go +++ b/pkg/ingress/model_build_frontend_nlb.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strconv" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -601,10 +602,10 @@ func (t *defaultModelBuildTask) buildFrontendNlbTargetGroupHealthCheckPort(_ con rawHealthCheckPort := "" exists := t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixFrontendNlbHealthCheckPort, &rawHealthCheckPort, svcAndIngAnnotations) if !exists { - return intstr.FromString(healthCheckPortTrafficPort), false, nil + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), false, nil } - if rawHealthCheckPort == healthCheckPortTrafficPort { - return intstr.FromString(healthCheckPortTrafficPort), true, nil + if rawHealthCheckPort == shared_constants.HealthCheckPortTrafficPort { + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), true, nil } healthCheckPort := intstr.Parse(rawHealthCheckPort) if healthCheckPort.Type == intstr.Int { diff --git a/pkg/ingress/model_build_target_group.go b/pkg/ingress/model_build_target_group.go index 381fd39721..e1e25fe9d2 100644 --- a/pkg/ingress/model_build_target_group.go +++ b/pkg/ingress/model_build_target_group.go @@ -7,6 +7,7 @@ import ( "fmt" awssdk "github.com/aws/aws-sdk-go-v2/aws" "regexp" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "strconv" "github.com/pkg/errors" @@ -21,10 +22,6 @@ import ( elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" ) -const ( - healthCheckPortTrafficPort = "traffic-port" -) - func (t *defaultModelBuildTask) buildTargetGroup(ctx context.Context, ing ClassifiedIngress, svc *corev1.Service, port intstr.IntOrString) (*elbv2model.TargetGroup, error) { tgResID := t.buildTargetGroupResourceID(k8s.NamespacedName(ing.Ing), k8s.NamespacedName(svc), port) @@ -63,7 +60,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingSpec(ctx context.Context, targetType := elbv2api.TargetType(tg.Spec.TargetType) targetPort := svcPort.TargetPort if targetType == elbv2api.TargetTypeInstance { - targetPort = intstr.FromInt(int(svcPort.NodePort)) + targetPort = intstr.FromInt32(svcPort.NodePort) } tgbNetworking := t.buildTargetGroupBindingNetworking(ctx, targetPort, *tg.Spec.HealthCheckConfig.Port) @@ -127,7 +124,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNetworking(ctx context.Co Protocol: &protocolTCP, Port: &targetPort, }) - if healthCheckPort.String() != healthCheckPortTrafficPort { + if healthCheckPort.String() != shared_constants.HealthCheckPortTrafficPort { networkingPorts = append(networkingPorts, elbv2api.NetworkingPort{ Protocol: &protocolTCP, Port: &healthCheckPort, @@ -339,10 +336,10 @@ func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfig(ctx context.Co func (t *defaultModelBuildTask) buildTargetGroupHealthCheckPort(_ context.Context, svc *corev1.Service, svcAndIngAnnotations map[string]string, targetType elbv2model.TargetType) (intstr.IntOrString, error) { rawHealthCheckPort := "" if exist := t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixHealthCheckPort, &rawHealthCheckPort, svcAndIngAnnotations); !exist { - return intstr.FromString(healthCheckPortTrafficPort), nil + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil } - if rawHealthCheckPort == healthCheckPortTrafficPort { - return intstr.FromString(healthCheckPortTrafficPort), nil + if rawHealthCheckPort == shared_constants.HealthCheckPortTrafficPort { + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil } healthCheckPort := intstr.Parse(rawHealthCheckPort) if healthCheckPort.Type == intstr.Int { diff --git a/pkg/service/model_build_load_balancer_test.go b/pkg/service/model_build_load_balancer_test.go index edc838f6c3..040a5099c7 100644 --- a/pkg/service/model_build_load_balancer_test.go +++ b/pkg/service/model_build_load_balancer_test.go @@ -181,7 +181,7 @@ func Test_defaultModelBuilderTask_buildLBAttributes(t *testing.T) { defaultLoadBalancingCrossZoneEnabled: false, defaultProxyProtocolV2Enabled: false, defaultHealthCheckProtocol: elbv2.ProtocolTCP, - defaultHealthCheckPort: healthCheckPortTrafficPort, + defaultHealthCheckPort: shared_constants.HealthCheckPortTrafficPort, defaultHealthCheckPath: "/", defaultHealthCheckInterval: 10, defaultHealthCheckTimeout: 10, @@ -1861,7 +1861,7 @@ func Test_defaultModelBuilderTask_buildLbCapacity(t *testing.T) { defaultLoadBalancingCrossZoneEnabled: false, defaultProxyProtocolV2Enabled: false, defaultHealthCheckProtocol: elbv2.ProtocolTCP, - defaultHealthCheckPort: healthCheckPortTrafficPort, + defaultHealthCheckPort: shared_constants.HealthCheckPortTrafficPort, defaultHealthCheckPath: "/", defaultHealthCheckInterval: 10, defaultHealthCheckTimeout: 10, diff --git a/pkg/service/model_build_target_group.go b/pkg/service/model_build_target_group.go index d566ad3b1d..3646bfc523 100644 --- a/pkg/service/model_build_target_group.go +++ b/pkg/service/model_build_target_group.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "regexp" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sort" "strconv" "strings" @@ -24,12 +25,6 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/networking" ) -const ( - tgAttrsProxyProtocolV2Enabled = "proxy_protocol_v2.enabled" - tgAttrsPreserveClientIPEnabled = "preserve_client_ip.enabled" - healthCheckPortTrafficPort = "traffic-port" -) - func (t *defaultModelBuildTask) buildTargetGroup(ctx context.Context, port corev1.ServicePort, tgProtocol elbv2model.Protocol, scheme elbv2model.LoadBalancerScheme) (*elbv2model.TargetGroup, error) { svcPort := intstr.FromInt(int(port.Port)) tgResourceID := t.buildTargetGroupResourceID(k8s.NamespacedName(t.service), svcPort) @@ -212,8 +207,8 @@ func (t *defaultModelBuildTask) buildTargetGroupAttributes(_ context.Context, po if rawAttributes == nil { rawAttributes = make(map[string]string) } - if _, ok := rawAttributes[tgAttrsProxyProtocolV2Enabled]; !ok { - rawAttributes[tgAttrsProxyProtocolV2Enabled] = strconv.FormatBool(t.defaultProxyProtocolV2Enabled) + if _, ok := rawAttributes[shared_constants.TGAttributeProxyProtocolV2Enabled]; !ok { + rawAttributes[shared_constants.TGAttributeProxyProtocolV2Enabled] = strconv.FormatBool(t.defaultProxyProtocolV2Enabled) } var proxyProtocolPerTG string @@ -232,9 +227,9 @@ func (t *defaultModelBuildTask) buildTargetGroupAttributes(_ context.Context, po currentPortStr := strconv.FormatInt(int64(port.Port), 10) if _, enabled := enabledPorts[currentPortStr]; enabled { - rawAttributes[tgAttrsProxyProtocolV2Enabled] = "true" + rawAttributes[shared_constants.TGAttributeProxyProtocolV2Enabled] = "true" } else { - rawAttributes[tgAttrsProxyProtocolV2Enabled] = "false" + rawAttributes[shared_constants.TGAttributeProxyProtocolV2Enabled] = "false" } } @@ -243,13 +238,13 @@ func (t *defaultModelBuildTask) buildTargetGroupAttributes(_ context.Context, po if proxyV2Annotation != "*" { return []elbv2model.TargetGroupAttribute{}, errors.Errorf("invalid value %v for Load Balancer proxy protocol v2 annotation, only value currently supported is *", proxyV2Annotation) } - rawAttributes[tgAttrsProxyProtocolV2Enabled] = "true" + rawAttributes[shared_constants.TGAttributeProxyProtocolV2Enabled] = "true" } - if rawPreserveIPEnabled, ok := rawAttributes[tgAttrsPreserveClientIPEnabled]; ok { + if rawPreserveIPEnabled, ok := rawAttributes[shared_constants.TGAttributePreserveClientIPEnabled]; ok { _, err := strconv.ParseBool(rawPreserveIPEnabled) if err != nil { - return nil, errors.Wrapf(err, "failed to parse attribute %v=%v", tgAttrsPreserveClientIPEnabled, rawPreserveIPEnabled) + return nil, errors.Wrapf(err, "failed to parse attribute %v=%v", shared_constants.TGAttributePreserveClientIPEnabled, rawPreserveIPEnabled) } } @@ -268,10 +263,10 @@ func (t *defaultModelBuildTask) buildTargetGroupAttributes(_ context.Context, po func (t *defaultModelBuildTask) buildPreserveClientIPFlag(_ context.Context, targetType elbv2model.TargetType, tgAttrs []elbv2model.TargetGroupAttribute) (bool, error) { for _, attr := range tgAttrs { - if attr.Key == tgAttrsPreserveClientIPEnabled { + if attr.Key == shared_constants.TGAttributePreserveClientIPEnabled { preserveClientIP, err := strconv.ParseBool(attr.Value) if err != nil { - return false, errors.Wrapf(err, "failed to parse attribute %v=%v", tgAttrsPreserveClientIPEnabled, attr.Value) + return false, errors.Wrapf(err, "failed to parse attribute %v=%v", shared_constants.TGAttributePreserveClientIPEnabled, attr.Value) } return preserveClientIP, nil } @@ -304,7 +299,7 @@ func (t *defaultModelBuildTask) buildTargetGroupPort(_ context.Context, targetTy func (t *defaultModelBuildTask) buildTargetGroupHealthCheckPort(_ context.Context, defaultHealthCheckPort string, targetType elbv2model.TargetType) (intstr.IntOrString, error) { rawHealthCheckPort := defaultHealthCheckPort t.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixHCPort, &rawHealthCheckPort, t.service.Annotations) - if rawHealthCheckPort == healthCheckPortTrafficPort { + if rawHealthCheckPort == shared_constants.HealthCheckPortTrafficPort { return intstr.FromString(rawHealthCheckPort), nil } healthCheckPort := intstr.Parse(rawHealthCheckPort) @@ -514,7 +509,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNetworking(_ context.Cont Protocol: &protocolUDP, Port: &tgPort, }) - if hcPort.String() == healthCheckPortTrafficPort || (hcPort.Type == intstr.Int && hcPort.IntValue() == tgPort.IntValue()) { + if hcPort.String() == shared_constants.HealthCheckPortTrafficPort || (hcPort.Type == intstr.Int && hcPort.IntValue() == tgPort.IntValue()) { ports = append(ports, elbv2api.NetworkingPort{ Protocol: &protocolTCP, Port: &tgPort, @@ -522,7 +517,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNetworking(_ context.Cont } } - if hcPort.String() != healthCheckPortTrafficPort && (hcPort.Type == intstr.Int && hcPort.IntValue() != tgPort.IntValue()) { + if hcPort.String() != shared_constants.HealthCheckPortTrafficPort && (hcPort.Type == intstr.Int && hcPort.IntValue() != tgPort.IntValue()) { ports = append(ports, elbv2api.NetworkingPort{ Protocol: &protocolTCP, Port: &hcPort, @@ -612,7 +607,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNetworkingLegacy(ctx cont if healthCheckSourceCIDRs := t.buildHealthCheckSourceCIDRs(trafficSource, loadBalancerSubnetCIDRs, tgPort, hcPort, tgProtocol, defaultRangeUsed); len(healthCheckSourceCIDRs) > 0 { networkingHealthCheckPort := hcPort - if hcPort.String() == healthCheckPortTrafficPort { + if hcPort.String() == shared_constants.HealthCheckPortTrafficPort { networkingHealthCheckPort = tgPort } tgbNetworking.Ingress = append(tgbNetworking.Ingress, elbv2model.NetworkingIngressRule{ @@ -698,7 +693,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNodeSelector(_ context.Co func (t *defaultModelBuildTask) buildHealthCheckSourceCIDRs(trafficSource, subnetCIDRs []string, tgPort, hcPort intstr.IntOrString, tgProtocol corev1.Protocol, defaultRangeUsed bool) []string { if tgProtocol != corev1.ProtocolUDP && - (hcPort.String() == healthCheckPortTrafficPort || hcPort.IntValue() == tgPort.IntValue()) { + (hcPort.String() == shared_constants.HealthCheckPortTrafficPort || hcPort.IntValue() == tgPort.IntValue()) { if !t.preserveClientIP { return nil } diff --git a/pkg/service/model_build_target_group_test.go b/pkg/service/model_build_target_group_test.go index 83ca491eb1..6ee5788a71 100644 --- a/pkg/service/model_build_target_group_test.go +++ b/pkg/service/model_build_target_group_test.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sort" "strconv" "testing" @@ -41,7 +42,7 @@ func Test_defaultModelBuilderTask_targetGroupAttrs(t *testing.T) { wantError: false, wantValue: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsProxyProtocolV2Enabled, + Key: shared_constants.TGAttributeProxyProtocolV2Enabled, Value: "false", }, }, @@ -58,7 +59,7 @@ func Test_defaultModelBuilderTask_targetGroupAttrs(t *testing.T) { wantError: false, wantValue: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsProxyProtocolV2Enabled, + Key: shared_constants.TGAttributeProxyProtocolV2Enabled, Value: "true", }, }, @@ -85,11 +86,11 @@ func Test_defaultModelBuilderTask_targetGroupAttrs(t *testing.T) { }, wantValue: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsProxyProtocolV2Enabled, + Key: shared_constants.TGAttributeProxyProtocolV2Enabled, Value: "false", }, { - Key: tgAttrsPreserveClientIPEnabled, + Key: shared_constants.TGAttributePreserveClientIPEnabled, Value: "true", }, { @@ -108,14 +109,14 @@ func Test_defaultModelBuilderTask_targetGroupAttrs(t *testing.T) { svc: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": tgAttrsProxyProtocolV2Enabled + "=false", + "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": shared_constants.TGAttributeProxyProtocolV2Enabled + "=false", "service.beta.kubernetes.io/aws-load-balancer-proxy-protocol": "*", }, }, }, wantValue: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsProxyProtocolV2Enabled, + Key: shared_constants.TGAttributeProxyProtocolV2Enabled, Value: "true", }, }, @@ -136,7 +137,7 @@ func Test_defaultModelBuilderTask_targetGroupAttrs(t *testing.T) { svc: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": tgAttrsPreserveClientIPEnabled + "= FalSe", + "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": shared_constants.TGAttributePreserveClientIPEnabled + "= FalSe", }, }, }, @@ -155,7 +156,7 @@ func Test_defaultModelBuilderTask_targetGroupAttrs(t *testing.T) { wantError: false, wantValue: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsProxyProtocolV2Enabled, + Key: shared_constants.TGAttributeProxyProtocolV2Enabled, Value: "true", }, }, @@ -174,7 +175,7 @@ func Test_defaultModelBuilderTask_targetGroupAttrs(t *testing.T) { wantError: false, wantValue: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsProxyProtocolV2Enabled, + Key: shared_constants.TGAttributeProxyProtocolV2Enabled, Value: "true", }, }, @@ -204,7 +205,7 @@ func Test_defaultModelBuilderTask_targetGroupAttrs(t *testing.T) { } func Test_defaultModelBuilderTask_buildTargetHealthCheck(t *testing.T) { - trafficPort := intstr.FromString(healthCheckPortTrafficPort) + trafficPort := intstr.FromString(shared_constants.HealthCheckPortTrafficPort) port8888 := intstr.FromInt(8888) port31223 := intstr.FromInt(31223) tests := []struct { @@ -410,7 +411,7 @@ func Test_defaultModelBuilderTask_buildTargetHealthCheck(t *testing.T) { defaultLoadBalancingCrossZoneEnabled: false, defaultProxyProtocolV2Enabled: false, defaultHealthCheckProtocol: elbv2.ProtocolTCP, - defaultHealthCheckPort: healthCheckPortTrafficPort, + defaultHealthCheckPort: shared_constants.HealthCheckPortTrafficPort, defaultHealthCheckPath: "/", defaultHealthCheckInterval: 10, defaultHealthCheckTimeout: 10, @@ -1498,7 +1499,7 @@ func Test_defaultModelBuilder_buildPreserveClientIPFlag(t *testing.T) { targetType: elbv2.TargetTypeIP, tgAttrs: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsProxyProtocolV2Enabled, + Key: shared_constants.TGAttributeProxyProtocolV2Enabled, Value: "false", }, { @@ -1521,7 +1522,7 @@ func Test_defaultModelBuilder_buildPreserveClientIPFlag(t *testing.T) { Value: "value", }, { - Key: tgAttrsPreserveClientIPEnabled, + Key: shared_constants.TGAttributePreserveClientIPEnabled, Value: "true", }, }, @@ -1537,7 +1538,7 @@ func Test_defaultModelBuilder_buildPreserveClientIPFlag(t *testing.T) { targetType: elbv2.TargetTypeInstance, tgAttrs: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsPreserveClientIPEnabled, + Key: shared_constants.TGAttributePreserveClientIPEnabled, Value: "false", }, { @@ -1552,7 +1553,7 @@ func Test_defaultModelBuilder_buildPreserveClientIPFlag(t *testing.T) { targetType: elbv2.TargetTypeInstance, tgAttrs: []elbv2.TargetGroupAttribute{ { - Key: tgAttrsPreserveClientIPEnabled, + Key: shared_constants.TGAttributePreserveClientIPEnabled, Value: " FalSe", }, { diff --git a/pkg/service/model_builder.go b/pkg/service/model_builder.go index 4b12004aeb..3f01b7b253 100644 --- a/pkg/service/model_builder.go +++ b/pkg/service/model_builder.go @@ -140,7 +140,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service defaultTargetType: b.defaultTargetType, defaultLoadBalancerScheme: b.defaultLoadBalancerScheme, defaultHealthCheckProtocol: elbv2model.ProtocolTCP, - defaultHealthCheckPort: healthCheckPortTrafficPort, + defaultHealthCheckPort: shared_constants.HealthCheckPortTrafficPort, defaultHealthCheckPath: "/", defaultHealthCheckInterval: 10, defaultHealthCheckTimeout: 10, diff --git a/pkg/shared_constants/attributes.go b/pkg/shared_constants/attributes.go index 73092d7746..ffcc177d32 100644 --- a/pkg/shared_constants/attributes.go +++ b/pkg/shared_constants/attributes.go @@ -4,3 +4,8 @@ const ( // LBAttributeDeletionProtection deletion protection attribute name LBAttributeDeletionProtection = "deletion_protection.enabled" ) + +const ( + TGAttributeProxyProtocolV2Enabled = "proxy_protocol_v2.enabled" + TGAttributePreserveClientIPEnabled = "preserve_client_ip.enabled" +) diff --git a/pkg/shared_constants/healthcheck.go b/pkg/shared_constants/healthcheck.go new file mode 100644 index 0000000000..14acf2ec7e --- /dev/null +++ b/pkg/shared_constants/healthcheck.go @@ -0,0 +1,6 @@ +package shared_constants + +const ( + // HealthCheckPortTrafficPort denotes that the target group health check should use the traffic port + HealthCheckPortTrafficPort = "traffic-port" +) From 700792713aba07e547be70716e538fe87c17e286 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Tue, 15 Apr 2025 14:27:16 -0700 Subject: [PATCH 31/40] [gw api] tg creation --- .../v1beta1/loadbalancerconfig_types.go | 5 ++ .../v1beta1/targetgroupconfig_types.go | 12 +-- pkg/gateway/model/base_model_builder.go | 2 + pkg/gateway/model/model_build_target_group.go | 85 ++++++++++--------- 4 files changed, 56 insertions(+), 48 deletions(-) diff --git a/apis/gateway/v1beta1/loadbalancerconfig_types.go b/apis/gateway/v1beta1/loadbalancerconfig_types.go index c9a3e164a5..8435bd9772 100644 --- a/apis/gateway/v1beta1/loadbalancerconfig_types.go +++ b/apis/gateway/v1beta1/loadbalancerconfig_types.go @@ -243,6 +243,11 @@ type LoadBalancerConfigurationSpec struct { // when you specify securityGroups // +optional ManageBackendSecurityGroupRules bool `json:"manageBackendSecurityGroupRules,omitempty"` + + // EnableMultiCluster [Application / Network LoadBalancer] + // All TargetGroupBindings attached to this Load Balancer will have multi cluster support enabled. + // +optional + EnableMultiCluster bool `json:"enableMultiCluster,omitempty"` } // TODO -- these can be used to set what generation the gateway is currently on to track progress on reconcile. diff --git a/apis/gateway/v1beta1/targetgroupconfig_types.go b/apis/gateway/v1beta1/targetgroupconfig_types.go index c5fe018f16..8794e6b520 100644 --- a/apis/gateway/v1beta1/targetgroupconfig_types.go +++ b/apis/gateway/v1beta1/targetgroupconfig_types.go @@ -112,9 +112,9 @@ const ( type TargetGroupHealthCheckProtocol string const ( - TargetGroupHealthCheckProtocolHTTP TargetGroupHealthCheckProtocol = "http" - TargetGroupHealthCheckProtocolHTTPS TargetGroupHealthCheckProtocol = "https" - TargetGroupHealthCheckProtocolTCP TargetGroupHealthCheckProtocol = "tcp" + TargetGroupHealthCheckProtocolHTTP TargetGroupHealthCheckProtocol = "HTTP" + TargetGroupHealthCheckProtocolHTTPS TargetGroupHealthCheckProtocol = "HTTPS" + TargetGroupHealthCheckProtocolTCP TargetGroupHealthCheckProtocol = "TCP" ) // +kubebuilder:validation:Enum=HTTP;HTTPS;TCP;TLS;UDP;TCP_UDP @@ -133,9 +133,9 @@ const ( type ProtocolVersion string const ( - ProtocolVersionHTTP1 ProtocolVersion = "http1" - ProtocolVersionHTTP2 ProtocolVersion = "http2" - ProtocolVersionGRPC ProtocolVersion = "grpc" + ProtocolVersionHTTP1 ProtocolVersion = "HTTP1" + ProtocolVersionHTTP2 ProtocolVersion = "HTTP2" + ProtocolVersionGRPC ProtocolVersion = "GRPC" ) // +kubebuilder:validation:Enum=none;prefer-route-specific;prefer-default diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go index b99d9888ee..cebfeafb0a 100644 --- a/pkg/gateway/model/base_model_builder.go +++ b/pkg/gateway/model/base_model_builder.go @@ -58,6 +58,8 @@ type baseModelBuilder struct { lbBuilder loadBalancerBuilder logger logr.Logger + tgByResID map[string]*elbv2model.TargetGroup + subnetBuilder subnetModelBuilder securityGroupBuilder securityGroupBuilder tgBuilder targetGroupBuilder diff --git a/pkg/gateway/model/model_build_target_group.go b/pkg/gateway/model/model_build_target_group.go index 0f5baafa4c..2288c32574 100644 --- a/pkg/gateway/model/model_build_target_group.go +++ b/pkg/gateway/model/model_build_target_group.go @@ -89,7 +89,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroup(tgByResID *map[string]bu return buildTargetGroupOutput{}, err } nodeSelector := builder.buildTargetGroupBindingNodeSelector(targetGroupProps, tgSpec.TargetType) - bindingSpec := builder.buildTargetGroupBindingSpec(lbConfig, targetGroupProps, tgSpec, nodeSelector, backend, backendSGIDToken) + bindingSpec := builder.buildTargetGroupBindingSpec(targetGroupProps, tgSpec, nodeSelector, backend, backendSGIDToken) output := buildTargetGroupOutput{ targetGroupSpec: tgSpec, @@ -109,7 +109,7 @@ func (builder *targetGroupBuilderImpl) getTargetGroupProps(routeDescriptor route return targetGroupProps } -func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(lbConfig *elbv2gw.LoadBalancerConfiguration, tgProps *elbv2gw.TargetGroupProps, tgSpec elbv2model.TargetGroupSpec, nodeSelector *metav1.LabelSelector, backend routeutils.Backend, backendSGIDToken core.StringToken) elbv2model.TargetGroupBindingResourceSpec { +func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(tgProps *elbv2gw.TargetGroupProps, tgSpec elbv2model.TargetGroupSpec, nodeSelector *metav1.LabelSelector, backend routeutils.Backend, backendSGIDToken core.StringToken) elbv2model.TargetGroupBindingResourceSpec { targetType := elbv2api.TargetType(tgSpec.TargetType) targetPort := backend.ServicePort.TargetPort if targetType == elbv2api.TargetTypeInstance { @@ -142,14 +142,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(lbConfig *elb } } -func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetPort intstr.IntOrString, healthCheckPort intstr.IntOrString, port corev1.ServicePort, backendSGIDToken core.StringToken) *elbv2model.TargetGroupBindingNetworking { +func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetPort intstr.IntOrString, healthCheckPort intstr.IntOrString, svcPort corev1.ServicePort, backendSGIDToken core.StringToken) *elbv2model.TargetGroupBindingNetworking { if backendSGIDToken == nil { return nil } protocolTCP := elbv2api.NetworkingProtocolTCP protocolUDP := elbv2api.NetworkingProtocolUDP - udpSupported := port.Protocol == corev1.ProtocolUDP + udpSupported := svcPort.Protocol == corev1.ProtocolUDP if builder.disableRestrictedSGRules { ports := []elbv2api.NetworkingPort{ @@ -183,7 +183,6 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetP } var networkingPorts []elbv2api.NetworkingPort - var networkingRules []elbv2model.NetworkingIngressRule protocolToUse := &protocolTCP if udpSupported { @@ -209,6 +208,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetP }) } + var networkingRules []elbv2model.NetworkingIngressRule for _, port := range networkingPorts { networkingRules = append(networkingRules, elbv2model.NetworkingIngressRule{ From: []elbv2model.NetworkingPeer{ @@ -232,7 +232,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro if err != nil { return elbv2model.TargetGroupSpec{}, err } - tgProtocolVersion := builder.buildTargetGroupProtocolVersion(targetGroupProps) + tgProtocolVersion := builder.buildTargetGroupProtocolVersion(targetGroupProps, route) healthCheckConfig, err := builder.buildTargetGroupHealthCheckConfig(targetGroupProps, tgProtocol, tgProtocolVersion, targetType, backend) if err != nil { @@ -249,8 +249,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro return elbv2model.TargetGroupSpec{}, err } tgPort := builder.buildTargetGroupPort(targetType, *backend.ServicePort) - // TODO - backend.ServicePort.TargetPort might not be correct. - name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), k8s.NamespacedName(backend.Service), backend.ServicePort.TargetPort, tgPort, targetType, tgProtocol, tgProtocolVersion) + name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), k8s.NamespacedName(backend.Service), tgPort, targetType, tgProtocol, tgProtocolVersion) return elbv2model.TargetGroupSpec{ Name: name, TargetType: targetType, @@ -268,7 +267,7 @@ var invalidTargetGroupNamePattern = regexp.MustCompile("[[:^alnum:]]") // buildTargetGroupName will calculate the targetGroup's name. func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *elbv2gw.TargetGroupProps, - gwKey types.NamespacedName, routeKey types.NamespacedName, svcKey types.NamespacedName, port intstr.IntOrString, tgPort int32, + gwKey types.NamespacedName, routeKey types.NamespacedName, svcKey types.NamespacedName, tgPort int32, targetType elbv2model.TargetType, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion) string { if targetGroupProps != nil && targetGroupProps.TargetGroupName != "" { @@ -283,7 +282,6 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *el _, _ = uuidHash.Write([]byte(routeKey.Name)) _, _ = uuidHash.Write([]byte(svcKey.Namespace)) _, _ = uuidHash.Write([]byte(svcKey.Name)) - _, _ = uuidHash.Write([]byte(port.String())) _, _ = uuidHash.Write([]byte(strconv.Itoa(int(tgPort)))) _, _ = uuidHash.Write([]byte(targetType)) _, _ = uuidHash.Write([]byte(tgProtocol)) @@ -365,10 +363,7 @@ func (builder *targetGroupBuilderImpl) buildL7TargetGroupProtocol(targetGroupPro } func (builder *targetGroupBuilderImpl) buildL4TargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { - // TODO, auto infer? if targetGroupProps == nil || targetGroupProps.Protocol == nil { - // infer this somehow!? - // use the backend config to get the protocol type. return builder.inferTargetGroupProtocolFromRoute(route), nil } @@ -406,7 +401,12 @@ func (builder *targetGroupBuilderImpl) inferTargetGroupProtocolFromRoute(route r return elbv2model.ProtocolTCP } -func (builder *targetGroupBuilderImpl) buildTargetGroupProtocolVersion(targetGroupProps *elbv2gw.TargetGroupProps) *elbv2model.ProtocolVersion { +var ( + http1 = elbv2model.ProtocolVersionHTTP1 + grpc = elbv2model.ProtocolVersionGRPC +) + +func (builder *targetGroupBuilderImpl) buildTargetGroupProtocolVersion(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) *elbv2model.ProtocolVersion { // NLB doesn't support protocol version if builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { return nil @@ -416,7 +416,11 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupProtocolVersion(targetGro pv := elbv2model.ProtocolVersion(*targetGroupProps.ProtocolVersion) return &pv } - http1 := elbv2model.ProtocolVersionHTTP1 + + if route.GetRouteKind() == routeutils.GRPCRouteKind { + return &grpc + } + return &http1 } @@ -425,13 +429,13 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetG // https://github.com/kubernetes-sigs/gateway-api/issues/451 // Gateway API doesn't have the same ServiceExternalTrafficPolicyLocal support. // TODO - Maybe a TargetGroupConfig attribute to support the same behavior? - healthCheckPort, err := builder.buildTargetGroupHealthCheckPort(targetGroupProps, targetType, backend) + healthCheckPort, err := builder.buildTargetGroupHealthCheckPort(targetGroupProps, targetType, backend.Service) if err != nil { return elbv2model.TargetGroupHealthCheckConfig{}, err } healthCheckProtocol := builder.buildTargetGroupHealthCheckProtocol(targetGroupProps, tgProtocol) healthCheckPath := builder.buildTargetGroupHealthCheckPath(targetGroupProps, tgProtocolVersion, healthCheckProtocol) - healthCheckMatcher := builder.buildTargetGroupHealthCheckMatcher(targetGroupProps, healthCheckProtocol) + healthCheckMatcher := builder.buildTargetGroupHealthCheckMatcher(targetGroupProps, tgProtocolVersion, healthCheckProtocol) healthCheckIntervalSeconds := builder.buildTargetGroupHealthCheckIntervalSeconds(targetGroupProps) healthCheckTimeoutSeconds := builder.buildTargetGroupHealthCheckTimeoutSeconds(targetGroupProps) healthCheckHealthyThresholdCount := builder.buildTargetGroupHealthCheckHealthyThresholdCount(targetGroupProps) @@ -450,8 +454,10 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetG return hcConfig, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckPort(targetGroupProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType, backend routeutils.Backend) (intstr.IntOrString, error) { - if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckPort == nil || *targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckPort(targetGroupProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType, svc *corev1.Service) (intstr.IntOrString, error) { + + portConfigNotExist := targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckPort == nil + if portConfigNotExist || *targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil } @@ -459,13 +465,17 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckPort(targetGro if healthCheckPort.Type == intstr.Int { return healthCheckPort, nil } + hcSvcPort, err := k8s.LookupServicePort(svc, healthCheckPort) + if err != nil { + return intstr.FromString(""), err + } - /* TODO - Zac revisit this? */ if targetType == elbv2model.TargetTypeInstance { - return intstr.FromInt(int(backend.ServicePort.NodePort)), nil + return intstr.FromInt(int(hcSvcPort.NodePort)), nil } - if backend.ServicePort.TargetPort.Type == intstr.Int { - return backend.ServicePort.TargetPort, nil + + if hcSvcPort.TargetPort.Type == intstr.Int { + return hcSvcPort.TargetPort, nil } return intstr.IntOrString{}, errors.New("cannot use named healthCheckPort for IP TargetType when service's targetPort is a named port") } @@ -487,7 +497,8 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckProtocol(targe case elbv2gw.TargetGroupHealthCheckProtocolHTTPS: return elbv2model.ProtocolHTTPS default: - return tgProtocol + // This should never happen, the CRD validation takes care of this. + return elbv2model.ProtocolHTTP } } @@ -507,15 +518,17 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckPath(targetGro return &builder.defaultHealthCheckPathHTTP } -func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckMatcher(targetGroupProps *elbv2gw.TargetGroupProps, hcProtocol elbv2model.Protocol) *elbv2model.HealthCheckMatcher { +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckMatcher(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocolVersion *elbv2model.ProtocolVersion, hcProtocol elbv2model.Protocol) *elbv2model.HealthCheckMatcher { if hcProtocol == elbv2model.ProtocolTCP { return nil } - if targetGroupProps != nil && targetGroupProps.ProtocolVersion != nil && string(*targetGroupProps.ProtocolVersion) == string(elbv2model.ProtocolVersionGRPC) { + useGRPC := tgProtocolVersion != nil && *tgProtocolVersion == elbv2model.ProtocolVersionGRPC + + if useGRPC { matcher := builder.defaultHealthCheckMatcherGRPCCode - if targetGroupProps.ProtocolVersion != nil && targetGroupProps.HealthCheckConfig != nil && targetGroupProps.HealthCheckConfig.Matcher != nil && targetGroupProps.HealthCheckConfig.Matcher.GRPCCode != nil { + if targetGroupProps != nil && targetGroupProps.HealthCheckConfig != nil && targetGroupProps.HealthCheckConfig.Matcher != nil && targetGroupProps.HealthCheckConfig.Matcher.GRPCCode != nil { matcher = *targetGroupProps.HealthCheckConfig.Matcher.GRPCCode } return &elbv2model.HealthCheckMatcher{ @@ -523,7 +536,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckMatcher(target } } matcher := builder.defaultHealthCheckMatcherHTTPCode - if targetGroupProps != nil && targetGroupProps.ProtocolVersion != nil && targetGroupProps.HealthCheckConfig != nil && targetGroupProps.HealthCheckConfig.Matcher != nil && targetGroupProps.HealthCheckConfig.Matcher.HTTPCode != nil { + if targetGroupProps != nil && targetGroupProps.HealthCheckConfig != nil && targetGroupProps.HealthCheckConfig.Matcher != nil && targetGroupProps.HealthCheckConfig.Matcher.HTTPCode != nil { matcher = *targetGroupProps.HealthCheckConfig.Matcher.HTTPCode } return &elbv2model.HealthCheckMatcher{ @@ -570,9 +583,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupAttributes(targetGroupPro attributeMap[attr.Key] = attr.Value } - if builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { - builder.buildL4TargetGroupAttributes(&attributeMap, targetGroupProps) - } + // TODO -- buildPreserveClientIPFlag Might need special logic return attributeMap } @@ -588,22 +599,12 @@ func (builder *targetGroupBuilderImpl) convertMapToAttributes(attributeMap map[s return convertedAttributes } -func (builder *targetGroupBuilderImpl) buildL4TargetGroupAttributes(attributeMap *map[string]string, targetGroupProps *elbv2gw.TargetGroupProps) { - if targetGroupProps == nil { - return - } - // TODO -- buildPreserveClientIPFlag -} - func (builder *targetGroupBuilderImpl) buildTargetGroupResourceID(gwKey types.NamespacedName, svcKey types.NamespacedName, routeKey types.NamespacedName, port intstr.IntOrString) string { return fmt.Sprintf("%s/%s:%s-%s:%s-%s:%s", gwKey.Namespace, gwKey.Name, routeKey.Namespace, routeKey.Name, svcKey.Namespace, svcKey.Name, port.String()) } func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNodeSelector(tgProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType) *metav1.LabelSelector { - if targetType != elbv2model.TargetTypeInstance { - return nil - } - if tgProps == nil { + if targetType != elbv2model.TargetTypeInstance || tgProps == nil { return nil } return tgProps.NodeSelector From c39ff253c67b0db35579f55a2c871b398369948a Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 17 Apr 2025 10:45:50 -0700 Subject: [PATCH 32/40] fixes to get tg + tgb working --- apis/gateway/v1beta1/zz_generated.deepcopy.go | 5 + ...ay.k8s.aws_loadbalancerconfigurations.yaml | 303 ++++ ...way.k8s.aws_targetgroupconfigurations.yaml | 498 ++++++ controllers/gateway/gateway_controller.go | 2 + pkg/gateway/model/base_model_builder.go | 2 - .../model/model_build_target_group_test.go | 1348 ++++++++++++++++- 6 files changed, 2148 insertions(+), 10 deletions(-) create mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml create mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml diff --git a/apis/gateway/v1beta1/zz_generated.deepcopy.go b/apis/gateway/v1beta1/zz_generated.deepcopy.go index fe1edd2f62..d5f95d1b74 100644 --- a/apis/gateway/v1beta1/zz_generated.deepcopy.go +++ b/apis/gateway/v1beta1/zz_generated.deepcopy.go @@ -696,6 +696,11 @@ func (in *TargetGroupProps) DeepCopyInto(out *TargetGroupProps) { *out = new(ProtocolVersion) **out = **in } + if in.EnableProxyProtocolV2 != nil { + in, out := &in.EnableProxyProtocolV2, &out.EnableProxyProtocolV2 + *out = new(bool) + **out = **in + } if in.VpcID != nil { in, out := &in.VpcID, &out.VpcID *out = new(string) diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml new file mode 100644 index 0000000000..aaa8a79ce3 --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -0,0 +1,303 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: loadbalancerconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: LoadBalancerConfiguration + listKind: LoadBalancerConfigurationList + plural: loadbalancerconfigurations + singular: loadbalancerconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LoadBalancerConfigurationSpec defines the desired state of + LoadBalancerConfiguration + properties: + customerOwnedIpv4Pool: + description: |- + customerOwnedIpv4Pool [Application LoadBalancer] + is the ID of the customer-owned address for Application Load Balancers on Outposts pool. + type: string + enableICMP: + description: |- + EnableICMP [Network LoadBalancer] + enables the creation of security group rules to the managed security group + to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs + type: boolean + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + All TargetGroupBindings attached to this Load Balancer will have multi cluster support enabled. + type: boolean + enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: + description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic + Indicates whether to evaluate inbound security group rules for traffic + sent to a Network Load Balancer through Amazon Web Services PrivateLink. + type: string + ipAddressType: + description: loadBalancerIPType defines what kind of load balancer + to provision (ipv4, dual stack) + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string + ipv4IPAMPoolId: + description: |- + IPv4IPAMPoolId [Application LoadBalancer] + defines the IPAM pool ID used for IPv4 Addresses on the ALB. + type: string + listenerConfigurations: + description: listenerConfigurations is an optional list of configurations + for each listener on LB + items: + properties: + alpnPolicy: + description: alpnPolicy an optional string that allows you to + configure ALPN policies on your Load Balancer + enum: + - HTTP1Only + - HTTP2Only + - HTTP2Optional + - HTTP2Preferred + - None + type: string + certificates: + description: certificates is the list of other certificates + to add to the listener. + items: + type: string + type: array + defaultCertificate: + description: |- + TODO: Add validation in admission webhook to make it required for secure protocols + defaultCertificate the cert arn to be used by default. + type: string + listenerAttributes: + description: listenerAttributes defines the attributes for the + listener + items: + description: ListenerAttribute defines listener attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + mutualAuthentication: + description: mutualAuthentication defines the mutual authentication + configuration information. + properties: + advertiseTrustStoreCaNames: + description: Indicates whether trust store CA certificate + names are advertised. + enum: + - "on" + - "off" + type: string + ignoreClientCertificateExpiry: + description: Indicates whether expired client certificates + are ignored. + type: boolean + mode: + description: The client certificate handling method. Options + are off , passthrough or verify + enum: + - "off" + - passthrough + - verify + type: string + trustStore: + description: The Name or ARN of the trust store. + type: string + required: + - mode + type: object + protocolPort: + description: protocolPort is identifier for the listener on + load balancer. It should be of the form PROTOCOL:PORT + pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ + type: string + sslPolicy: + description: sslPolicy is the security policy that defines which + protocols and ciphers are supported for secure listeners [HTTPS + or TLS listener]. + type: string + required: + - protocolPort + type: object + type: array + loadBalancerAttributes: + description: LoadBalancerAttributes defines the attribute of LB + items: + description: LoadBalancerAttribute defines LB attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + loadBalancerName: + description: loadBalancerName defines the name of the LB to provision. + If unspecified, it will be automatically generated. + maxLength: 32 + minLength: 1 + type: string + loadBalancerSubnets: + description: |- + loadBalancerSubnets is an optional list of subnet configurations to be used in the LB + This value takes precedence over loadBalancerSubnetsSelector if both are selected. + items: + description: SubnetConfiguration defines the subnet settings for + a Load Balancer. + properties: + eipAllocation: + description: eipAllocation [Network LoadBalancer] the EIP name + for this subnet. + type: string + identifier: + description: identifier [Application LoadBalancer / Network + LoadBalancer] name or id for the subnet + type: string + ipv6Allocation: + description: IPv6Allocation [Network LoadBalancer] the ipv6 + address to assign to this subnet. + type: string + privateIPv4Allocation: + description: privateIPv4Allocation [Network LoadBalancer] the + private ipv4 address to assign to this subnet. + type: string + sourceNatIPv6Prefix: + description: SourceNatIPv6Prefix [Network LoadBalancer] The + IPv6 prefix to use for source NAT. Specify an IPv6 prefix + (/80 netmask) from the subnet CIDR block or auto_assigned + to use an IPv6 prefix selected at random from the subnet CIDR + block. + type: string + type: object + type: array + loadBalancerSubnetsSelector: + additionalProperties: + items: + type: string + type: array + description: |- + LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each + tag specified in the map key contains one of the values in the corresponding + value list. + type: object + manageBackendSecurityGroupRules: + description: |- + ManageBackendSecurityGroupRules [Application / Network LoadBalancer] + specifies whether you want the controller to configure security group rules on Node/Pod for traffic access + when you specify securityGroups + type: boolean + scheme: + description: scheme defines the type of LB to provision. If unspecified, + it will be automatically inferred. + enum: + - internal + - internet-facing + type: string + securityGroupPrefixes: + description: securityGroupPrefixes an optional list of prefixes that + are allowed to access the LB. + items: + type: string + type: array + securityGroups: + description: securityGroups an optional list of security group ids + or names to apply to the LB + items: + type: string + type: array + sourceRanges: + description: sourceRanges an optional list of CIDRs that are allowed + to access the LB. + items: + type: string + type: array + tags: + description: Tags defines list of Tags on LB. + items: + description: AWSTag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + vpcId: + description: vpcId is the ID of the VPC for the load balancer. + type: string + type: object + status: + description: LoadBalancerConfigurationStatus defines the observed state + of TargetGroupBinding + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml new file mode 100644 index 0000000000..1ad36952b0 --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -0,0 +1,498 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: targetgroupconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: TargetGroupConfiguration + listKind: TargetGroupConfigurationList + plural: targetgroupconfigurations + singular: targetgroupconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The Kubernetes Service's name + jsonPath: .spec.targetReference.name + name: SERVICE-NAME + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TargetGroupConfiguration is the Schema for defining TargetGroups + with an AWS ELB Gateway + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TargetGroupConfigurationSpec defines the TargetGroup properties + for a route. + properties: + defaultConfiguration: + description: defaultRouteConfiguration fallback configuration applied + to all routes, unless overridden by route-specific configurations. + properties: + enableProxyProtocolV2: + description: |- + enableProxyProtocolV2 [Network LoadBalancers] Indicates whether proxy protocol version 2 is enabled. + By default, proxy protocol is disabled. + type: boolean + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount of + time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health checks + on the targets. + type: string + healthCheckPort: + description: |- + healthCheckPort The port the load balancer uses when performing health checks on targets. + The default is to use the port on which each target receives traffic from the load balancer. + type: string + healthCheckProtocol: + description: healthCheckProtocol The protocol to use to connect + with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols + are not supported for health checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, in seconds, + during which no response means a failed health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering an unhealthy + target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes to use + when checking for a successful response from a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering the target + unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target group + is of type IPv4 or IPv6. If unspecified, it will be automatically + inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups to + only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocol: + description: |- + Protocol [Application / Network Load Balancer] the protocol for the target group. + If unspecified, it will be automatically inferred. + enum: + - HTTP + - HTTPS + - TCP + - TLS + - UDP + - TCP_UDP + type: string + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute of target + group + items: + description: TargetGroupAttribute defines target group attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetGroupName: + description: targetGroupName specifies the name to assign to the + Target Group. If not defined, then one is generated. + type: string + targetType: + description: targetType is the TargetType of TargetGroup. If unspecified, + it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + routeConfigurations: + description: routeConfigurations the route configuration for specific + routes + items: + description: RouteConfiguration defines the per route configuration + properties: + identifier: + description: name the identifier of the route, it should be + in the form of ROUTE:NAMESPACE:NAME + pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ + type: string + targetGroupProps: + description: targetGroupProps the target group specific properties + properties: + enableProxyProtocolV2: + description: |- + enableProxyProtocolV2 [Network LoadBalancers] Indicates whether proxy protocol version 2 is enabled. + By default, proxy protocol is disabled. + type: boolean + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount + of time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health + checks on the targets. + type: string + healthCheckPort: + description: |- + healthCheckPort The port the load balancer uses when performing health checks on targets. + The default is to use the port on which each target receives traffic from the load balancer. + type: string + healthCheckProtocol: + description: healthCheckProtocol The protocol to use + to connect with the target. The GENEVE, TLS, UDP, + and TCP_UDP protocols are not supported for health + checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, + in seconds, during which no response means a failed + health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering + an unhealthy target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes + to use when checking for a successful response from + a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering + the target unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target + group is of type IPv4 or IPv6. If unspecified, it will + be automatically inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups + to only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocol: + description: |- + Protocol [Application / Network Load Balancer] the protocol for the target group. + If unspecified, it will be automatically inferred. + enum: + - HTTP + - HTTPS + - TCP + - TLS + - UDP + - TCP_UDP + type: string + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute + of target group + items: + description: TargetGroupAttribute defines target group + attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetGroupName: + description: targetGroupName specifies the name to assign + to the Target Group. If not defined, then one is generated. + type: string + targetType: + description: targetType is the TargetType of TargetGroup. + If unspecified, it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + required: + - identifier + - targetGroupProps + type: object + type: array + targetReference: + description: targetReference the kubernetes object to attach the Target + Group settings to. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + + Defaults to "Service" when not specified. + type: string + name: + description: Name is the name of the referent. + type: string + required: + - name + type: object + required: + - targetReference + type: object + status: + description: TargetGroupConfigurationStatus defines the observed state + of TargetGroupConfiguration + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 76e124e4ab..89947b8cc5 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -184,6 +184,8 @@ func (r *gatewayReconciler) reconcileHelper(ctx context.Context, req reconcile.R allRoutes, err := r.gatewayLoader.LoadRoutesForGateway(ctx, *gw, r.routeFilter) + r.logger.Info("In Gateway Controller - Got these routes", "routes", allRoutes) + if err != nil { return err } diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go index cebfeafb0a..b99d9888ee 100644 --- a/pkg/gateway/model/base_model_builder.go +++ b/pkg/gateway/model/base_model_builder.go @@ -58,8 +58,6 @@ type baseModelBuilder struct { lbBuilder loadBalancerBuilder logger logr.Logger - tgByResID map[string]*elbv2model.TargetGroup - subnetBuilder subnetModelBuilder securityGroupBuilder securityGroupBuilder tgBuilder targetGroupBuilder diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go index 292263c022..5c478df896 100644 --- a/pkg/gateway/model/model_build_target_group_test.go +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -5,9 +5,12 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" gwv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -67,7 +70,7 @@ func Test_buildTargetGroup(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-1949ae79d7", + Name: "k8s-myrouten-myroute-d02da2803b", TargetType: elbv2model.TargetTypeInstance, Port: awssdk.Int32(8080), Protocol: elbv2model.ProtocolTCP, @@ -90,7 +93,7 @@ func Test_buildTargetGroup(t *testing.T) { Template: elbv2model.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", - Name: "k8s-myrouten-myroute-1949ae79d7", + Name: "k8s-myrouten-myroute-d02da2803b", }, Spec: elbv2model.TargetGroupBindingSpec{ TargetType: &instanceType, @@ -139,7 +142,7 @@ func Test_buildTargetGroup(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-e99d898968", + Name: "k8s-myrouten-myroute-d146029dfb", TargetType: elbv2model.TargetTypeInstance, Port: awssdk.Int32(8080), Protocol: elbv2model.ProtocolHTTP, @@ -167,7 +170,7 @@ func Test_buildTargetGroup(t *testing.T) { Template: elbv2model.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", - Name: "k8s-myrouten-myroute-e99d898968", + Name: "k8s-myrouten-myroute-d146029dfb", }, Spec: elbv2model.TargetGroupBindingSpec{ TargetType: &instanceType, @@ -216,7 +219,7 @@ func Test_buildTargetGroup(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-7ac9e90fa0", + Name: "k8s-myrouten-myroute-d9d6c4e6eb", TargetType: elbv2model.TargetTypeIP, Port: awssdk.Int32(80), Protocol: elbv2model.ProtocolTCP, @@ -239,7 +242,7 @@ func Test_buildTargetGroup(t *testing.T) { Template: elbv2model.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", - Name: "k8s-myrouten-myroute-7ac9e90fa0", + Name: "k8s-myrouten-myroute-d9d6c4e6eb", }, Spec: elbv2model.TargetGroupBindingSpec{ TargetType: &ipType, @@ -288,7 +291,7 @@ func Test_buildTargetGroup(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-8a97d3dcbe", + Name: "k8s-myrouten-myroute-400113e816", TargetType: elbv2model.TargetTypeIP, Port: awssdk.Int32(80), Protocol: elbv2model.ProtocolHTTP, @@ -316,7 +319,7 @@ func Test_buildTargetGroup(t *testing.T) { Template: elbv2model.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", - Name: "k8s-myrouten-myroute-8a97d3dcbe", + Name: "k8s-myrouten-myroute-400113e816", }, Spec: elbv2model.TargetGroupBindingSpec{ TargetType: &ipType, @@ -360,3 +363,1332 @@ func Test_buildTargetGroup(t *testing.T) { }) } } + +func Test_getTargetGroupProps(t *testing.T) { + props := elbv2gw.TargetGroupProps{} + testCases := []struct { + name string + expected *elbv2gw.TargetGroupProps + backend routeutils.Backend + }{ + { + name: "no tg config", + }, + { + name: "with tg config", + backend: routeutils.Backend{ + ELBv2TargetGroupConfig: &elbv2gw.TargetGroupConfiguration{ + Spec: elbv2gw.TargetGroupConfigurationSpec{ + DefaultConfiguration: props, + }, + }, + }, + expected: &props, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := &targetGroupBuilderImpl{} + mockRoute := &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "my-route", + Namespace: "my-ns", + } + + result := builder.getTargetGroupProps(mockRoute, tc.backend) + assert.Equal(t, tc.expected, result) + }) + } +} + +func Test_buildTargetGroupBindingNetworking(t *testing.T) { + protocolTCP := elbv2api.NetworkingProtocolTCP + protocolUDP := elbv2api.NetworkingProtocolUDP + + intstr80 := intstr.FromInt32(80) + intstr85 := intstr.FromInt32(85) + intstrTrafficPort := intstr.FromString(shared_constants.HealthCheckPortTrafficPort) + + testCases := []struct { + name string + disableRestrictedSGRules bool + + targetPort intstr.IntOrString + healthCheckPort intstr.IntOrString + svcPort corev1.ServicePort + backendSGIDToken core.StringToken + + expected *elbv2model.TargetGroupBindingNetworking + }{ + { + name: "disable restricted sg rules", + disableRestrictedSGRules: true, + backendSGIDToken: core.LiteralStringToken("foo"), + expected: &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: nil, + }, + }, + }, + }, + }, + }, + { + name: "disable restricted sg rules - with udp", + disableRestrictedSGRules: true, + backendSGIDToken: core.LiteralStringToken("foo"), + svcPort: corev1.ServicePort{ + Protocol: corev1.ProtocolUDP, + }, + expected: &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: nil, + }, + { + Protocol: &protocolUDP, + Port: nil, + }, + }, + }, + }, + }, + }, + { + name: "use restricted sg rules - int hc port", + backendSGIDToken: core.LiteralStringToken("foo"), + svcPort: corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + }, + targetPort: intstr80, + healthCheckPort: intstr80, + expected: &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: &intstr80, + }, + }, + }, + }, + }, + }, + { + name: "use restricted sg rules - int hc port - udp traffic", + backendSGIDToken: core.LiteralStringToken("foo"), + svcPort: corev1.ServicePort{ + Protocol: corev1.ProtocolUDP, + }, + targetPort: intstr80, + healthCheckPort: intstr80, + expected: &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolUDP, + Port: &intstr80, + }, + }, + }, + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: &intstr80, + }, + }, + }, + }, + }, + }, + { + name: "use restricted sg rules - str hc port", + backendSGIDToken: core.LiteralStringToken("foo"), + svcPort: corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + }, + targetPort: intstr80, + healthCheckPort: intstrTrafficPort, + expected: &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: &intstr80, + }, + }, + }, + }, + }, + }, + { + name: "use restricted sg rules - str hc port - udp", + backendSGIDToken: core.LiteralStringToken("foo"), + svcPort: corev1.ServicePort{ + Protocol: corev1.ProtocolUDP, + }, + targetPort: intstr80, + healthCheckPort: intstrTrafficPort, + expected: &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolUDP, + Port: &intstr80, + }, + }, + }, + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: &intstr80, + }, + }, + }, + }, + }, + }, + { + name: "use restricted sg rules - diff hc port", + backendSGIDToken: core.LiteralStringToken("foo"), + svcPort: corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + }, + targetPort: intstr80, + healthCheckPort: intstr85, + expected: &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: &intstr80, + }, + }, + }, + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: &intstr85, + }, + }, + }, + }, + }, + }, + { + name: "use restricted sg rules - str hc port - udp", + backendSGIDToken: core.LiteralStringToken("foo"), + svcPort: corev1.ServicePort{ + Protocol: corev1.ProtocolUDP, + }, + targetPort: intstr80, + healthCheckPort: intstr85, + expected: &elbv2model.TargetGroupBindingNetworking{ + Ingress: []elbv2model.NetworkingIngressRule{ + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolUDP, + Port: &intstr80, + }, + }, + }, + { + From: []elbv2model.NetworkingPeer{ + { + SecurityGroup: &elbv2model.SecurityGroup{ + GroupID: core.LiteralStringToken("foo"), + }, + }, + }, + Ports: []elbv2api.NetworkingPort{ + { + Protocol: &protocolTCP, + Port: &intstr85, + }, + }, + }, + }, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := &targetGroupBuilderImpl{ + disableRestrictedSGRules: tc.disableRestrictedSGRules, + } + + result := builder.buildTargetGroupBindingNetworking(tc.targetPort, tc.healthCheckPort, tc.svcPort, tc.backendSGIDToken) + assert.Equal(t, tc.expected, result) + }) + } +} + +func Test_buildTargetGroupName(t *testing.T) { + http2 := elbv2model.ProtocolVersionHTTP2 + clusterName := "foo" + gwKey := types.NamespacedName{ + Namespace: "my-ns", + Name: "my-gw", + } + routeKey := types.NamespacedName{ + Namespace: "my-ns", + Name: "my-route", + } + svcKey := types.NamespacedName{ + Namespace: "my-ns", + Name: "my-svc", + } + testCases := []struct { + name string + targetGroupProps *elbv2gw.TargetGroupProps + protocolVersion *elbv2model.ProtocolVersion + expected string + }{ + { + name: "name override", + targetGroupProps: &elbv2gw.TargetGroupProps{TargetGroupName: "foobaz"}, + expected: "foobaz", + }, + { + name: "no name in props", + targetGroupProps: &elbv2gw.TargetGroupProps{}, + expected: "k8s-myns-myroute-719950e570", + }, + { + name: "no props", + expected: "k8s-myns-myroute-719950e570", + }, + { + name: "protocol specified props", + protocolVersion: &http2, + expected: "k8s-myns-myroute-ce262fa9fe", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{ + clusterName: clusterName, + } + + result := builder.buildTargetGroupName(tc.targetGroupProps, gwKey, routeKey, svcKey, 80, elbv2model.TargetTypeIP, elbv2model.ProtocolTCP, tc.protocolVersion) + assert.Equal(t, tc.expected, result) + }) + } +} + +func Test_buildTargetGroupTargetType(t *testing.T) { + builder := targetGroupBuilderImpl{ + defaultTargetType: elbv2model.TargetTypeIP, + } + + res := builder.buildTargetGroupTargetType(nil) + assert.Equal(t, elbv2model.TargetTypeIP, res) + + res = builder.buildTargetGroupTargetType(&elbv2gw.TargetGroupProps{}) + assert.Equal(t, elbv2model.TargetTypeIP, res) + + inst := elbv2gw.TargetTypeInstance + res = builder.buildTargetGroupTargetType(&elbv2gw.TargetGroupProps{ + TargetType: &inst, + }) + assert.Equal(t, elbv2model.TargetTypeInstance, res) +} + +func Test_buildTargetGroupIPAddressType(t *testing.T) { + testCases := []struct { + name string + svc *corev1.Service + loadBalancerIPAddressType elbv2model.IPAddressType + expectErr bool + expected elbv2model.TargetGroupIPAddressType + }{ + { + name: "no ip families", + svc: &corev1.Service{}, + loadBalancerIPAddressType: elbv2model.IPAddressTypeIPV4, + expected: elbv2model.TargetGroupIPAddressTypeIPv4, + }, + { + name: "ipv4 family", + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + IPFamilies: []corev1.IPFamily{ + corev1.IPv4Protocol, + }, + }, + }, + loadBalancerIPAddressType: elbv2model.IPAddressTypeIPV4, + expected: elbv2model.TargetGroupIPAddressTypeIPv4, + }, + { + name: "ipv6 family", + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + IPFamilies: []corev1.IPFamily{ + corev1.IPv6Protocol, + }, + }, + }, + loadBalancerIPAddressType: elbv2model.IPAddressTypeDualStack, + expected: elbv2model.TargetGroupIPAddressTypeIPv6, + }, + { + name: "ipv6 family - dual stack no ipv4", + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + IPFamilies: []corev1.IPFamily{ + corev1.IPv6Protocol, + }, + }, + }, + loadBalancerIPAddressType: elbv2model.IPAddressTypeDualStackWithoutPublicIPV4, + expected: elbv2model.TargetGroupIPAddressTypeIPv6, + }, + { + name: "ipv6 family - bad lb type", + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + IPFamilies: []corev1.IPFamily{ + corev1.IPv6Protocol, + }, + }, + }, + loadBalancerIPAddressType: elbv2model.IPAddressTypeIPV4, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{} + res, err := builder.buildTargetGroupIPAddressType(tc.svc, tc.loadBalancerIPAddressType) + if tc.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, res) + + }) + } +} + +func Test_buildTargetGroupPort(t *testing.T) { + testCases := []struct { + name string + targetType elbv2model.TargetType + svcPort corev1.ServicePort + expected int32 + }{ + { + name: "instance", + svcPort: corev1.ServicePort{ + NodePort: 8080, + }, + targetType: elbv2model.TargetTypeInstance, + expected: 8080, + }, + { + name: "instance - no node port", + svcPort: corev1.ServicePort{}, + targetType: elbv2model.TargetTypeInstance, + expected: 1, + }, + { + name: "ip", + svcPort: corev1.ServicePort{ + NodePort: 8080, + TargetPort: intstr.FromInt32(80), + }, + targetType: elbv2model.TargetTypeIP, + expected: 80, + }, + { + name: "ip - str port", + svcPort: corev1.ServicePort{ + NodePort: 8080, + TargetPort: intstr.FromString("foo"), + }, + targetType: elbv2model.TargetTypeIP, + expected: 1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{} + res := builder.buildTargetGroupPort(tc.targetType, tc.svcPort) + assert.Equal(t, tc.expected, res) + + }) + } +} + +func Test_buildTargetGroupProtocol(t *testing.T) { + testCases := []struct { + name string + lbType elbv2model.LoadBalancerType + targetGroupProps *elbv2gw.TargetGroupProps + route routeutils.RouteDescriptor + expected elbv2model.Protocol + expectErr bool + }{ + { + name: "alb - auto detect - http", + lbType: elbv2model.LoadBalancerTypeApplication, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolHTTP, + }, + { + name: "alb - auto detect - grpc", + lbType: elbv2model.LoadBalancerTypeApplication, + route: &routeutils.MockRoute{ + Kind: routeutils.GRPCRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolHTTP, + }, + { + name: "alb - auto detect - tls", + lbType: elbv2model.LoadBalancerTypeApplication, + route: &routeutils.MockRoute{ + Kind: routeutils.TLSRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolHTTPS, + }, + { + name: "nlb - auto detect - tcp", + lbType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolTCP, + }, + { + name: "alb - auto detect - udp", + lbType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{ + Kind: routeutils.UDPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolUDP, + }, + { + name: "nlb - auto detect - tls", + lbType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{ + Kind: routeutils.TLSRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolTLS, + }, + { + name: "alb - specified - http", + lbType: elbv2model.LoadBalancerTypeApplication, + targetGroupProps: &elbv2gw.TargetGroupProps{ + Protocol: protocolPtr(elbv2gw.ProtocolHTTP), + }, + route: &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolHTTP, + }, + { + name: "alb - specified - https", + lbType: elbv2model.LoadBalancerTypeApplication, + targetGroupProps: &elbv2gw.TargetGroupProps{ + Protocol: protocolPtr(elbv2gw.ProtocolHTTPS), + }, + route: &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolHTTPS, + }, + { + name: "alb - specified - invalid protocol", + lbType: elbv2model.LoadBalancerTypeApplication, + targetGroupProps: &elbv2gw.TargetGroupProps{ + Protocol: protocolPtr(elbv2gw.ProtocolTCP), + }, + route: &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expectErr: true, + }, + { + name: "nlb - auto detect - tcp", + lbType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolTCP, + }, + { + name: "alb - auto detect - udp", + lbType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{ + Kind: routeutils.UDPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolUDP, + }, + { + name: "nlb - auto detect - tls", + lbType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{ + Kind: routeutils.TLSRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolTLS, + }, + { + name: "nlb - specified - tcp protocol", + lbType: elbv2model.LoadBalancerTypeNetwork, + targetGroupProps: &elbv2gw.TargetGroupProps{ + Protocol: protocolPtr(elbv2gw.ProtocolTCP), + }, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolTCP, + }, + { + name: "nlb - specified - udp protocol", + lbType: elbv2model.LoadBalancerTypeNetwork, + targetGroupProps: &elbv2gw.TargetGroupProps{ + Protocol: protocolPtr(elbv2gw.ProtocolUDP), + }, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolUDP, + }, + { + name: "nlb - specified - tcpudp protocol", + lbType: elbv2model.LoadBalancerTypeNetwork, + targetGroupProps: &elbv2gw.TargetGroupProps{ + Protocol: protocolPtr(elbv2gw.ProtocolTCP_UDP), + }, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolTCP_UDP, + }, + { + name: "nlb - specified - tls protocol", + lbType: elbv2model.LoadBalancerTypeNetwork, + targetGroupProps: &elbv2gw.TargetGroupProps{ + Protocol: protocolPtr(elbv2gw.ProtocolTLS), + }, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolTLS, + }, + { + name: "nlb - specified - invalid protocol", + lbType: elbv2model.LoadBalancerTypeNetwork, + targetGroupProps: &elbv2gw.TargetGroupProps{ + Protocol: protocolPtr(elbv2gw.ProtocolHTTPS), + }, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{ + loadBalancerType: tc.lbType, + } + res, err := builder.buildTargetGroupProtocol(tc.targetGroupProps, tc.route) + if tc.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, res) + }) + } +} + +func Test_buildTargetGroupProtocolVersion(t *testing.T) { + http2Gw := elbv2gw.ProtocolVersionHTTP2 + http2Elb := elbv2model.ProtocolVersionHTTP2 + http1Elb := elbv2model.ProtocolVersionHTTP1 + grpcElb := elbv2model.ProtocolVersionGRPC + testCases := []struct { + name string + loadBalancerType elbv2model.LoadBalancerType + route routeutils.RouteDescriptor + targetGroupProps *elbv2gw.TargetGroupProps + expected *elbv2model.ProtocolVersion + }{ + { + name: "nlb - no props", + loadBalancerType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{Kind: routeutils.TCPRouteKind}, + }, + { + name: "nlb - with props", + loadBalancerType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{Kind: routeutils.TCPRouteKind}, + targetGroupProps: &elbv2gw.TargetGroupProps{ + ProtocolVersion: &http2Gw, + }, + }, + { + name: "alb - no props", + route: &routeutils.MockRoute{Kind: routeutils.HTTPRouteKind}, + loadBalancerType: elbv2model.LoadBalancerTypeApplication, + expected: &http1Elb, + }, + { + name: "alb - no props - grpc", + route: &routeutils.MockRoute{Kind: routeutils.GRPCRouteKind}, + loadBalancerType: elbv2model.LoadBalancerTypeApplication, + expected: &grpcElb, + }, + { + name: "alb - with props", + route: &routeutils.MockRoute{Kind: routeutils.HTTPRouteKind}, + loadBalancerType: elbv2model.LoadBalancerTypeApplication, + targetGroupProps: &elbv2gw.TargetGroupProps{ + ProtocolVersion: &http2Gw, + }, + expected: &http2Elb, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{ + loadBalancerType: tc.loadBalancerType, + } + res := builder.buildTargetGroupProtocolVersion(tc.targetGroupProps, tc.route) + assert.Equal(t, tc.expected, res) + }) + } +} + +func Test_buildTargetGroupHealthCheckPort(t *testing.T) { + testCases := []struct { + name string + targetGroupProps *elbv2gw.TargetGroupProps + targetType elbv2model.TargetType + svc *corev1.Service + expected intstr.IntOrString + expectErr bool + }{ + { + name: "nil props", + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "nil hc props", + targetGroupProps: &elbv2gw.TargetGroupProps{}, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "nil hc port", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{}, + }, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "explicit is use traffic port hc port", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String(shared_constants.HealthCheckPortTrafficPort), + }, + }, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "explicit port", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("80"), + }, + }, + expected: intstr.FromInt32(80), + }, + { + name: "resolve str port", + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromInt32(80), + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(80), + }, + { + name: "resolve str port - instance", + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromInt32(80), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(1000), + }, + { + name: "resolve str port - resolves to other str port (error)", + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expectErr: true, + }, + { + name: "resolve str port - resolves to other str port but instance mode", + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(1000), + }, + { + name: "resolve str port - cant find configured port", + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "baz", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{} + res, err := builder.buildTargetGroupHealthCheckPort(tc.targetGroupProps, tc.targetType, tc.svc) + if tc.expectErr { + assert.Error(t, err, res) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, res) + }) + } +} + +func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { + testCases := []struct { + name string + lbType elbv2model.LoadBalancerType + targetGroupProps *elbv2gw.TargetGroupProps + tgProtocol elbv2model.Protocol + expected elbv2model.Protocol + }{ + { + name: "nlb - default", + lbType: elbv2model.LoadBalancerTypeNetwork, + tgProtocol: elbv2model.ProtocolUDP, + expected: elbv2model.ProtocolTCP, + }, + { + name: "alb - default", + lbType: elbv2model.LoadBalancerTypeApplication, + tgProtocol: elbv2model.ProtocolHTTP, + expected: elbv2model.ProtocolHTTP, + }, + { + name: "specified http", + lbType: elbv2model.LoadBalancerTypeApplication, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolHTTP))), + }, + }, + tgProtocol: elbv2model.ProtocolHTTP, + expected: elbv2model.ProtocolHTTP, + }, + { + name: "specified https", + lbType: elbv2model.LoadBalancerTypeApplication, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolHTTPS))), + }, + }, + tgProtocol: elbv2model.ProtocolHTTP, + expected: elbv2model.ProtocolHTTPS, + }, + { + name: "specified tcp", + lbType: elbv2model.LoadBalancerTypeApplication, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolTCP))), + }, + }, + tgProtocol: elbv2model.ProtocolTCP, + expected: elbv2model.ProtocolTCP, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{ + loadBalancerType: tc.lbType, + } + + res := builder.buildTargetGroupHealthCheckProtocol(tc.targetGroupProps, tc.tgProtocol) + assert.Equal(t, tc.expected, res) + }) + } +} + +func Test_buildTargetGroupHealthCheckPath(t *testing.T) { + httpDefaultPath := "httpDefault" + grpcDefaultPath := "grpcDefault" + testCases := []struct { + name string + targetGroupProps *elbv2gw.TargetGroupProps + tgProtocolVersion *elbv2model.ProtocolVersion + hcProtocol elbv2model.Protocol + expected *string + }{ + { + name: "path specified", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPath: awssdk.String("foo"), + }, + }, + expected: awssdk.String("foo"), + }, + { + name: "default - tcp", + hcProtocol: elbv2model.ProtocolTCP, + }, + { + name: "default - http", + hcProtocol: elbv2model.ProtocolHTTP, + expected: &httpDefaultPath, + }, + { + name: "default - grpc", + hcProtocol: elbv2model.ProtocolHTTP, + tgProtocolVersion: (*elbv2model.ProtocolVersion)(awssdk.String(string(elbv2model.ProtocolVersionGRPC))), + expected: &grpcDefaultPath, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{ + defaultHealthCheckPathHTTP: httpDefaultPath, + defaultHealthCheckPathGRPC: grpcDefaultPath, + } + + res := builder.buildTargetGroupHealthCheckPath(tc.targetGroupProps, tc.tgProtocolVersion, tc.hcProtocol) + assert.Equal(t, tc.expected, res) + }) + } +} + +func Test_buildTargetGroupHealthCheckMatcher(t *testing.T) { + httpDefaultMatcher := "httpMatcher" + grpcDefaultMatcher := "grpcMatcher" + testCases := []struct { + name string + targetGroupProps *elbv2gw.TargetGroupProps + tgProtocolVersion *elbv2model.ProtocolVersion + hcProtocol elbv2model.Protocol + expected *elbv2model.HealthCheckMatcher + }{ + { + name: "default - tcp", + hcProtocol: elbv2model.ProtocolTCP, + }, + { + name: "specified - grpc", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + Matcher: &elbv2gw.HealthCheckMatcher{ + GRPCCode: awssdk.String("foo"), + }, + }, + }, + hcProtocol: elbv2model.ProtocolHTTP, + tgProtocolVersion: (*elbv2model.ProtocolVersion)(awssdk.String(string(elbv2model.ProtocolVersionGRPC))), + expected: &elbv2model.HealthCheckMatcher{ + GRPCCode: awssdk.String("foo"), + }, + }, + { + name: "specified - http", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + Matcher: &elbv2gw.HealthCheckMatcher{ + HTTPCode: awssdk.String("foo"), + }, + }, + }, + hcProtocol: elbv2model.ProtocolHTTP, + expected: &elbv2model.HealthCheckMatcher{ + HTTPCode: awssdk.String("foo"), + }, + }, + { + name: "default - grpc", + hcProtocol: elbv2model.ProtocolHTTP, + tgProtocolVersion: (*elbv2model.ProtocolVersion)(awssdk.String(string(elbv2model.ProtocolVersionGRPC))), + expected: &elbv2model.HealthCheckMatcher{ + GRPCCode: &grpcDefaultMatcher, + }, + }, + { + name: "default - http1", + hcProtocol: elbv2model.ProtocolHTTP, + tgProtocolVersion: (*elbv2model.ProtocolVersion)(awssdk.String(string(elbv2model.ProtocolVersionHTTP1))), + expected: &elbv2model.HealthCheckMatcher{ + HTTPCode: &httpDefaultMatcher, + }, + }, + { + name: "default - no protocol version", + hcProtocol: elbv2model.ProtocolHTTP, + tgProtocolVersion: (*elbv2model.ProtocolVersion)(awssdk.String(string(elbv2model.ProtocolVersionHTTP1))), + expected: &elbv2model.HealthCheckMatcher{ + HTTPCode: &httpDefaultMatcher, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{ + defaultHealthCheckMatcherHTTPCode: httpDefaultMatcher, + defaultHealthCheckMatcherGRPCCode: grpcDefaultMatcher, + } + + res := builder.buildTargetGroupHealthCheckMatcher(tc.targetGroupProps, tc.tgProtocolVersion, tc.hcProtocol) + assert.Equal(t, tc.expected, res) + }) + } +} + +func Test_basicHealthCheckParams(t *testing.T) { + builder := targetGroupBuilderImpl{ + defaultHealthCheckInterval: 1, + defaultHealthCheckTimeout: 2, + defaultHealthyThresholdCount: 3, + defaultHealthCheckUnhealthyThresholdCount: 4, + } + + defaultProps := []*elbv2gw.TargetGroupProps{ + nil, + {}, + { + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{}, + }, + } + + for _, prop := range defaultProps { + assert.Equal(t, int32(1), builder.buildTargetGroupHealthCheckIntervalSeconds(prop)) + assert.Equal(t, int32(2), builder.buildTargetGroupHealthCheckTimeoutSeconds(prop)) + assert.Equal(t, int32(3), builder.buildTargetGroupHealthCheckHealthyThresholdCount(prop)) + assert.Equal(t, int32(4), builder.buildTargetGroupHealthCheckUnhealthyThresholdCount(prop)) + } + + filledInProps := &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthyThresholdCount: awssdk.Int32(30), + HealthCheckInterval: awssdk.Int32(10), + HealthCheckPath: nil, + HealthCheckPort: nil, + HealthCheckProtocol: nil, + HealthCheckTimeout: awssdk.Int32(20), + UnhealthyThresholdCount: awssdk.Int32(40), + Matcher: nil, + }} + + assert.Equal(t, int32(10), builder.buildTargetGroupHealthCheckIntervalSeconds(filledInProps)) + assert.Equal(t, int32(20), builder.buildTargetGroupHealthCheckTimeoutSeconds(filledInProps)) + assert.Equal(t, int32(30), builder.buildTargetGroupHealthCheckHealthyThresholdCount(filledInProps)) + assert.Equal(t, int32(40), builder.buildTargetGroupHealthCheckUnhealthyThresholdCount(filledInProps)) +} + +func Test_targetGroupAttributes(t *testing.T) { + testCases := []struct { + name string + props *elbv2gw.TargetGroupProps + expected []elbv2model.TargetGroupAttribute + }{ + { + name: "no props - nil", + expected: make([]elbv2model.TargetGroupAttribute, 0), + }, + { + name: "no props", + props: &elbv2gw.TargetGroupProps{}, + expected: make([]elbv2model.TargetGroupAttribute, 0), + }, + { + name: "some props", + props: &elbv2gw.TargetGroupProps{ + TargetGroupAttributes: []elbv2gw.TargetGroupAttribute{ + { + Key: "foo", + Value: "bar", + }, + { + Key: "foo1", + Value: "bar1", + }, + { + Key: "foo2", + Value: "bar2", + }, + }, + }, + expected: []elbv2model.TargetGroupAttribute{ + { + Key: "foo", + Value: "bar", + }, + { + Key: "foo1", + Value: "bar1", + }, + { + Key: "foo2", + Value: "bar2", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := targetGroupBuilderImpl{} + + res := builder.convertMapToAttributes(builder.buildTargetGroupAttributes(tc.props)) + assert.ElementsMatch(t, tc.expected, res) + }) + } +} + +func Test_buildTargetGroupBindingNodeSelector(t *testing.T) { + builder := targetGroupBuilderImpl{} + + res := builder.buildTargetGroupBindingNodeSelector(nil, elbv2model.TargetTypeInstance) + assert.Nil(t, res) + + propWithSelector := &elbv2gw.TargetGroupProps{ + NodeSelector: &metav1.LabelSelector{}, + } + + res = builder.buildTargetGroupBindingNodeSelector(propWithSelector, elbv2model.TargetTypeIP) + assert.Nil(t, res) + + assert.NotNil(t, builder.buildTargetGroupBindingNodeSelector(propWithSelector, elbv2model.TargetTypeInstance)) +} + +func Test_buildTargetGroupBindingMultiClusterFlag(t *testing.T) { + builder := targetGroupBuilderImpl{} + + assert.False(t, builder.buildTargetGroupBindingMultiClusterFlag(nil)) + + props := &elbv2gw.TargetGroupProps{ + EnableMultiCluster: false, + } + + assert.False(t, builder.buildTargetGroupBindingMultiClusterFlag(props)) + props.EnableMultiCluster = true + assert.True(t, builder.buildTargetGroupBindingMultiClusterFlag(props)) +} + +func protocolPtr(protocol elbv2gw.Protocol) *elbv2gw.Protocol { + return &protocol +} From 5de3fec2587335871e7a50f4e3e67bdf4a40475b Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 17 Apr 2025 11:41:33 -0700 Subject: [PATCH 33/40] make logging less noisy --- controllers/gateway/gateway_controller.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 89947b8cc5..76e124e4ab 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -184,8 +184,6 @@ func (r *gatewayReconciler) reconcileHelper(ctx context.Context, req reconcile.R allRoutes, err := r.gatewayLoader.LoadRoutesForGateway(ctx, *gw, r.routeFilter) - r.logger.Info("In Gateway Controller - Got these routes", "routes", allRoutes) - if err != nil { return err } From ff496e4c45aa05dc56c5ca19e701e92e985e6659 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 21 Apr 2025 10:12:52 -0700 Subject: [PATCH 34/40] refactor multicluster to target group props --- apis/gateway/v1beta1/loadbalancerconfig_types.go | 5 ----- apis/gateway/v1beta1/targetgroupconfig_types.go | 5 +++++ .../gateway.k8s.aws_loadbalancerconfigurations.yaml | 5 ----- .../gateway.k8s.aws_targetgroupconfigurations.yaml | 10 ++++++++++ 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/apis/gateway/v1beta1/loadbalancerconfig_types.go b/apis/gateway/v1beta1/loadbalancerconfig_types.go index 8435bd9772..c9a3e164a5 100644 --- a/apis/gateway/v1beta1/loadbalancerconfig_types.go +++ b/apis/gateway/v1beta1/loadbalancerconfig_types.go @@ -243,11 +243,6 @@ type LoadBalancerConfigurationSpec struct { // when you specify securityGroups // +optional ManageBackendSecurityGroupRules bool `json:"manageBackendSecurityGroupRules,omitempty"` - - // EnableMultiCluster [Application / Network LoadBalancer] - // All TargetGroupBindings attached to this Load Balancer will have multi cluster support enabled. - // +optional - EnableMultiCluster bool `json:"enableMultiCluster,omitempty"` } // TODO -- these can be used to set what generation the gateway is currently on to track progress on reconcile. diff --git a/apis/gateway/v1beta1/targetgroupconfig_types.go b/apis/gateway/v1beta1/targetgroupconfig_types.go index 8794e6b520..7615fccd57 100644 --- a/apis/gateway/v1beta1/targetgroupconfig_types.go +++ b/apis/gateway/v1beta1/targetgroupconfig_types.go @@ -216,6 +216,11 @@ type TargetGroupProps struct { // +optional EnableMultiCluster bool `json:"enableMultiCluster,omitempty"` + // EnableMultiCluster [Application / Network LoadBalancer] + // Allows for multiple Clusters / Services to use the generated TargetGroup ARN + // +optional + EnableMultiCluster bool `json:"enableMultiCluster,omitempty"` + // vpcID is the VPC of the TargetGroup. If unspecified, it will be automatically inferred. // +optional VpcID *string `json:"vpcID,omitempty"` diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml index aaa8a79ce3..f146a9eb99 100644 --- a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -56,11 +56,6 @@ spec: enables the creation of security group rules to the managed security group to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs type: boolean - enableMultiCluster: - description: |- - EnableMultiCluster [Application / Network LoadBalancer] - All TargetGroupBindings attached to this Load Balancer will have multi cluster support enabled. - type: boolean enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic Indicates whether to evaluate inbound security group rules for traffic diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml index 1ad36952b0..4f4e4c62d1 100644 --- a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml +++ b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -53,6 +53,11 @@ spec: description: defaultRouteConfiguration fallback configuration applied to all routes, unless overridden by route-specific configurations. properties: + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + Allows for multiple Clusters / Services to use the generated TargetGroup ARN + type: boolean enableProxyProtocolV2: description: |- enableProxyProtocolV2 [Network LoadBalancers] Indicates whether proxy protocol version 2 is enabled. @@ -253,6 +258,11 @@ spec: targetGroupProps: description: targetGroupProps the target group specific properties properties: + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + Allows for multiple Clusters / Services to use the generated TargetGroup ARN + type: boolean enableProxyProtocolV2: description: |- enableProxyProtocolV2 [Network LoadBalancers] Indicates whether proxy protocol version 2 is enabled. From c35560ddf70d285911529038ea75ea61ac2c1e09 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 21 Apr 2025 11:51:53 -0700 Subject: [PATCH 35/40] refactor to use route kind enum --- apis/gateway/v1beta1/targetgroupconfig_types.go | 5 ----- apis/gateway/v1beta1/zz_generated.deepcopy.go | 5 ----- .../gateway.k8s.aws_targetgroupconfigurations.yaml | 10 ---------- 3 files changed, 20 deletions(-) diff --git a/apis/gateway/v1beta1/targetgroupconfig_types.go b/apis/gateway/v1beta1/targetgroupconfig_types.go index 7615fccd57..8794e6b520 100644 --- a/apis/gateway/v1beta1/targetgroupconfig_types.go +++ b/apis/gateway/v1beta1/targetgroupconfig_types.go @@ -216,11 +216,6 @@ type TargetGroupProps struct { // +optional EnableMultiCluster bool `json:"enableMultiCluster,omitempty"` - // EnableMultiCluster [Application / Network LoadBalancer] - // Allows for multiple Clusters / Services to use the generated TargetGroup ARN - // +optional - EnableMultiCluster bool `json:"enableMultiCluster,omitempty"` - // vpcID is the VPC of the TargetGroup. If unspecified, it will be automatically inferred. // +optional VpcID *string `json:"vpcID,omitempty"` diff --git a/apis/gateway/v1beta1/zz_generated.deepcopy.go b/apis/gateway/v1beta1/zz_generated.deepcopy.go index d5f95d1b74..fe1edd2f62 100644 --- a/apis/gateway/v1beta1/zz_generated.deepcopy.go +++ b/apis/gateway/v1beta1/zz_generated.deepcopy.go @@ -696,11 +696,6 @@ func (in *TargetGroupProps) DeepCopyInto(out *TargetGroupProps) { *out = new(ProtocolVersion) **out = **in } - if in.EnableProxyProtocolV2 != nil { - in, out := &in.EnableProxyProtocolV2, &out.EnableProxyProtocolV2 - *out = new(bool) - **out = **in - } if in.VpcID != nil { in, out := &in.VpcID, &out.VpcID *out = new(string) diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml index 4f4e4c62d1..b2b21ca7b2 100644 --- a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml +++ b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -58,11 +58,6 @@ spec: EnableMultiCluster [Application / Network LoadBalancer] Allows for multiple Clusters / Services to use the generated TargetGroup ARN type: boolean - enableProxyProtocolV2: - description: |- - enableProxyProtocolV2 [Network LoadBalancers] Indicates whether proxy protocol version 2 is enabled. - By default, proxy protocol is disabled. - type: boolean healthCheckConfig: description: healthCheckConfig The Health Check configuration for this backend. @@ -263,11 +258,6 @@ spec: EnableMultiCluster [Application / Network LoadBalancer] Allows for multiple Clusters / Services to use the generated TargetGroup ARN type: boolean - enableProxyProtocolV2: - description: |- - enableProxyProtocolV2 [Network LoadBalancers] Indicates whether proxy protocol version 2 is enabled. - By default, proxy protocol is disabled. - type: boolean healthCheckConfig: description: healthCheckConfig The Health Check configuration for this backend. From b9e922c1bb642df4ecfc34b4044d08688c718d2c Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 21 Apr 2025 12:01:07 -0700 Subject: [PATCH 36/40] infer target group type from route --- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 298 ----------- ...way.k8s.aws_targetgroupconfigurations.yaml | 498 ------------------ 2 files changed, 796 deletions(-) delete mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml delete mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml deleted file mode 100644 index f146a9eb99..0000000000 --- a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ /dev/null @@ -1,298 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: loadbalancerconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: LoadBalancerConfiguration - listKind: LoadBalancerConfigurationList - plural: loadbalancerconfigurations - singular: loadbalancerconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LoadBalancerConfigurationSpec defines the desired state of - LoadBalancerConfiguration - properties: - customerOwnedIpv4Pool: - description: |- - customerOwnedIpv4Pool [Application LoadBalancer] - is the ID of the customer-owned address for Application Load Balancers on Outposts pool. - type: string - enableICMP: - description: |- - EnableICMP [Network LoadBalancer] - enables the creation of security group rules to the managed security group - to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs - type: boolean - enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: - description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic - Indicates whether to evaluate inbound security group rules for traffic - sent to a Network Load Balancer through Amazon Web Services PrivateLink. - type: string - ipAddressType: - description: loadBalancerIPType defines what kind of load balancer - to provision (ipv4, dual stack) - enum: - - ipv4 - - dualstack - - dualstack-without-public-ipv4 - type: string - ipv4IPAMPoolId: - description: |- - IPv4IPAMPoolId [Application LoadBalancer] - defines the IPAM pool ID used for IPv4 Addresses on the ALB. - type: string - listenerConfigurations: - description: listenerConfigurations is an optional list of configurations - for each listener on LB - items: - properties: - alpnPolicy: - description: alpnPolicy an optional string that allows you to - configure ALPN policies on your Load Balancer - enum: - - HTTP1Only - - HTTP2Only - - HTTP2Optional - - HTTP2Preferred - - None - type: string - certificates: - description: certificates is the list of other certificates - to add to the listener. - items: - type: string - type: array - defaultCertificate: - description: |- - TODO: Add validation in admission webhook to make it required for secure protocols - defaultCertificate the cert arn to be used by default. - type: string - listenerAttributes: - description: listenerAttributes defines the attributes for the - listener - items: - description: ListenerAttribute defines listener attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - mutualAuthentication: - description: mutualAuthentication defines the mutual authentication - configuration information. - properties: - advertiseTrustStoreCaNames: - description: Indicates whether trust store CA certificate - names are advertised. - enum: - - "on" - - "off" - type: string - ignoreClientCertificateExpiry: - description: Indicates whether expired client certificates - are ignored. - type: boolean - mode: - description: The client certificate handling method. Options - are off , passthrough or verify - enum: - - "off" - - passthrough - - verify - type: string - trustStore: - description: The Name or ARN of the trust store. - type: string - required: - - mode - type: object - protocolPort: - description: protocolPort is identifier for the listener on - load balancer. It should be of the form PROTOCOL:PORT - pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ - type: string - sslPolicy: - description: sslPolicy is the security policy that defines which - protocols and ciphers are supported for secure listeners [HTTPS - or TLS listener]. - type: string - required: - - protocolPort - type: object - type: array - loadBalancerAttributes: - description: LoadBalancerAttributes defines the attribute of LB - items: - description: LoadBalancerAttribute defines LB attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - loadBalancerName: - description: loadBalancerName defines the name of the LB to provision. - If unspecified, it will be automatically generated. - maxLength: 32 - minLength: 1 - type: string - loadBalancerSubnets: - description: |- - loadBalancerSubnets is an optional list of subnet configurations to be used in the LB - This value takes precedence over loadBalancerSubnetsSelector if both are selected. - items: - description: SubnetConfiguration defines the subnet settings for - a Load Balancer. - properties: - eipAllocation: - description: eipAllocation [Network LoadBalancer] the EIP name - for this subnet. - type: string - identifier: - description: identifier [Application LoadBalancer / Network - LoadBalancer] name or id for the subnet - type: string - ipv6Allocation: - description: IPv6Allocation [Network LoadBalancer] the ipv6 - address to assign to this subnet. - type: string - privateIPv4Allocation: - description: privateIPv4Allocation [Network LoadBalancer] the - private ipv4 address to assign to this subnet. - type: string - sourceNatIPv6Prefix: - description: SourceNatIPv6Prefix [Network LoadBalancer] The - IPv6 prefix to use for source NAT. Specify an IPv6 prefix - (/80 netmask) from the subnet CIDR block or auto_assigned - to use an IPv6 prefix selected at random from the subnet CIDR - block. - type: string - type: object - type: array - loadBalancerSubnetsSelector: - additionalProperties: - items: - type: string - type: array - description: |- - LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each - tag specified in the map key contains one of the values in the corresponding - value list. - type: object - manageBackendSecurityGroupRules: - description: |- - ManageBackendSecurityGroupRules [Application / Network LoadBalancer] - specifies whether you want the controller to configure security group rules on Node/Pod for traffic access - when you specify securityGroups - type: boolean - scheme: - description: scheme defines the type of LB to provision. If unspecified, - it will be automatically inferred. - enum: - - internal - - internet-facing - type: string - securityGroupPrefixes: - description: securityGroupPrefixes an optional list of prefixes that - are allowed to access the LB. - items: - type: string - type: array - securityGroups: - description: securityGroups an optional list of security group ids - or names to apply to the LB - items: - type: string - type: array - sourceRanges: - description: sourceRanges an optional list of CIDRs that are allowed - to access the LB. - items: - type: string - type: array - tags: - description: Tags defines list of Tags on LB. - items: - description: AWSTag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - vpcId: - description: vpcId is the ID of the VPC for the load balancer. - type: string - type: object - status: - description: LoadBalancerConfigurationStatus defines the observed state - of TargetGroupBinding - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml deleted file mode 100644 index b2b21ca7b2..0000000000 --- a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml +++ /dev/null @@ -1,498 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: targetgroupconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: TargetGroupConfiguration - listKind: TargetGroupConfigurationList - plural: targetgroupconfigurations - singular: targetgroupconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: The Kubernetes Service's name - jsonPath: .spec.targetReference.name - name: SERVICE-NAME - type: string - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: TargetGroupConfiguration is the Schema for defining TargetGroups - with an AWS ELB Gateway - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TargetGroupConfigurationSpec defines the TargetGroup properties - for a route. - properties: - defaultConfiguration: - description: defaultRouteConfiguration fallback configuration applied - to all routes, unless overridden by route-specific configurations. - properties: - enableMultiCluster: - description: |- - EnableMultiCluster [Application / Network LoadBalancer] - Allows for multiple Clusters / Services to use the generated TargetGroup ARN - type: boolean - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount of - time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health checks - on the targets. - type: string - healthCheckPort: - description: |- - healthCheckPort The port the load balancer uses when performing health checks on targets. - The default is to use the port on which each target receives traffic from the load balancer. - type: string - healthCheckProtocol: - description: healthCheckProtocol The protocol to use to connect - with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols - are not supported for health checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, in seconds, - during which no response means a failed health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering an unhealthy - target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes to use - when checking for a successful response from a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering the target - unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target group - is of type IPv4 or IPv6. If unspecified, it will be automatically - inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups to - only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocol: - description: |- - Protocol [Application / Network Load Balancer] the protocol for the target group. - If unspecified, it will be automatically inferred. - enum: - - HTTP - - HTTPS - - TCP - - TLS - - UDP - - TCP_UDP - type: string - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute of target - group - items: - description: TargetGroupAttribute defines target group attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetGroupName: - description: targetGroupName specifies the name to assign to the - Target Group. If not defined, then one is generated. - type: string - targetType: - description: targetType is the TargetType of TargetGroup. If unspecified, - it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - routeConfigurations: - description: routeConfigurations the route configuration for specific - routes - items: - description: RouteConfiguration defines the per route configuration - properties: - identifier: - description: name the identifier of the route, it should be - in the form of ROUTE:NAMESPACE:NAME - pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ - type: string - targetGroupProps: - description: targetGroupProps the target group specific properties - properties: - enableMultiCluster: - description: |- - EnableMultiCluster [Application / Network LoadBalancer] - Allows for multiple Clusters / Services to use the generated TargetGroup ARN - type: boolean - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount - of time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health - checks on the targets. - type: string - healthCheckPort: - description: |- - healthCheckPort The port the load balancer uses when performing health checks on targets. - The default is to use the port on which each target receives traffic from the load balancer. - type: string - healthCheckProtocol: - description: healthCheckProtocol The protocol to use - to connect with the target. The GENEVE, TLS, UDP, - and TCP_UDP protocols are not supported for health - checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, - in seconds, during which no response means a failed - health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering - an unhealthy target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes - to use when checking for a successful response from - a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering - the target unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target - group is of type IPv4 or IPv6. If unspecified, it will - be automatically inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups - to only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocol: - description: |- - Protocol [Application / Network Load Balancer] the protocol for the target group. - If unspecified, it will be automatically inferred. - enum: - - HTTP - - HTTPS - - TCP - - TLS - - UDP - - TCP_UDP - type: string - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute - of target group - items: - description: TargetGroupAttribute defines target group - attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetGroupName: - description: targetGroupName specifies the name to assign - to the Target Group. If not defined, then one is generated. - type: string - targetType: - description: targetType is the TargetType of TargetGroup. - If unspecified, it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - required: - - identifier - - targetGroupProps - type: object - type: array - targetReference: - description: targetReference the kubernetes object to attach the Target - Group settings to. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - - Defaults to "Service" when not specified. - type: string - name: - description: Name is the name of the referent. - type: string - required: - - name - type: object - required: - - targetReference - type: object - status: - description: TargetGroupConfigurationStatus defines the observed state - of TargetGroupConfiguration - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} From 8bb0d0f01a57e9ec42637313c430f03756141622 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 21 Apr 2025 17:41:49 -0700 Subject: [PATCH 37/40] unit tests for target group builder --- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 298 +++++++++++ ...way.k8s.aws_targetgroupconfigurations.yaml | 498 ++++++++++++++++++ 2 files changed, 796 insertions(+) create mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml create mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml new file mode 100644 index 0000000000..f146a9eb99 --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -0,0 +1,298 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: loadbalancerconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: LoadBalancerConfiguration + listKind: LoadBalancerConfigurationList + plural: loadbalancerconfigurations + singular: loadbalancerconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LoadBalancerConfigurationSpec defines the desired state of + LoadBalancerConfiguration + properties: + customerOwnedIpv4Pool: + description: |- + customerOwnedIpv4Pool [Application LoadBalancer] + is the ID of the customer-owned address for Application Load Balancers on Outposts pool. + type: string + enableICMP: + description: |- + EnableICMP [Network LoadBalancer] + enables the creation of security group rules to the managed security group + to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs + type: boolean + enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: + description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic + Indicates whether to evaluate inbound security group rules for traffic + sent to a Network Load Balancer through Amazon Web Services PrivateLink. + type: string + ipAddressType: + description: loadBalancerIPType defines what kind of load balancer + to provision (ipv4, dual stack) + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string + ipv4IPAMPoolId: + description: |- + IPv4IPAMPoolId [Application LoadBalancer] + defines the IPAM pool ID used for IPv4 Addresses on the ALB. + type: string + listenerConfigurations: + description: listenerConfigurations is an optional list of configurations + for each listener on LB + items: + properties: + alpnPolicy: + description: alpnPolicy an optional string that allows you to + configure ALPN policies on your Load Balancer + enum: + - HTTP1Only + - HTTP2Only + - HTTP2Optional + - HTTP2Preferred + - None + type: string + certificates: + description: certificates is the list of other certificates + to add to the listener. + items: + type: string + type: array + defaultCertificate: + description: |- + TODO: Add validation in admission webhook to make it required for secure protocols + defaultCertificate the cert arn to be used by default. + type: string + listenerAttributes: + description: listenerAttributes defines the attributes for the + listener + items: + description: ListenerAttribute defines listener attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + mutualAuthentication: + description: mutualAuthentication defines the mutual authentication + configuration information. + properties: + advertiseTrustStoreCaNames: + description: Indicates whether trust store CA certificate + names are advertised. + enum: + - "on" + - "off" + type: string + ignoreClientCertificateExpiry: + description: Indicates whether expired client certificates + are ignored. + type: boolean + mode: + description: The client certificate handling method. Options + are off , passthrough or verify + enum: + - "off" + - passthrough + - verify + type: string + trustStore: + description: The Name or ARN of the trust store. + type: string + required: + - mode + type: object + protocolPort: + description: protocolPort is identifier for the listener on + load balancer. It should be of the form PROTOCOL:PORT + pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ + type: string + sslPolicy: + description: sslPolicy is the security policy that defines which + protocols and ciphers are supported for secure listeners [HTTPS + or TLS listener]. + type: string + required: + - protocolPort + type: object + type: array + loadBalancerAttributes: + description: LoadBalancerAttributes defines the attribute of LB + items: + description: LoadBalancerAttribute defines LB attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + loadBalancerName: + description: loadBalancerName defines the name of the LB to provision. + If unspecified, it will be automatically generated. + maxLength: 32 + minLength: 1 + type: string + loadBalancerSubnets: + description: |- + loadBalancerSubnets is an optional list of subnet configurations to be used in the LB + This value takes precedence over loadBalancerSubnetsSelector if both are selected. + items: + description: SubnetConfiguration defines the subnet settings for + a Load Balancer. + properties: + eipAllocation: + description: eipAllocation [Network LoadBalancer] the EIP name + for this subnet. + type: string + identifier: + description: identifier [Application LoadBalancer / Network + LoadBalancer] name or id for the subnet + type: string + ipv6Allocation: + description: IPv6Allocation [Network LoadBalancer] the ipv6 + address to assign to this subnet. + type: string + privateIPv4Allocation: + description: privateIPv4Allocation [Network LoadBalancer] the + private ipv4 address to assign to this subnet. + type: string + sourceNatIPv6Prefix: + description: SourceNatIPv6Prefix [Network LoadBalancer] The + IPv6 prefix to use for source NAT. Specify an IPv6 prefix + (/80 netmask) from the subnet CIDR block or auto_assigned + to use an IPv6 prefix selected at random from the subnet CIDR + block. + type: string + type: object + type: array + loadBalancerSubnetsSelector: + additionalProperties: + items: + type: string + type: array + description: |- + LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each + tag specified in the map key contains one of the values in the corresponding + value list. + type: object + manageBackendSecurityGroupRules: + description: |- + ManageBackendSecurityGroupRules [Application / Network LoadBalancer] + specifies whether you want the controller to configure security group rules on Node/Pod for traffic access + when you specify securityGroups + type: boolean + scheme: + description: scheme defines the type of LB to provision. If unspecified, + it will be automatically inferred. + enum: + - internal + - internet-facing + type: string + securityGroupPrefixes: + description: securityGroupPrefixes an optional list of prefixes that + are allowed to access the LB. + items: + type: string + type: array + securityGroups: + description: securityGroups an optional list of security group ids + or names to apply to the LB + items: + type: string + type: array + sourceRanges: + description: sourceRanges an optional list of CIDRs that are allowed + to access the LB. + items: + type: string + type: array + tags: + description: Tags defines list of Tags on LB. + items: + description: AWSTag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + vpcId: + description: vpcId is the ID of the VPC for the load balancer. + type: string + type: object + status: + description: LoadBalancerConfigurationStatus defines the observed state + of TargetGroupBinding + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml new file mode 100644 index 0000000000..b2b21ca7b2 --- /dev/null +++ b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -0,0 +1,498 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: targetgroupconfigurations.gateway.k8s.aws +spec: + group: gateway.k8s.aws + names: + kind: TargetGroupConfiguration + listKind: TargetGroupConfigurationList + plural: targetgroupconfigurations + singular: targetgroupconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The Kubernetes Service's name + jsonPath: .spec.targetReference.name + name: SERVICE-NAME + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TargetGroupConfiguration is the Schema for defining TargetGroups + with an AWS ELB Gateway + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TargetGroupConfigurationSpec defines the TargetGroup properties + for a route. + properties: + defaultConfiguration: + description: defaultRouteConfiguration fallback configuration applied + to all routes, unless overridden by route-specific configurations. + properties: + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + Allows for multiple Clusters / Services to use the generated TargetGroup ARN + type: boolean + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount of + time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health checks + on the targets. + type: string + healthCheckPort: + description: |- + healthCheckPort The port the load balancer uses when performing health checks on targets. + The default is to use the port on which each target receives traffic from the load balancer. + type: string + healthCheckProtocol: + description: healthCheckProtocol The protocol to use to connect + with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols + are not supported for health checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, in seconds, + during which no response means a failed health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering an unhealthy + target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes to use + when checking for a successful response from a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering the target + unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target group + is of type IPv4 or IPv6. If unspecified, it will be automatically + inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups to + only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocol: + description: |- + Protocol [Application / Network Load Balancer] the protocol for the target group. + If unspecified, it will be automatically inferred. + enum: + - HTTP + - HTTPS + - TCP + - TLS + - UDP + - TCP_UDP + type: string + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute of target + group + items: + description: TargetGroupAttribute defines target group attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetGroupName: + description: targetGroupName specifies the name to assign to the + Target Group. If not defined, then one is generated. + type: string + targetType: + description: targetType is the TargetType of TargetGroup. If unspecified, + it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + routeConfigurations: + description: routeConfigurations the route configuration for specific + routes + items: + description: RouteConfiguration defines the per route configuration + properties: + identifier: + description: name the identifier of the route, it should be + in the form of ROUTE:NAMESPACE:NAME + pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ + type: string + targetGroupProps: + description: targetGroupProps the target group specific properties + properties: + enableMultiCluster: + description: |- + EnableMultiCluster [Application / Network LoadBalancer] + Allows for multiple Clusters / Services to use the generated TargetGroup ARN + type: boolean + healthCheckConfig: + description: healthCheckConfig The Health Check configuration + for this backend. + properties: + healthCheckInterval: + description: healthCheckInterval The approximate amount + of time, in seconds, between health checks of an individual + target. + format: int32 + type: integer + healthCheckPath: + description: healthCheckPath The destination for health + checks on the targets. + type: string + healthCheckPort: + description: |- + healthCheckPort The port the load balancer uses when performing health checks on targets. + The default is to use the port on which each target receives traffic from the load balancer. + type: string + healthCheckProtocol: + description: healthCheckProtocol The protocol to use + to connect with the target. The GENEVE, TLS, UDP, + and TCP_UDP protocols are not supported for health + checks. + enum: + - http + - https + - tcp + type: string + healthCheckTimeout: + description: healthCheckTimeout The amount of time, + in seconds, during which no response means a failed + health check + format: int32 + type: integer + healthyThresholdCount: + description: healthyThresholdCount The number of consecutive + health checks successes required before considering + an unhealthy target healthy. + format: int32 + type: integer + matcher: + description: healthCheckCodes The HTTP or gRPC codes + to use when checking for a successful response from + a target + properties: + grpcCode: + description: The gRPC codes + type: string + httpCode: + description: The HTTP codes. + type: string + type: object + unhealthyThresholdCount: + description: unhealthyThresholdCount The number of consecutive + health check failures required before considering + the target unhealthy. + format: int32 + type: integer + type: object + ipAddressType: + description: ipAddressType specifies whether the target + group is of type IPv4 or IPv6. If unspecified, it will + be automatically inferred. + enum: + - ipv4 + - ipv6 + type: string + nodeSelector: + description: node selector for instance type target groups + to only register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + protocol: + description: |- + Protocol [Application / Network Load Balancer] the protocol for the target group. + If unspecified, it will be automatically inferred. + enum: + - HTTP + - HTTPS + - TCP + - TLS + - UDP + - TCP_UDP + type: string + protocolVersion: + description: protocolVersion [HTTP/HTTPS protocol] The protocol + version. The possible values are GRPC , HTTP1 and HTTP2 + enum: + - http1 + - http2 + - grpc + type: string + tags: + description: Tags defines list of Tags on target group. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + targetGroupAttributes: + description: targetGroupAttributes defines the attribute + of target group + items: + description: TargetGroupAttribute defines target group + attribute. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + targetGroupName: + description: targetGroupName specifies the name to assign + to the Target Group. If not defined, then one is generated. + type: string + targetType: + description: targetType is the TargetType of TargetGroup. + If unspecified, it will be automatically inferred as instance. + enum: + - instance + - ip + type: string + vpcID: + description: vpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + type: object + required: + - identifier + - targetGroupProps + type: object + type: array + targetReference: + description: targetReference the kubernetes object to attach the Target + Group settings to. + properties: + group: + default: "" + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. For example + "Service". + + + Defaults to "Service" when not specified. + type: string + name: + description: Name is the name of the referent. + type: string + required: + - name + type: object + required: + - targetReference + type: object + status: + description: TargetGroupConfigurationStatus defines the observed state + of TargetGroupConfiguration + properties: + observedGatewayClassConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the GatewayClass object. + format: int64 + type: integer + observedGatewayConfigurationGeneration: + description: The generation of the Gateway Configuration attached + to the Gateway object. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} From 1b544ea8f6fd4e63f0d1e86e75a8ccc5343a071d Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 21 Apr 2025 17:56:24 -0700 Subject: [PATCH 38/40] fix crds --- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 298 ----------- ...way.k8s.aws_targetgroupconfigurations.yaml | 498 ------------------ 2 files changed, 796 deletions(-) delete mode 100644 config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml delete mode 100644 config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml diff --git a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml deleted file mode 100644 index f146a9eb99..0000000000 --- a/config/crd/bases/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ /dev/null @@ -1,298 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: loadbalancerconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: LoadBalancerConfiguration - listKind: LoadBalancerConfigurationList - plural: loadbalancerconfigurations - singular: loadbalancerconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: LoadBalancerConfiguration is the Schema for the LoadBalancerConfiguration - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LoadBalancerConfigurationSpec defines the desired state of - LoadBalancerConfiguration - properties: - customerOwnedIpv4Pool: - description: |- - customerOwnedIpv4Pool [Application LoadBalancer] - is the ID of the customer-owned address for Application Load Balancers on Outposts pool. - type: string - enableICMP: - description: |- - EnableICMP [Network LoadBalancer] - enables the creation of security group rules to the managed security group - to allow explicit ICMP traffic for Path MTU discovery for IPv4 and dual-stack VPCs - type: boolean - enforceSecurityGroupInboundRulesOnPrivateLinkTraffic: - description: enforceSecurityGroupInboundRulesOnPrivateLinkTraffic - Indicates whether to evaluate inbound security group rules for traffic - sent to a Network Load Balancer through Amazon Web Services PrivateLink. - type: string - ipAddressType: - description: loadBalancerIPType defines what kind of load balancer - to provision (ipv4, dual stack) - enum: - - ipv4 - - dualstack - - dualstack-without-public-ipv4 - type: string - ipv4IPAMPoolId: - description: |- - IPv4IPAMPoolId [Application LoadBalancer] - defines the IPAM pool ID used for IPv4 Addresses on the ALB. - type: string - listenerConfigurations: - description: listenerConfigurations is an optional list of configurations - for each listener on LB - items: - properties: - alpnPolicy: - description: alpnPolicy an optional string that allows you to - configure ALPN policies on your Load Balancer - enum: - - HTTP1Only - - HTTP2Only - - HTTP2Optional - - HTTP2Preferred - - None - type: string - certificates: - description: certificates is the list of other certificates - to add to the listener. - items: - type: string - type: array - defaultCertificate: - description: |- - TODO: Add validation in admission webhook to make it required for secure protocols - defaultCertificate the cert arn to be used by default. - type: string - listenerAttributes: - description: listenerAttributes defines the attributes for the - listener - items: - description: ListenerAttribute defines listener attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - mutualAuthentication: - description: mutualAuthentication defines the mutual authentication - configuration information. - properties: - advertiseTrustStoreCaNames: - description: Indicates whether trust store CA certificate - names are advertised. - enum: - - "on" - - "off" - type: string - ignoreClientCertificateExpiry: - description: Indicates whether expired client certificates - are ignored. - type: boolean - mode: - description: The client certificate handling method. Options - are off , passthrough or verify - enum: - - "off" - - passthrough - - verify - type: string - trustStore: - description: The Name or ARN of the trust store. - type: string - required: - - mode - type: object - protocolPort: - description: protocolPort is identifier for the listener on - load balancer. It should be of the form PROTOCOL:PORT - pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ - type: string - sslPolicy: - description: sslPolicy is the security policy that defines which - protocols and ciphers are supported for secure listeners [HTTPS - or TLS listener]. - type: string - required: - - protocolPort - type: object - type: array - loadBalancerAttributes: - description: LoadBalancerAttributes defines the attribute of LB - items: - description: LoadBalancerAttribute defines LB attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - loadBalancerName: - description: loadBalancerName defines the name of the LB to provision. - If unspecified, it will be automatically generated. - maxLength: 32 - minLength: 1 - type: string - loadBalancerSubnets: - description: |- - loadBalancerSubnets is an optional list of subnet configurations to be used in the LB - This value takes precedence over loadBalancerSubnetsSelector if both are selected. - items: - description: SubnetConfiguration defines the subnet settings for - a Load Balancer. - properties: - eipAllocation: - description: eipAllocation [Network LoadBalancer] the EIP name - for this subnet. - type: string - identifier: - description: identifier [Application LoadBalancer / Network - LoadBalancer] name or id for the subnet - type: string - ipv6Allocation: - description: IPv6Allocation [Network LoadBalancer] the ipv6 - address to assign to this subnet. - type: string - privateIPv4Allocation: - description: privateIPv4Allocation [Network LoadBalancer] the - private ipv4 address to assign to this subnet. - type: string - sourceNatIPv6Prefix: - description: SourceNatIPv6Prefix [Network LoadBalancer] The - IPv6 prefix to use for source NAT. Specify an IPv6 prefix - (/80 netmask) from the subnet CIDR block or auto_assigned - to use an IPv6 prefix selected at random from the subnet CIDR - block. - type: string - type: object - type: array - loadBalancerSubnetsSelector: - additionalProperties: - items: - type: string - type: array - description: |- - LoadBalancerSubnetsSelector specifies subnets in the load balancer's VPC where each - tag specified in the map key contains one of the values in the corresponding - value list. - type: object - manageBackendSecurityGroupRules: - description: |- - ManageBackendSecurityGroupRules [Application / Network LoadBalancer] - specifies whether you want the controller to configure security group rules on Node/Pod for traffic access - when you specify securityGroups - type: boolean - scheme: - description: scheme defines the type of LB to provision. If unspecified, - it will be automatically inferred. - enum: - - internal - - internet-facing - type: string - securityGroupPrefixes: - description: securityGroupPrefixes an optional list of prefixes that - are allowed to access the LB. - items: - type: string - type: array - securityGroups: - description: securityGroups an optional list of security group ids - or names to apply to the LB - items: - type: string - type: array - sourceRanges: - description: sourceRanges an optional list of CIDRs that are allowed - to access the LB. - items: - type: string - type: array - tags: - description: Tags defines list of Tags on LB. - items: - description: AWSTag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - vpcId: - description: vpcId is the ID of the VPC for the load balancer. - type: string - type: object - status: - description: LoadBalancerConfigurationStatus defines the observed state - of TargetGroupBinding - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml deleted file mode 100644 index b2b21ca7b2..0000000000 --- a/config/crd/bases/gateway.k8s.aws_targetgroupconfigurations.yaml +++ /dev/null @@ -1,498 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: targetgroupconfigurations.gateway.k8s.aws -spec: - group: gateway.k8s.aws - names: - kind: TargetGroupConfiguration - listKind: TargetGroupConfigurationList - plural: targetgroupconfigurations - singular: targetgroupconfiguration - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: The Kubernetes Service's name - jsonPath: .spec.targetReference.name - name: SERVICE-NAME - type: string - - jsonPath: .metadata.creationTimestamp - name: AGE - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: TargetGroupConfiguration is the Schema for defining TargetGroups - with an AWS ELB Gateway - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TargetGroupConfigurationSpec defines the TargetGroup properties - for a route. - properties: - defaultConfiguration: - description: defaultRouteConfiguration fallback configuration applied - to all routes, unless overridden by route-specific configurations. - properties: - enableMultiCluster: - description: |- - EnableMultiCluster [Application / Network LoadBalancer] - Allows for multiple Clusters / Services to use the generated TargetGroup ARN - type: boolean - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount of - time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health checks - on the targets. - type: string - healthCheckPort: - description: |- - healthCheckPort The port the load balancer uses when performing health checks on targets. - The default is to use the port on which each target receives traffic from the load balancer. - type: string - healthCheckProtocol: - description: healthCheckProtocol The protocol to use to connect - with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols - are not supported for health checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, in seconds, - during which no response means a failed health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering an unhealthy - target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes to use - when checking for a successful response from a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering the target - unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target group - is of type IPv4 or IPv6. If unspecified, it will be automatically - inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups to - only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocol: - description: |- - Protocol [Application / Network Load Balancer] the protocol for the target group. - If unspecified, it will be automatically inferred. - enum: - - HTTP - - HTTPS - - TCP - - TLS - - UDP - - TCP_UDP - type: string - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute of target - group - items: - description: TargetGroupAttribute defines target group attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetGroupName: - description: targetGroupName specifies the name to assign to the - Target Group. If not defined, then one is generated. - type: string - targetType: - description: targetType is the TargetType of TargetGroup. If unspecified, - it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - routeConfigurations: - description: routeConfigurations the route configuration for specific - routes - items: - description: RouteConfiguration defines the per route configuration - properties: - identifier: - description: name the identifier of the route, it should be - in the form of ROUTE:NAMESPACE:NAME - pattern: ^(HTTPRoute|TLSRoute|TCPRoute|UDPRoute|GRPCRoute)?:([^:]+)?:([^:]+)?$ - type: string - targetGroupProps: - description: targetGroupProps the target group specific properties - properties: - enableMultiCluster: - description: |- - EnableMultiCluster [Application / Network LoadBalancer] - Allows for multiple Clusters / Services to use the generated TargetGroup ARN - type: boolean - healthCheckConfig: - description: healthCheckConfig The Health Check configuration - for this backend. - properties: - healthCheckInterval: - description: healthCheckInterval The approximate amount - of time, in seconds, between health checks of an individual - target. - format: int32 - type: integer - healthCheckPath: - description: healthCheckPath The destination for health - checks on the targets. - type: string - healthCheckPort: - description: |- - healthCheckPort The port the load balancer uses when performing health checks on targets. - The default is to use the port on which each target receives traffic from the load balancer. - type: string - healthCheckProtocol: - description: healthCheckProtocol The protocol to use - to connect with the target. The GENEVE, TLS, UDP, - and TCP_UDP protocols are not supported for health - checks. - enum: - - http - - https - - tcp - type: string - healthCheckTimeout: - description: healthCheckTimeout The amount of time, - in seconds, during which no response means a failed - health check - format: int32 - type: integer - healthyThresholdCount: - description: healthyThresholdCount The number of consecutive - health checks successes required before considering - an unhealthy target healthy. - format: int32 - type: integer - matcher: - description: healthCheckCodes The HTTP or gRPC codes - to use when checking for a successful response from - a target - properties: - grpcCode: - description: The gRPC codes - type: string - httpCode: - description: The HTTP codes. - type: string - type: object - unhealthyThresholdCount: - description: unhealthyThresholdCount The number of consecutive - health check failures required before considering - the target unhealthy. - format: int32 - type: integer - type: object - ipAddressType: - description: ipAddressType specifies whether the target - group is of type IPv4 or IPv6. If unspecified, it will - be automatically inferred. - enum: - - ipv4 - - ipv6 - type: string - nodeSelector: - description: node selector for instance type target groups - to only register certain nodes - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - protocol: - description: |- - Protocol [Application / Network Load Balancer] the protocol for the target group. - If unspecified, it will be automatically inferred. - enum: - - HTTP - - HTTPS - - TCP - - TLS - - UDP - - TCP_UDP - type: string - protocolVersion: - description: protocolVersion [HTTP/HTTPS protocol] The protocol - version. The possible values are GRPC , HTTP1 and HTTP2 - enum: - - http1 - - http2 - - grpc - type: string - tags: - description: Tags defines list of Tags on target group. - items: - description: Tag defines a AWS Tag on resources. - properties: - key: - description: The key of the tag. - type: string - value: - description: The value of the tag. - type: string - required: - - key - - value - type: object - type: array - targetGroupAttributes: - description: targetGroupAttributes defines the attribute - of target group - items: - description: TargetGroupAttribute defines target group - attribute. - properties: - key: - description: The key of the attribute. - type: string - value: - description: The value of the attribute. - type: string - required: - - key - - value - type: object - type: array - targetGroupName: - description: targetGroupName specifies the name to assign - to the Target Group. If not defined, then one is generated. - type: string - targetType: - description: targetType is the TargetType of TargetGroup. - If unspecified, it will be automatically inferred as instance. - enum: - - instance - - ip - type: string - vpcID: - description: vpcID is the VPC of the TargetGroup. If unspecified, - it will be automatically inferred. - type: string - type: object - required: - - identifier - - targetGroupProps - type: object - type: array - targetReference: - description: targetReference the kubernetes object to attach the Target - Group settings to. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - - Defaults to "Service" when not specified. - type: string - name: - description: Name is the name of the referent. - type: string - required: - - name - type: object - required: - - targetReference - type: object - status: - description: TargetGroupConfigurationStatus defines the observed state - of TargetGroupConfiguration - properties: - observedGatewayClassConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the GatewayClass object. - format: int64 - type: integer - observedGatewayConfigurationGeneration: - description: The generation of the Gateway Configuration attached - to the Gateway object. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} From a8d856a4875f615f5585b0279bf767bcdff26e4c Mon Sep 17 00:00:00 2001 From: niclask25 Date: Thu, 10 Apr 2025 11:45:56 +0200 Subject: [PATCH 39/40] feat: add healthProbeBindAddr --- helm/aws-load-balancer-controller/README.md | 165 ++++++++++---------- 1 file changed, 82 insertions(+), 83 deletions(-) diff --git a/helm/aws-load-balancer-controller/README.md b/helm/aws-load-balancer-controller/README.md index e023d215cb..8b9aa07b65 100644 --- a/helm/aws-load-balancer-controller/README.md +++ b/helm/aws-load-balancer-controller/README.md @@ -180,90 +180,89 @@ The following tables lists the configurable parameters of the chart and their de The default values set by the application itself can be confirmed [here](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/configurations/#controller-configuration-options). -| Parameter | Description | Default | -| ---------------------------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------| -| `image.repository` | image repository | `public.ecr.aws/eks/aws-load-balancer-controller` | -| `image.tag` | image tag | `` | -| `image.pullPolicy` | image pull policy | `IfNotPresent` | -| `clusterName` | Kubernetes cluster name | None | -| `cluster.dnsDomain` | DNS domain of the Kubernetes cluster, included in TLS certificate requests | `cluster.local` | -| `securityContext` | Set to security context for pod | `{}` | -| `resources` | Controller pod resource requests & limits | `{}` | -| `priorityClassName` | Controller pod priority class | system-cluster-critical | -| `nodeSelector` | Node labels for controller pod assignment | `{}` | -| `tolerations` | Controller pod toleration for taints | `{}` | -| `affinity` | Affinity for pod assignment | `{}` | -| `configureDefaultAffinity` | Configure soft pod anti-affinity if custom affinity is not configured | `true` | -| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment | `{}` | -| `deploymentAnnotations` | Annotations to add to deployment | `{}` | -| `podAnnotations` | Annotations to add to each pod | `{}` | -| `podLabels` | Labels to add to each pod | `{}` | -| `additionalLabels` | Labels to add to all components | `{}` | -| `rbac.create` | if `true`, create and use RBAC resources | `true` | -| `serviceAccount.annotations` | optional annotations to add to service account | None | -| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a Service Account | `true` | -| `serviceAccount.imagePullSecrets` | List of image pull secrets to add to the Service Account | `[]` | -| `serviceAccount.create` | If `true`, create a new service account | `true` | -| `serviceAccount.name` | Service account to be used | None | -| `terminationGracePeriodSeconds` | Time period for controller pod to do a graceful shutdown | 10 | -| `ingressClass` | The ingress class to satisfy | alb | -| `createIngressClassResource` | Create ingressClass resource | true | -| `ingressClassParams.name` | IngressClassParams resource's name, default to the aws load balancer controller's name | None | -| `ingressClassParams.create` | If `true`, create a new ingressClassParams | true | -| `ingressClassParams.spec` | IngressClassParams defined ingress specifications | {} | -| `region` | The AWS region for the kubernetes cluster | None | -| `vpcId` | The VPC ID for the Kubernetes cluster | None | -| `vpcTags` | This is alternative to vpcId. Set this when your pods are unable to use the metadata service to determine VPC automatically. | None -| `awsApiEndpoints` | Custom AWS API Endpoints | None | -| `awsApiThrottle` | Custom AWS API throttle settings | None | -| `awsMaxRetries` | Maximum retries for AWS APIs | None | -| `defaultTargetType` | Default target type. Used as the default value of the `alb.ingress.kubernetes.io/target-type` and `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" annotations.`Possible values are `ip` and `instance`. | `instance` | -| `enablePodReadinessGateInject` | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | None | -| `enableShield` | Enable Shield addon for ALB | None | -| `enableWaf` | Enable WAF addon for ALB | None | -| `enableWafv2` | Enable WAF V2 addon for ALB | None | -| `ingressMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for ingress | None | -| `logLevel` | Set the controller log level - info, debug | None | -| `metricsBindAddr` | The address the metric endpoint binds to | "" | -| `webhookConfig.disableIngressValidation` | Disables the validation of resources of kind Ingress | None | -| `webhookBindPort` | The TCP port the Webhook server binds to | None | -| `webhookTLS.caCert` | TLS CA certificate for webhook (auto-generated if not provided) | "" | -| `webhookTLS.cert` | TLS certificate for webhook (auto-generated if not provided) | "" | -| `webhookTLS.key` | TLS private key for webhook (auto-generated if not provided) | "" | -| `webhookNamespaceSelectors` | Namespace selectors for the wekbook | None | -| `keepTLSSecret` | Reuse existing TLS Secret during chart upgrade | `true` | -| `serviceAnnotations` | Annotations to be added to the provisioned webhook service resource | `{}` | -| `serviceMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for service | None | -| `targetgroupbindingMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for targetGroupBinding | None | -| `targetgroupbindingMaxExponentialBackoffDelay` | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | None | -| `syncPeriod` | Period at which the controller forces the repopulation of its local object stores | None | -| `watchNamespace` | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched | None | -| `disableIngressClassAnnotation` | Disables the usage of kubernetes.io/ingress.class annotation | None | -| `disableIngressGroupNameAnnotation` | Disables the usage of alb.ingress.kubernetes.io/group.name annotation | None | -| `tolerateNonExistentBackendService` | whether to allow rules that reference a backend service that does not exist. (When enabled, it will return 503 error if backend service not exist) | `true` | -| `tolerateNonExistentBackendAction` | whether to allow rules that reference a backend action that does not exist. (When enabled, it will return 503 error if backend action not exist) | `true` | -| `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | -| `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | -| `livenessProbe` | Liveness probe settings for the controller | `{}` (see `values.yaml`) | +| Parameter | Description | Default | +| ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | +| `image.repository` | image repository | `public.ecr.aws/eks/aws-load-balancer-controller` | +| `image.tag` | image tag | `` | +| `image.pullPolicy` | image pull policy | `IfNotPresent` | +| `clusterName` | Kubernetes cluster name | None | +| `cluster.dnsDomain` | DNS domain of the Kubernetes cluster, included in TLS certificate requests | `cluster.local` | +| `securityContext` | Set to security context for pod | `{}` | +| `resources` | Controller pod resource requests & limits | `{}` | +| `priorityClassName` | Controller pod priority class | system-cluster-critical | +| `nodeSelector` | Node labels for controller pod assignment | `{}` | +| `tolerations` | Controller pod toleration for taints | `{}` | +| `affinity` | Affinity for pod assignment | `{}` | +| `configureDefaultAffinity` | Configure soft pod anti-affinity if custom affinity is not configured | `true` | +| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment | `{}` | +| `deploymentAnnotations` | Annotations to add to deployment | `{}` | +| `podAnnotations` | Annotations to add to each pod | `{}` | +| `podLabels` | Labels to add to each pod | `{}` | +| `additionalLabels` | Labels to add to all components | `{}` | +| `rbac.create` | if `true`, create and use RBAC resources | `true` | +| `serviceAccount.annotations` | optional annotations to add to service account | None | +| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a Service Account | `true` | +| `serviceAccount.imagePullSecrets` | List of image pull secrets to add to the Service Account | `[]` | +| `serviceAccount.create` | If `true`, create a new service account | `true` | +| `serviceAccount.name` | Service account to be used | None | +| `terminationGracePeriodSeconds` | Time period for controller pod to do a graceful shutdown | 10 | +| `ingressClass` | The ingress class to satisfy | alb | +| `createIngressClassResource` | Create ingressClass resource | true | +| `ingressClassParams.name` | IngressClassParams resource's name, default to the aws load balancer controller's name | None | +| `ingressClassParams.create` | If `true`, create a new ingressClassParams | true | +| `ingressClassParams.spec` | IngressClassParams defined ingress specifications | {} | +| `region` | The AWS region for the kubernetes cluster | None | +| `vpcId` | The VPC ID for the Kubernetes cluster | None | +| `vpcTags` | This is alternative to vpcId. Set this when your pods are unable to use the metadata service to determine VPC automatically. | None +| `awsApiEndpoints` | Custom AWS API Endpoints | None | +| `awsApiThrottle` | Custom AWS API throttle settings | None | +| `awsMaxRetries` | Maximum retries for AWS APIs | None | +| `defaultTargetType` | Default target type. Used as the default value of the `alb.ingress.kubernetes.io/target-type` and `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" annotations.`Possible values are `ip` and `instance`. | `instance` | +| `enablePodReadinessGateInject` | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | None | +| `enableShield` | Enable Shield addon for ALB | None | +| `enableWaf` | Enable WAF addon for ALB | None | +| `enableWafv2` | Enable WAF V2 addon for ALB | None | +| `ingressMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for ingress | None | +| `logLevel` | Set the controller log level - info, debug | None | +| `metricsBindAddr` | The address the metric endpoint binds to | "" | +| `webhookConfig.disableIngressValidation` | Disables the validation of resources of kind Ingress | None | +| `webhookBindPort` | The TCP port the Webhook server binds to | None | +| `webhookTLS.caCert` | TLS CA certificate for webhook (auto-generated if not provided) | "" | +| `webhookTLS.cert` | TLS certificate for webhook (auto-generated if not provided) | "" | +| `webhookTLS.key` | TLS private key for webhook (auto-generated if not provided) | "" | +| `webhookNamespaceSelectors` | Namespace selectors for the wekbook | None | +| `keepTLSSecret` | Reuse existing TLS Secret during chart upgrade | `true` | +| `serviceAnnotations` | Annotations to be added to the provisioned webhook service resource | `{}` | +| `serviceMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for service | None | +| `targetgroupbindingMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for targetGroupBinding | None | +| `targetgroupbindingMaxExponentialBackoffDelay` | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | None | +| `syncPeriod` | Period at which the controller forces the repopulation of its local object stores | None | +| `watchNamespace` | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched | None | +| `disableIngressClassAnnotation` | Disables the usage of kubernetes.io/ingress.class annotation | None | +| `disableIngressGroupNameAnnotation` | Disables the usage of alb.ingress.kubernetes.io/group.name annotation | None | +| `tolerateNonExistentBackendService` | whether to allow rules that reference a backend service that does not exist. (When enabled, it will return 503 error if backend service not exist) | `true` | +| `tolerateNonExistentBackendAction` | whether to allow rules that reference a backend action that does not exist. (When enabled, it will return 503 error if backend action not exist) | `true` | +| `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | +| `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | +| `livenessProbe` | Liveness probe settings for the controller | `{}` (see `values.yaml`) | | `readinessProbe` | Readiness probe settings for the controller | `{}` (see `values.yaml`) | -| `env` | Environment variables to set for aws-load-balancer-controller pod | None | -| `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | -| `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | -| `hostNetwork` | If `true`, use hostNetwork | `false` | -| `dnsPolicy` | Set dnsPolicy if required | `ClusterFirst` | -| `extraVolumeMounts` | Extra volume mounts for the pod | `[]` | -| `extraVolumes` | Extra volumes for the pod | `[]` | -| `defaultTags` | Default tags to apply to all AWS resources managed by this controller | `{}` | -| `replicaCount` | Number of controller pods to run, only one will be active due to leader election | `2` | -| `revisionHistoryLimit` | Number of revisions to keep | `10` | -| `podDisruptionBudget` | Limit the disruption for controller pods. Require at least 2 controller replicas and 3 worker nodes | `{}` | -| `updateStrategy` | Defines the update strategy for the deployment | `{}` | -| `enableCertManager` | If enabled, cert-manager issues the webhook certificates instead of the helm template, requires cert-manager and it's CRDs to be installed | `false` | -| `certManager.duration` | Overrides the default expiry duration for the webhook certificates. defaults to `90d` | `""` | -| `certManager.renewBefore` | Overrides the renewal time period duration for the webhook certificates. defaults to `60d` | `""` | -| `enableEndpointSlices` | If enabled, controller uses k8s EndpointSlices instead of Endpoints for IP targets | `false` | -| `enableBackendSecurityGroup` | If enabled, controller uses shared security group for backend traffic | `true` | -| `enableManageBackendSecurityGroupRules` | If enabled, controller will manage security group rules | `false` | +| `env` | Environment variables to set for aws-load-balancer-controller pod | None | +| `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | +| `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | +| `hostNetwork` | If `true`, use hostNetwork | `false` | +| `dnsPolicy` | Set dnsPolicy if required | `ClusterFirst` | +| `extraVolumeMounts` | Extra volume mounts for the pod | `[]` | +| `extraVolumes` | Extra volumes for the pod | `[]` | +| `defaultTags` | Default tags to apply to all AWS resources managed by this controller | `{}` | +| `replicaCount` | Number of controller pods to run, only one will be active due to leader election | `2` | +| `revisionHistoryLimit` | Number of revisions to keep | `10` | +| `podDisruptionBudget` | Limit the disruption for controller pods. Require at least 2 controller replicas and 3 worker nodes | `{}` | +| `updateStrategy` | Defines the update strategy for the deployment | `{}` | +| `enableCertManager` | If enabled, cert-manager issues the webhook certificates instead of the helm template, requires cert-manager and it's CRDs to be installed | `false` | +| `certManager.duration` | Overrides the default expiry duration for the webhook certificates. defaults to `90d` | `""` | +| `certManager.renewBefore` | Overrides the renewal time period duration for the webhook certificates. defaults to `60d` | `""` | +| `enableEndpointSlices` | If enabled, controller uses k8s EndpointSlices instead of Endpoints for IP targets | `false` | +| `enableBackendSecurityGroup` | If enabled, controller uses shared security group for backend traffic | `true` | | `backendSecurityGroup` | Backend security group to use instead of auto created one if the feature is enabled | `` | | `disableRestrictedSecurityGroupRules` | If disabled, controller will not specify port range restriction in the backend security group rules | `false` | | `objectSelector.matchExpressions` | Webhook configuration to select specific pods by specifying the expression to be matched | None | From a27cd0ef67bc48b1ef05177f7966c937d936c28d Mon Sep 17 00:00:00 2001 From: niclask25 Date: Tue, 22 Apr 2025 20:58:31 +0200 Subject: [PATCH 40/40] Revert "feat: add healthProbeBindAddr" This reverts commit a8d856a4875f615f5585b0279bf767bcdff26e4c. --- helm/aws-load-balancer-controller/README.md | 165 ++++++++++---------- 1 file changed, 83 insertions(+), 82 deletions(-) diff --git a/helm/aws-load-balancer-controller/README.md b/helm/aws-load-balancer-controller/README.md index 8b9aa07b65..e023d215cb 100644 --- a/helm/aws-load-balancer-controller/README.md +++ b/helm/aws-load-balancer-controller/README.md @@ -180,89 +180,90 @@ The following tables lists the configurable parameters of the chart and their de The default values set by the application itself can be confirmed [here](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/configurations/#controller-configuration-options). -| Parameter | Description | Default | -| ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | -| `image.repository` | image repository | `public.ecr.aws/eks/aws-load-balancer-controller` | -| `image.tag` | image tag | `` | -| `image.pullPolicy` | image pull policy | `IfNotPresent` | -| `clusterName` | Kubernetes cluster name | None | -| `cluster.dnsDomain` | DNS domain of the Kubernetes cluster, included in TLS certificate requests | `cluster.local` | -| `securityContext` | Set to security context for pod | `{}` | -| `resources` | Controller pod resource requests & limits | `{}` | -| `priorityClassName` | Controller pod priority class | system-cluster-critical | -| `nodeSelector` | Node labels for controller pod assignment | `{}` | -| `tolerations` | Controller pod toleration for taints | `{}` | -| `affinity` | Affinity for pod assignment | `{}` | -| `configureDefaultAffinity` | Configure soft pod anti-affinity if custom affinity is not configured | `true` | -| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment | `{}` | -| `deploymentAnnotations` | Annotations to add to deployment | `{}` | -| `podAnnotations` | Annotations to add to each pod | `{}` | -| `podLabels` | Labels to add to each pod | `{}` | -| `additionalLabels` | Labels to add to all components | `{}` | -| `rbac.create` | if `true`, create and use RBAC resources | `true` | -| `serviceAccount.annotations` | optional annotations to add to service account | None | -| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a Service Account | `true` | -| `serviceAccount.imagePullSecrets` | List of image pull secrets to add to the Service Account | `[]` | -| `serviceAccount.create` | If `true`, create a new service account | `true` | -| `serviceAccount.name` | Service account to be used | None | -| `terminationGracePeriodSeconds` | Time period for controller pod to do a graceful shutdown | 10 | -| `ingressClass` | The ingress class to satisfy | alb | -| `createIngressClassResource` | Create ingressClass resource | true | -| `ingressClassParams.name` | IngressClassParams resource's name, default to the aws load balancer controller's name | None | -| `ingressClassParams.create` | If `true`, create a new ingressClassParams | true | -| `ingressClassParams.spec` | IngressClassParams defined ingress specifications | {} | -| `region` | The AWS region for the kubernetes cluster | None | -| `vpcId` | The VPC ID for the Kubernetes cluster | None | -| `vpcTags` | This is alternative to vpcId. Set this when your pods are unable to use the metadata service to determine VPC automatically. | None -| `awsApiEndpoints` | Custom AWS API Endpoints | None | -| `awsApiThrottle` | Custom AWS API throttle settings | None | -| `awsMaxRetries` | Maximum retries for AWS APIs | None | -| `defaultTargetType` | Default target type. Used as the default value of the `alb.ingress.kubernetes.io/target-type` and `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" annotations.`Possible values are `ip` and `instance`. | `instance` | -| `enablePodReadinessGateInject` | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | None | -| `enableShield` | Enable Shield addon for ALB | None | -| `enableWaf` | Enable WAF addon for ALB | None | -| `enableWafv2` | Enable WAF V2 addon for ALB | None | -| `ingressMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for ingress | None | -| `logLevel` | Set the controller log level - info, debug | None | -| `metricsBindAddr` | The address the metric endpoint binds to | "" | -| `webhookConfig.disableIngressValidation` | Disables the validation of resources of kind Ingress | None | -| `webhookBindPort` | The TCP port the Webhook server binds to | None | -| `webhookTLS.caCert` | TLS CA certificate for webhook (auto-generated if not provided) | "" | -| `webhookTLS.cert` | TLS certificate for webhook (auto-generated if not provided) | "" | -| `webhookTLS.key` | TLS private key for webhook (auto-generated if not provided) | "" | -| `webhookNamespaceSelectors` | Namespace selectors for the wekbook | None | -| `keepTLSSecret` | Reuse existing TLS Secret during chart upgrade | `true` | -| `serviceAnnotations` | Annotations to be added to the provisioned webhook service resource | `{}` | -| `serviceMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for service | None | -| `targetgroupbindingMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for targetGroupBinding | None | -| `targetgroupbindingMaxExponentialBackoffDelay` | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | None | -| `syncPeriod` | Period at which the controller forces the repopulation of its local object stores | None | -| `watchNamespace` | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched | None | -| `disableIngressClassAnnotation` | Disables the usage of kubernetes.io/ingress.class annotation | None | -| `disableIngressGroupNameAnnotation` | Disables the usage of alb.ingress.kubernetes.io/group.name annotation | None | -| `tolerateNonExistentBackendService` | whether to allow rules that reference a backend service that does not exist. (When enabled, it will return 503 error if backend service not exist) | `true` | -| `tolerateNonExistentBackendAction` | whether to allow rules that reference a backend action that does not exist. (When enabled, it will return 503 error if backend action not exist) | `true` | -| `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | -| `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | -| `livenessProbe` | Liveness probe settings for the controller | `{}` (see `values.yaml`) | +| Parameter | Description | Default | +| ---------------------------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------| +| `image.repository` | image repository | `public.ecr.aws/eks/aws-load-balancer-controller` | +| `image.tag` | image tag | `` | +| `image.pullPolicy` | image pull policy | `IfNotPresent` | +| `clusterName` | Kubernetes cluster name | None | +| `cluster.dnsDomain` | DNS domain of the Kubernetes cluster, included in TLS certificate requests | `cluster.local` | +| `securityContext` | Set to security context for pod | `{}` | +| `resources` | Controller pod resource requests & limits | `{}` | +| `priorityClassName` | Controller pod priority class | system-cluster-critical | +| `nodeSelector` | Node labels for controller pod assignment | `{}` | +| `tolerations` | Controller pod toleration for taints | `{}` | +| `affinity` | Affinity for pod assignment | `{}` | +| `configureDefaultAffinity` | Configure soft pod anti-affinity if custom affinity is not configured | `true` | +| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment | `{}` | +| `deploymentAnnotations` | Annotations to add to deployment | `{}` | +| `podAnnotations` | Annotations to add to each pod | `{}` | +| `podLabels` | Labels to add to each pod | `{}` | +| `additionalLabels` | Labels to add to all components | `{}` | +| `rbac.create` | if `true`, create and use RBAC resources | `true` | +| `serviceAccount.annotations` | optional annotations to add to service account | None | +| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a Service Account | `true` | +| `serviceAccount.imagePullSecrets` | List of image pull secrets to add to the Service Account | `[]` | +| `serviceAccount.create` | If `true`, create a new service account | `true` | +| `serviceAccount.name` | Service account to be used | None | +| `terminationGracePeriodSeconds` | Time period for controller pod to do a graceful shutdown | 10 | +| `ingressClass` | The ingress class to satisfy | alb | +| `createIngressClassResource` | Create ingressClass resource | true | +| `ingressClassParams.name` | IngressClassParams resource's name, default to the aws load balancer controller's name | None | +| `ingressClassParams.create` | If `true`, create a new ingressClassParams | true | +| `ingressClassParams.spec` | IngressClassParams defined ingress specifications | {} | +| `region` | The AWS region for the kubernetes cluster | None | +| `vpcId` | The VPC ID for the Kubernetes cluster | None | +| `vpcTags` | This is alternative to vpcId. Set this when your pods are unable to use the metadata service to determine VPC automatically. | None +| `awsApiEndpoints` | Custom AWS API Endpoints | None | +| `awsApiThrottle` | Custom AWS API throttle settings | None | +| `awsMaxRetries` | Maximum retries for AWS APIs | None | +| `defaultTargetType` | Default target type. Used as the default value of the `alb.ingress.kubernetes.io/target-type` and `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" annotations.`Possible values are `ip` and `instance`. | `instance` | +| `enablePodReadinessGateInject` | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | None | +| `enableShield` | Enable Shield addon for ALB | None | +| `enableWaf` | Enable WAF addon for ALB | None | +| `enableWafv2` | Enable WAF V2 addon for ALB | None | +| `ingressMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for ingress | None | +| `logLevel` | Set the controller log level - info, debug | None | +| `metricsBindAddr` | The address the metric endpoint binds to | "" | +| `webhookConfig.disableIngressValidation` | Disables the validation of resources of kind Ingress | None | +| `webhookBindPort` | The TCP port the Webhook server binds to | None | +| `webhookTLS.caCert` | TLS CA certificate for webhook (auto-generated if not provided) | "" | +| `webhookTLS.cert` | TLS certificate for webhook (auto-generated if not provided) | "" | +| `webhookTLS.key` | TLS private key for webhook (auto-generated if not provided) | "" | +| `webhookNamespaceSelectors` | Namespace selectors for the wekbook | None | +| `keepTLSSecret` | Reuse existing TLS Secret during chart upgrade | `true` | +| `serviceAnnotations` | Annotations to be added to the provisioned webhook service resource | `{}` | +| `serviceMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for service | None | +| `targetgroupbindingMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for targetGroupBinding | None | +| `targetgroupbindingMaxExponentialBackoffDelay` | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | None | +| `syncPeriod` | Period at which the controller forces the repopulation of its local object stores | None | +| `watchNamespace` | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched | None | +| `disableIngressClassAnnotation` | Disables the usage of kubernetes.io/ingress.class annotation | None | +| `disableIngressGroupNameAnnotation` | Disables the usage of alb.ingress.kubernetes.io/group.name annotation | None | +| `tolerateNonExistentBackendService` | whether to allow rules that reference a backend service that does not exist. (When enabled, it will return 503 error if backend service not exist) | `true` | +| `tolerateNonExistentBackendAction` | whether to allow rules that reference a backend action that does not exist. (When enabled, it will return 503 error if backend action not exist) | `true` | +| `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | +| `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | +| `livenessProbe` | Liveness probe settings for the controller | `{}` (see `values.yaml`) | | `readinessProbe` | Readiness probe settings for the controller | `{}` (see `values.yaml`) | -| `env` | Environment variables to set for aws-load-balancer-controller pod | None | -| `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | -| `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | -| `hostNetwork` | If `true`, use hostNetwork | `false` | -| `dnsPolicy` | Set dnsPolicy if required | `ClusterFirst` | -| `extraVolumeMounts` | Extra volume mounts for the pod | `[]` | -| `extraVolumes` | Extra volumes for the pod | `[]` | -| `defaultTags` | Default tags to apply to all AWS resources managed by this controller | `{}` | -| `replicaCount` | Number of controller pods to run, only one will be active due to leader election | `2` | -| `revisionHistoryLimit` | Number of revisions to keep | `10` | -| `podDisruptionBudget` | Limit the disruption for controller pods. Require at least 2 controller replicas and 3 worker nodes | `{}` | -| `updateStrategy` | Defines the update strategy for the deployment | `{}` | -| `enableCertManager` | If enabled, cert-manager issues the webhook certificates instead of the helm template, requires cert-manager and it's CRDs to be installed | `false` | -| `certManager.duration` | Overrides the default expiry duration for the webhook certificates. defaults to `90d` | `""` | -| `certManager.renewBefore` | Overrides the renewal time period duration for the webhook certificates. defaults to `60d` | `""` | -| `enableEndpointSlices` | If enabled, controller uses k8s EndpointSlices instead of Endpoints for IP targets | `false` | -| `enableBackendSecurityGroup` | If enabled, controller uses shared security group for backend traffic | `true` | +| `env` | Environment variables to set for aws-load-balancer-controller pod | None | +| `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | +| `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | +| `hostNetwork` | If `true`, use hostNetwork | `false` | +| `dnsPolicy` | Set dnsPolicy if required | `ClusterFirst` | +| `extraVolumeMounts` | Extra volume mounts for the pod | `[]` | +| `extraVolumes` | Extra volumes for the pod | `[]` | +| `defaultTags` | Default tags to apply to all AWS resources managed by this controller | `{}` | +| `replicaCount` | Number of controller pods to run, only one will be active due to leader election | `2` | +| `revisionHistoryLimit` | Number of revisions to keep | `10` | +| `podDisruptionBudget` | Limit the disruption for controller pods. Require at least 2 controller replicas and 3 worker nodes | `{}` | +| `updateStrategy` | Defines the update strategy for the deployment | `{}` | +| `enableCertManager` | If enabled, cert-manager issues the webhook certificates instead of the helm template, requires cert-manager and it's CRDs to be installed | `false` | +| `certManager.duration` | Overrides the default expiry duration for the webhook certificates. defaults to `90d` | `""` | +| `certManager.renewBefore` | Overrides the renewal time period duration for the webhook certificates. defaults to `60d` | `""` | +| `enableEndpointSlices` | If enabled, controller uses k8s EndpointSlices instead of Endpoints for IP targets | `false` | +| `enableBackendSecurityGroup` | If enabled, controller uses shared security group for backend traffic | `true` | +| `enableManageBackendSecurityGroupRules` | If enabled, controller will manage security group rules | `false` | | `backendSecurityGroup` | Backend security group to use instead of auto created one if the feature is enabled | `` | | `disableRestrictedSecurityGroupRules` | If disabled, controller will not specify port range restriction in the backend security group rules | `false` | | `objectSelector.matchExpressions` | Webhook configuration to select specific pods by specifying the expression to be matched | None |