diff --git a/go.mod b/go.mod index a5817308..8941694a 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( k8s.io/code-generator v0.29.4 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.17.5 + sigs.k8s.io/gateway-api v1.0.0 sigs.k8s.io/kustomize/kustomize/v5 v5.4.2 ) diff --git a/go.sum b/go.sum index 81219840..8225b89a 100644 --- a/go.sum +++ b/go.sum @@ -268,6 +268,8 @@ knative.dev/pkg v0.0.0-20240423132823-3c6badc82748 h1:0X8ZtnOZqGPjauVLLvOyMaBOMX knative.dev/pkg v0.0.0-20240423132823-3c6badc82748/go.mod h1:Y/ufiCvMogYcpDwZJPcTRBYeBo57RaEQhY0Lq/9RKmU= sigs.k8s.io/controller-runtime v0.17.5 h1:1FI9Lm7NiOOmBsgTV36/s2XrEFXnO2C4sbg/Zme72Rw= sigs.k8s.io/controller-runtime v0.17.5/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/gateway-api v1.0.0 h1:iPTStSv41+d9p0xFydll6d7f7MOBGuqXM6p2/zVYMAs= +sigs.k8s.io/gateway-api v1.0.0/go.mod h1:4cUgr0Lnp5FZ0Cdq8FdRwCvpiWws7LVhLHGIudLlf4c= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g= diff --git a/tests/checks/httproute_in_app_namespace/httproute_in_app_namespace_test.go b/tests/checks/httproute_in_app_namespace/httproute_in_app_namespace_test.go new file mode 100644 index 00000000..0a665851 --- /dev/null +++ b/tests/checks/httproute_in_app_namespace/httproute_in_app_namespace_test.go @@ -0,0 +1,255 @@ +//go:build e2e +// +build e2e + +package httproute_in_app_namespace_test + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/http-add-on/tests/helper" +) + +const ( + testName = "httproute-in-app-namespace-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + serviceName = fmt.Sprintf("%s-service", testName) + httprouteName = fmt.Sprintf("%s-httproute", testName) + httpScaledObjectName = fmt.Sprintf("%s-http-so", testName) + referenceGrantName = fmt.Sprintf("%s-rg", testName) + gatewayHostPattern = "http://%v.envoy-gateway-system.svc.cluster.local" + host = testName + minReplicaCount = 0 + maxReplicaCount = 1 +) + +type templateData struct { + TestNamespace string + DeploymentName string + ServiceName string + HTTPRouteName string + ReferenceGrantName string + KEDANamespace string + GatewayHost string + HTTPScaledObjectName string + Host string + MinReplicas int + MaxReplicas int +} + +const ( + httprouteTemplate = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{.HTTPRouteName}} + namespace: {{.TestNamespace}} +spec: + parentRefs: + - name: eg + namespace: envoy-gateway-system + hostnames: + - {{.Host}} + rules: + - backendRefs: + - kind: Service + name: keda-http-add-on-interceptor-proxy + namespace: keda + port: 8080 + matches: + - path: + type: PathPrefix + value: / +` + + referenceGrantTemplate = ` +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: {{.ReferenceGrantName}} + namespace: keda +spec: + from: + - group: gateway.networking.k8s.io + kind: HTTPRoute + namespace: {{.TestNamespace}} + to: + - group: "" + kind: Service + name: keda-http-add-on-interceptor-proxy +` + serviceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: {{.ServiceName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + ports: + - port: 8080 + targetPort: http + protocol: TCP + name: http + selector: + app: {{.DeploymentName}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}} + image: registry.k8s.io/e2e-test-images/agnhost:2.45 + args: + - netexec + ports: + - name: http + containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: / + port: http +` + + loadJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-request + namespace: {{.TestNamespace}} +spec: + template: + spec: + containers: + - name: curl-client + image: curlimages/curl + imagePullPolicy: Always + command: ["curl", "-H", "Host: {{.Host}}", "{{.GatewayHost}}"] + restartPolicy: Never + activeDeadlineSeconds: 600 + backoffLimit: 5 +` + + httpScaledObjectTemplate = ` +kind: HTTPScaledObject +apiVersion: http.keda.sh/v1alpha1 +metadata: + name: {{.HTTPScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + hosts: + - {{.Host}} + targetPendingRequests: 100 + scaledownPeriod: 10 + scaleTargetRef: + name: {{.DeploymentName}} + service: {{.ServiceName}} + port: 8080 + replicas: + min: {{ .MinReplicas }} + max: {{ .MaxReplicas }} +` +) + +func TestCheckHTTPRoute(t *testing.T) { + // setup + t.Log("--- setting up ---") + // Create kubernetes resources + kc := GetKubernetesClient(t) + gc := GetGatewayClient(t) + data, templates := getTemplateData(t) + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10), + "replica count should be %d after 1 minutes", minReplicaCount) + assert.True(t, WaitForHTTPRouteAccepted(t, gc, httprouteName, testNamespace, 12, 10), + "HTTPRoute should be accepted after 2 minutes") + + testScaleOut(t, kc, data) + testScaleIn(t, kc, data) + + // cleanup + DeleteKubernetesResources(t, testNamespace, data, templates) +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale out ---") + + KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10), + "replica count should be %d after 1 minutes", maxReplicaCount) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale in ---") + + KubectlDeleteWithTemplate(t, data, "loadJobTemplate", loadJobTemplate) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 12, 10), + "replica count should be %d after 2 minutes", minReplicaCount) +} + +func getTemplateData(t *testing.T) (templateData, []Template) { + kc := GetKubernetesClient(t) + services, err := kc.CoreV1().Services(EnvoyNamespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("failed to list services in %s namespace: %v", EnvoyNamespace, err) + } + gatewayHost := "" + for _, svc := range services.Items { + if svc.Spec.Type == corev1.ServiceTypeLoadBalancer { + gatewayHost = fmt.Sprintf(gatewayHostPattern, svc.Name) + break + } + } + if gatewayHost == "" { + t.Fatalf("failed to find gateway host, no LB service found in %s namespace", EnvoyNamespace) + } + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentName, + ServiceName: serviceName, + HTTPRouteName: httprouteName, + ReferenceGrantName: referenceGrantName, + KEDANamespace: KEDANamespace, + HTTPScaledObjectName: httpScaledObjectName, + GatewayHost: gatewayHost, + Host: host, + MinReplicas: minReplicaCount, + MaxReplicas: maxReplicaCount, + }, []Template{ + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "serviceTemplate", Config: serviceTemplate}, + {Name: "httprouteTemplate", Config: httprouteTemplate}, + {Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate}, + {Name: "referenceGrantTemplate", Config: referenceGrantTemplate}, + } +} diff --git a/tests/helper/helper.go b/tests/helper/helper.go index 3d15bb4b..27f314ac 100644 --- a/tests/helper/helper.go +++ b/tests/helper/helper.go @@ -28,6 +28,8 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" "sigs.k8s.io/controller-runtime/pkg/client/config" + gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayapiv1clientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1" ) const ( @@ -36,6 +38,8 @@ const ( ArgoRolloutsName = "argo-rollouts" IngressNamespace = "ingress" IngressReleaseName = "ingress" + EnvoyNamespace = "envoy-gateway-system" + EnvoyReleaseName = "eg" StringFalse = "false" StringTrue = "true" @@ -46,6 +50,7 @@ var random = rand.New(rand.NewSource(time.Now().UnixNano())) var ( KubeClient *kubernetes.Clientset KedaKubeClient *v1alpha1.KedaV1alpha1Client + GWClient *gatewayapiv1clientset.GatewayV1Client KubeConfig *rest.Config ) @@ -168,6 +173,21 @@ func GetKubernetesClient(t *testing.T) *kubernetes.Clientset { return KubeClient } +func GetGatewayClient(t *testing.T) *gatewayapiv1clientset.GatewayV1Client { + if GWClient != nil && KubeConfig != nil { + return GWClient + } + + var err error + KubeConfig, err = config.GetConfig() + assert.NoErrorf(t, err, "cannot fetch kube config file - %s", err) + + GWClient, err = gatewayapiv1clientset.NewForConfig(KubeConfig) + assert.NoErrorf(t, err, "cannot create gateway client - %s", err) + + return GWClient +} + func GetKedaKubernetesClient(t *testing.T) *v1alpha1.KedaV1alpha1Client { if KedaKubeClient != nil && KubeConfig != nil { return KedaKubeClient @@ -388,6 +408,23 @@ func WaitForIngressReady(t *testing.T, kc *kubernetes.Clientset, name, namespace return false } +func WaitForHTTPRouteAccepted(t *testing.T, gc *gatewayapiv1clientset.GatewayV1Client, name, namespace string, iterations, intervalSeconds int) bool { + for i := 0; i < iterations; i++ { + httpRoute, _ := gc.HTTPRoutes(namespace).Get(context.Background(), name, metav1.GetOptions{}) + for _, parent := range httpRoute.Status.Parents { + for _, condition := range parent.Conditions { + if condition.Type == string(gatewayapiv1.RouteConditionAccepted) && condition.Status == metav1.ConditionTrue { + return true + } + } + } + t.Log("Waiting for accepted HTTPRoute") + + time.Sleep(time.Duration(intervalSeconds) * time.Second) + } + return false +} + // Waits until statefulset count hits target or number of iterations are done. func WaitForStatefulsetReplicaReadyCount(t *testing.T, kc *kubernetes.Clientset, name, namespace string, target, iterations, intervalSeconds int) bool { diff --git a/tests/utils/cleanup_test.go b/tests/utils/cleanup_test.go index 8a9fd464..e6a1393e 100644 --- a/tests/utils/cleanup_test.go +++ b/tests/utils/cleanup_test.go @@ -54,3 +54,26 @@ func TestCleanUpCerts(t *testing.T) { t.Log(string(out)) t.Log("test certificates successfully cleaned up") } + +func TestRemoveEnvoyGateway(t *testing.T) { + gatewayClass := ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +` + + gateway := ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: eg + namespace: envoy-gateway-system +` + + KubectlDeleteWithTemplate(t, nil, "gateway", gateway) + KubectlDeleteWithTemplate(t, nil, "gatewayClass", gatewayClass) + _, err := ExecuteCommand(fmt.Sprintf("helm uninstall %s --namespace %s", EnvoyReleaseName, EnvoyNamespace)) + require.NoErrorf(t, err, "cannot uninstall envoy gateway - %s", err) + DeleteNamespace(t, EnvoyNamespace) +} diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index f7f24fca..848cb071 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -69,6 +69,32 @@ spec: name: prometheus protocol: TCP ` + + gatewayClass = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +` + + gateway = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: eg + namespace: envoy-gateway-system +spec: + gatewayClassName: eg + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +` ) func TestVerifyCommands(t *testing.T) { @@ -149,6 +175,20 @@ func TestSetupIngress(t *testing.T) { require.NoErrorf(t, err, "cannot install ingress - %s", err) } +func TestSetupEnvoyGateway(t *testing.T) { + KubeClient = GetKubernetesClient(t) + _, err := ExecuteCommand("helm version") + require.NoErrorf(t, err, "helm is not installed - %s", err) + + _, err = ExecuteCommand(fmt.Sprintf("helm install %s oci://docker.io/envoyproxy/gateway-helm --version v1.0.1 -n %s --create-namespace", EnvoyReleaseName, EnvoyNamespace)) + require.NoErrorf(t, err, "cannot install envoy gateway - %s", err) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, KubeClient, "envoy-gateway", "envoy-gateway-system", 1, 30, 6)) + + KubectlApplyWithTemplate(t, nil, "gatewayClass", gatewayClass) + KubectlApplyWithTemplate(t, nil, "gateway", gateway) +} + func TestSetupKEDA(t *testing.T) { _, err := ExecuteCommand("helm version") require.NoErrorf(t, err, "helm is not installed - %s", err)