Skip to content

Commit

Permalink
GatewayAPI e2e tests (#1052)
Browse files Browse the repository at this point in the history
Signed-off-by: Jan Wozniak <wozniak.jan@gmail.com>
  • Loading branch information
wozniakjan authored Jun 27, 2024
1 parent a86b134 commit 1e42612
Show file tree
Hide file tree
Showing 6 changed files with 358 additions and 0 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ require (
k8s.io/code-generator v0.29.4
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
sigs.k8s.io/controller-runtime v0.17.5
sigs.k8s.io/gateway-api v1.0.0
sigs.k8s.io/kustomize/kustomize/v5 v5.4.2
)

Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,8 @@ knative.dev/pkg v0.0.0-20240423132823-3c6badc82748 h1:0X8ZtnOZqGPjauVLLvOyMaBOMX
knative.dev/pkg v0.0.0-20240423132823-3c6badc82748/go.mod h1:Y/ufiCvMogYcpDwZJPcTRBYeBo57RaEQhY0Lq/9RKmU=
sigs.k8s.io/controller-runtime v0.17.5 h1:1FI9Lm7NiOOmBsgTV36/s2XrEFXnO2C4sbg/Zme72Rw=
sigs.k8s.io/controller-runtime v0.17.5/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY=
sigs.k8s.io/gateway-api v1.0.0 h1:iPTStSv41+d9p0xFydll6d7f7MOBGuqXM6p2/zVYMAs=
sigs.k8s.io/gateway-api v1.0.0/go.mod h1:4cUgr0Lnp5FZ0Cdq8FdRwCvpiWws7LVhLHGIudLlf4c=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g=
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
//go:build e2e
// +build e2e

package httproute_in_app_namespace_test

import (
"context"
"fmt"
"testing"

"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"

. "github.com/kedacore/http-add-on/tests/helper"
)

const (
testName = "httproute-in-app-namespace-test"
)

var (
testNamespace = fmt.Sprintf("%s-ns", testName)
deploymentName = fmt.Sprintf("%s-deployment", testName)
serviceName = fmt.Sprintf("%s-service", testName)
httprouteName = fmt.Sprintf("%s-httproute", testName)
httpScaledObjectName = fmt.Sprintf("%s-http-so", testName)
referenceGrantName = fmt.Sprintf("%s-rg", testName)
gatewayHostPattern = "http://%v.envoy-gateway-system.svc.cluster.local"
host = testName
minReplicaCount = 0
maxReplicaCount = 1
)

type templateData struct {
TestNamespace string
DeploymentName string
ServiceName string
HTTPRouteName string
ReferenceGrantName string
KEDANamespace string
GatewayHost string
HTTPScaledObjectName string
Host string
MinReplicas int
MaxReplicas int
}

const (
httprouteTemplate = `
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: {{.HTTPRouteName}}
namespace: {{.TestNamespace}}
spec:
parentRefs:
- name: eg
namespace: envoy-gateway-system
hostnames:
- {{.Host}}
rules:
- backendRefs:
- kind: Service
name: keda-http-add-on-interceptor-proxy
namespace: keda
port: 8080
matches:
- path:
type: PathPrefix
value: /
`

referenceGrantTemplate = `
apiVersion: gateway.networking.k8s.io/v1beta1
kind: ReferenceGrant
metadata:
name: {{.ReferenceGrantName}}
namespace: keda
spec:
from:
- group: gateway.networking.k8s.io
kind: HTTPRoute
namespace: {{.TestNamespace}}
to:
- group: ""
kind: Service
name: keda-http-add-on-interceptor-proxy
`
serviceTemplate = `
apiVersion: v1
kind: Service
metadata:
name: {{.ServiceName}}
namespace: {{.TestNamespace}}
labels:
app: {{.DeploymentName}}
spec:
ports:
- port: 8080
targetPort: http
protocol: TCP
name: http
selector:
app: {{.DeploymentName}}
`

deploymentTemplate = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{.DeploymentName}}
namespace: {{.TestNamespace}}
labels:
app: {{.DeploymentName}}
spec:
replicas: 0
selector:
matchLabels:
app: {{.DeploymentName}}
template:
metadata:
labels:
app: {{.DeploymentName}}
spec:
containers:
- name: {{.DeploymentName}}
image: registry.k8s.io/e2e-test-images/agnhost:2.45
args:
- netexec
ports:
- name: http
containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /
port: http
`

loadJobTemplate = `
apiVersion: batch/v1
kind: Job
metadata:
name: generate-request
namespace: {{.TestNamespace}}
spec:
template:
spec:
containers:
- name: curl-client
image: curlimages/curl
imagePullPolicy: Always
command: ["curl", "-H", "Host: {{.Host}}", "{{.GatewayHost}}"]
restartPolicy: Never
activeDeadlineSeconds: 600
backoffLimit: 5
`

httpScaledObjectTemplate = `
kind: HTTPScaledObject
apiVersion: http.keda.sh/v1alpha1
metadata:
name: {{.HTTPScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
hosts:
- {{.Host}}
targetPendingRequests: 100
scaledownPeriod: 10
scaleTargetRef:
name: {{.DeploymentName}}
service: {{.ServiceName}}
port: 8080
replicas:
min: {{ .MinReplicas }}
max: {{ .MaxReplicas }}
`
)

func TestCheckHTTPRoute(t *testing.T) {
// setup
t.Log("--- setting up ---")
// Create kubernetes resources
kc := GetKubernetesClient(t)
gc := GetGatewayClient(t)
data, templates := getTemplateData(t)
CreateKubernetesResources(t, kc, testNamespace, data, templates)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10),
"replica count should be %d after 1 minutes", minReplicaCount)
assert.True(t, WaitForHTTPRouteAccepted(t, gc, httprouteName, testNamespace, 12, 10),
"HTTPRoute should be accepted after 2 minutes")

testScaleOut(t, kc, data)
testScaleIn(t, kc, data)

// cleanup
DeleteKubernetesResources(t, testNamespace, data, templates)
}

func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) {
t.Log("--- testing scale out ---")

KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10),
"replica count should be %d after 1 minutes", maxReplicaCount)
}

func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) {
t.Log("--- testing scale in ---")

KubectlDeleteWithTemplate(t, data, "loadJobTemplate", loadJobTemplate)
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 12, 10),
"replica count should be %d after 2 minutes", minReplicaCount)
}

func getTemplateData(t *testing.T) (templateData, []Template) {
kc := GetKubernetesClient(t)
services, err := kc.CoreV1().Services(EnvoyNamespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
t.Fatalf("failed to list services in %s namespace: %v", EnvoyNamespace, err)
}
gatewayHost := ""
for _, svc := range services.Items {
if svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
gatewayHost = fmt.Sprintf(gatewayHostPattern, svc.Name)
break
}
}
if gatewayHost == "" {
t.Fatalf("failed to find gateway host, no LB service found in %s namespace", EnvoyNamespace)
}
return templateData{
TestNamespace: testNamespace,
DeploymentName: deploymentName,
ServiceName: serviceName,
HTTPRouteName: httprouteName,
ReferenceGrantName: referenceGrantName,
KEDANamespace: KEDANamespace,
HTTPScaledObjectName: httpScaledObjectName,
GatewayHost: gatewayHost,
Host: host,
MinReplicas: minReplicaCount,
MaxReplicas: maxReplicaCount,
}, []Template{
{Name: "deploymentTemplate", Config: deploymentTemplate},
{Name: "serviceTemplate", Config: serviceTemplate},
{Name: "httprouteTemplate", Config: httprouteTemplate},
{Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate},
{Name: "referenceGrantTemplate", Config: referenceGrantTemplate},
}
}
37 changes: 37 additions & 0 deletions tests/helper/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ import (
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/remotecommand"
"sigs.k8s.io/controller-runtime/pkg/client/config"
gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1"
gatewayapiv1clientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1"
)

const (
Expand All @@ -36,6 +38,8 @@ const (
ArgoRolloutsName = "argo-rollouts"
IngressNamespace = "ingress"
IngressReleaseName = "ingress"
EnvoyNamespace = "envoy-gateway-system"
EnvoyReleaseName = "eg"

StringFalse = "false"
StringTrue = "true"
Expand All @@ -46,6 +50,7 @@ var random = rand.New(rand.NewSource(time.Now().UnixNano()))
var (
KubeClient *kubernetes.Clientset
KedaKubeClient *v1alpha1.KedaV1alpha1Client
GWClient *gatewayapiv1clientset.GatewayV1Client
KubeConfig *rest.Config
)

Expand Down Expand Up @@ -168,6 +173,21 @@ func GetKubernetesClient(t *testing.T) *kubernetes.Clientset {
return KubeClient
}

func GetGatewayClient(t *testing.T) *gatewayapiv1clientset.GatewayV1Client {
if GWClient != nil && KubeConfig != nil {
return GWClient
}

var err error
KubeConfig, err = config.GetConfig()
assert.NoErrorf(t, err, "cannot fetch kube config file - %s", err)

GWClient, err = gatewayapiv1clientset.NewForConfig(KubeConfig)
assert.NoErrorf(t, err, "cannot create gateway client - %s", err)

return GWClient
}

func GetKedaKubernetesClient(t *testing.T) *v1alpha1.KedaV1alpha1Client {
if KedaKubeClient != nil && KubeConfig != nil {
return KedaKubeClient
Expand Down Expand Up @@ -388,6 +408,23 @@ func WaitForIngressReady(t *testing.T, kc *kubernetes.Clientset, name, namespace
return false
}

func WaitForHTTPRouteAccepted(t *testing.T, gc *gatewayapiv1clientset.GatewayV1Client, name, namespace string, iterations, intervalSeconds int) bool {
for i := 0; i < iterations; i++ {
httpRoute, _ := gc.HTTPRoutes(namespace).Get(context.Background(), name, metav1.GetOptions{})
for _, parent := range httpRoute.Status.Parents {
for _, condition := range parent.Conditions {
if condition.Type == string(gatewayapiv1.RouteConditionAccepted) && condition.Status == metav1.ConditionTrue {
return true
}
}
}
t.Log("Waiting for accepted HTTPRoute")

time.Sleep(time.Duration(intervalSeconds) * time.Second)
}
return false
}

// Waits until statefulset count hits target or number of iterations are done.
func WaitForStatefulsetReplicaReadyCount(t *testing.T, kc *kubernetes.Clientset, name, namespace string,
target, iterations, intervalSeconds int) bool {
Expand Down
23 changes: 23 additions & 0 deletions tests/utils/cleanup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,26 @@ func TestCleanUpCerts(t *testing.T) {
t.Log(string(out))
t.Log("test certificates successfully cleaned up")
}

func TestRemoveEnvoyGateway(t *testing.T) {
gatewayClass := `
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: eg
`

gateway := `
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: eg
namespace: envoy-gateway-system
`

KubectlDeleteWithTemplate(t, nil, "gateway", gateway)
KubectlDeleteWithTemplate(t, nil, "gatewayClass", gatewayClass)
_, err := ExecuteCommand(fmt.Sprintf("helm uninstall %s --namespace %s", EnvoyReleaseName, EnvoyNamespace))
require.NoErrorf(t, err, "cannot uninstall envoy gateway - %s", err)
DeleteNamespace(t, EnvoyNamespace)
}
Loading

0 comments on commit 1e42612

Please sign in to comment.