Add VM and VMI persistent IPs e2e tests

.github/workflows/checks.yaml

@@ -38,3 +38,30 @@ jobs:
 
     - name: Test
       run: ENVTEST_VERSION="release-0.17" make test
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version-file: 'go.mod'
+
+    - name: Start cluster
+      env:
+        KIND_ALLOW_SYSTEM_WRITES: true
+      run: make cluster-up
+
+    - name: Sync cluster
+      run: make cluster-sync
+
+    - name: Run e2e tests
+      run: make test-e2e
+
+    - uses: actions/upload-artifact@v4  # upload test results
+      if: success() || failure()        # run this step even if previous step failed
+      with:
+        name: test-e2e-results
+        path: .output/*.xml

.gitignore

@@ -26,3 +26,6 @@ go.work
 *.swp
 *.swo
 *~
+
+# output dir for the hack/cluster.sh artifacts and e2e tests
+.output

Makefile

@@ -22,6 +22,9 @@ CONTAINER_TOOL ?= docker
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+E2E_TEST_TIMEOUT ?= "1h"
+E2E_TEST_ARGS ?= ""
+
 .PHONY: all
 all: build
 
@@ -67,7 +70,11 @@ test: manifests generate fmt vet envtest ## Run tests.
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e:
-	go test ./test/e2e/ -v -ginkgo.v
+	export KUBECONFIG=$$(pwd)/.output/kubeconfig && \
+	export PATH=$$(pwd)/.output/ovn-kubernetes/bin:$${PATH} && \
+	export REPORT_PATH=$$(pwd)/.output/ && \
+	cd test/e2e && \
+	go test -test.v --ginkgo.v --test.timeout=${E2E_TEST_TIMEOUT} ${E2E_TEST_ARGS} --ginkgo.junit-report=$${REPORT_PATH}/test-e2e.junit.xml
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter & yamllint
@@ -177,6 +184,18 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION})
 
+.PHONY: cluster-up
+cluster-up:
+	./hack/cluster.sh up
+
+.PHONY: cluster-down
+cluster-down:
+	./hack/cluster.sh down
+
+.PHONY: cluster-sync
+cluster-sync:
+	./hack/cluster.sh sync
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary (ideally with version)
 # $2 - package url which can be installed

hack/cluster.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -xe
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+
+OUTPUT_DIR=${OUTPUT_DIR:-${SCRIPT_DIR}/../.output}
+
+OVN_KUBERNETES_REPO="${OVN_KUBERNETES_REPO:-https://github.com/ovn-org/ovn-kubernetes}"
+OVN_KUBERNETES_BRANCH="${OVN_KUBERNETES_BRANCH:-master}"
+OVN_KUBERNETES_DIR=${OUTPUT_DIR}/ovn-kubernetes
+
+# from https://github.com/kubernetes-sigs/kind/releases
+KIND_BIN=${OUTPUT_DIR}/kind
+
+export KUBECONFIG=${OUTPUT_DIR}/kubeconfig
+
+cluster_name=virt-ipam
+op=$1
+shift
+
+function ensure_ovn_kubernetes() {
+    if [ -d "${OVN_KUBERNETES_DIR}" ]; then
+        return 0
+    fi
+    (   
+        cd ${OUTPUT_DIR}
+        git clone --depth 1 --single-branch ${OVN_KUBERNETES_REPO} -b ${OVN_KUBERNETES_BRANCH}
+    )
+}
+
+function ensure_kind() {
+    if [ -f ${KIND_BIN} ]; then
+        return 0
+    fi
+    local kind_version=v0.20.0
+    local arch=""
+    case $(uname -m) in
+        x86_64)  arch="amd64";;
+        aarch64) arch="arm64" ;;
+    esac
+    local kind_url=https://kind.sigs.k8s.io/dl/$kind_version/kind-linux-${arch}
+    curl --retry 5 -Lo ${KIND_BIN} ${kind_url}
+	chmod +x ${KIND_BIN}
+}
+
+function up() {
+    mkdir -p ${OUTPUT_DIR}
+    ensure_ovn_kubernetes
+    ensure_kind
+    kind delete cluster --name $cluster_name
+    (
+        cd ${OVN_KUBERNETES_DIR}
+        ./contrib/kind.sh --local-kind-registry -ic -ikv -i6 -mne -cn ${cluster_name}
+    )
+}
+
+function down() {
+    ensure_kind
+    ${KIND_BIN} delete cluster --name ${cluster_name}
+    echo "down"
+}
+
+function sync() {
+    local img=localhost:5000/kubevirt-ipam-controller
+    local tag=latest
+    IMG=$img:$tag make \
+        build \
+        docker-build \
+        docker-push
+
+    # Generate the manifest with the "sha256" to force kubernetes to reload the image
+    sha=$(skopeo inspect --tls-verify=false docker://$img:$tag |jq -r .Digest)
+    IMG=$img@$sha make deploy
+}
+
+
+
+$op $@

test/e2e/e2e_suite_test.go

@@ -21,8 +21,19 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	testenv "github.com/maiqueb/kubevirt-ipam-claims/test/env"
 )
 
+var _ = BeforeSuite(func() {
+	ctrl.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+	testenv.Start()
+})
+
 // Run e2e tests using the Ginkgo runner.
 func TestE2E(t *testing.T) {
 	RegisterFailHandler(Fail)

test/e2e/persistentips_test.go

@@ -0,0 +1,175 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2024 Red Hat, Inc.
+ *
+ */
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+
+	kubevirtv1 "kubevirt.io/api/core/v1"
+
+	testenv "github.com/maiqueb/kubevirt-ipam-claims/test/env"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("Persistent IPs", func() {
+	When("network attachment definition created with allowPersistentIPs=true", func() {
+		var (
+			td                   testenv.TestData
+			networkInterfaceName = "multus"
+			vm                   *kubevirtv1.VirtualMachine
+			vmi                  *kubevirtv1.VirtualMachineInstance
+			nad                  *nadv1.NetworkAttachmentDefinition
+		)
+		BeforeEach(func() {
+			td = testenv.GenerateTestData()
+			td.SetUp()
+			DeferCleanup(func() {
+				td.TearDown()
+			})
+
+			nad = testenv.GenerateLayer2WithSubnetNAD(td.Namespace)
+			vmi = testenv.GenerateAlpineWithMultusVMI(td.Namespace, networkInterfaceName, nad.Name)
+			vm = testenv.NewVirtualMachine(vmi, testenv.WithRunning())
+
+			By("Create NetworkAttachmentDefinition")
+			Expect(testenv.Client.Create(context.Background(), nad)).To(Succeed())
+		})
+		Context("and a virtual machine using it is also created", func() {
+			BeforeEach(func() {
+				By("Creating VM using the nad")
+				Expect(testenv.Client.Create(context.Background(), vm)).To(Succeed())
+
+				By(fmt.Sprintf("Waiting for readiness at virtual machine %s", vm.Name))
+				Eventually(testenv.ThisVMReadiness(vm)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(BeTrue())
+
+				By("Wait for IPAMClaim to get created")
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					ShouldNot(BeEmpty())
+
+				Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed())
+
+				Expect(vmi.Status.Interfaces).NotTo(BeEmpty())
+				Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty())
+			})
+
+			It("should keep ips after live migration", func() {
+				vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs
+
+				testenv.LiveMigrateVirtualMachine(td.Namespace, vm.Name)
+				testenv.CheckLiveMigrationSucceeded(td.Namespace, vm.Name)
+
+				By("Wait for VMI to be ready after live migration")
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(testenv.ContainConditionVMIReady())
+
+				Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration)))
+
+			})
+
+			It("should garbage collect IPAMClaims after virtual machine deletion", func() {
+				Expect(testenv.Client.Delete(context.Background(), vm)).To(Succeed())
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeEmpty())
+			})
+
+			It("should keep ips after restart", func() {
+				vmiIPsBeforeRestart := vmi.Status.Interfaces[0].IPs
+				vmiUUIDBeforeRestart := vmi.UID
+
+				By("Re-starting the VM")
+				output, err := exec.Command("virtctl", "restart", "-n", td.Namespace, vmi.Name).CombinedOutput()
+				Expect(err).NotTo(HaveOccurred(), output)
+
+				By("Wait for a new VMI to be re-started")
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(90 * time.Second).
+					Should(testenv.BeRestarted(vmiUUIDBeforeRestart))
+
+				By("Wait for VMI to be ready after restart")
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(testenv.ContainConditionVMIReady())
+
+				Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeRestart)))
+			})
+		})
+		Context("and a virtual machine instance using it is also created", func() {
+			BeforeEach(func() {
+				By("Creating VMI using the nad")
+				Expect(testenv.Client.Create(context.Background(), vmi)).To(Succeed())
+
+				By(fmt.Sprintf("Waiting for readiness at virtual machine instance %s", vmi.Name))
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(testenv.ContainConditionVMIReady())
+
+				By("Wait for IPAMClaim to get created")
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					ShouldNot(BeEmpty())
+
+				Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed())
+
+				Expect(vmi.Status.Interfaces).NotTo(BeEmpty())
+				Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty())
+			})
+
+			It("should keep ips after live migration", func() {
+				vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs
+
+				testenv.LiveMigrateVirtualMachine(td.Namespace, vmi.Name)
+				testenv.CheckLiveMigrationSucceeded(td.Namespace, vmi.Name)
+
+				Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration)))
+
+			})
+
+			It("should garbage collect IPAMClaims after virtual machine deletion", func() {
+				Expect(testenv.Client.Delete(context.Background(), vmi)).To(Succeed())
+				Eventually(testenv.IPAMClaimsFromNamespace(vmi.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeEmpty())
+			})
+		})
+
+	})
+})