Merge pull request #1593 from jglick/JenkinsRuleNonLocalhost-JENKINS-…

…73138

[JENKINS-73138] Replace `JenkinsRuleNonLocalhost` and `test-in-k8s` with `ktunnel`

Jenkinsfile

@@ -27,9 +27,7 @@ stage('Tests') {
                         writeFile file: (split.includes ? "$WORKSPACE_TMP/includes.txt" : "$WORKSPACE_TMP/excludes.txt"), text: split.list.join("\n")
                         writeFile file: (split.includes ? "$WORKSPACE_TMP/excludes.txt" : "$WORKSPACE_TMP/includes.txt"), text: ''
                         sh './kind.sh -Dsurefire.includesFile="$WORKSPACE_TMP/includes.txt" -Dsurefire.excludesFile="$WORKSPACE_TMP/excludes.txt"'
-                        dir(env.WORKSPACE_TMP) {
-                            junit 'surefire-reports/*.xml'
-                        }
+                        junit 'target/surefire-reports/*.xml'
                     } finally {
                         dir(env.WORKSPACE_TMP) {
                             if (fileExists('kindlogs/docker-info.txt')) {

README.md

@@ -1006,72 +1006,31 @@ Run `mvn clean install` and copy `target/kubernetes.hpi` to Jenkins plugins fold
 
 ## Running Kubernetes Integration Tests
 
-### Integration tests with Kind (recommended)
+Set up your `$KUBECONFIG` however you like, for example
 
 ```bash
-export MOUNT_M2=true
-./kind-mount-m2.sh
-./kind-preload.sh
-./test-in-k8s.sh -Dtest=KubernetesPipelineTest#runInPod
-kind delete cluster
+kind create cluster
 ```
 
-### Integration tests in a remote cluster
-
-Set up your `$KUBECONFIG` as usual, then
+then
 
 ```bash
-./test-in-k8s.sh -Dtest=KubernetesPipelineTest#runInPod
+kubectl krew install tunnel # as needed; install Krew first
+kubectl tunnel expose jenkins 8000:8000 8001:8001 &
+mvn test -Djenkins.host.address=jenkins.default -Dport=8000 -DslaveAgentPort=8001 -Dtest=KubernetesPipelineTest#runInPod
 ```
 
-### Integration tests with Minikube
-
-The system you run `mvn` on needs to be reachable from the cluster.
-If you see the agents happen to connect to the wrong host, see you can use
-`jenkins.host.address` as mentioned above.
-
-For integration tests install and start [minikube](https://github.com/kubernetes/minikube).
-Tests will detect it and run a set of integration tests in a new namespace.
-
-Some integration tests run a local jenkins, so the host that runs them needs
-to be accessible from the kubernetes cluster.
-By default Jenkins will listen on `192.168.64.1` interface only, for security reasons.
-If your minikube is not running in that network, pass `connectorHost` to maven, ie.
-
-    mvn clean install -DconnectorHost=$(minikube ip | sed -e 's/\([0-9]*\.[0-9]*\.[0-9]*\).*/\1.1/')
-
-If you don't mind others in your network being able to use your test jenkins you could just use this:
-
-    mvn clean install -DconnectorHost=0.0.0.0
-
-Then your test jenkins will listen on all ip addresses so that the build pods will be able to connect from the pods in your minikube VM to your host.  
-
-If your minikube is running in a VM (e.g. on virtualbox) and the host running `mvn`
-does not have a public hostname for the VM to access, you can set the `jenkins.host.address`
-system property to the (host-only or NAT) IP of your host:
-
-    mvn clean install -Djenkins.host.address=192.168.99.1
-
-### Integration tests with Microk8s
-
-If [Microk8s](https://microk8s.io/) is running and is the default context in your `~/.kube/config`,
-just run as
-
-    mvn clean install -Pmicrok8s
-
-This assumes that from a pod, the host system is accessible as IP address `10.1.1.1`.
-It might be some variant such as `10.1.37.1`,
-in which case you would need to set `-DconnectorHost=… -Djenkins.host.address=…` instead.
-To see the actual address, try:
+Alternately, you can run everything like in CI:
 
 ```bash
-ifdata -pa cni0
+export KIND_PRELOAD=true # optionally
+./kind.sh -Dtest=KubernetesPipelineTest#runInPod
 ```
 
-Or to verify the networking inside a pod:
+You can also run interactively after setting up the tunnel:
 
 ```bash
-kubectl run --rm --image=praqma/network-multitool --restart=Never --attach sh ip route | fgrep 'default via'
+mvn hpi:run -Djenkins.host.address=jenkins.default -Dport=8000 -Djenkins.model.Jenkins.slaveAgentPort=8001
 ```
 
 # Docker image

kind-preload.sh

@@ -3,7 +3,7 @@ set -euxo pipefail
 cd $(dirname $0)
 
 PRE_LOAD_IMAGES=()
-PRE_LOAD_IMAGES+=($(grep -e image: test-in-k8s.yaml | cut -d ':' -f 2- | xargs))
+PRE_LOAD_IMAGES+=(omrieival/ktunnel:v1.6.1)
 PRE_LOAD_IMAGES+=($(grep -h --include="*.groovy" -e "^\s*image: .*$" -R src/test/resources | sed -e "s/^[[:space:]]*image: //" | sort | uniq | grep -v "windows" | grep -v "nonexistent" | grep -v "invalid" | xargs))
 PRE_LOAD_IMAGES+=($(grep -e FROM src/main/resources/org/csanchez/jenkins/plugins/kubernetes/Dockerfile | cut -d ' ' -f 2-))
 if [[ -v cluster ]]

kind.sh

@@ -2,6 +2,8 @@
 set -euxo pipefail
 cd $(dirname $0)
 
+: ${WORKSPACE_TMP:=/tmp}
+
 export PATH=$WORKSPACE_TMP:$PATH
 if [ \! -x "$WORKSPACE_TMP/kind" ]
 then
@@ -13,25 +15,41 @@ then
     curl -sLo "$WORKSPACE_TMP/kubectl" https://storage.googleapis.com/kubernetes-release/release/v1.30.1/bin/$(uname | tr '[:upper:]' '[:lower:]')/amd64/kubectl
     chmod +x "$WORKSPACE_TMP/kubectl"
 fi
+if [ \! -x "$WORKSPACE_TMP/ktunnel" ]
+then
+    (cd "$WORKSPACE_TMP"; curl -sL https://github.com/omrikiei/ktunnel/releases/download/v1.6.1/ktunnel_1.6.1_Linux_x86_64.tar.gz | tar xvfz - ktunnel)
+fi
 
 export cluster=ci$RANDOM
 export KUBECONFIG="$WORKSPACE_TMP/kubeconfig-$cluster"
-if ${MOUNT_M2:-false}
-then
-  ./kind-mount-m2.sh
-else
-  kind create cluster --name $cluster --wait 5m
-fi
+kind create cluster --name $cluster --wait 5m
 function cleanup() {
-    kind export logs --name $cluster "$WORKSPACE_TMP/kindlogs" || :
-    kind delete cluster --name $cluster || :
+    set +e
+    if [ -v ktunnel_pid ]
+    then
+        kill $ktunnel_pid
+    fi
+    kind export logs --name $cluster "$WORKSPACE_TMP/kindlogs"
+    kind delete cluster --name $cluster
     rm "$KUBECONFIG"
 }
 trap cleanup EXIT
 kubectl cluster-info
 
-./kind-preload.sh
+if ${KIND_PRELOAD:-false}
+then
+   ./kind-preload.sh
+fi
+
+ktunnel expose jenkins 8000:8000 8001:8001 &
+ktunnel_pid=$!
 
-./test-in-k8s.sh "$@"
-rm -rf "$WORKSPACE_TMP/surefire-reports"
-kubectl cp jenkins:/checkout/target/surefire-reports "$WORKSPACE_TMP/surefire-reports"
+mvn \
+    -B \
+    -ntp \
+    -Djenkins.host.address=jenkins.default \
+    -Dport=8000 \
+    -DslaveAgentPort=8001 \
+    -Dmaven.test.failure.ignore \
+    verify \
+    "$@"

pom.xml

@@ -41,9 +41,6 @@
 
   <properties>
     <changelist>999999-SNAPSHOT</changelist>
-    <!-- in minikube
-    minikube ip | sed -e 's/\([0-9]*\.[0-9]*\.[0-9]*\).*/\1.1/' -->
-    <connectorHost />
     <jenkins.host.address />
     <slaveAgentPort />
     <jenkins.version>2.426.3</jenkins.version>
@@ -296,8 +293,6 @@
             <hudson.slaves.NodeProvisioner.initialDelay>0</hudson.slaves.NodeProvisioner.initialDelay>
             <hudson.slaves.NodeProvisioner.recurrencePeriod>3000</hudson.slaves.NodeProvisioner.recurrencePeriod>
             <org.jenkinsci.plugins.workflow.support.pickles.ExecutorPickle.timeoutForNodeMillis>60000</org.jenkinsci.plugins.workflow.support.pickles.ExecutorPickle.timeoutForNodeMillis>
-            <!-- listen in this interface for connections from kubernetes pods -->
-            <connectorHost>${connectorHost}</connectorHost>
             <!-- have pods connect to this address for Jenkins -->
             <jenkins.host.address>${jenkins.host.address}</jenkins.host.address>
             <slaveAgentPort>${slaveAgentPort}</slaveAgentPort>
@@ -313,6 +308,7 @@
             <hudson.slaves.NodeProvisioner.MARGIN>50</hudson.slaves.NodeProvisioner.MARGIN>
             <hudson.slaves.NodeProvisioner.MARGIN0>0.85</hudson.slaves.NodeProvisioner.MARGIN0>
             <jenkins.host.address>${jenkins.host.address}</jenkins.host.address>
+            <port>${port}</port>
             <org.csanchez.jenkins.plugins.kubernetes.pipeline.PodTemplateStepExecution.verbose>true</org.csanchez.jenkins.plugins.kubernetes.pipeline.PodTemplateStepExecution.verbose>
           </systemProperties>
         </configuration>
@@ -348,28 +344,4 @@
     </plugins>
   </build>
 
-  <profiles>
-    <profile>
-      <id>docker</id>
-      <properties>
-        <connectorHost>192.168.1.27</connectorHost>
-        <jenkins.host.address>192.168.1.27</jenkins.host.address>
-      </properties>
-    </profile>
-    <profile>
-      <id>microk8s</id>
-      <properties>
-        <connectorHost>10.1.1.1</connectorHost>
-        <jenkins.host.address>10.1.1.1</jenkins.host.address>
-      </properties>
-    </profile>
-    <profile>
-      <id>docker-desktop</id>
-      <properties>
-        <connectorHost>127.0.0.1</connectorHost>
-        <jenkins.host.address>host.docker.internal</jenkins.host.address>
-      </properties>
-    </profile>
-  </profiles>
-
 </project>

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java

@@ -55,6 +55,7 @@
 import jenkins.metrics.api.Metrics;
 import jenkins.model.Jenkins;
 import jenkins.model.JenkinsLocationConfiguration;
+import jenkins.util.SystemProperties;
 import jenkins.websocket.WebSockets;
 import net.sf.json.JSONObject;
 import org.apache.commons.lang.StringUtils;
@@ -1171,7 +1172,8 @@ public static void hpiRunInit() {
             if (hostAddress != null
                     && jenkins.clouds.getAll(KubernetesCloud.class).isEmpty()) {
                 KubernetesCloud cloud = new KubernetesCloud("kubernetes");
-                cloud.setJenkinsUrl("http://" + hostAddress + ":8080/jenkins/");
+                cloud.setJenkinsUrl(
+                        "http://" + hostAddress + ":" + SystemProperties.getInteger("port", 8080) + "/jenkins/");
                 jenkins.clouds.add(cloud);
             }
         }

src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesTestUtil.java

@@ -135,6 +135,7 @@ public static void setupHost(KubernetesCloud cloud) throws Exception {
         URL nonLocalhostUrl = new URL(url.getProtocol(), hostAddress, url.getPort(), url.getFile());
         cloud.setJenkinsUrl(nonLocalhostUrl.toString());
 
+        // TODO could just use standard jenkins.model.Jenkins.slaveAgentPort and skip this code (also in SetupCloud):
         Integer slaveAgentPort = Integer.getInteger("slaveAgentPort");
         if (slaveAgentPort != null) {
             Jenkins.get().setSlaveAgentPort(slaveAgentPort);

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/AbstractKubernetesPipelineRJRTest.java

@@ -23,11 +23,6 @@ public abstract class AbstractKubernetesPipelineRJRTest {
 
     {
         rjr = new RealJenkinsRule();
-        String connectorHost = System.getProperty("connectorHost");
-        if (StringUtils.isNotBlank(connectorHost)) {
-            System.err.println("Listening on host address: " + connectorHost);
-            rjr.withHttpListenAddress(connectorHost);
-        }
         String port = System.getProperty("port");
         if (StringUtils.isNotBlank(port)) {
             System.err.println("Overriding port using system property: " + port);

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/AbstractKubernetesPipelineTest.java

@@ -62,7 +62,7 @@
 import org.junit.Rule;
 import org.junit.rules.TestName;
 import org.jvnet.hudson.test.BuildWatcher;
-import org.jvnet.hudson.test.JenkinsRuleNonLocalhost;
+import org.jvnet.hudson.test.JenkinsRule;
 import org.jvnet.hudson.test.LoggerRule;
 
 public abstract class AbstractKubernetesPipelineTest {
@@ -76,7 +76,7 @@ public abstract class AbstractKubernetesPipelineTest {
     protected KubernetesCloud cloud;
 
     @Rule
-    public JenkinsRuleNonLocalhost r = new JenkinsRuleNonLocalhost();
+    public JenkinsRule r = new JenkinsRule();
 
     @Rule
     public LoggerRule logs = new LoggerRule()

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/DirectConnectionTest.java

@@ -16,6 +16,7 @@
 
 package org.csanchez.jenkins.plugins.kubernetes.pipeline;
 
+import hudson.TcpSlaveAgentListener;
 import java.util.logging.Level;
 import org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil;
 import org.csanchez.jenkins.plugins.kubernetes.PodTemplateBuilder;
@@ -24,6 +25,11 @@
 
 public final class DirectConnectionTest extends AbstractKubernetesPipelineTest {
 
+    static {
+        System.setProperty(
+                TcpSlaveAgentListener.class.getName() + ".hostName", System.getProperty("jenkins.host.address"));
+    }
+
     @Before
     public void setUp() throws Exception {
         KubernetesTestUtil.deletePods(cloud.connect(), KubernetesTestUtil.getLabels(cloud, this, name), false);

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java

@@ -105,7 +105,6 @@
 import org.jvnet.hudson.test.FlagRule;
 import org.jvnet.hudson.test.Issue;
 import org.jvnet.hudson.test.JenkinsRule;
-import org.jvnet.hudson.test.JenkinsRuleNonLocalhost;
 import org.jvnet.hudson.test.LoggerRule;
 import org.jvnet.hudson.test.MockAuthorizationStrategy;
 import org.jvnet.hudson.test.TestExtension;
@@ -436,7 +435,7 @@ public void runWithEnvVariablesInContext() throws Exception {
         r.assertLogContains("The value of WILL.NOT is ", b);
     }
 
-    private void assertEnvVars(JenkinsRuleNonLocalhost r2, WorkflowRun b) throws Exception {
+    private void assertEnvVars(JenkinsRule r2, WorkflowRun b) throws Exception {
         r.assertLogNotContains(POD_ENV_VAR_FROM_SECRET_VALUE, b);
         r.assertLogNotContains(CONTAINER_ENV_VAR_FROM_SECRET_VALUE, b);
 

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/PodTemplateStepExecutionTest.java

@@ -38,12 +38,12 @@
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
-import org.jvnet.hudson.test.JenkinsRuleNonLocalhost;
+import org.jvnet.hudson.test.JenkinsRule;
 
 public class PodTemplateStepExecutionTest {
 
     @Rule
-    public JenkinsRuleNonLocalhost r = new JenkinsRuleNonLocalhost();
+    public JenkinsRule r = new JenkinsRule();
 
     protected KubernetesCloud cloud;