Retry example integration tests that didn't finish after 5 minutes (#…

…3125)

The idea is, that our example integration tests should finish after a
short period of time. But sometimes then hang in CI, so we simply abort
and retry them automatically, instaed of manually.

The hardcoded timeout of 5 minutes is just a guess. I want to see the CI
results. Maybe, we can make it configurable per test suite.

Pull request: #3125

example/src/mill/integration/ExampleTestSuite.scala

@@ -1,8 +1,11 @@
 package mill.example
 import mill.integration.{BashTokenizer, IntegrationTestSuite}
-import utest._
 import mill.util.Util
+import utest._
 
+import java.util.concurrent.{Executors, TimeoutException}
+import scala.annotation.tailrec
+import scala.concurrent.duration.DurationInt
 import scala.util.chaining.scalaUtilChainingOps
 
 /**
@@ -46,18 +49,41 @@ object ExampleTestSuite extends IntegrationTestSuite {
   val tests: Tests = Tests {
     val workspaceRoot = initWorkspace()
 
-    test("exampleUsage") {
-      try {
-        val parsed = upickle.default.read[Seq[(String, String)]](sys.env("MILL_EXAMPLE_PARSED"))
-        val usageComment = parsed.collect { case ("example", txt) => txt }.mkString("\n\n")
-        val commandBlocks = ("\n" + usageComment.trim).split("\n> ").filter(_.nonEmpty)
+    val testTimeout = 5.minutes
+
+    // Integration tests sometime hang on CI
+    // The idea is to just abort and retry them after a reasonable amount of time
+    @tailrec def retryOnTimeout[T](n: Int)(body: => T): T = {
 
-        for (commandBlock <- commandBlocks) processCommandBlock(workspaceRoot, commandBlock)
+      // We use Java Future here, as it supports cancellation
+      val executor = Executors.newFixedThreadPool(1)
+      val fut = executor.submit { () => body }
 
-        if (integrationTestMode != "fork") evalStdout("shutdown")
-      } finally {
-        try os.remove.all(workspaceRoot / "out")
-        catch { case e: Throwable => /*do nothing*/ }
+      try fut.get(testTimeout.length, testTimeout.unit)
+      catch {
+        case e: TimeoutException =>
+          fut.cancel(true)
+          if (n > 0) {
+            Console.err.println(s"Timeout occurred (${testTimeout}). Retrying...")
+            retryOnTimeout(n - 1)(body)
+          } else throw e
+      }
+
+    }
+
+    test("exampleUsage") {
+      val parsed = upickle.default.read[Seq[(String, String)]](sys.env("MILL_EXAMPLE_PARSED"))
+      val usageComment = parsed.collect { case ("example", txt) => txt }.mkString("\n\n")
+      val commandBlocks = ("\n" + usageComment.trim).split("\n> ").filter(_.nonEmpty)
+
+      retryOnTimeout(3) {
+        try {
+          for (commandBlock <- commandBlocks) processCommandBlock(workspaceRoot, commandBlock)
+          if (integrationTestMode != "fork") evalStdout("shutdown")
+        } finally {
+          try os.remove.all(workspaceRoot / "out")
+          catch { case e: Throwable => /*do nothing*/ }
+        }
       }
     }
   }