Retry example integration tests that didn't finish after 5 minutes

The idea is, that our example integration tests should finish after a short period of time. But sometimes then hang in CI, so we simply abort and retry them automatically, instaed of manually.

The hardcoded timeout of 5 minutes is just a guess. I want to see the CI results. Maybe, we can make it configurable per test suite.

example/src/mill/integration/ExampleTestSuite.scala

@@ -3,6 +3,10 @@ import mill.integration.{BashTokenizer, IntegrationTestSuite}
 import utest._
 import mill.util.Util
 
+import java.util.concurrent.TimeoutException
+import scala.annotation.tailrec
+import scala.concurrent.duration.DurationInt
+import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.util.chaining.scalaUtilChainingOps
 
 /**
@@ -46,15 +50,36 @@ object ExampleTestSuite extends IntegrationTestSuite {
   val tests: Tests = Tests {
     val workspaceRoot = initWorkspace()
 
+    val testTimeout = 5.minutes
+
+    // Integration tests sometime hang on CI
+    // The idea is to just abort and retry them after a reasonable amount of time
+    @tailrec def retryOnTimeout[T](n: Int)(body: => T): T = {
+      val fut = Future { body }(ExecutionContext.global)
+
+      try Await.result(fut, testTimeout)
+      catch {
+        case e: TimeoutException =>
+          if (n > 0) {
+            Console.err.println(s"Timeout occurred (${testTimeout}). Retrying...")
+            retryOnTimeout(n - 1)(body)
+          } else throw e
+      }
+
+    }
+
     test("exampleUsage") {
       try {
-        val parsed = upickle.default.read[Seq[(String, String)]](sys.env("MILL_EXAMPLE_PARSED"))
-        val usageComment = parsed.collect { case ("example", txt) => txt }.mkString("\n\n")
-        val commandBlocks = ("\n" + usageComment.trim).split("\n> ").filter(_.nonEmpty)
+        def body = {
+          val parsed = upickle.default.read[Seq[(String, String)]](sys.env("MILL_EXAMPLE_PARSED"))
+          val usageComment = parsed.collect { case ("example", txt) => txt }.mkString("\n\n")
+          val commandBlocks = ("\n" + usageComment.trim).split("\n> ").filter(_.nonEmpty)
 
-        for (commandBlock <- commandBlocks) processCommandBlock(workspaceRoot, commandBlock)
+          for (commandBlock <- commandBlocks) processCommandBlock(workspaceRoot, commandBlock)
 
-        if (integrationTestMode != "fork") evalStdout("shutdown")
+          if (integrationTestMode != "fork") evalStdout("shutdown")
+        }
+        retryOnTimeout(3)(body)
       } finally {
         try os.remove.all(workspaceRoot / "out")
         catch { case e: Throwable => /*do nothing*/ }