Merge pull request #361 from Lombiq/issue/OSOE-844

OSOE-844: Don't include a file called .htmlvalidate.json by default in Lombiq.UITestingToolbox

Lombiq.Tests.UI.Samples/.htmlvalidate.json

@@ -1,19 +1,12 @@
 {
   "extends": [
-    "html-validate:recommended"
+    "./default.htmlvalidate.json"
   ],
 
   "rules": {
-    "attribute-boolean-style": "off",
     "long-title": "off",
-    "no-trailing-whitespace": "off",
-    "no-inline-style": "off",
-    "no-dup-class": "off",
-    "wcag/h30": "off",
-    "wcag/h32": "off",
-    "wcag/h36": "off",
-    "wcag/h37": "off",
-    "wcag/h67": "off",
-    "wcag/h71": "off"
-  }
+    "no-dup-class": "off"
+  },
+
+  "root": true
 }

Lombiq.Tests.UI.Tests.UI/TestCases/TimeoutTestCases.cs

@@ -0,0 +1,25 @@
+using Lombiq.Tests.UI.Services;
+using Shouldly;
+
+namespace Lombiq.Tests.UI.Tests.UI.TestCases;
+
+public static class TimeoutTestCases
+{
+    public static Task TestRunTimeoutShouldThrowAsync(
+        ExecuteTestAfterSetupAsync executeTestAfterSetupAsync,
+        Browser browser = default) =>
+        Should.ThrowAsync(
+            async () => await executeTestAfterSetupAsync(
+                context => Task.Delay(TimeSpan.FromSeconds(1)),
+                browser,
+                configuration =>
+                {
+                    configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
+                    configuration.MaxRetryCount = 0;
+
+                    configuration.TimeoutConfiguration.TestRunTimeout = TimeSpan.FromMilliseconds(10);
+
+                    return Task.CompletedTask;
+                }),
+            typeof(TimeoutException));
+}

Lombiq.Tests.UI/Docs/Configuration.md

@@ -59,42 +59,42 @@ Recommendations and notes for such configuration:
 
 ### HTML validation configuration
 
-If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). For example:
+If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). This should extend the [default.htmlvalidate.json](../default.htmlvalidate.json) file (which is always copied into the build directory) by setting the value of `"extends"` to a relative path pointing to it and declaring `"root": true`. For example:
 
 ```json
 {
-  "extends": [
-    "html-validate:recommended"
-  ],
-
-  "rules": {
-    "attribute-boolean-style": "off",
-    "element-required-attributes": "off",
-    "no-trailing-whitespace": "off",
-    "no-inline-style": "off",
-    "no-implicit-button-type": "off",
-    "wcag/h30": "off",
-    "wcag/h32": "off",
-    "wcag/h36": "off",
-    "wcag/h37": "off",
-    "wcag/h67": "off",
-    "wcag/h71": "off"
-  },
+    "extends": [
+        "./default.htmlvalidate.json"
+    ],
+
+    "rules": {
+        "element-required-attributes": "off",
+        "no-implicit-button-type": "off"
+    },
 
-  "root":  true
+    "root": true
 }
 ```
 
-Then you can change the configuration to use that:
+You can also create a completely standalone config file too, without any inheritance, but we'd recommend against that.
+
+You can change the configuration to use the above file as follows:
 
 ```cs
- changeConfiguration: configuration => configuration.HtmlValidationConfiguration.HtmlValidationOptions =
-                configuration.HtmlValidationConfiguration.HtmlValidationOptions
-                    .CloneWith(validationOptions => validationOptions.ConfigPath =
-                        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestName.htmlvalidate.json")));
+changeConfiguration: configuration =>
+    configuration.HtmlValidationConfiguration.HtmlValidationOptions.ConfigPath =
+        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestName.htmlvalidate.json");
 ```
 
-Make sure to also include the `root` attribute and set it to `true` inside the custom _.htmlvalidate.json_ file and include it in the test project like this:
+Though if the file is in the base directory like above, then it can be simplified using the `WithRelativeConfigPath(params string[] pathSegments)` method:
+
+```cs
+changeConfiguration: configuration => configuration.HtmlValidationConfiguration.WithRelativeConfigPath("TestName.htmlvalidate.json");
+```
+
+If you want to do this for all tests in the project, just put an _.htmlvalidate.json_ file into the project root and it will be picked up without further configuration.
+
+Include it in the test project like this:
 
 ```xml
   <ItemGroup>

Lombiq.Tests.UI/Lombiq.Tests.UI.csproj

@@ -40,7 +40,7 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <PackageCopyToOutput>true</PackageCopyToOutput>
     </Content>
-    <Content Include=".htmlvalidate.json">
+    <Content Include="default.htmlvalidate.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <PackageCopyToOutput>true</PackageCopyToOutput>
     </Content>

Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs

@@ -2,6 +2,7 @@
 using Lombiq.Tests.UI.Helpers;
 using Shouldly;
 using System;
+using System.IO;
 using System.Threading.Tasks;
 
 namespace Lombiq.Tests.UI.Services;
@@ -33,6 +34,13 @@ public class HtmlValidationConfiguration
         // This is necessary so no long folder names will be generated, see:
         // https://github.com/atata-framework/atata-htmlvalidation/issues/5
         WorkingDirectory = "HtmlValidationTemp",
+        // If a consuming project adds a ".htmlvalidate.json" config file then use it, otherwise fall back to the
+        // "default.htmlvalidate.json" which always exists because Lombiq.Tests.UI copies it into the directory during
+        // build.
+        ConfigPath =
+            Path.Combine(AppDomain.CurrentDomain.BaseDirectory, ".htmlvalidate.json") is { } rootConfiguration && File.Exists(rootConfiguration)
+            ? rootConfiguration
+            : Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "default.htmlvalidate.json"),
     };
 
     /// <summary>
@@ -61,6 +69,22 @@ public class HtmlValidationConfiguration
     public Predicate<UITestContext> HtmlValidationAndAssertionOnPageChangeRule { get; set; } =
         EnableOnValidatablePagesHtmlValidationAndAssertionOnPageChangeRule;
 
+    /// <summary>
+    /// Updates the <see cref="HtmlValidationOptions"/>.<see cref="HtmlValidationOptions.ConfigPath"/> with a path
+    /// relative to the <see cref="AppDomain.BaseDirectory"/> of the <see cref="AppDomain.CurrentDomain"/> (i.e. the
+    /// build directory).
+    /// </summary>
+    /// <param name="pathSegments">
+    /// Directory and file names which are joined together using <see cref="Path.Combine(string[])"/>.
+    /// </param>
+    public HtmlValidationConfiguration WithRelativeConfigPath(params string[] pathSegments)
+    {
+        string[] path = [AppDomain.CurrentDomain.BaseDirectory, .. pathSegments];
+        HtmlValidationOptions.ConfigPath = Path.Combine(path);
+
+        return this;
+    }
+
     public static readonly Func<HtmlValidationResult, Task> AssertHtmlValidationOutputIsEmptyAsync =
         validationResult =>
         {

Lombiq.Tests.UI/Services/TimeoutConfiguration.cs

@@ -4,6 +4,14 @@ namespace Lombiq.Tests.UI.Services;
 
 public class TimeoutConfiguration
 {
+    private static readonly TimeoutConfiguration _default = new()
+    {
+        RetryTimeout = GetTimeoutConfiguration(nameof(RetryTimeout), 10),
+        RetryInterval = GetTimeoutConfiguration(nameof(RetryInterval), 500, useMilliseconds: true),
+        PageLoadTimeout = GetTimeoutConfiguration(nameof(PageLoadTimeout), 180),
+        TestRunTimeout = GetTimeoutConfiguration(nameof(TestRunTimeout), 600),
+    };
+
     /// <summary>
     /// Gets or sets how long to wait for an operation to finish when it's retried. Defaults to 10s. Higher,
     /// paradoxically, is usually less safe.
@@ -27,13 +35,27 @@ public class TimeoutConfiguration
     /// </summary>
     public TimeSpan PageLoadTimeout { get; set; }
 
-    public static readonly TimeoutConfiguration Default = new()
+    /// <summary>
+    /// Gets or sets how long an individual test can run to prevent hanging indefinitely. Defaults to 10 minutes.
+    /// </summary>
+    public TimeSpan TestRunTimeout { get; set; }
+
+    /// <summary>
+    /// Gets a copy of the timeout configuration derived from the values in the <see cref="TestConfigurationManager"/>.
+    /// </summary>
+    public static TimeoutConfiguration Default => new()
     {
-        RetryTimeout = TimeSpan
-            .FromSeconds(TestConfigurationManager.GetIntConfiguration("TimeoutConfiguration:RetryTimeoutSeconds", 10)),
-        RetryInterval = TimeSpan
-            .FromMilliseconds(TestConfigurationManager.GetIntConfiguration("TimeoutConfiguration:RetryIntervalMillisecondSeconds", 500)),
-        PageLoadTimeout = TimeSpan
-            .FromSeconds(TestConfigurationManager.GetIntConfiguration("TimeoutConfiguration:PageLoadTimeoutSeconds", 180)),
+        RetryTimeout = _default.RetryTimeout,
+        RetryInterval = _default.RetryInterval,
+        PageLoadTimeout = _default.PageLoadTimeout,
+        TestRunTimeout = _default.TestRunTimeout,
     };
+
+    private static TimeSpan GetTimeoutConfiguration(string name, int defaultValue, bool useMilliseconds = false)
+    {
+        var suffix = useMilliseconds ? "Milliseconds" : "Seconds";
+        var key = $"{nameof(TimeoutConfiguration)}:{name}{suffix}";
+        var value = TestConfigurationManager.GetIntConfiguration(key, defaultValue);
+        return useMilliseconds ? TimeSpan.FromMilliseconds(value) : TimeSpan.FromSeconds(value);
+    }
 }

Lombiq.Tests.UI/UITestBase.cs

@@ -21,6 +21,8 @@ protected async Task ExecuteOrchardCoreTestAsync(
         OrchardCoreUITestExecutorConfiguration configuration)
     {
         var originalTestOutputHelper = _testOutputHelper;
+        var timeout = configuration.TimeoutConfiguration.TestRunTimeout;
+
         Action afterTest = null;
         if (configuration.ExtendGitHubActionsOutput &&
             configuration.GitHubActionsOutputConfiguration.EnablePerTestOutputGrouping &&
@@ -33,7 +35,23 @@ protected async Task ExecuteOrchardCoreTestAsync(
 
         try
         {
-            await UITestExecutor.ExecuteOrchardCoreTestAsync(webApplicationInstanceFactory, testManifest, configuration);
+            var testTask = UITestExecutor.ExecuteOrchardCoreTestAsync(
+                webApplicationInstanceFactory,
+                testManifest,
+                configuration);
+            var timeoutTask = Task.Delay(timeout);
+
+            await Task.WhenAny(testTask, timeoutTask);
+
+            if (timeoutTask.IsCompleted)
+            {
+                throw new TimeoutException($"The time allotted for the test ({timeout}) was exceeded.");
+            }
+
+            // Since the timeout task is not yet completed but the Task.WhenAny has finished, the test task is done in
+            // some way. So it's safe to await it here. It's also necessary to cleanly propagate any exceptions that may
+            // have been thrown inside it.
+            await testTask;
         }
         finally
         {