Merge pull request #364 from Lombiq/issue/OSOE-770-re

OSOE-770: Add support for structured html-validate output

Lombiq.Tests.UI/Docs/Configuration.md

@@ -59,7 +59,22 @@ Recommendations and notes for such configuration:
 
 ### HTML validation configuration
 
-If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). This should extend the [default.htmlvalidate.json](../default.htmlvalidate.json) file (which is always copied into the build directory) by setting the value of `"extends"` to a relative path pointing to it and declaring `"root": true`. For example:
+If you want to filter out certain html validation errors for a specific test you can simply filter them out of the error results by their rule ID. For example:
+
+```c#
+configuration => configuration.HtmlValidationConfiguration.AssertHtmlValidationResultAsync =
+    validationResult =>
+    {
+        var errors = validationResult.GetParsedErrors()
+            .Where(error => error.RuleId is not "prefer-native-element");
+        errors.ShouldBeEmpty(HtmlValidationResultExtensions.GetParsedErrorMessageString(errors));
+        return Task.CompletedTask;
+    });
+```
+
+Note that the `RuleId` is the identifier of the rule that you want to exclude from the results. The custom string formatter in the call to `errors.ShouldBeEmpty` is used to display the errors in a more readable way and is not strictly necessary.
+
+If you want to change some HTML validation rules for multiple tests, you can also create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). This should extend the [default.htmlvalidate.json](../default.htmlvalidate.json) file (which is always copied into the build directory) by setting the value of `"extends"` to a relative path pointing to it and declaring `"root": true`. For example:
 
 ```json
 {

Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs

@@ -1,8 +1,11 @@
 using Atata.HtmlValidation;
+using Lombiq.Tests.UI.Models;
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Text.Json;
 using System.Threading.Tasks;
 
 namespace Lombiq.Tests.UI.Extensions;
@@ -22,4 +25,46 @@ public static async Task<IEnumerable<string>> GetErrorsAsync(this HtmlValidation
             .Select(error =>
                 (error.StartsWith("error:", StringComparison.OrdinalIgnoreCase) ? string.Empty : "error:") + error);
     }
+
+    /// <summary>
+    /// Gets the parsed errors from the HTML validation result.
+    /// Can only be used if the output formatter is set to JSON.
+    /// </summary>
+    public static IEnumerable<JsonHtmlValidationError> GetParsedErrors(this HtmlValidationResult result) => ParseOutput(result.Output);
+
+    public static string GetParsedErrorMessageString(IEnumerable<JsonHtmlValidationError> errors) =>
+        string.Join(
+            '\n', errors.Select(error =>
+                $"{error.Line.ToString(CultureInfo.InvariantCulture)}:{error.Column.ToString(CultureInfo.InvariantCulture)} - " +
+                $"{error.Message} - " +
+                $"{error.RuleId}"));
+
+    private static IEnumerable<JsonHtmlValidationError> ParseOutput(string output)
+    {
+        try
+        {
+            // In some cases the output is too large and is not a valid JSON anymore. In this case we need to fix it.
+            // tracking issue: https://github.com/atata-framework/atata-htmlvalidation/issues/9
+            if (output.Trim().StartsWith('[') && !output.Trim().EndsWith(']'))
+            {
+                output += "\"}]";
+            }
+
+            var document = JsonDocument.Parse(output);
+            return document.RootElement.EnumerateArray()
+                .SelectMany(element => element.GetProperty("messages").EnumerateArray())
+                .Select(message =>
+                {
+                    var rawMessageText = message.GetRawText();
+                    return JsonSerializer.Deserialize<JsonHtmlValidationError>(rawMessageText);
+                });
+        }
+        catch (JsonException exception)
+        {
+            throw new JsonException(
+                $"Unable to parse output, was OutputFormatter set to JSON? Length: {output.Length} " +
+                $"Output: {output}",
+                exception);
+        }
+    }
 }

Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs

@@ -0,0 +1,28 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Lombiq.Tests.UI.Models;
+
+public class JsonHtmlValidationError
+{
+    [JsonPropertyName("ruleId")]
+    public string RuleId { get; set; }
+    [JsonPropertyName("severity")]
+    public int Severity { get; set; }
+    [JsonPropertyName("message")]
+    public string Message { get; set; }
+    [JsonPropertyName("offset")]
+    public int Offset { get; set; }
+    [JsonPropertyName("line")]
+    public int Line { get; set; }
+    [JsonPropertyName("column")]
+    public int Column { get; set; }
+    [JsonPropertyName("size")]
+    public int Size { get; set; }
+    [JsonPropertyName("selector")]
+    public string Selector { get; set; }
+    [JsonPropertyName("ruleUrl")]
+    public string RuleUrl { get; set; }
+    [JsonPropertyName("context")]
+    public JsonElement Context { get; set; }
+}

Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs

@@ -1,4 +1,6 @@
+using Atata.Cli.HtmlValidate;
 using Atata.HtmlValidation;
+using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Helpers;
 using Shouldly;
 using System;
@@ -29,6 +31,7 @@ public class HtmlValidationConfiguration
     /// </summary>
     public HtmlValidationOptions HtmlValidationOptions { get; set; } = new()
     {
+        OutputFormatter = HtmlValidateFormatter.Names.Json,
         SaveHtmlToFile = HtmlSaveCondition.Never,
         SaveResultToFile = true,
         // This is necessary so no long folder names will be generated, see:
@@ -88,7 +91,18 @@ public HtmlValidationConfiguration WithRelativeConfigPath(params string[] pathSe
     public static readonly Func<HtmlValidationResult, Task> AssertHtmlValidationOutputIsEmptyAsync =
         validationResult =>
         {
-            validationResult.Output.ShouldBeEmpty();
+            // Keep supporting cases where output format is not set to JSON.
+            if (validationResult.Output.Trim().StartsWith('[') ||
+                validationResult.Output.Trim().StartsWith('{'))
+            {
+                var errors = validationResult.GetParsedErrors();
+                errors.ShouldBeEmpty(HtmlValidationResultExtensions.GetParsedErrorMessageString(errors));
+            }
+            else
+            {
+                validationResult.Output.ShouldBeEmpty();
+            }
+
             return Task.CompletedTask;
         };