From 5a078b513fd4955306ecd5c9d9418bcf87416d09 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 2 Nov 2021 20:16:25 +0000 Subject: [PATCH] [wasm] Re-enable source generator tests failing due to OOM (#60701) This adds support for setting per-project optimization flags, and sets them for these projects so they don't OOM. Microsoft.Extensions.Logging.Generators.Roslyn3.11.Tests Microsoft.Extensions.Logging.Generators.Roslyn4.0.Tests System.Text.Json.SourceGeneration.Roslyn3.11.Unit.Tests System.Text.Json.SourceGeneration.Roslyn4.0.Unit.Tests System.Text.RegularExpressions.Generators.Tests Some individual ones are disabled due to #58226, and #60899 . Additionally, if a AOT build fails, then on linux it dumps the last few lines from dmesg, to help identify it was an oom-kill. Fixes #51961 . Co-authored-by: Larry Ewing --- eng/testing/WasmRunnerAOTTemplate.sh | 57 +++++++++++++++++++ eng/testing/tests.targets | 3 +- eng/testing/tests.wasm.targets | 29 +++++++--- ...Logging.Generators.Roslyn3.11.Tests.csproj | 1 + ....Logging.Generators.Roslyn4.0.Tests.csproj | 3 + .../JsonSourceGeneratorDiagnosticsTests.cs | 4 ++ .../JsonSourceGeneratorTests.cs | 1 + ...rceGeneration.Roslyn3.11.Unit.Tests.csproj | 1 + ...urceGeneration.Roslyn4.0.Unit.Tests.csproj | 1 + .../TypeWrapperTests.cs | 2 + .../tests/RegexCultureTests.cs | 1 + ...RegularExpressions.Generators.Tests.csproj | 1 + src/libraries/tests.proj | 9 --- .../Wasm.Browser.Bench.Sample.csproj | 3 +- src/mono/wasm/build/WasmApp.Native.targets | 10 +++- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 1 - src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 17 ++++++ 17 files changed, 120 insertions(+), 24 deletions(-) create mode 100644 eng/testing/WasmRunnerAOTTemplate.sh diff --git a/eng/testing/WasmRunnerAOTTemplate.sh b/eng/testing/WasmRunnerAOTTemplate.sh new file mode 100644 index 0000000000000..6922b550551b8 --- /dev/null +++ b/eng/testing/WasmRunnerAOTTemplate.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +EXECUTION_DIR=$(dirname $0) +SCENARIO=$3 + +cd $EXECUTION_DIR + +if [ -z "$HELIX_WORKITEM_UPLOAD_ROOT" ]; then + XHARNESS_OUT="$EXECUTION_DIR/xharness-output" +else + XHARNESS_OUT="$HELIX_WORKITEM_UPLOAD_ROOT/xharness-output" +fi + +if [ ! -z "$XHARNESS_CLI_PATH" ]; then + # When running in CI, we only have the .NET runtime available + # We need to call the XHarness CLI DLL directly via dotnet exec + HARNESS_RUNNER="dotnet exec $XHARNESS_CLI_PATH" +else + HARNESS_RUNNER="dotnet xharness" +fi + +if [ "$SCENARIO" == "WasmTestOnBrowser" ]; then + XHARNESS_COMMAND="test-browser" +elif [ -z "$XHARNESS_COMMAND" ]; then + XHARNESS_COMMAND="test" +fi + +function _buildAOTFunc() +{ + local projectFile=$1 + local binLog=$2 + shift 2 + + time dotnet msbuild $projectFile /bl:$binLog $* + local buildExitCode=$? + + echo "\n** Performance summary for the build **\n" + dotnet msbuild $binLog -clp:PerformanceSummary -v:q -nologo + if [[ "$(uname -s)" == "Linux" && $buildExitCode -ne 0 ]]; then + echo "\nLast few messages from dmesg:\n" + dmesg | tail -n 20 + fi + + echo + echo + + return $buildExitCode +} + +# RunCommands defined in tests.mobile.targets +[[RunCommands]] + +_exitCode=$? + +echo "XHarness artifacts: $XHARNESS_OUT" + +exit $_exitCode diff --git a/eng/testing/tests.targets b/eng/testing/tests.targets index c59ecf9514735..1d14775dadd90 100644 --- a/eng/testing/tests.targets +++ b/eng/testing/tests.targets @@ -5,7 +5,8 @@ AppleHelixRunnerTemplate.sh AppleRunnerTemplate.sh AndroidRunnerTemplate.sh - WasmRunnerTemplate.sh + WasmRunnerAOTTemplate.sh + WasmRunnerTemplate.sh WasmRunnerTemplate.cmd diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index f68b4f9889a79..00aa7b19f1701 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -38,12 +38,12 @@ - <_AOTBuildCommand>dotnet msbuild publish/ProxyProjectForAOTOnHelix.proj /bl:$XHARNESS_OUT/AOTBuild.binlog + <_AOTBuildCommand>_buildAOTFunc publish/ProxyProjectForAOTOnHelix.proj $XHARNESS_OUT/AOTBuild.binlog <_AOTBuildCommand Condition="'$(ContinuousIntegrationBuild)' != 'true'">$(_AOTBuildCommand) /p:RuntimeSrcDir=$(RepoRoot) /p:RuntimeConfig=$(Configuration) - <_AOTBuildCommand>$(_AOTBuildCommand) /p:RunAOTCompilation=$(RunAOTCompilation) /p:EmccLinkOptimizationFlag='-Oz -Wl%252C-O0 -Wl%252C-lto-O0' + <_AOTBuildCommand>$(_AOTBuildCommand) /p:RunAOTCompilation=$(RunAOTCompilation) <_AOTBuildCommand>$(_AOTBuildCommand) && cd wasm_build/AppBundle $(_AOTBuildCommand) @@ -87,9 +87,15 @@ AssemblyFile="$(WasmBuildTasksAssemblyPath)" /> + + true + -O2 + + <_MainAssemblyPath Condition="'%(WasmAssembliesToBundle.FileName)' == $(AssemblyName) and '%(WasmAssembliesToBundle.Extension)' == '.dll'">%(WasmAssembliesToBundle.Identity) $([System.IO.Path]::ChangeExtension($(_MainAssemblyPath), '.runtimeconfig.json')) + -Oz -Wl%252C-O0 -Wl%252C-lto-O0 @@ -111,15 +117,20 @@ - <_WasmPropertyNames Include="InvariantGlobalization" /> <_WasmPropertyNames Include="AOTMode" /> - <_WasmPropertyNames Include="WasmDebugLevel" /> + <_WasmPropertyNames Include="AssemblyName" /> + <_WasmPropertyNames Include="DisableParallelAot" /> + <_WasmPropertyNames Include="DisableParallelEmccCompile" /> + <_WasmPropertyNames Include="EmccCompileOptimizationFlag" /> + <_WasmPropertyNames Include="EmccLinkOptimizationFlag" /> + <_WasmPropertyNames Include="IncludeSatelliteAssembliesInVFS" /> + <_WasmPropertyNames Include="InvariantGlobalization" /> <_WasmPropertyNames Include="WasmBuildNative" /> - <_WasmPropertyNames Include="_WasmDevel" /> - <_WasmPropertyNames Include="WasmLinkIcalls" /> + <_WasmPropertyNames Include="WasmDebugLevel" /> <_WasmPropertyNames Include="WasmDedup" /> - <_WasmPropertyNames Include="IncludeSatelliteAssembliesInVFS" /> - <_WasmPropertyNames Include="AssemblyName" /> + <_WasmPropertyNames Include="WasmLinkIcalls" /> + <_WasmPropertyNames Include="WasmNativeStrip" /> + <_WasmPropertyNames Include="_WasmDevel" /> <_WasmPropertiesToPass Include="$(%(_WasmPropertyNames.Identity))" @@ -159,7 +170,7 @@ -1 - + <_SatelliteAssemblies Include="$(PublishDir)*\*.resources.dll" /> <_SatelliteAssemblies CultureName="$([System.IO.Directory]::GetParent('%(Identity)').Name)" /> <_SatelliteAssemblies TargetPath="%(CultureName)\%(FileName)%(Extension)" /> diff --git a/src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn3.11.Tests.csproj b/src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn3.11.Tests.csproj index bb4f032a25357..e3ac6c40c52f2 100644 --- a/src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn3.11.Tests.csproj +++ b/src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn3.11.Tests.csproj @@ -2,6 +2,7 @@ $(MicrosoftCodeAnalysisCSharpWorkspacesVersion_3_11) + true diff --git a/src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn4.0.Tests.csproj b/src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn4.0.Tests.csproj index 1c826c0b30a3f..074da27c19e02 100644 --- a/src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn4.0.Tests.csproj +++ b/src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn4.0.Tests.csproj @@ -3,6 +3,9 @@ $(MicrosoftCodeAnalysisCSharpWorkspacesVersion) $(DefineConstants);ROSLYN4_0_OR_GREATER + true + -O1 + false diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs index 470c83408c2c1..a25196a34fe07 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs @@ -125,6 +125,7 @@ public class IndexViewModel } [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] public void NameClashSourceGeneration() { // Without resolution. @@ -149,6 +150,7 @@ public void NameClashSourceGeneration() } [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] public void ProgramsThatDontUseGeneratorCompile() { // No STJ usage. @@ -192,6 +194,7 @@ public static void Main() } [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] public void WarnOnClassesWithInitOnlyProperties() { Compilation compilation = CompilationHelper.CreateCompilationWithInitOnlyProperties(); @@ -206,6 +209,7 @@ public void WarnOnClassesWithInitOnlyProperties() } [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] public void WarnOnClassesWithInaccessibleJsonIncludeProperties() { Compilation compilation = CompilationHelper.CreateCompilationWithInaccessibleJsonIncludeProperties(); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs index 105dc91af17b3..08519e16fde39 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs @@ -11,6 +11,7 @@ namespace System.Text.Json.SourceGeneration.UnitTests { + [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] public class GeneratorTests { [Fact] diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Roslyn3.11.Unit.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Roslyn3.11.Unit.Tests.csproj index 5a99fa881f97f..af08a0ae33acc 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Roslyn3.11.Unit.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Roslyn3.11.Unit.Tests.csproj @@ -1,6 +1,7 @@ $(MicrosoftCodeAnalysisCSharpWorkspacesVersion_3_11) + true diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Roslyn4.0.Unit.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Roslyn4.0.Unit.Tests.csproj index 073d9ca3126c7..709f667c267a2 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Roslyn4.0.Unit.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/System.Text.Json.SourceGeneration.Roslyn4.0.Unit.Tests.csproj @@ -2,6 +2,7 @@ $(MicrosoftCodeAnalysisCSharpWorkspacesVersion) $(DefineConstants);ROSLYN4_0_OR_GREATER + true diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/TypeWrapperTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/TypeWrapperTests.cs index 24a86137bc175..ca6d9dce1bd1e 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/TypeWrapperTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/TypeWrapperTests.cs @@ -14,6 +14,7 @@ namespace System.Text.Json.SourceGeneration.UnitTests public class TypeWrapperTests { [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] public void MetadataLoadFilePathHandle() { // Create a MetadataReference from new code. @@ -79,6 +80,7 @@ public void MySecondMethod() { } } [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] public void CanGetAttributes() { string source = @" diff --git a/src/libraries/System.Text.RegularExpressions/tests/RegexCultureTests.cs b/src/libraries/System.Text.RegularExpressions/tests/RegexCultureTests.cs index ac0bc82bff7a7..adcde90c42b97 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/RegexCultureTests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/RegexCultureTests.cs @@ -325,6 +325,7 @@ public static IEnumerable Match_In_Different_Cultures_CriticalCases_Te public static IEnumerable Match_In_Different_Cultures_CriticalCases_TestData() => Match_In_Different_Cultures_CriticalCases_TestData_For(RegexOptions.None).Union(Match_In_Different_Cultures_CriticalCases_TestData_For(RegexOptions.Compiled)); + [ActiveIssue("https://github.com/dotnet/runtime/issues/60899", TestPlatforms.Browser)] [Theory] [MemberData(nameof(Match_In_Different_Cultures_TestData))] [ActiveIssue("https://github.com/dotnet/runtime/issues/60697", TestPlatforms.iOS | TestPlatforms.tvOS)] diff --git a/src/libraries/System.Text.RegularExpressions/tests/System.Text.RegularExpressions.Generators.Tests/System.Text.RegularExpressions.Generators.Tests.csproj b/src/libraries/System.Text.RegularExpressions/tests/System.Text.RegularExpressions.Generators.Tests/System.Text.RegularExpressions.Generators.Tests.csproj index 82dfb654c2e4a..17a78af31cc11 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/System.Text.RegularExpressions.Generators.Tests/System.Text.RegularExpressions.Generators.Tests.csproj +++ b/src/libraries/System.Text.RegularExpressions/tests/System.Text.RegularExpressions.Generators.Tests/System.Text.RegularExpressions.Generators.Tests.csproj @@ -7,6 +7,7 @@ enable $(NoWarn);xUnit2008 + true diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index d748ceaba2fb8..ce8b72544865a 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -14,9 +14,6 @@ - - - @@ -272,12 +269,6 @@ - - - - - - diff --git a/src/mono/sample/wasm/browser-bench/Wasm.Browser.Bench.Sample.csproj b/src/mono/sample/wasm/browser-bench/Wasm.Browser.Bench.Sample.csproj index 01b36c00dc7ce..ff9f990c78caf 100644 --- a/src/mono/sample/wasm/browser-bench/Wasm.Browser.Bench.Sample.csproj +++ b/src/mono/sample/wasm/browser-bench/Wasm.Browser.Bench.Sample.csproj @@ -1,6 +1,7 @@ - true + + false runtime.js diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 69facc5d7204a..8476977afef42 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -152,6 +152,8 @@ <_WasmPInvokeTablePath>$(_WasmIntermediateOutputPath)pinvoke-table.h <_WasmPInvokeHPath>$(_WasmRuntimePackIncludeDir)wasm\pinvoke.h <_DriverGenCPath>$(_WasmIntermediateOutputPath)driver-gen.c + false + $(DisableParallelAot) <_DriverGenCNeeded Condition="'$(_DriverGenCNeeded)' == '' and '$(_WasmShouldAOT)' == 'true'">true @@ -283,7 +285,7 @@ - + <_WasmSourceFileToCompile Remove="@(_WasmSourceFileToCompile)" /> <_WasmSourceFileToCompile Include="@(_WasmRuntimePackSrcFile)" Dependencies="%(_WasmRuntimePackSrcFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileRsp)" /> @@ -292,6 +294,7 @@ SourceFiles="@(_WasmSourceFileToCompile)" Arguments='"@$(_EmccDefaultFlagsRsp)" "@$(_EmccCompileRsp)"' EnvironmentVariables="@(EmscriptenEnvVars)" + DisableParallelCompile="$(DisableParallelEmccCompile)" OutputMessageImportance="$(_EmccCompileOutputMessageImportance)"> @@ -308,11 +311,12 @@ <_BitCodeFile Dependencies="%(_BitCodeFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)" /> - + @@ -371,7 +375,7 @@ DependsOnTargets="_WasmSelectRuntimeComponentsForLinking;_WasmCompileAssemblyBitCodeFilesForAOT;_WasmWriteRspFilesForLinking" Returns="@(FileWrites)" > - + diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index e8c3680dd6e4f..dee6b537b7bd5 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -395,7 +395,6 @@ private bool ExecuteInternal() new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, (args, state) => PrecompileLibraryParallel(args, state)); - Log.LogMessage(MessageImportance.High, $"result: {result.IsCompleted}"); if (result.IsCompleted) { int numUnchanged = _totalNumAssemblies - _numCompiled; diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index e356291d83a31..4c3054cad30e9 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Collections.Specialized; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; @@ -222,6 +223,7 @@ private bool ExecuteInternal () Directory.CreateDirectory(supportFilesDir); var i = 0; + StringDictionary targetPathTable = new(); foreach (var item in FilesToIncludeInFileSystem) { string? targetPath = item.GetMetadata("TargetPath"); @@ -232,6 +234,21 @@ private bool ExecuteInternal () // We normalize paths from `\` to `/` as MSBuild items could use `\`. targetPath = targetPath.Replace('\\', '/'); + if (targetPathTable.ContainsKey(targetPath)) + { + string firstPath = Path.GetFullPath(targetPathTable[targetPath]!); + string secondPath = Path.GetFullPath(item.ItemSpec); + + if (firstPath == secondPath) + { + Log.LogWarning($"Found identical vfs mappings for target path: {targetPath}, source file: {firstPath}. Ignoring."); + continue; + } + + throw new LogAsErrorException($"Found more than one file mapping to the target VFS path: {targetPath}. Source files: {firstPath}, and {secondPath}"); + } + + targetPathTable[targetPath] = item.ItemSpec; var generatedFileName = $"{i++}_{Path.GetFileName(item.ItemSpec)}";