AppDomain configuration is serialized without using BinFmt

documentation/wiki/ChangeWaves.md

@@ -23,6 +23,9 @@ A wave of features is set to "rotate out" (i.e. become standard functionality) t
 
 ## Current Rotation of Change Waves
 
+### 17.10
+- [AppDomain configuration is serialized without using BinFmt](https://github.com/dotnet/msbuild/pull/9320) - feature can be opted out only if [BinaryFormatter](https://learn.microsoft.com/en-us/dotnet/api/system.runtime.serialization.formatters.binary.binaryformatter) is allowed at runtime by editing `MSBuild.runtimeconfig.json`
+
 ### 17.8
 - [[RAR] Don't do I/O on SDK-provided references](https://github.com/dotnet/msbuild/pull/8688)
 - [Delete destination file before copy](https://github.com/dotnet/msbuild/pull/8685)

src/Build.UnitTests/BackEnd/NodeConfiguration_Tests.cs

@@ -0,0 +1,59 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Build.BackEnd;
+using Microsoft.Build.Execution;
+using Microsoft.Build.Logging;
+using Microsoft.Build.UnitTests.BackEnd;
+using Shouldly;
+using Xunit;
+
+namespace Microsoft.Build.Engine.UnitTests.BackEnd
+{
+    public class NodeConfiguration_Tests
+    {
+#if FEATURE_APPDOMAIN
+        /// <summary>
+        /// Test serialization / deserialization of the AppDomainSetup instance. 
+        /// </summary>
+        [Theory]
+        [InlineData(new byte[] { 1, 2, 3 })]
+        [InlineData(null)]
+        public void TestTranslationWithAppDomainSetup(byte[] configBytes)
+        {
+            AppDomainSetup setup = new AppDomainSetup();
+
+            NodeConfiguration config = new NodeConfiguration(
+                nodeId: 1,
+                buildParameters: new BuildParameters(),
+                forwardingLoggers: Array.Empty<LoggerDescription>(),
+                appDomainSetup: setup,
+                loggingNodeConfiguration: new LoggingNodeConfiguration());
+
+            setup.SetConfigurationBytes(configBytes);
+
+            ((ITranslatable)config).Translate(TranslationHelpers.GetWriteTranslator());
+            INodePacket packet = NodeConfiguration.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
+
+            packet.ShouldBeOfType<NodeConfiguration>();
+            NodeConfiguration deserializedConfig = (NodeConfiguration)packet;
+
+            deserializedConfig.AppDomainSetup.ShouldNotBeNull();
+
+            if (configBytes is null)
+            {
+                deserializedConfig.AppDomainSetup.GetConfigurationBytes().ShouldBeNull();
+            }
+            else
+            {
+                deserializedConfig.AppDomainSetup.GetConfigurationBytes().SequenceEqual(configBytes).ShouldBeTrue();
+            }
+        }
+#endif
+    }
+}

src/Build.UnitTests/BackEnd/TaskHostConfiguration_Tests.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 
 
@@ -376,6 +377,57 @@ public void TestTranslationWithNullDictionary()
             Assert.Equal(expectedGlobalProperties, deserializedConfig.GlobalProperties);
         }
 
+#if FEATURE_APPDOMAIN
+        /// <summary>
+        /// Test serialization / deserialization of the AppDomainSetup instance. 
+        /// </summary>
+        [Theory]
+        [InlineData(new byte[] { 1, 2, 3 })]
+        [InlineData(null)]
+        public void TestTranslationWithAppDomainSetup(byte[] configBytes)
+        {
+            AppDomainSetup setup = new AppDomainSetup();
+
+            TaskHostConfiguration config = new TaskHostConfiguration(
+                nodeId: 1,
+                startupDirectory: Directory.GetCurrentDirectory(),
+                buildProcessEnvironment: null,
+                culture: Thread.CurrentThread.CurrentCulture,
+                uiCulture: Thread.CurrentThread.CurrentUICulture,
+                appDomainSetup: setup,
+                lineNumberOfTask: 1,
+                columnNumberOfTask: 1,
+                projectFileOfTask: @"c:\my project\myproj.proj",
+                continueOnError: _continueOnErrorDefault,
+                taskName: "TaskName",
+                taskLocation: @"c:\MyTasks\MyTask.dll",
+                isTaskInputLoggingEnabled: false,
+                taskParameters: new Dictionary<string, object>(),
+                globalParameters: new Dictionary<string, string>(),
+                warningsAsErrors: null,
+                warningsNotAsErrors: null,
+                warningsAsMessages: null);
+
+            setup.SetConfigurationBytes(configBytes);
+
+            ((ITranslatable)config).Translate(TranslationHelpers.GetWriteTranslator());
+            INodePacket packet = TaskHostConfiguration.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
+
+            TaskHostConfiguration deserializedConfig = packet as TaskHostConfiguration;
+
+            deserializedConfig.AppDomainSetup.ShouldNotBeNull();
+
+            if (configBytes is null)
+            {
+                deserializedConfig.AppDomainSetup.GetConfigurationBytes().ShouldBeNull();
+            }
+            else
+            {
+                deserializedConfig.AppDomainSetup.GetConfigurationBytes().SequenceEqual(configBytes).ShouldBeTrue();
+            }
+        }
+#endif
+
         /// <summary>
         /// Test serialization / deserialization when the parameter dictionary is empty. 
         /// </summary>

src/Build/BackEnd/Node/NodeConfiguration.cs

@@ -7,6 +7,7 @@
 using System.Diagnostics;
 
 using Microsoft.Build.Execution;
+using Microsoft.Build.Framework;
 using Microsoft.Build.Logging;
 #nullable disable
 
@@ -161,7 +162,28 @@ public void Translate(ITranslator translator)
             translator.Translate(ref _buildParameters, BuildParameters.FactoryForDeserialization);
             translator.TranslateArray(ref _forwardingLoggers, LoggerDescription.FactoryForTranslation);
 #if FEATURE_APPDOMAIN
-            translator.TranslateDotNet(ref _appDomainSetup);
+            if (ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_10) || !Traits.Instance.EscapeHatches.IsBinaryFormatterSerializationAllowed)
+            {
+                byte[] appDomainConfigBytes = null;
+
+                // Set the configuration bytes just before serialization in case the SetConfigurationBytes was invoked during lifetime of this instance.
+                if (translator.Mode == TranslationDirection.WriteToStream)
+                {
+                    appDomainConfigBytes = _appDomainSetup?.GetConfigurationBytes();
+                }
+
+                translator.Translate(ref appDomainConfigBytes);
+
+                if (translator.Mode == TranslationDirection.ReadFromStream)
+                {
+                    _appDomainSetup = new AppDomainSetup();
+                    _appDomainSetup.SetConfigurationBytes(appDomainConfigBytes);
+                }
+            }
+            else
+            {
+                translator.TranslateDotNet(ref _appDomainSetup);
+            }
 #endif
             translator.Translate(ref _loggingNodeConfiguration);
         }
@@ -173,6 +195,7 @@ internal static INodePacket FactoryForDeserialization(ITranslator translator)
         {
             NodeConfiguration configuration = new NodeConfiguration();
             configuration.Translate(translator);
+
             return configuration;
         }
         #endregion

src/Framework/ChangeWaves.cs

@@ -27,7 +27,8 @@ internal class ChangeWaves
         internal static readonly Version Wave17_4 = new Version(17, 4);
         internal static readonly Version Wave17_6 = new Version(17, 6);
         internal static readonly Version Wave17_8 = new Version(17, 8);
-        internal static readonly Version[] AllWaves = { Wave17_4, Wave17_6, Wave17_8 };
+        internal static readonly Version Wave17_10 = new Version(17, 10);
+        internal static readonly Version[] AllWaves = { Wave17_4, Wave17_6, Wave17_8, Wave17_10 };
 
         /// <summary>
         /// Special value indicating that all features behind all Change Waves should be enabled.

src/Shared/TaskHostConfiguration.cs

@@ -5,7 +5,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
-
+using Microsoft.Build.Framework;
 using Microsoft.Build.Shared;
 
 #nullable disable
@@ -417,7 +417,28 @@ public void Translate(ITranslator translator)
             translator.TranslateCulture(ref _culture);
             translator.TranslateCulture(ref _uiCulture);
 #if FEATURE_APPDOMAIN
-            translator.TranslateDotNet(ref _appDomainSetup);
+            if (ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_10) || !Traits.Instance.EscapeHatches.IsBinaryFormatterSerializationAllowed)
+            {
+                byte[] appDomainConfigBytes = null;
+
+                // Set the configuration bytes just before serialization in case the SetConfigurationBytes was invoked during lifetime of this instance.
+                if (translator.Mode == TranslationDirection.WriteToStream)
+                {
+                    appDomainConfigBytes = _appDomainSetup?.GetConfigurationBytes();
+                }
+
+                translator.Translate(ref appDomainConfigBytes);
+
+                if (translator.Mode == TranslationDirection.ReadFromStream)
+                {
+                    _appDomainSetup = new AppDomainSetup();
+                    _appDomainSetup.SetConfigurationBytes(appDomainConfigBytes);
+                }
+            }
+            else
+            {
+                translator.TranslateDotNet(ref _appDomainSetup);
+            }
 #endif
             translator.Translate(ref _lineNumberOfTask);
             translator.Translate(ref _columnNumberOfTask);
@@ -458,6 +479,7 @@ internal static INodePacket FactoryForDeserialization(ITranslator translator)
         {
             TaskHostConfiguration configuration = new TaskHostConfiguration();
             configuration.Translate(translator);
+
             return configuration;
         }
     }