From 7aba2caa536e168f93da3a23e427887521f57a4d Mon Sep 17 00:00:00 2001 From: Raul Santos Date: Wed, 6 Mar 2024 05:10:50 +0100 Subject: [PATCH 01/30] Add ClassName property to ResxSourceGenerator --- .../Resources.Designer.cs | 21 +++++++ .../Resources.Designer.cs | 23 +++++++ .../Resources.Designer.cs | 24 +++++++ .../Resources.Designer.vb | 33 ++++++++++ .../Resources.Designer.vb | 36 +++++++++++ .../Resources.Designer.vb | 36 +++++++++++ .../ResxGeneratorTests.cs | 62 +++++++++++++++++++ .../AbstractResxGenerator.cs | 4 +- .../Program.cs | 1 + 9 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync/Resources.Designer.cs create mode 100644 src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync_NS/Resources.Designer.cs create mode 100644 src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync_NS1.NS2/Resources.Designer.cs create mode 100644 src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync/Resources.Designer.vb create mode 100644 src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync_NS/Resources.Designer.vb create mode 100644 src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync_NS1.NS2/Resources.Designer.vb diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync/Resources.Designer.cs b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync/Resources.Designer.cs new file mode 100644 index 0000000000..c1c925afa5 --- /dev/null +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync/Resources.Designer.cs @@ -0,0 +1,21 @@ +// + +#nullable enable +using System.Reflection; + + +namespace TestProject +{ + internal static partial class Resources + { + private static global::System.Resources.ResourceManager? s_resourceManager; + public static global::System.Resources.ResourceManager ResourceManager => s_resourceManager ?? (s_resourceManager = new global::System.Resources.ResourceManager(typeof(Resources))); + public static global::System.Globalization.CultureInfo? Culture { get; set; } + [global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + [return: global::System.Diagnostics.CodeAnalysis.NotNullIfNotNull("defaultValue")] + internal static string? GetResourceString(string resourceKey, string? defaultValue = null) => ResourceManager.GetString(resourceKey, Culture) ?? defaultValue; + /// value + public static string @Name => GetResourceString("Name")!; + + } +} diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync_NS/Resources.Designer.cs b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync_NS/Resources.Designer.cs new file mode 100644 index 0000000000..88c5228568 --- /dev/null +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync_NS/Resources.Designer.cs @@ -0,0 +1,23 @@ +// + +#nullable enable +using System.Reflection; + +namespace TestProject +{ + internal static class Resources { } +} + +internal static partial class NS +{ + private static global::System.Resources.ResourceManager? s_resourceManager; + public static global::System.Resources.ResourceManager ResourceManager => s_resourceManager ?? (s_resourceManager = new global::System.Resources.ResourceManager(typeof(TestProject.Resources))); + public static global::System.Globalization.CultureInfo? Culture { get; set; } + [global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + [return: global::System.Diagnostics.CodeAnalysis.NotNullIfNotNull("defaultValue")] + internal static string? GetResourceString(string resourceKey, string? defaultValue = null) => ResourceManager.GetString(resourceKey, Culture) ?? defaultValue; + /// value + public static string @Name => GetResourceString("Name")!; + +} + diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync_NS1.NS2/Resources.Designer.cs b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync_NS1.NS2/Resources.Designer.cs new file mode 100644 index 0000000000..4ded4b004c --- /dev/null +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameCSharpAsync_NS1.NS2/Resources.Designer.cs @@ -0,0 +1,24 @@ +// + +#nullable enable +using System.Reflection; + +namespace TestProject +{ + internal static class Resources { } +} +namespace NS1 +{ + internal static partial class NS2 + { + private static global::System.Resources.ResourceManager? s_resourceManager; + public static global::System.Resources.ResourceManager ResourceManager => s_resourceManager ?? (s_resourceManager = new global::System.Resources.ResourceManager(typeof(TestProject.Resources))); + public static global::System.Globalization.CultureInfo? Culture { get; set; } + [global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)] + [return: global::System.Diagnostics.CodeAnalysis.NotNullIfNotNull("defaultValue")] + internal static string? GetResourceString(string resourceKey, string? defaultValue = null) => ResourceManager.GetString(resourceKey, Culture) ?? defaultValue; + /// value + public static string @Name => GetResourceString("Name")!; + + } +} diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync/Resources.Designer.vb b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync/Resources.Designer.vb new file mode 100644 index 0000000000..7e5e0abe93 --- /dev/null +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync/Resources.Designer.vb @@ -0,0 +1,33 @@ +' + +Imports System.Reflection + + +Namespace Global.TestProject + Friend Partial Class Resources + Private Sub New + End Sub + + Private Shared s_resourceManager As Global.System.Resources.ResourceManager + Public Shared ReadOnly Property ResourceManager As Global.System.Resources.ResourceManager + Get + If s_resourceManager Is Nothing Then + s_resourceManager = New Global.System.Resources.ResourceManager(GetType(Resources)) + End If + Return s_resourceManager + End Get + End Property + Public Shared Property Culture As Global.System.Globalization.CultureInfo + + Friend Shared Function GetResourceString(ByVal resourceKey As String, Optional ByVal defaultValue As String = Nothing) As String + Return ResourceManager.GetString(resourceKey, Culture) + End Function + ''' value + Public Shared ReadOnly Property [Name] As String + Get + Return GetResourceString("Name") + End Get + End Property + + End Class +End Namespace diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync_NS/Resources.Designer.vb b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync_NS/Resources.Designer.vb new file mode 100644 index 0000000000..84a8ca15a6 --- /dev/null +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync_NS/Resources.Designer.vb @@ -0,0 +1,36 @@ +' + +Imports System.Reflection + +Namespace TestProject + Friend Class Resources + End Class +End Namespace + +Friend Partial Class NS + Private Sub New + End Sub + + Private Shared s_resourceManager As Global.System.Resources.ResourceManager + Public Shared ReadOnly Property ResourceManager As Global.System.Resources.ResourceManager + Get + If s_resourceManager Is Nothing Then + s_resourceManager = New Global.System.Resources.ResourceManager(GetType(TestProject.Resources)) + End If + Return s_resourceManager + End Get + End Property + Public Shared Property Culture As Global.System.Globalization.CultureInfo + + Friend Shared Function GetResourceString(ByVal resourceKey As String, Optional ByVal defaultValue As String = Nothing) As String + Return ResourceManager.GetString(resourceKey, Culture) + End Function + ''' value + Public Shared ReadOnly Property [Name] As String + Get + Return GetResourceString("Name") + End Get + End Property + +End Class + diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync_NS1.NS2/Resources.Designer.vb b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync_NS1.NS2/Resources.Designer.vb new file mode 100644 index 0000000000..866c481c2e --- /dev/null +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Resources/SingleString_ClassNameVisualBasicAsync_NS1.NS2/Resources.Designer.vb @@ -0,0 +1,36 @@ +' + +Imports System.Reflection + +Namespace TestProject + Friend Class Resources + End Class +End Namespace +Namespace Global.NS1 + Friend Partial Class NS2 + Private Sub New + End Sub + + Private Shared s_resourceManager As Global.System.Resources.ResourceManager + Public Shared ReadOnly Property ResourceManager As Global.System.Resources.ResourceManager + Get + If s_resourceManager Is Nothing Then + s_resourceManager = New Global.System.Resources.ResourceManager(GetType(TestProject.Resources)) + End If + Return s_resourceManager + End Get + End Property + Public Shared Property Culture As Global.System.Globalization.CultureInfo + + Friend Shared Function GetResourceString(ByVal resourceKey As String, Optional ByVal defaultValue As String = Nothing) As String + Return ResourceManager.GetString(resourceKey, Culture) + End Function + ''' value + Public Shared ReadOnly Property [Name] As String + Get + Return GetResourceString("Name") + End Get + End Property + + End Class +End Namespace diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/ResxGeneratorTests.cs b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/ResxGeneratorTests.cs index 30e2bd9fe0..47edb82fca 100644 --- a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/ResxGeneratorTests.cs +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/ResxGeneratorTests.cs @@ -364,6 +364,68 @@ public async Task SingleString_RelativeDirVisualBasicAsync(string relativeDir) }.AddGeneratedSources().RunAsync(); } + [Theory] + [InlineData("")] + [InlineData("NS")] + [InlineData("NS1.NS2")] + public async Task SingleString_ClassNameCSharpAsync(string className) + { + var code = ResxHeader + + @" + value + comment + " + + ResxFooter; + + await new VerifyCS.Test(identifier: className) + { + TestState = + { + AdditionalFiles = { ("/0/Resources.resx", code) }, + AnalyzerConfigFiles = + { + ("/.globalconfig", $@" +is_global = true + +[/0/Resources.resx] +build_metadata.AdditionalFiles.ClassName = {className} +"), + }, + }, + }.AddGeneratedSources().RunAsync(); + } + + [Theory] + [InlineData("")] + [InlineData("NS")] + [InlineData("NS1.NS2")] + public async Task SingleString_ClassNameVisualBasicAsync(string className) + { + var code = ResxHeader + + @" + value + comment + " + + ResxFooter; + + await new VerifyVB.Test(identifier: className) + { + TestState = + { + AdditionalFiles = { ("/0/Resources.resx", code) }, + AnalyzerConfigFiles = + { + ("/.globalconfig", $@" +is_global = true + +[/0/Resources.resx] +build_metadata.AdditionalFiles.ClassName = {className} +"), + }, + }, + }.AddGeneratedSources().RunAsync(); + } + [Theory] [CombinatorialData] public async Task SingleString_OmitGetResourceStringCSharpAsync(bool omitGetResourceString) diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator/AbstractResxGenerator.cs b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator/AbstractResxGenerator.cs index 9646708469..8cabf7b520 100644 --- a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator/AbstractResxGenerator.cs +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator/AbstractResxGenerator.cs @@ -83,6 +83,8 @@ public void Initialize(IncrementalGeneratorInitializationContext context) resourceName = relativeDir.Replace(Path.DirectorySeparatorChar, '.').Replace(Path.AltDirectorySeparatorChar, '.') + resourceName; } + options.TryGetValue("build_metadata.AdditionalFiles.ClassName", out var resourceClassName); + if (!options.TryGetValue("build_metadata.AdditionalFiles.OmitGetResourceString", out var omitGetResourceStringText) || !bool.TryParse(omitGetResourceStringText, out var omitGetResourceString)) { @@ -120,7 +122,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context) ResourceFile: resourceFile, ResourceName: string.Join(".", rootNamespace, resourceName), ResourceHintName: resourceHintName, - ResourceClassName: null, + ResourceClassName: resourceClassName, OmitGetResourceString: omitGetResourceString, AsConstants: asConstants, IncludeDefaultValues: includeDefaultValues, diff --git a/src/Tools/GenerateDocumentationAndConfigFiles/Program.cs b/src/Tools/GenerateDocumentationAndConfigFiles/Program.cs index 4bbe407afc..d14521efe2 100644 --- a/src/Tools/GenerateDocumentationAndConfigFiles/Program.cs +++ b/src/Tools/GenerateDocumentationAndConfigFiles/Program.cs @@ -343,6 +343,7 @@ string getCompilerVisibleProperties() + From aba7cf1bdb0040d22e11407f266cdd469e9db2ec Mon Sep 17 00:00:00 2001 From: DoctorKrolic Date: Wed, 17 Apr 2024 17:32:52 +0300 Subject: [PATCH 02/30] Avoid direct descriptor comparison, so the fix can be shown --- .../CompareSymbolsCorrectlyAnalyzer.cs | 18 ++++++---- .../Fixers/CompareSymbolsCorrectlyFix.cs | 36 ++++++++++--------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/CompareSymbolsCorrectlyAnalyzer.cs b/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/CompareSymbolsCorrectlyAnalyzer.cs index f0e3be553e..f3a99ee10f 100644 --- a/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/CompareSymbolsCorrectlyAnalyzer.cs +++ b/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/CompareSymbolsCorrectlyAnalyzer.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. +using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.Linq; @@ -26,7 +27,9 @@ public class CompareSymbolsCorrectlyAnalyzer : DiagnosticAnalyzer private static readonly string s_symbolTypeFullName = typeof(ISymbol).FullName; private const string s_symbolEqualsName = nameof(ISymbol.Equals); private const string s_HashCodeCombineName = "Combine"; + public const string SymbolEqualityComparerName = "Microsoft.CodeAnalysis.SymbolEqualityComparer"; + public const string RulePropertyName = "Rule"; public static readonly DiagnosticDescriptor EqualityRule = new( DiagnosticIds.CompareSymbolsCorrectlyRuleId, @@ -142,7 +145,7 @@ private static void HandleBinaryOperator(in OperationAnalysisContext context, IN return; } - context.ReportDiagnostic(binary.Syntax.GetLocation().CreateDiagnostic(EqualityRule)); + context.ReportDiagnostic(binary.Syntax.GetLocation().CreateDiagnostic(EqualityRule, MakeProperties(nameof(EqualityRule)))); } private static void HandleInvocationOperation( @@ -163,7 +166,7 @@ private static void HandleInvocationOperation( // without the correct arguments if (IsSymbolType(invocationOperation.Instance, symbolType)) { - context.ReportDiagnostic(invocationOperation.CreateDiagnostic(GetHashCodeRule)); + context.ReportDiagnostic(invocationOperation.CreateDiagnostic(GetHashCodeRule, MakeProperties(nameof(GetHashCode)))); } break; @@ -174,7 +177,7 @@ private static void HandleInvocationOperation( var parameters = invocationOperation.Arguments; if (parameters.All(p => IsSymbolType(p.Value, symbolType))) { - context.ReportDiagnostic(invocationOperation.Syntax.GetLocation().CreateDiagnostic(EqualityRule)); + context.ReportDiagnostic(invocationOperation.Syntax.GetLocation().CreateDiagnostic(EqualityRule, MakeProperties(nameof(EqualityRule)))); } } @@ -187,7 +190,7 @@ invocationOperation.Instance is null && systemHashCodeType.Equals(method.ContainingType, SymbolEqualityComparer.Default) && invocationOperation.Arguments.Any(arg => IsSymbolType(arg.Value, symbolType))) { - context.ReportDiagnostic(invocationOperation.CreateDiagnostic(GetHashCodeRule)); + context.ReportDiagnostic(invocationOperation.CreateDiagnostic(GetHashCodeRule, MakeProperties(nameof(GetHashCodeRule)))); } break; @@ -199,7 +202,7 @@ invocationOperation.Instance is null && IsBehavingOnSymbolType(method, symbolType) && !invocationOperation.Arguments.Any(arg => IsSymbolType(arg.Value, iEqualityComparer))) { - context.ReportDiagnostic(invocationOperation.CreateDiagnostic(CollectionRule)); + context.ReportDiagnostic(invocationOperation.CreateDiagnostic(CollectionRule, MakeProperties(nameof(CollectionRule)))); } break; @@ -249,7 +252,7 @@ private static void HandleObjectCreation(in OperationAnalysisContext context, IN IsSymbolType(createdType.TypeArguments[0], symbolType) && !objectCreation.Arguments.Any(arg => IsSymbolType(arg.Value, iEqualityComparerType))) { - context.ReportDiagnostic(objectCreation.CreateDiagnostic(CollectionRule)); + context.ReportDiagnostic(objectCreation.CreateDiagnostic(CollectionRule, MakeProperties(nameof(CollectionRule)))); } } @@ -354,6 +357,9 @@ void AddOrUpdate(string methodName, INamedTypeSymbol typeSymbol) } } + private static ImmutableDictionary MakeProperties(string rule) + => ImmutableDictionary.CreateRange([new KeyValuePair(RulePropertyName, rule)]); + public static bool UseSymbolEqualityComparer(Compilation compilation) => compilation.GetOrCreateTypeByMetadataName(SymbolEqualityComparerName) is object; } diff --git a/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/Fixers/CompareSymbolsCorrectlyFix.cs b/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/Fixers/CompareSymbolsCorrectlyFix.cs index fe9a8fc585..4a1a007df6 100644 --- a/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/Fixers/CompareSymbolsCorrectlyFix.cs +++ b/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/Fixers/CompareSymbolsCorrectlyFix.cs @@ -32,23 +32,27 @@ public sealed override Task RegisterCodeFixesAsync(CodeFixContext context) { foreach (var diagnostic in context.Diagnostics) { - if (diagnostic.Descriptor == CompareSymbolsCorrectlyAnalyzer.EqualityRule) + if (diagnostic.Properties.TryGetValue(CompareSymbolsCorrectlyAnalyzer.RulePropertyName, out var rule)) { - context.RegisterCodeFix( - CodeAction.Create( - CodeAnalysisDiagnosticsResources.CompareSymbolsCorrectlyCodeFix, - cancellationToken => ConvertToEqualsAsync(context.Document, diagnostic.Location.SourceSpan, cancellationToken), - equivalenceKey: nameof(CompareSymbolsCorrectlyFix)), - diagnostic); - } - else if (diagnostic.Descriptor == CompareSymbolsCorrectlyAnalyzer.CollectionRule) - { - context.RegisterCodeFix( - CodeAction.Create( - CodeAnalysisDiagnosticsResources.CompareSymbolsCorrectlyCodeFix, - cancellationToken => CallOverloadWithEqualityComparerAsync(context.Document, diagnostic.Location.SourceSpan, cancellationToken), - equivalenceKey: nameof(CompareSymbolsCorrectlyFix)), - diagnostic); + switch (rule) + { + case nameof(CompareSymbolsCorrectlyAnalyzer.EqualityRule): + context.RegisterCodeFix( + CodeAction.Create( + CodeAnalysisDiagnosticsResources.CompareSymbolsCorrectlyCodeFix, + cancellationToken => ConvertToEqualsAsync(context.Document, diagnostic.Location.SourceSpan, cancellationToken), + equivalenceKey: nameof(CompareSymbolsCorrectlyFix)), + diagnostic); + break; + case nameof(CompareSymbolsCorrectlyAnalyzer.CollectionRule): + context.RegisterCodeFix( + CodeAction.Create( + CodeAnalysisDiagnosticsResources.CompareSymbolsCorrectlyCodeFix, + cancellationToken => CallOverloadWithEqualityComparerAsync(context.Document, diagnostic.Location.SourceSpan, cancellationToken), + equivalenceKey: nameof(CompareSymbolsCorrectlyFix)), + diagnostic); + break; + } } } From a647f3a4b2fb2c3ce9baf9aa6d750a9f80662f6b Mon Sep 17 00:00:00 2001 From: Raul Santos Date: Thu, 18 Apr 2024 17:22:06 +0200 Subject: [PATCH 03/30] Fix some typos --- .../ImplementStandardExceptionConstructors.cs | 2 +- .../AvoidMultipleEnumerationsHelpers.cs | 4 ++-- .../FlowAnalysis/GlobalFlowStateDictionaryAnalysis.cs | 2 +- .../InteropServices/PlatformCompatibilityAnalyzer.cs | 2 +- .../Security/SetHttpOnlyForHttpCookie.cs | 2 +- .../DoNotUseInsecureDtdProcessing.cs | 2 +- .../PerformanceSensitiveAnalyzers.md | 2 +- .../CopyAnalysis/SetCopyAbstractValuePredicateKind.cs | 2 +- ...ionAnalysis.ParameterValidationDataFlowOperationVisitor.cs | 2 +- src/Utilities/Refactoring/AbstractRefactoringHelpers`3.cs | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ImplementStandardExceptionConstructors.cs b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ImplementStandardExceptionConstructors.cs index d1fbfe702a..c27c224f95 100644 --- a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ImplementStandardExceptionConstructors.cs +++ b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ImplementStandardExceptionConstructors.cs @@ -130,7 +130,7 @@ private static void ReportDiagnostic(SymbolAnalysisContext context, INamedTypeSy ImmutableDictionary.Builder builder = ImmutableDictionary.CreateBuilder(); builder.Add("Signature", missingCtorSignature.ToString()); - //create dignostic and store signature into diagnostic property for fixer + //create diagnostic and store signature into diagnostic property for fixer Diagnostic diagnostic = namedTypeSymbol.Locations.CreateDiagnostic(MissingConstructorRule, builder.ToImmutableDictionary(), namedTypeSymbol.Name, constructorSignature); //report diagnostic diff --git a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/QualityGuidelines/AvoidMultipleEnumerations/AvoidMultipleEnumerationsHelpers.cs b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/QualityGuidelines/AvoidMultipleEnumerations/AvoidMultipleEnumerationsHelpers.cs index 1fe1d68508..b44ca9eb8e 100644 --- a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/QualityGuidelines/AvoidMultipleEnumerations/AvoidMultipleEnumerationsHelpers.cs +++ b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/QualityGuidelines/AvoidMultipleEnumerations/AvoidMultipleEnumerationsHelpers.cs @@ -89,7 +89,7 @@ public static bool IsValidImplicitConversion([NotNullWhen(true)] IOperation? ope } /// - /// Check if the operation is deferred type and also it is a collecton enumerated by a for each loop. + /// Check if the operation is deferred type and also it is a collection enumerated by a for each loop. /// public static bool IsOperationEnumeratedByForEachLoop( IOperation operation, @@ -399,7 +399,7 @@ private static void GetImmutableCollectionConversionMethods( foreach (var method in methods) { // Usually there are two overloads for these methods, like ToImmutableArray, - // it has two overloads, one convert from ImmutableArray.Builder and one covert from IEnumerable + // it has two overloads, one convert from ImmutableArray.Builder and one convert from IEnumerable // and we only want the last one if (method is IMethodSymbol { Parameters: { Length: > 0 } parameters, IsExtensionMethod: true } methodSymbol && parameters[0].Type.OriginalDefinition.SpecialType == SpecialType.System_Collections_Generic_IEnumerable_T) diff --git a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/QualityGuidelines/AvoidMultipleEnumerations/FlowAnalysis/GlobalFlowStateDictionaryAnalysis.cs b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/QualityGuidelines/AvoidMultipleEnumerations/FlowAnalysis/GlobalFlowStateDictionaryAnalysis.cs index 4467dbcc4d..f144e032c5 100644 --- a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/QualityGuidelines/AvoidMultipleEnumerations/FlowAnalysis/GlobalFlowStateDictionaryAnalysis.cs +++ b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/QualityGuidelines/AvoidMultipleEnumerations/FlowAnalysis/GlobalFlowStateDictionaryAnalysis.cs @@ -96,7 +96,7 @@ public GlobalFlowStateDictionaryAnalysis( protected override GlobalFlowStateDictionaryAnalysisResult ToResult(GlobalFlowStateDictionaryAnalysisContext analysisContext, GlobalFlowStateDictionaryAnalysisResult dataFlowAnalysisResult) { - // Use the global values map. Drop the per-operation inforamation + // Use the global values map. Drop the per-operation information var operationVisitor = (GlobalFlowStateDictionaryFlowOperationVisitor)OperationVisitor; return dataFlowAnalysisResult.With(operationVisitor.GetGlobalValuesMap()); } diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs index 697058c66f..d38d19b817 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs @@ -2021,7 +2021,7 @@ static Versions NormalizeAttribute(Versions attributes) attributes.SupportedFirst = null; } } - // For deny list UnsupportedSecond should only set if there is SupportedFirst verison between UnsupportedSecond and UnsupportedFirst + // For deny list UnsupportedSecond should only set if there is SupportedFirst version between UnsupportedSecond and UnsupportedFirst else if (attributes.SupportedFirst == null || (attributes.UnsupportedSecond != null && attributes.SupportedFirst > attributes.UnsupportedSecond)) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Security/SetHttpOnlyForHttpCookie.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Security/SetHttpOnlyForHttpCookie.cs index cc344ea78d..5465d23d24 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Security/SetHttpOnlyForHttpCookie.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Security/SetHttpOnlyForHttpCookie.cs @@ -46,7 +46,7 @@ internal class SetHttpOnlyForHttpCookie : DiagnosticAnalyzer }); // If HttpOnly is set explicitly, the callbacks of OperationKind.SimpleAssignment can cover that case. - // Otherwise, using PropertySetAnalysis to cover the case where HttpCookie object is returned without initializing or assgining HttpOnly property. + // Otherwise, using PropertySetAnalysis to cover the case where HttpCookie object is returned without initializing or assigning HttpOnly property. private static readonly PropertyMapperCollection PropertyMappers = new( new PropertyMapper( "HttpOnly", diff --git a/src/NetAnalyzers/Core/Microsoft.NetFramework.Analyzers/DoNotUseInsecureDtdProcessing.cs b/src/NetAnalyzers/Core/Microsoft.NetFramework.Analyzers/DoNotUseInsecureDtdProcessing.cs index 62f7d4e479..43aecd1bbc 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetFramework.Analyzers/DoNotUseInsecureDtdProcessing.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetFramework.Analyzers/DoNotUseInsecureDtdProcessing.cs @@ -164,7 +164,7 @@ private class XmlReaderSettingsEnvironment internal bool IsSecureResolver { get; set; } internal bool IsConstructedInCodeBlock { get; set; } - // this constructor is used for keep track of XmlReaderSettings craeted in the code block + // this constructor is used for keep track of XmlReaderSettings created in the code block internal XmlReaderSettingsEnvironment(bool isTargetFrameworkSecure) { IsConstructedInCodeBlock = true; diff --git a/src/PerformanceSensitiveAnalyzers/PerformanceSensitiveAnalyzers.md b/src/PerformanceSensitiveAnalyzers/PerformanceSensitiveAnalyzers.md index 1f6d96a447..3c4d74bc80 100644 --- a/src/PerformanceSensitiveAnalyzers/PerformanceSensitiveAnalyzers.md +++ b/src/PerformanceSensitiveAnalyzers/PerformanceSensitiveAnalyzers.md @@ -7,7 +7,7 @@ Because we inject the source of `PerformanceSensitiveAttribute` into projects re One way to resolve this issue is setting `GeneratePerformanceSensitiveAttribute` property to true in the project at the root of your IVT tree, and false otherwise. -For example, given the dependecy graph below, if project A has IVT for project B, and project C has IVT for project D and project E. You need to set `GeneratePerformanceSensitiveAttribute` to true in A and C, and false in B, D and E. +For example, given the dependency graph below, if project A has IVT for project B, and project C has IVT for project D and project E. You need to set `GeneratePerformanceSensitiveAttribute` to true in A and C, and false in B, D and E. ```text A diff --git a/src/Utilities/FlowAnalysis/FlowAnalysis/Analysis/CopyAnalysis/SetCopyAbstractValuePredicateKind.cs b/src/Utilities/FlowAnalysis/FlowAnalysis/Analysis/CopyAnalysis/SetCopyAbstractValuePredicateKind.cs index 1069bdf183..e8a506260a 100644 --- a/src/Utilities/FlowAnalysis/FlowAnalysis/Analysis/CopyAnalysis/SetCopyAbstractValuePredicateKind.cs +++ b/src/Utilities/FlowAnalysis/FlowAnalysis/Analysis/CopyAnalysis/SetCopyAbstractValuePredicateKind.cs @@ -8,7 +8,7 @@ public partial class CopyAnalysis : ForwardDataFlowAnalysis /// Predicate kind for - /// to indicte if the copy equality check for the SetAbstractValue call is a reference compare or a value compare operation. + /// to indicate if the copy equality check for the SetAbstractValue call is a reference compare or a value compare operation. /// internal enum SetCopyAbstractValuePredicateKind { diff --git a/src/Utilities/FlowAnalysis/FlowAnalysis/Analysis/ParameterValidationAnalysis/ParameterValidationAnalysis.ParameterValidationDataFlowOperationVisitor.cs b/src/Utilities/FlowAnalysis/FlowAnalysis/Analysis/ParameterValidationAnalysis/ParameterValidationAnalysis.ParameterValidationDataFlowOperationVisitor.cs index 9302ea5e34..6408e417c6 100644 --- a/src/Utilities/FlowAnalysis/FlowAnalysis/Analysis/ParameterValidationAnalysis/ParameterValidationAnalysis.ParameterValidationDataFlowOperationVisitor.cs +++ b/src/Utilities/FlowAnalysis/FlowAnalysis/Analysis/ParameterValidationAnalysis/ParameterValidationAnalysis.ParameterValidationDataFlowOperationVisitor.cs @@ -312,7 +312,7 @@ private void ProcessRegularInvocationOrCreation(IMethodSymbol? targetMethod, Imm !targetMethod.IsExternallyVisible() && TryGetInterproceduralAnalysisResult(operation, out var invokedMethodAnalysisResult)) { - // Check if this private/interal method that has hazardous usages of non-validated argument. + // Check if this private/internal method that has hazardous usages of non-validated argument. Debug.Assert(!targetMethod.IsVirtual && !targetMethod.IsOverride); var hazardousParameterUsagesInInvokedMethod = invokedMethodAnalysisResult.HazardousParameterUsages; diff --git a/src/Utilities/Refactoring/AbstractRefactoringHelpers`3.cs b/src/Utilities/Refactoring/AbstractRefactoringHelpers`3.cs index b15407708f..f44c4dd047 100644 --- a/src/Utilities/Refactoring/AbstractRefactoringHelpers`3.cs +++ b/src/Utilities/Refactoring/AbstractRefactoringHelpers`3.cs @@ -225,7 +225,7 @@ static bool IsAcceptableLineDistanceAway( return locationLine.IsEmptyOrWhitespace(); } - // On hte same line. This position is acceptable. + // On the same line. This position is acceptable. return true; } } From 2cd9d4564a6b117b2531747efa73f43bfa4e35fd Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 22 Apr 2024 12:17:34 +0000 Subject: [PATCH 04/30] Update dependencies from https://github.com/dotnet/arcade build 20240419.5 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24212.4 -> To Version 9.0.0-beta.24219.5 --- eng/Version.Details.xml | 12 +++++------ eng/common/build.ps1 | 4 +--- eng/common/build.sh | 8 -------- eng/common/cross/toolchain.cmake | 20 +++++++++++++++++++ eng/common/templates-official/job/job.yml | 2 +- .../templates-official/job/onelocbuild.yml | 2 +- .../templates-official/job/source-build.yml | 2 +- .../post-build/post-build.yml | 6 +++--- .../variables/pool-providers.yml | 2 +- global.json | 6 +++--- 10 files changed, 37 insertions(+), 27 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 28451ec569..84e3e69cb4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,14 +19,14 @@ - + https://github.com/dotnet/arcade - 87b015b938e5400d6e57afd7650348c17a764b73 + 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc - + https://github.com/dotnet/arcade - 87b015b938e5400d6e57afd7650348c17a764b73 + 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc @@ -34,9 +34,9 @@ https://github.com/dotnet/runtime 84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f - + https://github.com/dotnet/arcade - 87b015b938e5400d6e57afd7650348c17a764b73 + 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc https://github.com/dotnet/roslyn diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 83e6d82e02..438f9920c4 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -19,7 +19,6 @@ Param( [switch] $pack, [switch] $publish, [switch] $clean, - [switch] $verticalBuild, [switch][Alias('pb')]$productBuild, [switch][Alias('bl')]$binaryLog, [switch][Alias('nobl')]$excludeCIBinarylog, @@ -60,7 +59,6 @@ function Print-Usage() { Write-Host " -sign Sign build outputs" Write-Host " -publish Publish artifacts (e.g. symbols)" Write-Host " -clean Clean the solution" - Write-Host " -verticalBuild Run in 'vertical build' infra mode." Write-Host " -productBuild Build the solution in the way it will be built in the full .NET product (VMR) build (short: -pb)" Write-Host "" @@ -124,7 +122,7 @@ function Build { /p:Deploy=$deploy ` /p:Test=$test ` /p:Pack=$pack ` - /p:DotNetBuildRepo=$($productBuild -or $verticalBuild) ` + /p:DotNetBuildRepo=$productBuild ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` diff --git a/eng/common/build.sh b/eng/common/build.sh index d82ebf7428..ac1ee8620c 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -62,7 +62,6 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" restore=false build=false source_build=false -vertical_build=false product_build=false rebuild=false test=false @@ -141,13 +140,6 @@ while [[ $# > 0 ]]; do restore=true pack=true ;; - -verticalbuild|-vb) - build=true - vertical_build=true - product_build=true - restore=true - pack=true - ;; -test|-t) test=true ;; diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 3762640fdc..9a4e285a5a 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -382,6 +382,26 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") endif() endif() +# Set C++ standard library options if specified +set(CLR_CMAKE_CXX_STANDARD_LIBRARY "" CACHE STRING "Standard library flavor to link against. Only supported with the Clang compiler.") +if (CLR_CMAKE_CXX_STANDARD_LIBRARY) + add_compile_options($<$:--stdlib=${CLR_CMAKE_CXX_STANDARD_LIBRARY}>) + add_link_options($<$:--stdlib=${CLR_CMAKE_CXX_STANDARD_LIBRARY}>) +endif() + +option(CLR_CMAKE_CXX_STANDARD_LIBRARY_STATIC "Statically link against the C++ standard library" OFF) +if(CLR_CMAKE_CXX_STANDARD_LIBRARY_STATIC) + add_link_options($<$:-static-libstdc++>) +endif() + +set(CLR_CMAKE_CXX_ABI_LIBRARY "" CACHE STRING "C++ ABI implementation library to link against. Only supported with the Clang compiler.") +if (CLR_CMAKE_CXX_ABI_LIBRARY) + # The user may specify the ABI library with the 'lib' prefix, like 'libstdc++'. Strip the prefix here so the linker finds the right library. + string(REGEX REPLACE "^lib(.+)" "\\1" CLR_CMAKE_CXX_ABI_LIBRARY ${CLR_CMAKE_CXX_ABI_LIBRARY}) + # We need to specify this as a linker-backend option as Clang will filter this option out when linking to libc++. + add_link_options("LINKER:-l${CLR_CMAKE_CXX_ABI_LIBRARY}") +endif() + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index c63e17e863..761acc5eb6 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -210,7 +210,7 @@ jobs: - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: 'artifacts/log' - artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)_Attempt$(System.JobAttempt)') }} displayName: 'Publish logs' continueOnError: true condition: always() diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index ba9ba49303..52b4d05d3f 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -56,7 +56,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index c918720931..2180e97a28 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -52,7 +52,7 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2-pt + image: 1es-mariner-2 os: linux ${{ if ne(parameters.platform.pool, '') }}: diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index d286e956bd..da1f40958b 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -110,7 +110,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: @@ -150,7 +150,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: - template: setup-maestro-vars.yml @@ -208,7 +208,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: - template: setup-maestro-vars.yml diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml index beab7d1bfb..1f308b24ef 100644 --- a/eng/common/templates-official/variables/pool-providers.yml +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022-pt +# image: 1es-windows-2022 variables: # Coalesce the target and source branches so we know when a PR targets a release branch diff --git a/global.json b/global.json index 1fe8e0644e..2791735b3f 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "9.0.100-preview.1.24101.2", + "dotnet": "9.0.100-preview.3.24204.13", "runtimes": { "dotnet": [ "3.1.7", @@ -13,11 +13,11 @@ "xcopy-msbuild": "17.7.4" }, "sdk": { - "version": "9.0.100-preview.1.24101.2", + "version": "9.0.100-preview.3.24204.13", "allowPrerelease": true, "rollForward": "patch" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24212.4" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24219.5" } } From abb24db0f01f5a489d90c82e8a0366841372c2dd Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 29 Apr 2024 12:17:43 +0000 Subject: [PATCH 05/30] Update dependencies from https://github.com/dotnet/arcade build 20240426.3 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24212.4 -> To Version 9.0.0-beta.24226.3 --- eng/Version.Details.xml | 12 +- eng/common/core-templates/job/job.yml | 266 ++++++++++++++ eng/common/core-templates/job/onelocbuild.yml | 121 +++++++ .../job/publish-build-assets.yml | 172 ++++++++++ .../core-templates/job/source-build.yml | 80 +++++ .../job/source-index-stage1.yml | 73 ++++ .../core-templates/jobs/codeql-build.yml | 33 ++ eng/common/core-templates/jobs/jobs.yml | 119 +++++++ .../core-templates/jobs/source-build.yml | 50 +++ .../post-build/common-variables.yml | 24 ++ .../core-templates/post-build/post-build.yml | 298 ++++++++++++++++ .../post-build/setup-maestro-vars.yml | 74 ++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../steps/component-governance.yml | 14 + .../core-templates/steps/generate-sbom.yml | 54 +++ .../steps/publish-build-artifacts.yml | 20 ++ .../core-templates/steps/publish-logs.yml | 59 ++++ .../steps/publish-pipeline-artifacts.yml | 20 ++ .../core-templates/steps/retain-build.yml | 28 ++ .../core-templates/steps/send-to-helix.yml | 93 +++++ .../core-templates/steps/source-build.yml | 134 ++++++++ .../variables/pool-providers.yml | 8 + eng/common/sdk-task.ps1 | 2 +- eng/common/template-guidance.md | 137 ++++++++ eng/common/templates-official/job/job.yml | 324 ++++-------------- .../templates-official/job/onelocbuild.yml | 115 +------ .../job/publish-build-assets.yml | 162 +-------- .../templates-official/job/source-build.yml | 70 +--- .../job/source-index-stage1.yml | 70 +--- .../templates-official/jobs/codeql-build.yml | 32 +- eng/common/templates-official/jobs/jobs.yml | 100 +----- .../templates-official/jobs/source-build.yml | 49 +-- .../post-build/common-variables.yml | 28 +- .../post-build/post-build.yml | 289 +--------------- .../post-build/setup-maestro-vars.yml | 74 +--- .../steps/add-build-to-channel.yml | 18 +- .../steps/component-governance.yml | 18 +- .../steps/generate-sbom.yml | 51 +-- .../steps/publish-build-artifacts.yml | 41 +++ .../templates-official/steps/publish-logs.yml | 54 +-- .../steps/publish-pipeline-artifacts.yml | 26 ++ .../templates-official/steps/retain-build.yml | 33 +- .../steps/send-to-helix.yml | 98 +----- .../templates-official/steps/source-build.yml | 134 +------- eng/common/templates/job/execute-sdl.yml | 139 -------- eng/common/templates/job/job.yml | 314 ++++------------- eng/common/templates/job/onelocbuild.yml | 112 +----- .../templates/job/publish-build-assets.yml | 158 +-------- eng/common/templates/job/source-build.yml | 69 +--- .../templates/job/source-index-stage1.yml | 70 +--- eng/common/templates/jobs/codeql-build.yml | 32 +- eng/common/templates/jobs/jobs.yml | 100 +----- eng/common/templates/jobs/source-build.yml | 49 +-- .../templates/post-build/common-variables.yml | 28 +- .../templates/post-build/post-build.yml | 286 +--------------- .../post-build/setup-maestro-vars.yml | 74 +--- .../templates/steps/add-build-to-channel.yml | 18 +- eng/common/templates/steps/build-reason.yml | 12 - .../templates/steps/component-governance.yml | 18 +- eng/common/templates/steps/execute-codeql.yml | 32 -- eng/common/templates/steps/execute-sdl.yml | 88 ----- eng/common/templates/steps/generate-sbom.yml | 51 +-- .../steps/publish-build-artifacts.yml | 40 +++ eng/common/templates/steps/publish-logs.yml | 54 +-- .../steps/publish-pipeline-artifacts.yml | 34 ++ eng/common/templates/steps/retain-build.yml | 33 +- eng/common/templates/steps/run-on-unix.yml | 7 - eng/common/templates/steps/run-on-windows.yml | 7 - .../steps/run-script-ifequalelse.yml | 33 -- eng/common/templates/steps/send-to-helix.yml | 98 +----- eng/common/templates/steps/source-build.yml | 134 +------- eng/common/templates/steps/telemetry-end.yml | 102 ------ .../templates/steps/telemetry-start.yml | 241 ------------- .../templates/variables/pool-providers.yml | 54 +-- .../templates/variables/sdl-variables.yml | 7 - eng/common/tools.ps1 | 4 +- global.json | 2 +- 78 files changed, 2385 insertions(+), 3818 deletions(-) create mode 100644 eng/common/core-templates/job/job.yml create mode 100644 eng/common/core-templates/job/onelocbuild.yml create mode 100644 eng/common/core-templates/job/publish-build-assets.yml create mode 100644 eng/common/core-templates/job/source-build.yml create mode 100644 eng/common/core-templates/job/source-index-stage1.yml create mode 100644 eng/common/core-templates/jobs/codeql-build.yml create mode 100644 eng/common/core-templates/jobs/jobs.yml create mode 100644 eng/common/core-templates/jobs/source-build.yml create mode 100644 eng/common/core-templates/post-build/common-variables.yml create mode 100644 eng/common/core-templates/post-build/post-build.yml create mode 100644 eng/common/core-templates/post-build/setup-maestro-vars.yml create mode 100644 eng/common/core-templates/post-build/trigger-subscription.yml create mode 100644 eng/common/core-templates/steps/add-build-to-channel.yml create mode 100644 eng/common/core-templates/steps/component-governance.yml create mode 100644 eng/common/core-templates/steps/generate-sbom.yml create mode 100644 eng/common/core-templates/steps/publish-build-artifacts.yml create mode 100644 eng/common/core-templates/steps/publish-logs.yml create mode 100644 eng/common/core-templates/steps/publish-pipeline-artifacts.yml create mode 100644 eng/common/core-templates/steps/retain-build.yml create mode 100644 eng/common/core-templates/steps/send-to-helix.yml create mode 100644 eng/common/core-templates/steps/source-build.yml create mode 100644 eng/common/core-templates/variables/pool-providers.yml create mode 100644 eng/common/template-guidance.md create mode 100644 eng/common/templates-official/steps/publish-build-artifacts.yml create mode 100644 eng/common/templates-official/steps/publish-pipeline-artifacts.yml delete mode 100644 eng/common/templates/job/execute-sdl.yml delete mode 100644 eng/common/templates/steps/build-reason.yml delete mode 100644 eng/common/templates/steps/execute-codeql.yml delete mode 100644 eng/common/templates/steps/execute-sdl.yml create mode 100644 eng/common/templates/steps/publish-build-artifacts.yml create mode 100644 eng/common/templates/steps/publish-pipeline-artifacts.yml delete mode 100644 eng/common/templates/steps/run-on-unix.yml delete mode 100644 eng/common/templates/steps/run-on-windows.yml delete mode 100644 eng/common/templates/steps/run-script-ifequalelse.yml delete mode 100644 eng/common/templates/steps/telemetry-end.yml delete mode 100644 eng/common/templates/steps/telemetry-start.yml delete mode 100644 eng/common/templates/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 84e3e69cb4..cea1bc13cb 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,14 +19,14 @@ - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + 97e2f41e909dcabb1103fe98ba4540a246662187 - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + 97e2f41e909dcabb1103fe98ba4540a246662187 @@ -34,9 +34,9 @@ https://github.com/dotnet/runtime 84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + 97e2f41e909dcabb1103fe98ba4540a246662187 https://github.com/dotnet/roslyn diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml new file mode 100644 index 0000000000..dc3bd560a5 --- /dev/null +++ b/eng/common/core-templates/job/job.yml @@ -0,0 +1,266 @@ +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: {} + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + # publishing defaults + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + artifactPublishSteps: [] + runAsPublic: false + +# Sbom related params + enableSbom: true + PackageVersion: 9.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +# 1es specific parameters + is1ESPipeline: '' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + publishArtifacts: false + + # Publish test results + - ${{ if and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')) }}: + - ${{ if eq(parameters.testResultsFormat, 'xunit') }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if eq(parameters.testResultsFormat, 'vstest') }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + # gather artifacts + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: CopyFiles@2 + displayName: Gather logs for publish to artifacts + inputs: + SourceFolder: 'artifacts/log' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - task: CopyFiles@2 + displayName: Gather logs for publish to artifacts + inputs: + SourceFolder: 'artifacts/log/$(_BuildConfig)' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - task: CopyFiles@2 + displayName: Gather buildconfiguration for build retry + inputs: + SourceFolder: '$(Build.SourcesDirectory)/eng/common/BuildConfiguration' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/eng/common/BuildConfiguration' + + - ${{ each step in parameters.artifactPublishSteps }}: + - ${{ step }} diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml new file mode 100644 index 0000000000..00feec8ebb --- /dev/null +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -0,0 +1,121 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + is1ESPipeline: '' +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish Localization Files + pathToPublish: '$(Build.ArtifactStagingDirectory)/loc' + publishLocation: Container + artifactName: Loc + condition: ${{ parameters.condition }} + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish LocProject.json + pathToPublish: '$(Build.SourcesDirectory)/eng/Localize/' + publishLocation: Container + artifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml new file mode 100644 index 0000000000..8fe9299542 --- /dev/null +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -0,0 +1,172 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + + is1ESPipeline: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/core-templates/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - checkout: self + fetchDepth: 3 + clean: true + + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro.dot.net + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish ReleaseConfigs Artifact + pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs' + publishLocation: Container + artifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exist" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + pathToPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + publishLocation: Container + artifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml new file mode 100644 index 0000000000..c0ce4b3c86 --- /dev/null +++ b/eng/common/core-templates/job/source-build.yml @@ -0,0 +1,80 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + + is1ESPipeline: '' + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/core-templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + ${{ if eq(parameters.is1ESPipeline, 'true') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals build.ubuntu.2004.amd64 + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2 + os: linux + ${{ else }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + platform: ${{ parameters.platform }} diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml new file mode 100644 index 0000000000..9c6e5ae3c3 --- /dev/null +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -0,0 +1,73 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20240129.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + is1ESPipeline: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: windows.vs2022.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022.amd64 + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET 8 SDK + inputs: + packageType: sdk + version: 8.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml new file mode 100644 index 0000000000..f2144252cc --- /dev/null +++ b/eng/common/core-templates/jobs/codeql-build.yml @@ -0,0 +1,33 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + is1ESPipeline: '' + +jobs: +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml new file mode 100644 index 0000000000..ea69be4341 --- /dev/null +++ b/eng/common/core-templates/jobs/jobs.yml @@ -0,0 +1,119 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/core-templates/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + + artifacts: {} + is1ESPipeline: '' + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - ${{ if eq(parameters.is1ESPipeline, 'true') }}: + - template: /eng/common/templates-official/job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + + - ${{ else }}: + - template: /eng/common/templates/job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/core-templates/jobs/source-build.yml b/eng/common/core-templates/jobs/source-build.yml new file mode 100644 index 0000000000..d8e5d00852 --- /dev/null +++ b/eng/common/core-templates/jobs/source-build.yml @@ -0,0 +1,50 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/core-templates/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + + is1ESPipeline: '' + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml new file mode 100644 index 0000000000..b9ede10bf0 --- /dev/null +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -0,0 +1,24 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro.dot.net" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.11 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml new file mode 100644 index 0000000000..ed1e6692f7 --- /dev/null +++ b/eng/common/core-templates/post-build/post-build.yml @@ -0,0 +1,298 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + + - name: is1ESPipeline + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml new file mode 100644 index 0000000000..8d56b57267 --- /dev/null +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,74 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + is1ESPipeline: '' + +steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/core-templates/post-build/trigger-subscription.yml b/eng/common/core-templates/post-build/trigger-subscription.yml new file mode 100644 index 0000000000..da669030da --- /dev/null +++ b/eng/common/core-templates/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/core-templates/steps/add-build-to-channel.yml b/eng/common/core-templates/steps/add-build-to-channel.yml new file mode 100644 index 0000000000..f67a210d62 --- /dev/null +++ b/eng/common/core-templates/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/core-templates/steps/component-governance.yml b/eng/common/core-templates/steps/component-governance.yml new file mode 100644 index 0000000000..df449a34c1 --- /dev/null +++ b/eng/common/core-templates/steps/component-governance.yml @@ -0,0 +1,14 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + is1ESPipeline: false + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml new file mode 100644 index 0000000000..d938b60e1b --- /dev/null +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -0,0 +1,54 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 9.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + is1ESPipeline: false + # disable publishArtifacts if some other step is publishing the artifacts (like job.yml). + publishArtifacts: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- ${{ if eq(parameters.publishArtifacts, 'true')}}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + targetPath: '${{ parameters.manifestDirPath }}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/core-templates/steps/publish-build-artifacts.yml b/eng/common/core-templates/steps/publish-build-artifacts.yml new file mode 100644 index 0000000000..f24ce34668 --- /dev/null +++ b/eng/common/core-templates/steps/publish-build-artifacts.yml @@ -0,0 +1,20 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false +- name: args + type: object + default: {} +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - template: /eng/common/templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ each parameter in parameters.args }}: + ${{ parameter.key }}: ${{ parameter.value }} +- ${{ else }}: + - template: /eng/common/templates-official/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ each parameter in parameters.args }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml new file mode 100644 index 0000000000..8c5ea77b58 --- /dev/null +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -0,0 +1,59 @@ +parameters: + StageLabel: '' + JobLabel: '' + CustomSensitiveDataList: '' + # A default - in case value from eng/common/core-templates/post-build/common-variables.yml is not passed + BinlogToolVersion: '1.0.11' + is1ESPipeline: false + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: PowerShell@2 + displayName: Redact Logs + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(akams-client-secret)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} + continueOnError: true + condition: always() + +- task: CopyFiles@2 + displayName: Gather post build logs + inputs: + SourceFolder: '$(Build.SourcesDirectory)/PostBuildLogs' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + +- template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish Logs + pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + publishLocation: Container + artifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/core-templates/steps/publish-pipeline-artifacts.yml b/eng/common/core-templates/steps/publish-pipeline-artifacts.yml new file mode 100644 index 0000000000..2efec04dc2 --- /dev/null +++ b/eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,20 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: args + type: object + default: {} + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - template: /eng/common/templates/steps/publish-pipeline-artifacts.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} +- ${{ else }}: + - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/core-templates/steps/retain-build.yml b/eng/common/core-templates/steps/retain-build.yml new file mode 100644 index 0000000000..83d97a26a0 --- /dev/null +++ b/eng/common/core-templates/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml new file mode 100644 index 0000000000..68fa739c4a --- /dev/null +++ b/eng/common/core-templates/steps/send-to-helix.yml @@ -0,0 +1,93 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY + HelixProjectArguments: '' # optional -- arguments passed to the build command + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml new file mode 100644 index 0000000000..bdd725b496 --- /dev/null +++ b/eng/common/core-templates/steps/source-build.yml @@ -0,0 +1,134 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/core-templates/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + is1ESPipeline: false + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:DotNetBuildSourceOnly=true \ + /p:DotNetBuildRepo=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/sb/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish BuildLogs + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/core-templates/variables/pool-providers.yml b/eng/common/core-templates/variables/pool-providers.yml new file mode 100644 index 0000000000..41053d382a --- /dev/null +++ b/eng/common/core-templates/variables/pool-providers.yml @@ -0,0 +1,8 @@ +parameters: + is1ESPipeline: false + +variables: + - ${{ if eq(parameters.is1ESPipeline, 'true') }}: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ else }}: + - template: /eng/common/templates/variables/pool-providers.yml \ No newline at end of file diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 091023970f..aab40de3fd 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.5" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.10.0-pre.4.0" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md new file mode 100644 index 0000000000..c114bc28dc --- /dev/null +++ b/eng/common/template-guidance.md @@ -0,0 +1,137 @@ +# Overview + +Arcade provides templates for public (`/templates`) and 1ES pipeline templates (`/templates-official`) scenarios. Pipelines which are required to be managed by 1ES pipeline templates should reference `/templates-offical`, all other pipelines may reference `/templates`. + +## How to use + +Basic guidance is: + +- 1ES Pipeline Template or 1ES Microbuild template runs should reference `eng/common/templates-official`. Any internal production-graded pipeline should use these templates. + +- All other runs should reference `eng/common/templates`. + +See [azure-pipelines.yml](../../azure-pipelines.yml) (templates-official example) or [azure-pipelines-pr.yml](../../azure-pipelines-pr.yml) (templates example) for examples. + +#### The `templateIs1ESManaged` parameter + +The `templateIs1ESManaged` is available on most templates and affects which of the variants is used for nested templates. See [Development Notes](#development-notes) below for more information on the `templateIs1ESManaged1 parameter. + +- For templates under `job/`, `jobs/`, `steps`, or `post-build/`, this parameter must be explicitly set. + +## Multiple outputs + +1ES pipeline templates impose a policy where every publish artifact execution results in additional security scans being injected into your pipeline. When using `templates-official/jobs/jobs.yml`, Arcade reduces the number of additional security injections by gathering all publishing outputs into the [Build.ArtifactStagingDirectory](https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services), and utilizing the [outputParentDirectory](https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs#multiple-outputs) feature of 1ES pipeline templates. When implementing your pipeline, if you ensure publish artifacts are located in the `$(Build.ArtifactStagingDirectory)`, and utilize the 1ES provided template context, then you can reduce the number of security scans for your pipeline. + +Example: +``` yaml +# azure-pipelines.yml +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + stages: + - stage: build + jobs: + - template: /eng/common/templates-official/jobs/jobs.yml@self + parameters: + # 1ES makes use of outputs to reduce security task injection overhead + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish logs from source' + continueOnError: true + condition: always() + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/log + artifactName: Logs + jobs: + - job: Windows + steps: + - script: echo "friendly neighborhood" > artifacts/marvel/spiderman.txt + # copy build outputs to artifact staging directory for publishing + - task: CopyFiles@2 + displayName: Gather build output + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/marvel' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/marvel' +``` + +Note: Multiple outputs are ONLY applicable to 1ES PT publishing (only usable when referencing `templates-official`). + +# Development notes + +**Folder / file structure** + +``` text +eng\common\ + [templates || templates-official]\ + job\ + job.yml (shim + artifact publishing logic) + onelocbuild.yml (shim) + publish-build-assets.yml (shim) + source-build.yml (shim) + source-index-stage1.yml (shim) + jobs\ + codeql-build.yml (shim) + jobs.yml (shim) + source-build.yml (shim) + post-build\ + post-build.yml (shim) + trigger-subscription.yml (shim) + common-variabls.yml (shim) + setup-maestro-vars.yml (shim) + steps\ + publish-build-artifacts.yml (logic) + publish-pipeline-artifacts.yml (logic) + add-build-channel.yml (shim) + component-governance.yml (shim) + generate-sbom.yml (shim) + publish-logs.yml (shim) + retain-build.yml (shim) + send-to-helix.yml (shim) + source-build.yml (shim) + variables\ + pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project + sdl-variables.yml (logic) + core-templates\ + job\ + job.yml (logic) + onelocbuild.yml (logic) + publish-build-assets.yml (logic) + source-build.yml (logic) + source-index-stage1.yml (logic) + jobs\ + codeql-build.yml (logic) + jobs.yml (logic) + source-build.yml (logic) + post-build\ + common-variabls.yml (logic) + post-build.yml (logic) + setup-maestro-vars.yml (logic) + trigger-subscription.yml (logic) + steps\ + add-build-to-channel.yml (logic) + component-governance.yml (logic) + generate-sbom.yml (logic) + publish-build-artifacts.yml (redirect) + publish-logs.yml (logic) + publish-pipeline-artifacts.yml (redirect) + retain-build.yml (logic) + send-to-helix.yml (logic) + source-build.yml (logic) + variables\ + pool-providers.yml (redirect) +``` + +In the table above, a file is designated as "shim", "logic", or "redirect". + +- shim - represents a yaml file which is an intermediate step between pipeline logic and .Net Core Engineering's templates (`core-templates`) and defines the `is1ESPipeline` parameter value. + +- logic - represents actual base template logic. + +- redirect- represents a file in `core-templates` which redirects to the "logic" file in either `templates` or `templates-official`. + +Logic for Arcade's templates live **primarily** in the `core-templates` folder. The exceptions to the location of the logic files are around artifact publishing, which is handled differently between 1es pipeline templates and standard templates. `templates` and `templates-official` provide shim entry points which redirect to `core-templates` while also defining the `is1ESPipeline` parameter. If a shim is referenced in `templates`, then `is1ESPipeline` is set to `false`. If a shim is referenced in `templates-official`, then `is1ESPipeline` is set to `true`. + +Within `templates` and `templates-official`, the templates at the "stages", and "jobs" / "job" level have been replaced with shims. Templates at the "steps" and "variables" level are typically too granular to be replaced with shims and instead persist logic which is directly applicable to either scenario. + +Within `core-templates`, there are a handful of places where logic is dependent on which shim entry point was used. In those places, we redirect back to the respective logic file in `templates` or `templates-official`. diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 761acc5eb6..4724e9aaa8 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -1,264 +1,62 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false -# Sbom related params - enableSbom: true - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} - richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates-official/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: 'artifacts/log' - artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)_Attempt$(System.JobAttempt)') }} - displayName: 'Publish logs' - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates-official/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' - artifactName: 'BuildConfiguration' - displayName: 'Publish build retry configuration' - continueOnError: true \ No newline at end of file +- template: /eng/common/core-templates/job/job.yml + parameters: + is1ESPipeline: true + + # publish artifacts + # for 1ES managed templates, use the templateContext.output to handle multiple outputs. + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - output: buildArtifacts + displayName: Publish pipeline artifacts + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + condition: always() + continueOnError: true + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)_Attempt$(System.JobAttempt)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}: + - output: buildArtifacts + displayName: Publish Logs + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + publishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/eng/common/BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - output: pipelineArtifact + displayName: Publish SBOM manifest + continueOnError: true + targetPath: $(Build.ArtifactStagingDirectory)/sbom + artifactName: $(ARTIFACT_NAME) + + # add any outputs provided via root yaml + - ${{ if ne(parameters.templateContext.outputs, '') }}: + - ${{ each output in parameters.templateContext.outputs }}: + - ${{ output }} + + # add any remaining templateContext properties + ${{ each context in parameters.templateContext }}: + ${{ if and(ne(context.key, 'outputParentDirectory'), ne(context.key, 'outputs')) }}: + ${{ context.key }}: ${{ context.value }} + + ${{ each parameter in parameters }}: + ${{ if and(ne(parameter.key, 'templateContext'), ne(parameter.key, 'is1ESPipeline')) }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index 52b4d05d3f..0f0c514b91 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -1,112 +1,7 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(Build.SourcesDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} +- template: /eng/common/core-templates/job/onelocbuild.yml + parameters: + is1ESPipeline: true - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 38340d3e38..d667a70e8d 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -1,159 +1,7 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - # unconditional - needed for logs publishing (redactor tool version) - - template: /eng/common/templates-official/post-build/common-variables.yml - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 - os: windows - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - checkout: self - fetchDepth: 3 - clean: true - - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(Build.BuildNumber) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force - $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" - Add-Content -Path $filePath -Value $(BARBuildId) - Add-Content -Path $filePath -Value "$(DefaultChannels)" - Add-Content -Path $filePath -Value $(IsStableBuild) - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' +- template: /eng/common/core-templates/job/publish-build-assets.yml + parameters: + is1ESPipeline: true - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates-official/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index 2180e97a28..1a480034b6 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -1,67 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals build.ubuntu.2004.amd64 - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2 - os: linux - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all +- template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: true - steps: - - template: /eng/common/templates-official/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 53a9ef51fd..6d5ead316f 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,67 +1,7 @@ -parameters: - runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240129.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: '' - dependsOn: '' - pool: '' - jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln +- template: /eng/common/core-templates/job/source-index-stage1.yml + parameters: + is1ESPipeline: true - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml index b68d3c2f31..a726322ecf 100644 --- a/eng/common/templates-official/jobs/codeql-build.yml +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -1,31 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - jobs: -- template: /eng/common/templates-official/jobs/jobs.yml +- template: /eng/common/core-templates/jobs/codeql-build.yml parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true + is1ESPipeline: true - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml index 857a0f8ba4..007deddaea 100644 --- a/eng/common/templates-official/jobs/jobs.yml +++ b/eng/common/templates-official/jobs/jobs.yml @@ -1,97 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates-official/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates-official/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: true - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 2076f4e25b..483e7b611f 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -1,46 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates-official/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - jobs: +- template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: true -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index b9ede10bf0..c32fc49233 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -1,24 +1,8 @@ variables: - - group: Publish-Build-Assets +- template: /eng/common/core-templates/post-build/common-variables.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - name: BinlogToolVersion - value: 1.0.11 - - - name: runCodesignValidationInjection - value: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index da1f40958b..2364c0fd4a 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -1,285 +1,8 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 +- template: /eng/common/core-templates/post-build/post-build.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml index 0c87f149a4..024397d878 100644 --- a/eng/common/templates-official/post-build/setup-maestro-vars.yml +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -1,70 +1,8 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" +- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml index f67a210d62..543dea8c69 100644 --- a/eng/common/templates-official/steps/add-build-to-channel.yml +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -1,13 +1,7 @@ -parameters: - ChannelId: 0 - steps: -- task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) +- template: /eng/common/core-templates/steps/add-build-to-channel.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml index 0ecec47b0c..30bb3985ca 100644 --- a/eng/common/templates-official/steps/component-governance.yml +++ b/eng/common/templates-official/steps/component-governance.yml @@ -1,13 +1,7 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml index 488b560e8b..9a89a4706d 100644 --- a/eng/common/templates-official/steps/generate-sbom.yml +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -1,48 +1,7 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) +- template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: true + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/publish-build-artifacts.yml b/eng/common/templates-official/steps/publish-build-artifacts.yml new file mode 100644 index 0000000000..100a3fc984 --- /dev/null +++ b/eng/common/templates-official/steps/publish-build-artifacts.yml @@ -0,0 +1,41 @@ +parameters: +- name: displayName + type: string + default: 'Publish to Build Artifact' + +- name: condition + type: string + default: succeeded() + +- name: artifactName + type: string + +- name: pathToPublish + type: string + +- name: continueOnError + type: boolean + default: false + +- name: publishLocation + type: string + default: 'Container' + +- name: is1ESPipeline + type: boolean + default: true + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates-official cannot be referenced from a non-1ES managed template': error +- task: 1ES.PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition }} + ${{ if parameters.continueOnError }}: + continueOnError: ${{ parameters.continueOnError }} + inputs: + PublishLocation: ${{ parameters.publishLocation }} + PathtoPublish: ${{ parameters.pathToPublish }} + ${{ if parameters.artifactName }}: + ArtifactName: ${{ parameters.artifactName }} + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml index 84b2f559c5..579fd531e9 100644 --- a/eng/common/templates-official/steps/publish-logs.yml +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -1,49 +1,7 @@ -parameters: - StageLabel: '' - JobLabel: '' - CustomSensitiveDataList: '' - # A default - in case value from eng/common/templates-official/post-build/common-variables.yml is not passed - BinlogToolVersion: '1.0.11' - steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() - -- task: PowerShell@2 - displayName: Redact Logs - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 - # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml - # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - # If the file exists - sensitive data for redaction will be sourced from it - # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} - -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' - '$(dn-bot-all-orgs-artifact-feeds-rw)' - '$(akams-client-id)' - '$(akams-client-secret)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' - '$(dn-bot-all-orgs-build-rw-code-rw)' - ${{parameters.CustomSensitiveDataList}} - continueOnError: true - condition: always() - -- task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() +- template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml new file mode 100644 index 0000000000..15d74a9d91 --- /dev/null +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,26 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: true + +- name: args + type: object + default: {} + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates-official cannot be referenced from a non-1ES managed template': error +- task: 1ES.PublishPipelineArtifact@1 + displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} + ${{ if parameters.args.condition }}: + condition: ${{ parameters.args.condition }} + ${{ else }}: + condition: succeeded() + ${{ if parameters.args.continueOnError }}: + continueOnError: ${{ parameters.args.continueOnError }} + inputs: + targetPath: ${{ parameters.args.targetPath }} + ${{ if parameters.args.artifactName }}: + artifactName: ${{ parameters.args.artifactName }} + ${{ if parameters.args.properties }}: + properties: ${{ properties.args.properties }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml index 83d97a26a0..5594551508 100644 --- a/eng/common/templates-official/steps/retain-build.yml +++ b/eng/common/templates-official/steps/retain-build.yml @@ -1,28 +1,7 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file +- template: /eng/common/core-templates/steps/retain-build.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml index 68fa739c4a..6500f21bf8 100644 --- a/eng/common/templates-official/steps/send-to-helix.yml +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -1,93 +1,7 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY - HelixProjectArguments: '' # optional -- arguments passed to the build command - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} +- template: /eng/common/core-templates/steps/send-to-helix.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml index 53ed57b6d4..8f92c49e7b 100644 --- a/eng/common/templates-official/steps/source-build.yml +++ b/eng/common/templates-official/steps/source-build.yml @@ -1,131 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:DotNetBuildSourceOnly=true \ - /p:DotNetBuildRepo=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/sb/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() +- template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: true -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml deleted file mode 100644 index 7870f93bc1..0000000000 --- a/eng/common/templates/job/execute-sdl.yml +++ /dev/null @@ -1,139 +0,0 @@ -parameters: - enable: 'false' # Whether the SDL validation job should execute or not - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth - # diagnosis of problems with specific tool configurations. - publishGuardianDirectoryToPipeline: false - # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL - # parameters rather than relying on YAML. It may be better to use a local script, because you can - # reproduce results locally without piecing together a command based on the YAML. - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named - # 'continueOnError', the parameter value is not correctly picked up. - # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter - sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; - # optional: determines if build artifacts should be downloaded. - downloadArtifacts: true - # optional: determines if this job should search the directory of downloaded artifacts for - # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks. - extractArchiveArtifacts: false - dependsOn: '' # Optional: dependencies of the job - artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts - # Usage: - # artifactNames: - # - 'BlobArtifacts' - # - 'Artifacts_Windows_NT_Release' - # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts, - # not pipeline artifacts, so doesn't affect the use of this parameter. - pipelineArtifactNames: [] - -jobs: -- job: Run_SDL - dependsOn: ${{ parameters.dependsOn }} - displayName: Run SDL tool - condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) - variables: - - group: DotNet-VSTS-Bot - - name: AzDOProjectName - value: ${{ parameters.AzDOProjectName }} - - name: AzDOPipelineId - value: ${{ parameters.AzDOPipelineId }} - - name: AzDOBuildId - value: ${{ parameters.AzDOBuildId }} - - template: /eng/common/templates/variables/sdl-variables.yml - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - template: /eng/common/templates/variables/pool-providers.yml - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - checkout: self - clean: true - - # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars. - - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - - - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - - ${{ if ne(parameters.artifactNames, '') }}: - - ${{ each artifactName in parameters.artifactNames }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.artifactNames, '') }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: specific files - itemPattern: "**" - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - - ${{ each artifactName in parameters.pipelineArtifactNames }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - - powershell: eng/common/sdl/trim-assets-version.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts - displayName: Trim the version from the NuGet packages - continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - displayName: Extract Package Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: - - powershell: eng/common/sdl/extract-artifact-archives.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts - displayName: Extract Archive Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - template: /eng/common/templates/steps/execute-sdl.yml - parameters: - overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} - executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} - overrideParameters: ${{ parameters.overrideParameters }} - additionalParameters: ${{ parameters.additionalParameters }} - publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index a3277bf15c..1cf9a6d481 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,259 +1,61 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false +parameters: enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false -# Sbom related params - enableSbom: true - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@3 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} - richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - publish: artifacts/log - artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: Publish logs - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish build retry configuration - continueOnError: true +- template: /eng/common/core-templates/job/job.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ if and(ne(parameter.key, 'steps'), ne(parameter.key, 'is1ESPipeline')) }}: + ${{ parameter.key }}: ${{ parameter.value }} + + steps: + - ${{ each step in parameters.steps }}: + - ${{ step }} + + artifactPublishSteps: + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish pipeline artifacts + pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + publishLocation: Container + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish Logs + pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + publishLocation: Container + artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 60ab00c4de..ff829dc4c7 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -1,109 +1,7 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(Build.SourcesDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} +- template: /eng/common/core-templates/job/onelocbuild.yml + parameters: + is1ESPipeline: false - - task: PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index bb42240f86..ab2edec2ad 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -1,155 +1,7 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - # unconditional - needed for logs publishing (redactor tool version) - - template: /eng/common/templates/post-build/common-variables.yml - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - checkout: self - fetchDepth: 3 - clean: true - - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(Build.BuildNumber) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - - - task: PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' +- template: /eng/common/core-templates/job/publish-build-assets.yml + parameters: + is1ESPipeline: false - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index d7ed209494..e44d47b1d7 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -1,66 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all +- template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: false - steps: - - template: /eng/common/templates/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b5a3e5c4a6..89f3291593 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,67 +1,7 @@ -parameters: - runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240129.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: '' - dependsOn: '' - pool: '' - jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln +- template: /eng/common/core-templates/job/source-index-stage1.yml + parameters: + is1ESPipeline: false - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml index f7dc5ea4aa..517f24d6a5 100644 --- a/eng/common/templates/jobs/codeql-build.yml +++ b/eng/common/templates/jobs/codeql-build.yml @@ -1,31 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - jobs: -- template: /eng/common/templates/jobs/jobs.yml +- template: /eng/common/core-templates/jobs/codeql-build.yml parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true + is1ESPipeline: false - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 289bb2396c..388e9037b3 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -1,97 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: false - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index da91481ff1..818d4c326d 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -1,46 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - jobs: +- template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: false -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index b9ede10bf0..7fa1058755 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,24 +1,8 @@ variables: - - group: Publish-Build-Assets +- template: /eng/common/core-templates/post-build/common-variables.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - name: BinlogToolVersion - value: 1.0.11 - - - name: runCodesignValidationInjection - value: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ee70e2b399..53ede714bd 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,282 +1,8 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} - publishGuardianDirectoryToPipeline: ${{ parameters.SDLValidationParameters.publishGdn }} - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 +- template: /eng/common/core-templates/post-build/post-build.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0c87f149a4..a79fab5b44 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,70 +1,8 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" +- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/steps/add-build-to-channel.yml b/eng/common/templates/steps/add-build-to-channel.yml index f67a210d62..42bbba161b 100644 --- a/eng/common/templates/steps/add-build-to-channel.yml +++ b/eng/common/templates/steps/add-build-to-channel.yml @@ -1,13 +1,7 @@ -parameters: - ChannelId: 0 - steps: -- task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) +- template: /eng/common/core-templates/steps/add-build-to-channel.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/build-reason.yml b/eng/common/templates/steps/build-reason.yml deleted file mode 100644 index eba58109b5..0000000000 --- a/eng/common/templates/steps/build-reason.yml +++ /dev/null @@ -1,12 +0,0 @@ -# build-reason.yml -# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons -# to include steps (',' separated). -parameters: - conditions: '' - steps: [] - -steps: - - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: - - ${{ parameters.steps }} - - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index 0ecec47b0c..c12a5f8d21 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -1,13 +1,7 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/execute-codeql.yml b/eng/common/templates/steps/execute-codeql.yml deleted file mode 100644 index 3930b16302..0000000000 --- a/eng/common/templates/steps/execute-codeql.yml +++ /dev/null @@ -1,32 +0,0 @@ -parameters: - # Language that should be analyzed. Defaults to csharp - language: csharp - # Build Commands - buildCommands: '' - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth - # diagnosis of problems with specific tool configurations. - publishGuardianDirectoryToPipeline: false - # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL - # parameters rather than relying on YAML. It may be better to use a local script, because you can - # reproduce results locally without piecing together a command based on the YAML. - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named - # 'continueOnError', the parameter value is not correctly picked up. - # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter - # optional: determines whether to continue the build if the step errors; - sdlContinueOnError: false - -steps: -- template: /eng/common/templates/steps/execute-sdl.yml - parameters: - overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} - executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} - overrideParameters: ${{ parameters.overrideParameters }} - additionalParameters: '${{ parameters.additionalParameters }} - -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' - publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml deleted file mode 100644 index 07426fde05..0000000000 --- a/eng/common/templates/steps/execute-sdl.yml +++ /dev/null @@ -1,88 +0,0 @@ -parameters: - overrideGuardianVersion: '' - executeAllSdlToolsScript: '' - overrideParameters: '' - additionalParameters: '' - publishGuardianDirectoryToPipeline: false - sdlContinueOnError: false - condition: '' - -steps: -- task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - -- task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - -- ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 - $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} - Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" - displayName: Install Guardian (Overridden) - -- ${{ if eq(parameters.overrideGuardianVersion, '') }}: - - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 - $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts - Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" - displayName: Install Guardian - -- ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - displayName: Execute SDL (Overridden) - continueOnError: ${{ parameters.sdlContinueOnError }} - condition: ${{ parameters.condition }} - -- ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianCliLocation $(GuardianCliLocation) - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) - ${{ parameters.additionalParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - condition: ${{ parameters.condition }} - -- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - # We want to publish the Guardian results and configuration for easy diagnosis. However, the - # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default - # tooling files. Some of these files are large and aren't useful during an investigation, so - # exclude them by simply deleting them before publishing. (As of writing, there is no documented - # way to selectively exclude a dir from the pipeline artifact publish task.) - - task: DeleteFiles@1 - displayName: Delete Guardian dependencies to avoid uploading - inputs: - SourceFolder: $(Agent.BuildDirectory)/.gdn - Contents: | - c - i - condition: succeededOrFailed() - - - publish: $(Agent.BuildDirectory)/.gdn - artifact: GuardianConfiguration - displayName: Publish GuardianConfiguration - condition: succeededOrFailed() - - # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration - # with the "SARIF SAST Scans Tab" Azure DevOps extension - - task: CopyFiles@2 - displayName: Copy SARIF files - inputs: - flattenFolders: true - sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ - contents: '**/*.sarif' - targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs - condition: succeededOrFailed() - - # Use PublishBuildArtifacts because the SARIF extension only checks this case - # see microsoft/sarif-azuredevops-extension#4 - - task: PublishBuildArtifacts@1 - displayName: Publish SARIF files to CodeAnalysisLogs container - inputs: - pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs - artifactName: CodeAnalysisLogs - condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38f..26dc00a2e0 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -1,48 +1,7 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) +- template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/publish-build-artifacts.yml b/eng/common/templates/steps/publish-build-artifacts.yml new file mode 100644 index 0000000000..6428a98dfe --- /dev/null +++ b/eng/common/templates/steps/publish-build-artifacts.yml @@ -0,0 +1,40 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: displayName + type: string + default: 'Publish to Build Artifact' + +- name: condition + type: string + default: succeeded() + +- name: artifactName + type: string + +- name: pathToPublish + type: string + +- name: continueOnError + type: boolean + default: false + +- name: publishLocation + type: string + default: 'Container' + +steps: +- ${{ if eq(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates cannot be referenced from a 1ES managed template': error +- task: PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition }} + ${{ if parameters.continueOnError }}: + continueOnError: ${{ parameters.continueOnError }} + inputs: + PublishLocation: ${{ parameters.publishLocation }} + PathtoPublish: ${{ parameters.pathToPublish }} + ${{ if parameters.artifactName }}: + ArtifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 80861297dd..4ea86bd882 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -1,49 +1,7 @@ -parameters: - StageLabel: '' - JobLabel: '' - CustomSensitiveDataList: '' - # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed - BinlogToolVersion: '1.0.11' - steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() - -- task: PowerShell@2 - displayName: Redact Logs - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 - # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml - # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - # If the file exists - sensitive data for redaction will be sourced from it - # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} - -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' - '$(dn-bot-all-orgs-artifact-feeds-rw)' - '$(akams-client-id)' - '$(akams-client-secret)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' - '$(dn-bot-all-orgs-build-rw-code-rw)' - ${{parameters.CustomSensitiveDataList}} - continueOnError: true - condition: always() - -- task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() +- template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/publish-pipeline-artifacts.yml b/eng/common/templates/steps/publish-pipeline-artifacts.yml new file mode 100644 index 0000000000..88c5635412 --- /dev/null +++ b/eng/common/templates/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,34 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: args + type: object + default: {} + +steps: +- ${{ if eq(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates cannot be referenced from a 1ES managed template': error +- task: PublishPipelineArtifact@1 + displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} + ${{ if parameters.args.condition }}: + condition: ${{ parameters.args.condition }} + ${{ else }}: + condition: succeeded() + ${{ if parameters.args.continueOnError }}: + continueOnError: ${{ parameters.args.continueOnError }} + inputs: + TargetPath: ${{ parameters.args.TargetPath }} + ${{ if parameters.args.ArtifactName }}: + ArtifactName: ${{ parameters.args.ArtifactName }} + ${{ if parameters.args.PublishLocation }}: + PublishLocation: ${{ parameters.args.PublishLocation }} + ${{ if parameters.args.FileSharePath }}: + FileSharePath: ${{ parameters.args.FileSharePath }} + ${{ if parameters.args.Parallel }}: + Parallel: ${{ parameters.args.Parallel }} + ${{ if parameters.args.ParallelCount }}: + ParallelCount: ${{ parameters.args.ParallelCount }} + ${{ if parameters.args.Properties }}: + Properties: ${{ properties.args.Properties }} \ No newline at end of file diff --git a/eng/common/templates/steps/retain-build.yml b/eng/common/templates/steps/retain-build.yml index 83d97a26a0..8e841ace3d 100644 --- a/eng/common/templates/steps/retain-build.yml +++ b/eng/common/templates/steps/retain-build.yml @@ -1,28 +1,7 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file +- template: /eng/common/core-templates/steps/retain-build.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/run-on-unix.yml b/eng/common/templates/steps/run-on-unix.yml deleted file mode 100644 index e1733814f6..0000000000 --- a/eng/common/templates/steps/run-on-unix.yml +++ /dev/null @@ -1,7 +0,0 @@ -parameters: - agentOs: '' - steps: [] - -steps: -- ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-on-windows.yml b/eng/common/templates/steps/run-on-windows.yml deleted file mode 100644 index 73e7e9c275..0000000000 --- a/eng/common/templates/steps/run-on-windows.yml +++ /dev/null @@ -1,7 +0,0 @@ -parameters: - agentOs: '' - steps: [] - -steps: -- ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-script-ifequalelse.yml b/eng/common/templates/steps/run-script-ifequalelse.yml deleted file mode 100644 index 3d1242f558..0000000000 --- a/eng/common/templates/steps/run-script-ifequalelse.yml +++ /dev/null @@ -1,33 +0,0 @@ -parameters: - # if parameter1 equals parameter 2, run 'ifScript' command, else run 'elsescript' command - parameter1: '' - parameter2: '' - ifScript: '' - elseScript: '' - - # name of script step - name: Script - - # display name of script step - displayName: If-Equal-Else Script - - # environment - env: {} - - # conditional expression for step execution - condition: '' - -steps: -- ${{ if and(ne(parameters.ifScript, ''), eq(parameters.parameter1, parameters.parameter2)) }}: - - script: ${{ parameters.ifScript }} - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - env: ${{ parameters.env }} - condition: ${{ parameters.condition }} - -- ${{ if and(ne(parameters.elseScript, ''), ne(parameters.parameter1, parameters.parameter2)) }}: - - script: ${{ parameters.elseScript }} - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - env: ${{ parameters.env }} - condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 68fa739c4a..39f99fc276 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -1,93 +1,7 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY - HelixProjectArguments: '' # optional -- arguments passed to the build command - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} +- template: /eng/common/core-templates/steps/send-to-helix.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 32738aa938..23c1d6f4e9 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -1,131 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:DotNetBuildSourceOnly=true \ - /p:DotNetBuildRepo=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/sb/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() +- template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: false -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/telemetry-end.yml b/eng/common/templates/steps/telemetry-end.yml deleted file mode 100644 index fadc04ca1b..0000000000 --- a/eng/common/templates/steps/telemetry-end.yml +++ /dev/null @@ -1,102 +0,0 @@ -parameters: - maxRetries: 5 - retryDelay: 10 # in seconds - -steps: -- bash: | - if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then - errorCount=0 - else - errorCount=1 - fi - warningCount=0 - - curlStatus=1 - retryCount=0 - # retry loop to harden against spotty telemetry connections - # we don't retry successes and 4xx client errors - until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]] - do - if [ $retryCount -gt 0 ]; then - echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..." - sleep $RetryDelay - fi - - # create a temporary file for curl output - res=`mktemp` - - curlResult=` - curl --verbose --output $res --write-out "%{http_code}"\ - -H 'Content-Type: application/json' \ - -H "X-Helix-Job-Token: $Helix_JobToken" \ - -H 'Content-Length: 0' \ - -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \ - --data-urlencode "errorCount=$errorCount" \ - --data-urlencode "warningCount=$warningCount"` - curlStatus=$? - - if [ $curlStatus -eq 0 ]; then - if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then - curlStatus=$curlResult - fi - fi - - let retryCount++ - done - - if [ $curlStatus -ne 0 ]; then - echo "Failed to Send Build Finish information after $retryCount retries" - vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus" - echo "##$vstsLogOutput" - exit 1 - fi - displayName: Send Unix Build End Telemetry - env: - # defined via VSTS variables in start-job.sh - Helix_JobToken: $(Helix_JobToken) - Helix_WorkItemId: $(Helix_WorkItemId) - MaxRetries: ${{ parameters.maxRetries }} - RetryDelay: ${{ parameters.retryDelay }} - condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT')) -- powershell: | - if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) { - $ErrorCount = 0 - } else { - $ErrorCount = 1 - } - $WarningCount = 0 - - # Basic retry loop to harden against server flakiness - $retryCount = 0 - while ($retryCount -lt $env:MaxRetries) { - try { - Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" ` - -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } - break - } - catch { - $statusCode = $_.Exception.Response.StatusCode.value__ - if ($statusCode -ge 400 -and $statusCode -le 499) { - Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)" - Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message - exit 1 - } - Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..." - $retryCount++ - sleep $env:RetryDelay - continue - } - } - - if ($retryCount -ge $env:MaxRetries) { - Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries." - exit 1 - } - displayName: Send Windows Build End Telemetry - env: - # defined via VSTS variables in start-job.ps1 - Helix_JobToken: $(Helix_JobToken) - Helix_WorkItemId: $(Helix_WorkItemId) - MaxRetries: ${{ parameters.maxRetries }} - RetryDelay: ${{ parameters.retryDelay }} - condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml deleted file mode 100644 index 32c01ef0b5..0000000000 --- a/eng/common/templates/steps/telemetry-start.yml +++ /dev/null @@ -1,241 +0,0 @@ -parameters: - helixSource: 'undefined_defaulted_in_telemetry.yml' - helixType: 'undefined_defaulted_in_telemetry.yml' - buildConfig: '' - runAsPublic: false - maxRetries: 5 - retryDelay: 10 # in seconds - -steps: -- ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@1 - inputs: - azureSubscription: 'HelixProd_KeyVault' - KeyVaultName: HelixProdKV - SecretsFilter: 'HelixApiAccessToken' - condition: always() -- bash: | - # create a temporary file - jobInfo=`mktemp` - - # write job info content to temporary file - cat > $jobInfo < Date: Mon, 29 Apr 2024 19:13:57 +0300 Subject: [PATCH 06/30] Feedback and some refactorings --- .../CompareSymbolsCorrectlyAnalyzer.cs | 36 ++++++++++++------- .../Fixers/CompareSymbolsCorrectlyFix.cs | 4 +-- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/CompareSymbolsCorrectlyAnalyzer.cs b/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/CompareSymbolsCorrectlyAnalyzer.cs index f3a99ee10f..e2e30d64b1 100644 --- a/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/CompareSymbolsCorrectlyAnalyzer.cs +++ b/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/CompareSymbolsCorrectlyAnalyzer.cs @@ -31,7 +31,11 @@ public class CompareSymbolsCorrectlyAnalyzer : DiagnosticAnalyzer public const string SymbolEqualityComparerName = "Microsoft.CodeAnalysis.SymbolEqualityComparer"; public const string RulePropertyName = "Rule"; - public static readonly DiagnosticDescriptor EqualityRule = new( + public const string EqualityRuleName = "EqualityRule"; + public const string GetHashCodeRuleName = "GetHashCodeRule"; + public const string CollectionRuleName = "CollectionRule"; + + private static readonly DiagnosticDescriptor s_equalityRule = new( DiagnosticIds.CompareSymbolsCorrectlyRuleId, s_localizableTitle, s_localizableMessage, @@ -41,7 +45,7 @@ public class CompareSymbolsCorrectlyAnalyzer : DiagnosticAnalyzer description: s_localizableDescription, customTags: WellKnownDiagnosticTagsExtensions.Telemetry); - public static readonly DiagnosticDescriptor GetHashCodeRule = new( + private static readonly DiagnosticDescriptor s_getHashCodeRule = new( DiagnosticIds.CompareSymbolsCorrectlyRuleId, s_localizableTitle, s_localizableMessage, @@ -51,7 +55,7 @@ public class CompareSymbolsCorrectlyAnalyzer : DiagnosticAnalyzer description: CreateLocalizableResourceString(nameof(CompareSymbolsCorrectlyDescriptionGetHashCode)), customTags: WellKnownDiagnosticTagsExtensions.Telemetry); - public static readonly DiagnosticDescriptor CollectionRule = new( + private static readonly DiagnosticDescriptor s_collectionRule = new( DiagnosticIds.CompareSymbolsCorrectlyRuleId, s_localizableTitle, s_localizableMessage, @@ -61,7 +65,16 @@ public class CompareSymbolsCorrectlyAnalyzer : DiagnosticAnalyzer description: s_localizableDescription, customTags: WellKnownDiagnosticTagsExtensions.Telemetry); - public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(EqualityRule); + private static readonly ImmutableDictionary s_EqualityRuleProperties = + ImmutableDictionary.CreateRange([new KeyValuePair(RulePropertyName, EqualityRuleName)]); + + private static readonly ImmutableDictionary s_GetHashCodeRuleProperties = + ImmutableDictionary.CreateRange([new KeyValuePair(RulePropertyName, GetHashCodeRuleName)]); + + private static readonly ImmutableDictionary s_CollectionRuleProperties = + ImmutableDictionary.CreateRange([new KeyValuePair(RulePropertyName, CollectionRuleName)]); + + public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(s_equalityRule); public override void Initialize(AnalysisContext context) { @@ -145,7 +158,7 @@ private static void HandleBinaryOperator(in OperationAnalysisContext context, IN return; } - context.ReportDiagnostic(binary.Syntax.GetLocation().CreateDiagnostic(EqualityRule, MakeProperties(nameof(EqualityRule)))); + context.ReportDiagnostic(binary.Syntax.GetLocation().CreateDiagnostic(s_equalityRule, s_EqualityRuleProperties)); } private static void HandleInvocationOperation( @@ -166,7 +179,7 @@ private static void HandleInvocationOperation( // without the correct arguments if (IsSymbolType(invocationOperation.Instance, symbolType)) { - context.ReportDiagnostic(invocationOperation.CreateDiagnostic(GetHashCodeRule, MakeProperties(nameof(GetHashCode)))); + context.ReportDiagnostic(invocationOperation.CreateDiagnostic(s_getHashCodeRule, s_GetHashCodeRuleProperties)); } break; @@ -177,7 +190,7 @@ private static void HandleInvocationOperation( var parameters = invocationOperation.Arguments; if (parameters.All(p => IsSymbolType(p.Value, symbolType))) { - context.ReportDiagnostic(invocationOperation.Syntax.GetLocation().CreateDiagnostic(EqualityRule, MakeProperties(nameof(EqualityRule)))); + context.ReportDiagnostic(invocationOperation.Syntax.GetLocation().CreateDiagnostic(s_equalityRule, s_EqualityRuleProperties)); } } @@ -190,7 +203,7 @@ invocationOperation.Instance is null && systemHashCodeType.Equals(method.ContainingType, SymbolEqualityComparer.Default) && invocationOperation.Arguments.Any(arg => IsSymbolType(arg.Value, symbolType))) { - context.ReportDiagnostic(invocationOperation.CreateDiagnostic(GetHashCodeRule, MakeProperties(nameof(GetHashCodeRule)))); + context.ReportDiagnostic(invocationOperation.CreateDiagnostic(s_getHashCodeRule, s_GetHashCodeRuleProperties)); } break; @@ -202,7 +215,7 @@ invocationOperation.Instance is null && IsBehavingOnSymbolType(method, symbolType) && !invocationOperation.Arguments.Any(arg => IsSymbolType(arg.Value, iEqualityComparer))) { - context.ReportDiagnostic(invocationOperation.CreateDiagnostic(CollectionRule, MakeProperties(nameof(CollectionRule)))); + context.ReportDiagnostic(invocationOperation.CreateDiagnostic(s_collectionRule, s_CollectionRuleProperties)); } break; @@ -252,7 +265,7 @@ private static void HandleObjectCreation(in OperationAnalysisContext context, IN IsSymbolType(createdType.TypeArguments[0], symbolType) && !objectCreation.Arguments.Any(arg => IsSymbolType(arg.Value, iEqualityComparerType))) { - context.ReportDiagnostic(objectCreation.CreateDiagnostic(CollectionRule, MakeProperties(nameof(CollectionRule)))); + context.ReportDiagnostic(objectCreation.CreateDiagnostic(s_collectionRule, s_CollectionRuleProperties)); } } @@ -357,9 +370,6 @@ void AddOrUpdate(string methodName, INamedTypeSymbol typeSymbol) } } - private static ImmutableDictionary MakeProperties(string rule) - => ImmutableDictionary.CreateRange([new KeyValuePair(RulePropertyName, rule)]); - public static bool UseSymbolEqualityComparer(Compilation compilation) => compilation.GetOrCreateTypeByMetadataName(SymbolEqualityComparerName) is object; } diff --git a/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/Fixers/CompareSymbolsCorrectlyFix.cs b/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/Fixers/CompareSymbolsCorrectlyFix.cs index 4a1a007df6..1114be3838 100644 --- a/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/Fixers/CompareSymbolsCorrectlyFix.cs +++ b/src/Microsoft.CodeAnalysis.Analyzers/Core/MetaAnalyzers/Fixers/CompareSymbolsCorrectlyFix.cs @@ -36,7 +36,7 @@ public sealed override Task RegisterCodeFixesAsync(CodeFixContext context) { switch (rule) { - case nameof(CompareSymbolsCorrectlyAnalyzer.EqualityRule): + case CompareSymbolsCorrectlyAnalyzer.EqualityRuleName: context.RegisterCodeFix( CodeAction.Create( CodeAnalysisDiagnosticsResources.CompareSymbolsCorrectlyCodeFix, @@ -44,7 +44,7 @@ public sealed override Task RegisterCodeFixesAsync(CodeFixContext context) equivalenceKey: nameof(CompareSymbolsCorrectlyFix)), diagnostic); break; - case nameof(CompareSymbolsCorrectlyAnalyzer.CollectionRule): + case CompareSymbolsCorrectlyAnalyzer.CollectionRuleName: context.RegisterCodeFix( CodeAction.Create( CodeAnalysisDiagnosticsResources.CompareSymbolsCorrectlyCodeFix, From 7c9087012f99c9c2e219d73deed232e208b76375 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 30 Apr 2024 20:22:35 +0000 Subject: [PATCH 07/30] Update dependencies from https://github.com/dotnet/arcade build 20240429.1 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24212.4 -> To Version 9.0.0-beta.24229.1 --- eng/Version.Details.xml | 12 +++++----- .../steps/publish-pipeline-artifacts.yml | 24 +++++++++---------- global.json | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index cea1bc13cb..4d0f6df4fe 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,14 +19,14 @@ - + https://github.com/dotnet/arcade - 97e2f41e909dcabb1103fe98ba4540a246662187 + be933308b9024d798a9a22c0b8f3c8e3616ffbd8 - + https://github.com/dotnet/arcade - 97e2f41e909dcabb1103fe98ba4540a246662187 + be933308b9024d798a9a22c0b8f3c8e3616ffbd8 @@ -34,9 +34,9 @@ https://github.com/dotnet/runtime 84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f - + https://github.com/dotnet/arcade - 97e2f41e909dcabb1103fe98ba4540a246662187 + be933308b9024d798a9a22c0b8f3c8e3616ffbd8 https://github.com/dotnet/roslyn diff --git a/eng/common/templates/steps/publish-pipeline-artifacts.yml b/eng/common/templates/steps/publish-pipeline-artifacts.yml index 88c5635412..815c86038e 100644 --- a/eng/common/templates/steps/publish-pipeline-artifacts.yml +++ b/eng/common/templates/steps/publish-pipeline-artifacts.yml @@ -19,16 +19,16 @@ steps: ${{ if parameters.args.continueOnError }}: continueOnError: ${{ parameters.args.continueOnError }} inputs: - TargetPath: ${{ parameters.args.TargetPath }} - ${{ if parameters.args.ArtifactName }}: - ArtifactName: ${{ parameters.args.ArtifactName }} - ${{ if parameters.args.PublishLocation }}: - PublishLocation: ${{ parameters.args.PublishLocation }} - ${{ if parameters.args.FileSharePath }}: - FileSharePath: ${{ parameters.args.FileSharePath }} + targetPath: ${{ parameters.args.targetPath }} + ${{ if parameters.args.artifactName }}: + artifactName: ${{ parameters.args.artifactName }} + ${{ if parameters.args.publishLocation }}: + publishLocation: ${{ parameters.args.publishLocation }} + ${{ if parameters.args.fileSharePath }}: + fileSharePath: ${{ parameters.args.fileSharePath }} ${{ if parameters.args.Parallel }}: - Parallel: ${{ parameters.args.Parallel }} - ${{ if parameters.args.ParallelCount }}: - ParallelCount: ${{ parameters.args.ParallelCount }} - ${{ if parameters.args.Properties }}: - Properties: ${{ properties.args.Properties }} \ No newline at end of file + parallel: ${{ parameters.args.Parallel }} + ${{ if parameters.args.parallelCount }}: + parallelCount: ${{ parameters.args.parallelCount }} + ${{ if parameters.args.properties }}: + properties: ${{ properties.args.properties }} \ No newline at end of file diff --git a/global.json b/global.json index 1808e75b9e..d251491e5c 100644 --- a/global.json +++ b/global.json @@ -18,6 +18,6 @@ "rollForward": "patch" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24226.3" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24229.1" } } From 638ddbd4e960c37c10d10a559f7b35554f678d3e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 6 May 2024 12:19:32 +0000 Subject: [PATCH 08/30] Update dependencies from https://github.com/dotnet/arcade build 20240503.1 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24212.4 -> To Version 9.0.0-beta.24253.1 --- eng/Version.Details.xml | 12 ++++++------ eng/common/native/init-compiler.sh | 2 +- .../steps/publish-pipeline-artifacts.yml | 2 +- .../templates/steps/publish-pipeline-artifacts.yml | 2 +- global.json | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4d0f6df4fe..fdaff02e28 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,14 +19,14 @@ - + https://github.com/dotnet/arcade - be933308b9024d798a9a22c0b8f3c8e3616ffbd8 + 020255bcf7d0b8beed7de05338d97396982ae527 - + https://github.com/dotnet/arcade - be933308b9024d798a9a22c0b8f3c8e3616ffbd8 + 020255bcf7d0b8beed7de05338d97396982ae527 @@ -34,9 +34,9 @@ https://github.com/dotnet/runtime 84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f - + https://github.com/dotnet/arcade - be933308b9024d798a9a22c0b8f3c8e3616ffbd8 + 020255bcf7d0b8beed7de05338d97396982ae527 https://github.com/dotnet/roslyn diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index afdeb7a4d5..ccd3a17268 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -64,7 +64,7 @@ if [ -z "$CLR_CC" ]; then if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" - elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi + elif [ "$compiler" = "gcc" ]; then versions="14 13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do _major="${version%%.*}" diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml index 15d74a9d91..d71eb0c743 100644 --- a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -23,4 +23,4 @@ steps: ${{ if parameters.args.artifactName }}: artifactName: ${{ parameters.args.artifactName }} ${{ if parameters.args.properties }}: - properties: ${{ properties.args.properties }} \ No newline at end of file + properties: ${{ parameters.args.properties }} \ No newline at end of file diff --git a/eng/common/templates/steps/publish-pipeline-artifacts.yml b/eng/common/templates/steps/publish-pipeline-artifacts.yml index 815c86038e..5dd698b212 100644 --- a/eng/common/templates/steps/publish-pipeline-artifacts.yml +++ b/eng/common/templates/steps/publish-pipeline-artifacts.yml @@ -31,4 +31,4 @@ steps: ${{ if parameters.args.parallelCount }}: parallelCount: ${{ parameters.args.parallelCount }} ${{ if parameters.args.properties }}: - properties: ${{ properties.args.properties }} \ No newline at end of file + properties: ${{ parameters.args.properties }} \ No newline at end of file diff --git a/global.json b/global.json index d251491e5c..939c18594e 100644 --- a/global.json +++ b/global.json @@ -18,6 +18,6 @@ "rollForward": "patch" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24229.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24253.1" } } From 6d91d1b35aead1772bd344cef5aa3f9ae1764708 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 9 May 2024 19:15:09 +0000 Subject: [PATCH 09/30] Update dependencies from https://github.com/dotnet/arcade build 20240508.1 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24212.4 -> To Version 9.0.0-beta.24258.1 --- eng/Version.Details.xml | 12 +- .../job/source-index-stage1.yml | 40 ++++-- .../core-templates/post-build/post-build.yml | 40 ++++-- eng/common/cross/build-rootfs.sh | 122 +++++++++++++++--- eng/common/native/init-distro-rid.sh | 18 +-- global.json | 2 +- 6 files changed, 167 insertions(+), 67 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index fdaff02e28..b0129c658d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,14 +19,14 @@ - + https://github.com/dotnet/arcade - 020255bcf7d0b8beed7de05338d97396982ae527 + a62b4639193b96a1a848ccddaf9fe421c3d3f21b - + https://github.com/dotnet/arcade - 020255bcf7d0b8beed7de05338d97396982ae527 + a62b4639193b96a1a848ccddaf9fe421c3d3f21b @@ -34,9 +34,9 @@ https://github.com/dotnet/runtime 84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f - + https://github.com/dotnet/arcade - 020255bcf7d0b8beed7de05338d97396982ae527 + a62b4639193b96a1a848ccddaf9fe421c3d3f21b https://github.com/dotnet/roslyn diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 9c6e5ae3c3..f1938eec10 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240129.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -15,14 +16,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - template: /eng/common/core-templates/variables/pool-providers.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} @@ -54,8 +55,8 @@ jobs: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -67,7 +68,24 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index ed1e6692f7..865bc1ecb4 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -115,9 +115,13 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows + ${{ if eq(parameters.is1ESPipeline, true) }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022.amd64 + os: windows + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -156,9 +160,13 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows + ${{ if eq(parameters.is1ESPipeline, true) }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -216,9 +224,13 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows + ${{ if eq(parameters.is1ESPipeline, true) }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -273,9 +285,13 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 - os: windows + ${{ if eq(parameters.is1ESPipeline, true) }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + ${{ else }}: + name: NetCore1ESPool-Publishing-Internal + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index ed1bfeda3c..a8e35df7ce 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -451,13 +451,32 @@ fi mkdir -p "$__RootfsDir" __RootfsDir="$( cd "$__RootfsDir" && pwd )" +__hasWget= +ensureDownloadTool() +{ + if command -v wget &> /dev/null; then + __hasWget=1 + elif command -v curl &> /dev/null; then + __hasWget=0 + else + >&2 echo "ERROR: either wget or curl is required by this script." + exit 1 + fi +} + if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.12.11 __ApkToolsDir="$(mktemp -d)" __ApkKeysDir="$(mktemp -d)" - arch="$(uname -m)" - wget "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" -P "$__ApkToolsDir" + + ensureDownloadTool + + if [[ "$__hasWget" == 1 ]]; then + wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" + else + curl -SLO --create-dirs --output-dir "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" + fi if [[ "$arch" == "x86_64" ]]; then __ApkToolsSHA512SUM="53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33" elif [[ "$arch" == "aarch64" ]]; then @@ -520,12 +539,23 @@ if [[ "$__CodeName" == "alpine" ]]; then elif [[ "$__CodeName" == "freebsd" ]]; then mkdir -p "$__RootfsDir"/usr/local/etc JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + + ensureDownloadTool + + if [[ "$__hasWget" == 1 ]]; then + wget -O- "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + else + curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + fi echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf mkdir -p "$__RootfsDir"/tmp # get and build package manager - wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - + if [[ "$__hasWget" == 1 ]]; then + wget -O- "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - + else + curl -SL "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - + fi cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # needed for install to succeed mkdir -p "$__RootfsDir"/host/etc @@ -539,15 +569,30 @@ elif [[ "$__CodeName" == "illumos" ]]; then mkdir "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp" JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} + + ensureDownloadTool + echo "Downloading sysroot." - wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + if [[ "$__hasWget" == 1 ]]; then + wget -O- https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + else + curl -SL https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + fi echo "Building binutils. Please wait.." - wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - + if [[ "$__hasWget" == 1 ]]; then + wget -O- https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - + else + curl -SL https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - + fi mkdir build-binutils && cd build-binutils ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" make -j "$JOBS" && make install && cd .. echo "Building gcc. Please wait.." - wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - + if [[ "$__hasWget" == 1 ]]; then + wget -O- https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - + else + curl -SL https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - + fi CFLAGS="-fPIC" CXXFLAGS="-fPIC" CXXFLAGS_FOR_TARGET="-fPIC" @@ -564,7 +609,11 @@ elif [[ "$__CodeName" == "illumos" ]]; then fi BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All" echo "Downloading manifest" - wget "$BaseUrl" + if [[ "$__hasWget" == 1 ]]; then + wget "$BaseUrl" + else + curl -SLO "$BaseUrl" + fi echo "Downloading dependencies." read -ra array <<<"$__IllumosPackages" for package in "${array[@]}"; do @@ -572,7 +621,11 @@ elif [[ "$__CodeName" == "illumos" ]]; then # find last occurrence of package in listing and extract its name package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)" echo "Resolved name '$package'" - wget "$BaseUrl"/"$package".tgz + if [[ "$__hasWget" == 1 ]]; then + wget "$BaseUrl"/"$package".tgz + else + curl -SLO "$BaseUrl"/"$package".tgz + fi ar -x "$package".tgz tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null done @@ -581,10 +634,17 @@ elif [[ "$__CodeName" == "illumos" ]]; then rm -rf "$__RootfsDir"/{tmp,+*} mkdir -p "$__RootfsDir"/usr/include/net mkdir -p "$__RootfsDir"/usr/include/netpacket - wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h - wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h - wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h - wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h + if [[ "$__hasWget" == 1 ]]; then + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h + wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h + wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h + else + curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h + curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h + curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h + curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h + fi elif [[ "$__CodeName" == "haiku" ]]; then JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} @@ -594,9 +654,16 @@ elif [[ "$__CodeName" == "haiku" ]]; then mkdir "$__RootfsDir/tmp/download" + ensureDownloadTool + echo "Downloading Haiku package tool" git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script" - wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)" + if [[ "$__hasWget" == 1 ]]; then + wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)" + else + curl -SLo "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)" + fi + unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" @@ -609,14 +676,25 @@ elif [[ "$__CodeName" == "haiku" ]]; then echo "Downloading $package..." # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 - hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" - wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + if [[ "$__hasWget" == 1 ]]; then + hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + else + hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + fi done for package in haiku haiku_devel; do echo "Downloading $package..." - hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" - wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + if [[ "$__hasWget" == 1 ]]; then + hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + else + hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + fi done # Set up the sysroot @@ -629,7 +707,11 @@ elif [[ "$__CodeName" == "haiku" ]]; then # Download buildtools echo "Downloading Haiku buildtools" - wget -O "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)" + if [[ "$__hasWget" == 1 ]]; then + wget -O "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)" + else + curl -SLo "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)" + fi unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir" # Cleaning up temporary files diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index 228be0b159..83ea7aab0e 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -20,10 +20,6 @@ getNonPortableDistroRid() # shellcheck disable=SC1091 if [ -e "${rootfsDir}/etc/os-release" ]; then . "${rootfsDir}/etc/os-release" - if [ "${ID}" = "rhel" ] || [ "${ID}" = "rocky" ] || [ "${ID}" = "alpine" ] || [ "${ID}" = "ol" ]; then - VERSION_ID="${VERSION_ID%.*}" # Remove the last version digit for these distros - fi - if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" else @@ -48,19 +44,7 @@ getNonPortableDistroRid() nonPortableRid="android.$__android_sdk_version-${targetArch}" elif [ "$targetOs" = "illumos" ]; then __uname_version=$(uname -v) - case "$__uname_version" in - omnios-*) - __omnios_major_version=$(echo "$__uname_version" | cut -c9-10) - nonPortableRid="omnios.$__omnios_major_version-${targetArch}" - ;; - joyent_*) - __smartos_major_version=$(echo "$__uname_version" | cut -c9-10) - nonPortableRid="smartos.$__smartos_major_version-${targetArch}" - ;; - *) - nonPortableRid="illumos-${targetArch}" - ;; - esac + nonPortableRid="illumos-${targetArch}" elif [ "$targetOs" = "solaris" ]; then __uname_version=$(uname -v) __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) diff --git a/global.json b/global.json index 939c18594e..5baaf5c864 100644 --- a/global.json +++ b/global.json @@ -18,6 +18,6 @@ "rollForward": "patch" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24253.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24258.1" } } From 6a4cd458fd8012f68a007de6d9debee232de402e Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Fri, 10 May 2024 19:53:18 +0200 Subject: [PATCH 10/30] Prevent SemaphoreSlim.Wait(0) from triggering CA1849 --- .../Runtime/UseAsyncMethodInAsyncContext.cs | 35 ++- .../UseAsyncMethodInAsyncContextTests.cs | 212 ++++++++++++++++++ src/Utilities/Compiler/WellKnownTypeNames.cs | 1 + 3 files changed, 247 insertions(+), 1 deletion(-) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs index a720990d51..e1bfd6e857 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs @@ -5,6 +5,8 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Analyzer.Utilities; using Analyzer.Utilities.Extensions; using Microsoft.CodeAnalysis; @@ -83,6 +85,20 @@ public override void Initialize(AnalysisContext context) return; } + INamedTypeSymbol? semaphoreSlimType = wellKnownTypeProvider.GetOrCreateTypeByMetadataName(WellKnownTypeNames.SystemThreadingSemaphoreSlim); + INamedTypeSymbol? timeSpanType = wellKnownTypeProvider.GetOrCreateTypeByMetadataName(WellKnownTypeNames.SystemTimeSpan); + INamedTypeSymbol intType = context.Compilation.GetSpecialType(SpecialType.System_Int32); + IFieldSymbol? timeSpanZero = timeSpanType?.GetMembers(nameof(TimeSpan.Zero)) + .OfType() + .FirstOrDefault(); + ISet semaphoreSlimWaitMethods = semaphoreSlimType + ?.GetMembers(nameof(SemaphoreSlim.Wait)) + .OfType() + .Where(m => m.Parameters.Length > 0 + && (SymbolEqualityComparer.Default.Equals(m.Parameters[0].Type, intType) + || SymbolEqualityComparer.Default.Equals(m.Parameters[0].Type, timeSpanType))) + .ToSet() ?? ImmutableHashSet.Empty; + ImmutableArray excludedMethods = GetExcludedMethods(wellKnownTypeProvider); context.RegisterOperationAction(context => { @@ -91,7 +107,9 @@ public override void Initialize(AnalysisContext context) if (context.Operation is IInvocationOperation invocationOperation) { var methodSymbol = invocationOperation.TargetMethod; - if (excludedMethods.Contains(methodSymbol.OriginalDefinition, SymbolEqualityComparer.Default) || InspectAndReportBlockingMemberAccess(context, methodSymbol, syncBlockingSymbols, SymbolKind.Method)) + if (excludedMethods.Contains(methodSymbol.OriginalDefinition, SymbolEqualityComparer.Default) + || InspectAndReportBlockingMemberAccess(context, methodSymbol, syncBlockingSymbols, SymbolKind.Method) + || IsSemaphoreSlimWaitWithZeroArgumentInvocation(invocationOperation, timeSpanZero, semaphoreSlimWaitMethods)) { // Don't return double-diagnostics. return; @@ -148,6 +166,21 @@ public override void Initialize(AnalysisContext context) }); } + private static bool IsSemaphoreSlimWaitWithZeroArgumentInvocation(IInvocationOperation invocation, IFieldSymbol? timeSpanZero, ISet semaphoreSlimWaitMethods) + { + if(!semaphoreSlimWaitMethods.Contains(invocation.TargetMethod, SymbolEqualityComparer.Default) || invocation.Arguments.IsEmpty) + { + return false; + } + + IOperation argumentValue = invocation.Arguments[0].Value; + + return argumentValue.HasConstantValue(0) + || timeSpanZero is not null + && argumentValue is IFieldReferenceOperation fieldReference + && SymbolEqualityComparer.Default.Equals(fieldReference.Field, timeSpanZero); + } + private static SymbolDisplayFormat GetLanguageSpecificFormat(IOperation operation) => operation.Language == LanguageNames.CSharp ? SymbolDisplayFormat.CSharpShortErrorMessageFormat : SymbolDisplayFormat.VisualBasicShortErrorMessageFormat; diff --git a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContextTests.cs b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContextTests.cs index f9e050ca07..10cadbcd90 100644 --- a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContextTests.cs +++ b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContextTests.cs @@ -1343,6 +1343,218 @@ public async Task Foo() return VerifyCS.VerifyAnalyzerAsync(code); } + [Fact, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + public Task WhenPassingZeroToSemaphoreSlimWait_NoDiagnostic() + { + const string code = """ + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + s.Wait(0); + } + } + """; + + return VerifyCS.VerifyAnalyzerAsync(code); + } + + [Fact, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + public Task WhenPassingZeroWithCancellationTokenToSemaphoreSlimWait_NoDiagnostic() + { + const string code = """ + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + s.Wait(0, CancellationToken.None); + } + } + """; + + return VerifyCS.VerifyAnalyzerAsync(code); + } + + [Fact, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + public Task WhenPassingTimeSpanZeroToSemaphoreSlimWait_NoDiagnostic() + { + const string code = """ + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + s.Wait(TimeSpan.Zero); + } + } + """; + + return VerifyCS.VerifyAnalyzerAsync(code); + } + + [Fact, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + public Task WhenPassingTimeSpanZeroWithCancellationTokenToSemaphoreSlimWait_NoDiagnostic() + { + const string code = """ + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + s.Wait(TimeSpan.Zero, CancellationToken.None); + } + } + """; + + return VerifyCS.VerifyAnalyzerAsync(code); + } + + [Theory, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + [InlineData("1")] + [InlineData("500")] + public Task WhenPassingNonZeroToSemaphoreSlimWait_Diagnostic(string nonZero) + { + var code = $$""" + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + {|#0:s.Wait({{nonZero}})|}; + } + } + """; + var result = new DiagnosticResult(UseAsyncMethodInAsyncContext.Descriptor) + .WithLocation(0) + .WithArguments("SemaphoreSlim.Wait(int)", "SemaphoreSlim.WaitAsync()"); + + return VerifyCS.VerifyAnalyzerAsync(code, result); + } + + [Theory, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + [InlineData("1")] + [InlineData("500")] + public Task WhenPassingNonZeroWithCancellationTokenToSemaphoreSlimWait_Diagnostic(string nonZero) + { + var code = $$""" + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + {|#0:s.Wait({{nonZero}}, CancellationToken.None)|}; + } + } + """; + var result = new DiagnosticResult(UseAsyncMethodInAsyncContext.Descriptor) + .WithLocation(0) + .WithArguments("SemaphoreSlim.Wait(int, CancellationToken)", "SemaphoreSlim.WaitAsync()"); + + return VerifyCS.VerifyAnalyzerAsync(code, result); + } + + [Theory, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + [InlineData("TimeSpan.FromSeconds(30)")] + [InlineData("TimeSpan.Parse(\"0:32:0\")")] + public Task WhenPassingNonZeroTimeSpanToSemaphoreSlimWait_Diagnostic(string nonZero) + { + var code = $$""" + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + {|#0:s.Wait({{nonZero}})|}; + } + } + """; + var result = new DiagnosticResult(UseAsyncMethodInAsyncContext.Descriptor) + .WithLocation(0) + .WithArguments("SemaphoreSlim.Wait(TimeSpan)", "SemaphoreSlim.WaitAsync()"); + + return VerifyCS.VerifyAnalyzerAsync(code, result); + } + + [Theory, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + [InlineData("TimeSpan.FromSeconds(30)")] + [InlineData("TimeSpan.Parse(\"0:32:0\")")] + public Task WhenPassingNonZeroTimeSpanWithCancellationTokenToSemaphoreSlimWait_Diagnostic(string nonZero) + { + var code = $$""" + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + {|#0:s.Wait({{nonZero}}, CancellationToken.None)|}; + } + } + """; + var result = new DiagnosticResult(UseAsyncMethodInAsyncContext.Descriptor) + .WithLocation(0) + .WithArguments("SemaphoreSlim.Wait(TimeSpan, CancellationToken)", "SemaphoreSlim.WaitAsync()"); + + return VerifyCS.VerifyAnalyzerAsync(code, result); + } + + [Fact, WorkItem(7271, "https://github.com/dotnet/roslyn-analyzers/issues/7271")] + public Task WhenPassingCancellationTokenToSemaphoreSlimWait_Diagnostic() + { + const string code = """ + using System; + using System.Threading; + using System.Threading.Tasks; + + class Test + { + async Task M() + { + SemaphoreSlim s = new SemaphoreSlim(0); + {|#0:s.Wait(CancellationToken.None)|}; + } + } + """; + var result = new DiagnosticResult(UseAsyncMethodInAsyncContext.Descriptor) + .WithLocation(0) + .WithArguments("SemaphoreSlim.Wait(CancellationToken)", "SemaphoreSlim.WaitAsync()"); + + return VerifyCS.VerifyAnalyzerAsync(code, result); + } + private static async Task CreateCSTestAndRunAsync(string testCS) { var csTestVerify = new VerifyCS.Test diff --git a/src/Utilities/Compiler/WellKnownTypeNames.cs b/src/Utilities/Compiler/WellKnownTypeNames.cs index 9fa8662c79..304ae70813 100644 --- a/src/Utilities/Compiler/WellKnownTypeNames.cs +++ b/src/Utilities/Compiler/WellKnownTypeNames.cs @@ -426,6 +426,7 @@ internal static class WellKnownTypeNames public const string SystemThreadingCancellationToken = "System.Threading.CancellationToken"; public const string SystemThreadingInterlocked = "System.Threading.Interlocked"; public const string SystemThreadingMonitor = "System.Threading.Monitor"; + public const string SystemThreadingSemaphoreSlim = "System.Threading.SemaphoreSlim"; public const string SystemThreadingSpinLock = "System.Threading.SpinLock"; public const string SystemThreadingTasksConfigureAwaitOptions = "System.Threading.Tasks.ConfigureAwaitOptions"; public const string SystemThreadingTasksTaskAsyncEnumerableExtensions = "System.Threading.Tasks.TaskAsyncEnumerableExtensions"; From 2ab7ba1f63083587c9ce3cf27c6d93fe49bd5d9a Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Fri, 10 May 2024 19:53:58 +0200 Subject: [PATCH 11/30] Remove unnecessary using --- .../Runtime/UseAsyncMethodInAsyncContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs index e1bfd6e857..4415394dfe 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs @@ -6,7 +6,6 @@ using System.Collections.Immutable; using System.Linq; using System.Threading; -using System.Threading.Tasks; using Analyzer.Utilities; using Analyzer.Utilities.Extensions; using Microsoft.CodeAnalysis; From c71561df03e7d06d92fb1aa050dd87e252eb1686 Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Fri, 10 May 2024 20:13:02 +0200 Subject: [PATCH 12/30] Fix formatting --- .../Runtime/UseAsyncMethodInAsyncContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs index 4415394dfe..a65ca6979c 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs @@ -167,7 +167,7 @@ public override void Initialize(AnalysisContext context) private static bool IsSemaphoreSlimWaitWithZeroArgumentInvocation(IInvocationOperation invocation, IFieldSymbol? timeSpanZero, ISet semaphoreSlimWaitMethods) { - if(!semaphoreSlimWaitMethods.Contains(invocation.TargetMethod, SymbolEqualityComparer.Default) || invocation.Arguments.IsEmpty) + if (!semaphoreSlimWaitMethods.Contains(invocation.TargetMethod, SymbolEqualityComparer.Default) || invocation.Arguments.IsEmpty) { return false; } From b27caf431eafeadd56246ff6b75066850f046df7 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 13 May 2024 12:19:28 +0000 Subject: [PATCH 13/30] Update dependencies from https://github.com/dotnet/arcade build 20240510.2 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24212.4 -> To Version 9.0.0-beta.24260.2 --- eng/Version.Details.xml | 12 ++++++------ global.json | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b0129c658d..915ccd64c2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,14 +19,14 @@ - + https://github.com/dotnet/arcade - a62b4639193b96a1a848ccddaf9fe421c3d3f21b + 480401b003bfd2eb989c315da5d6b99ad13a968c - + https://github.com/dotnet/arcade - a62b4639193b96a1a848ccddaf9fe421c3d3f21b + 480401b003bfd2eb989c315da5d6b99ad13a968c @@ -34,9 +34,9 @@ https://github.com/dotnet/runtime 84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f - + https://github.com/dotnet/arcade - a62b4639193b96a1a848ccddaf9fe421c3d3f21b + 480401b003bfd2eb989c315da5d6b99ad13a968c https://github.com/dotnet/roslyn diff --git a/global.json b/global.json index 5baaf5c864..fb5422373f 100644 --- a/global.json +++ b/global.json @@ -18,6 +18,6 @@ "rollForward": "patch" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24258.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24260.2" } } From c455f63e592972cda73bfbdc41a72fbca36eda57 Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Sat, 18 May 2024 14:18:38 +0200 Subject: [PATCH 14/30] Apply code review suggestions. --- .../Runtime/UseAsyncMethodInAsyncContext.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs index a65ca6979c..e56cb7736f 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs @@ -4,6 +4,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics; using System.Linq; using System.Threading; using Analyzer.Utilities; @@ -90,13 +91,13 @@ public override void Initialize(AnalysisContext context) IFieldSymbol? timeSpanZero = timeSpanType?.GetMembers(nameof(TimeSpan.Zero)) .OfType() .FirstOrDefault(); - ISet semaphoreSlimWaitMethods = semaphoreSlimType + ImmutableArray semaphoreSlimWaitWithTimeoutMethods = semaphoreSlimType ?.GetMembers(nameof(SemaphoreSlim.Wait)) .OfType() .Where(m => m.Parameters.Length > 0 && (SymbolEqualityComparer.Default.Equals(m.Parameters[0].Type, intType) || SymbolEqualityComparer.Default.Equals(m.Parameters[0].Type, timeSpanType))) - .ToSet() ?? ImmutableHashSet.Empty; + .ToImmutableArray() ?? ImmutableArray.Empty; ImmutableArray excludedMethods = GetExcludedMethods(wellKnownTypeProvider); context.RegisterOperationAction(context => @@ -108,7 +109,7 @@ public override void Initialize(AnalysisContext context) var methodSymbol = invocationOperation.TargetMethod; if (excludedMethods.Contains(methodSymbol.OriginalDefinition, SymbolEqualityComparer.Default) || InspectAndReportBlockingMemberAccess(context, methodSymbol, syncBlockingSymbols, SymbolKind.Method) - || IsSemaphoreSlimWaitWithZeroArgumentInvocation(invocationOperation, timeSpanZero, semaphoreSlimWaitMethods)) + || IsSemaphoreSlimWaitWithZeroArgumentInvocation(invocationOperation, timeSpanZero, semaphoreSlimWaitWithTimeoutMethods)) { // Don't return double-diagnostics. return; @@ -165,9 +166,9 @@ public override void Initialize(AnalysisContext context) }); } - private static bool IsSemaphoreSlimWaitWithZeroArgumentInvocation(IInvocationOperation invocation, IFieldSymbol? timeSpanZero, ISet semaphoreSlimWaitMethods) + private static bool IsSemaphoreSlimWaitWithZeroArgumentInvocation(IInvocationOperation invocation, IFieldSymbol? timeSpanZero, ImmutableArray semaphoreSlimWaitWithTimeoutMethods) { - if (!semaphoreSlimWaitMethods.Contains(invocation.TargetMethod, SymbolEqualityComparer.Default) || invocation.Arguments.IsEmpty) + if (!semaphoreSlimWaitWithTimeoutMethods.Contains(invocation.TargetMethod, SymbolEqualityComparer.Default)) { return false; } From e264af8ea77a8544aaafed73220ed27f801abf2e Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Sat, 18 May 2024 14:18:52 +0200 Subject: [PATCH 15/30] Remove unnecessary using. --- .../Runtime/UseAsyncMethodInAsyncContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs index e56cb7736f..3a90a31fd3 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs @@ -4,7 +4,6 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; -using System.Diagnostics; using System.Linq; using System.Threading; using Analyzer.Utilities; From 6d45c87b6a307d537903608ec33126523ab3bfd2 Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Sat, 18 May 2024 14:20:35 +0200 Subject: [PATCH 16/30] Add Debug.Assert --- .../Runtime/UseAsyncMethodInAsyncContext.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs index 3a90a31fd3..1e29c872f5 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/UseAsyncMethodInAsyncContext.cs @@ -4,6 +4,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics; using System.Linq; using System.Threading; using Analyzer.Utilities; @@ -172,6 +173,8 @@ private static bool IsSemaphoreSlimWaitWithZeroArgumentInvocation(IInvocationOpe return false; } + Debug.Assert(!invocation.Arguments.IsEmpty); + IOperation argumentValue = invocation.Arguments[0].Value; return argumentValue.HasConstantValue(0) From fbd78b34c2a8438d59f046792dd41094db5f3b0c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 20 May 2024 12:19:09 +0000 Subject: [PATCH 17/30] Update dependencies from https://github.com/dotnet/arcade build 20240516.1 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24212.4 -> To Version 9.0.0-beta.24266.1 --- eng/Version.Details.xml | 12 ++++++------ global.json | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 915ccd64c2..168912faae 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,14 +19,14 @@ - + https://github.com/dotnet/arcade - 480401b003bfd2eb989c315da5d6b99ad13a968c + ed14da5934ffb536cff8f41f8b5719334524cbed - + https://github.com/dotnet/arcade - 480401b003bfd2eb989c315da5d6b99ad13a968c + ed14da5934ffb536cff8f41f8b5719334524cbed @@ -34,9 +34,9 @@ https://github.com/dotnet/runtime 84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f - + https://github.com/dotnet/arcade - 480401b003bfd2eb989c315da5d6b99ad13a968c + ed14da5934ffb536cff8f41f8b5719334524cbed https://github.com/dotnet/roslyn diff --git a/global.json b/global.json index fb5422373f..72f8b2c61c 100644 --- a/global.json +++ b/global.json @@ -18,6 +18,6 @@ "rollForward": "patch" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24260.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24266.1" } } From d633dd2992968839b709c52b87ebc6a00a5042fe Mon Sep 17 00:00:00 2001 From: Arun Chander Date: Tue, 21 May 2024 17:10:17 -0700 Subject: [PATCH 18/30] Monthly run of the CI build to make sure all rel branches are buildable --- azure-pipelines-microbuild.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-microbuild.yml b/azure-pipelines-microbuild.yml index 2825adb3b6..4b52706688 100644 --- a/azure-pipelines-microbuild.yml +++ b/azure-pipelines-microbuild.yml @@ -12,6 +12,18 @@ trigger: - features/* - 2.9.x +schedules: + - cron: "0 8 22-28 * 0" + displayName: "Monthly smoke test" + branches: + include: + - main + - release/* + exclude: + - "" + always: true # Run even if there have been no source code changes since the last successful scheduled run + batch: false # Do not run the pipeline if the previously scheduled run is in-progress + variables: - name: TeamName value: Roslyn @@ -116,4 +128,4 @@ extends: enableSymbolValidation: false enableSigningValidation: false enableSourceLinkValidation: false - enableNugetValidation: false \ No newline at end of file + enableNugetValidation: false From ff2edbc421765f4283d63496f6953ce0ae93b6e4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 27 May 2024 12:17:29 +0000 Subject: [PATCH 19/30] Update dependencies from https://github.com/dotnet/arcade build 20240522.5 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24212.4 -> To Version 9.0.0-beta.24272.5 --- eng/Version.Details.xml | 12 +-- .../job/source-index-stage1.yml | 6 +- eng/common/internal/Directory.Build.props | 5 ++ eng/common/internal/Tools.csproj | 1 - eng/common/native/init-compiler.sh | 85 ++++++++++++------- .../templates/steps/get-delegate-sas.yml | 43 ++++++++++ eng/common/tools.ps1 | 6 +- global.json | 2 +- 8 files changed, 117 insertions(+), 43 deletions(-) create mode 100644 eng/common/templates/steps/get-delegate-sas.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 168912faae..f3eadf8fe2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,14 +19,14 @@ - + https://github.com/dotnet/arcade - ed14da5934ffb536cff8f41f8b5719334524cbed + 2001d73c8ff942331a73300ba61fa6164805b231 - + https://github.com/dotnet/arcade - ed14da5934ffb536cff8f41f8b5719334524cbed + 2001d73c8ff942331a73300ba61fa6164805b231 @@ -34,9 +34,9 @@ https://github.com/dotnet/runtime 84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f - + https://github.com/dotnet/arcade - ed14da5934ffb536cff8f41f8b5719334524cbed + 2001d73c8ff942331a73300ba61fa6164805b231 https://github.com/dotnet/roslyn diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index f1938eec10..8328e52ab1 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -1,7 +1,7 @@ parameters: runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20240502.12 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 + sourceIndexUploadPackageVersion: 2.0.0-20240522.1 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -88,4 +88,4 @@ jobs: displayName: "Login to Azure" - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index \ No newline at end of file + displayName: Upload stage1 artifacts to source index diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props index a735fe9a13..f1d041c33d 100644 --- a/eng/common/internal/Directory.Build.props +++ b/eng/common/internal/Directory.Build.props @@ -1,6 +1,11 @@ + + false + false + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 8fa77e5b18..e925952d56 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -3,7 +3,6 @@ net472 - false false diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index ccd3a17268..62900e12b2 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -2,7 +2,9 @@ # # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # -# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! +# NOTE: some scripts source this file and rely on stdout being empty, make sure +# to not output *anything* here, unless it is an error message that fails the +# build. if [ -z "$build_arch" ] || [ -z "$compiler" ]; then echo "Usage..." @@ -58,6 +60,26 @@ check_version_exists() { echo "$desired_version" } +__baseOS="$(uname)" +set_compiler_version_from_CC() { + if [ "$__baseOS" = "Darwin" ]; then + # On Darwin, the versions from -version/-dumpversion refer to Xcode + # versions, not llvm versions, so we can't rely on them. + return + fi + + version="$("$CC" -dumpversion)" + if [ -z "$version" ]; then + echo "Error: $CC -dumpversion didn't provide a version" + exit 1 + fi + + # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments. + IFS=. read -r majorVersion minorVersion _ < /dev/null; then - if [ "$(uname)" != "Darwin" ]; then - echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH." - fi - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" - else - echo "No usable version of $compiler found." + if ! command -v "$compiler" > /dev/null; then + echo "Error: No usable version of $compiler found." exit 1 fi + + CC="$(command -v "$compiler" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler" 2> /dev/null)" + set_compiler_version_from_CC else - if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then - if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then - if command -v "$compiler" > /dev/null; then - echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" - else - echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." - exit 1 - fi + if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ] && { [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; }; then + # If a major version was provided explicitly, and it was too old, find a newer compiler instead + if ! command -v "$compiler" > /dev/null; then + echo "Error: Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." + exit 1 fi + + CC="$(command -v "$compiler" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler" 2> /dev/null)" + set_compiler_version_from_CC fi fi else desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" if [ "$desired_version" = "-1" ]; then - echo "Could not find specific version of $compiler: $majorVersion $minorVersion." + echo "Error: Could not find specific version of $compiler: $majorVersion $minorVersion." exit 1 fi fi if [ -z "$CC" ]; then - CC="$(command -v "$compiler$desired_version")" - CXX="$(command -v "$cxxCompiler$desired_version")" - if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi + CC="$(command -v "$compiler$desired_version" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler$desired_version" 2> /dev/null)" + if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler" 2> /dev/null)"; fi + set_compiler_version_from_CC fi else if [ ! -f "$CLR_CC" ]; then - echo "CLR_CC is set but path '$CLR_CC' does not exist" + echo "Error: CLR_CC is set but path '$CLR_CC' does not exist" exit 1 fi CC="$CLR_CC" CXX="$CLR_CXX" + set_compiler_version_from_CC fi if [ -z "$CC" ]; then - echo "Unable to find $compiler." + echo "Error: Unable to find $compiler." exit 1 fi -# Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0. -if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && ([ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]); then - if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then - LDFLAGS="-fuse-ld=lld" +if [ "$__baseOS" != "Darwin" ]; then + # On Darwin, we always want to use the Apple linker. + + # Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0. + if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && { [ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]; }; then + if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then + LDFLAGS="-fuse-ld=lld" + fi fi fi -SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" +SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version" 2> /dev/null)" export CC CXX LDFLAGS SCAN_BUILD_COMMAND diff --git a/eng/common/templates/steps/get-delegate-sas.yml b/eng/common/templates/steps/get-delegate-sas.yml new file mode 100644 index 0000000000..c0e8f91317 --- /dev/null +++ b/eng/common/templates/steps/get-delegate-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 7fc78a0851..0febe696db 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -254,7 +254,6 @@ function Retry($downloadBlock, $maxRetries = 5) { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." break } - } } @@ -424,7 +423,6 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] @@ -504,6 +502,10 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath }) + if (!(Test-Path $packagePath)) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild" + throw + } Unzip $packagePath $packageDir } diff --git a/global.json b/global.json index 72f8b2c61c..dd02aae890 100644 --- a/global.json +++ b/global.json @@ -18,6 +18,6 @@ "rollForward": "patch" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24266.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24272.5" } } From 43aac771900aa28f2bf564e7eab4072111a6a866 Mon Sep 17 00:00:00 2001 From: Arun Chander Date: Tue, 28 May 2024 10:17:17 -0700 Subject: [PATCH 20/30] Update azure-pipelines-microbuild.yml --- azure-pipelines-microbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-microbuild.yml b/azure-pipelines-microbuild.yml index 4b52706688..f918bd2326 100644 --- a/azure-pipelines-microbuild.yml +++ b/azure-pipelines-microbuild.yml @@ -13,7 +13,7 @@ trigger: - 2.9.x schedules: - - cron: "0 8 22-28 * 0" + - cron: "0 8 23-29 * 0" displayName: "Monthly smoke test" branches: include: From be1b0f08d89b7ca5606239fadb84d4c86c8eda05 Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Fri, 7 Jun 2024 05:30:46 +0200 Subject: [PATCH 21/30] Mark type as instantiated when used with a collection expression initializer (#7229) --- .../AvoidUninstantiatedInternalClasses.cs | 6 +-- ...AvoidUninstantiatedInternalClassesTests.cs | 42 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/AvoidUninstantiatedInternalClasses.cs b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/AvoidUninstantiatedInternalClasses.cs index 70cf9d0add..1b9c320268 100644 --- a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/AvoidUninstantiatedInternalClasses.cs +++ b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/AvoidUninstantiatedInternalClasses.cs @@ -7,6 +7,7 @@ using System.Linq; using Analyzer.Utilities; using Analyzer.Utilities.Extensions; +using Analyzer.Utilities.Lightup; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Operations; @@ -78,12 +79,11 @@ public sealed override void Initialize(AnalysisContext context) startContext.RegisterOperationAction(context => { - var expr = (IObjectCreationOperation)context.Operation; - if (expr.Type is INamedTypeSymbol namedType && namedType.ContainingAssembly.Equals(context.ContainingSymbol.ContainingAssembly)) + if (context.Operation.Type is INamedTypeSymbol namedType && namedType.ContainingAssembly.Equals(context.ContainingSymbol.ContainingAssembly)) { instantiatedTypes.TryAdd(namedType, null); } - }, OperationKind.ObjectCreation); + }, OperationKind.ObjectCreation, OperationKindEx.CollectionExpression); startContext.RegisterSymbolAction(context => { diff --git a/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/Maintainability/AvoidUninstantiatedInternalClassesTests.cs b/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/Maintainability/AvoidUninstantiatedInternalClassesTests.cs index 865b1f22fc..f459c98f24 100644 --- a/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/Maintainability/AvoidUninstantiatedInternalClassesTests.cs +++ b/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/Maintainability/AvoidUninstantiatedInternalClassesTests.cs @@ -2,6 +2,7 @@ using System.Threading.Tasks; using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.Testing; using Test.Utilities; @@ -1656,6 +1657,47 @@ internal class {|#0:C|} { }"; await test.RunAsync(); } + [Fact, WorkItem(7223, "https://github.com/dotnet/roslyn-analyzers/issues/7223")] + public Task CA1812_CollectionExpressionsInField_NoDiagnostic() + { + const string code = """ + using System.Collections.ObjectModel; + + public class Class1 + { + private readonly MyCollection _items = []; + } + internal sealed class MyCollection : Collection; + """; + return new VerifyCS.Test + { + TestCode = code, + LanguageVersion = LanguageVersion.CSharp12 + }.RunAsync(); + } + + [Fact, WorkItem(7223, "https://github.com/dotnet/roslyn-analyzers/issues/7223")] + public Task CA1812_CollectionExpressionsInLocalVariable_NoDiagnostic() + { + const string code = """ + using System.Collections.ObjectModel; + + public class Class1 + { + private void M() + { + MyCollection c = []; + } + } + internal sealed class MyCollection : Collection; + """; + return new VerifyCS.Test + { + TestCode = code, + LanguageVersion = LanguageVersion.CSharp12 + }.RunAsync(); + } + private static DiagnosticResult GetCSharpResultAt(int line, int column, string className) #pragma warning disable RS0030 // Do not use banned APIs => VerifyCS.Diagnostic() From 4d5fd9da36d64d4c3370b8813122e226844fc6ed Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Fri, 7 Jun 2024 05:49:55 +0200 Subject: [PATCH 22/30] Generate field for const arrays in constructor declarations (#7231) --- .../Runtime/AvoidConstArrays.Fixer.cs | 1 + .../Runtime/AvoidConstArraysTests.cs | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/AvoidConstArrays.Fixer.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/AvoidConstArrays.Fixer.cs index 99e9a69fa5..d675d297e2 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/AvoidConstArrays.Fixer.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/Runtime/AvoidConstArrays.Fixer.cs @@ -74,6 +74,7 @@ private static async Task ExtractConstArrayAsync(Document document, Sy // Get method containing the symbol that is being diagnosed IOperation? methodContext = arrayArgument.GetAncestor(OperationKind.MethodBody); methodContext ??= arrayArgument.GetAncestor(OperationKind.Block); // VB methods have a different structure than CS methods + methodContext ??= arrayArgument.GetAncestor(OperationKind.ConstructorBody); // Create the new member SyntaxNode newMember = generator.FieldDeclaration( diff --git a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/AvoidConstArraysTests.cs b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/AvoidConstArraysTests.cs index 26c95331a0..c189f175db 100644 --- a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/AvoidConstArraysTests.cs +++ b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/AvoidConstArraysTests.cs @@ -1053,5 +1053,38 @@ partial class Program } }.RunAsync(); } + + [Fact, WorkItem(7216, "https://github.com/dotnet/roslyn-analyzers/issues/7216")] + public Task BaseDeclaration_Diagnostic() + { + const string code = """ + public class Class1 + { + public Class1(int[] arr) { } + } + + public class Class2 : Class1 + { + public Class2() + : base([|new int[] { 1, 2, 3 }|]) { } + } + """; + const string fixedCode = """ + public class Class1 + { + public Class1(int[] arr) { } + } + + public class Class2 : Class1 + { + private static readonly int[] arr = new int[] { 1, 2, 3 }; + + public Class2() + : base(arr) { } + } + """; + + return VerifyCS.VerifyCodeFixAsync(code, fixedCode); + } } } From 52913b28e20a224a5f6d56fe0b84b13210e274b0 Mon Sep 17 00:00:00 2001 From: Buyaa Namnan Date: Fri, 14 Jun 2024 00:30:19 +0000 Subject: [PATCH 23/30] Do not warn when invoking APIs that has SupportedOSPlatformGuard attribute (#7326) * Do not warn when invoking APIs that has SupportedOSPlatformGuard attribute * dotnet build /t:pack update * Add more scenarios in the test * Again doc updated --- .../PlatformCompatibilityAnalyzer.cs | 12 +-- src/NetAnalyzers/RulesMissingDocumentation.md | 3 +- ...tibilityAnalyzerTests.GuardedCallsTests.cs | 88 ++++++++++++++++++- 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs index d38d19b817..1bf6950d50 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs @@ -1407,7 +1407,8 @@ void CheckOperationAttributes(ISymbol symbol, bool checkParents) platformSpecificOperations.TryAdd(new KeyValuePair(operation, symbol), (notSuppressedAttributes, callSiteAttributes.Platforms)); } } - else if (TryCopyAttributesNotSuppressedByMsBuild(operationAttributes.Platforms!, msBuildPlatforms, out var copiedAttributes)) + else if (!OperationHasOnlyAssemblyAttributesAndCalledFromSameAssembly(operationAttributes, symbol, containingSymbol) && + TryCopyAttributesNotSuppressedByMsBuild(operationAttributes.Platforms!, msBuildPlatforms, out var copiedAttributes)) { platformSpecificOperations.TryAdd(new KeyValuePair(operation, symbol), (copiedAttributes, null)); } @@ -1415,6 +1416,9 @@ void CheckOperationAttributes(ISymbol symbol, bool checkParents) } } + private static bool OperationHasOnlyAssemblyAttributesAndCalledFromSameAssembly(PlatformAttributes operationAttributes, ISymbol symbol, ISymbol containingSymbol) => + operationAttributes.IsAssemblyAttribute && containingSymbol.ContainingAssembly == symbol.ContainingAssembly; + private static bool UsedInCreatingNotSupportedException(IArgumentOperation operation, ITypeSymbol? notSupportedExceptionType) { if (operation.Parent is IObjectCreationOperation creation && @@ -1799,11 +1803,7 @@ static void MergePlatformAttributes(ImmutableArray immediateAttri if (attribute.AttributeClass.Name is SupportedOSPlatformGuardAttribute or UnsupportedOSPlatformGuardAttribute) { - if (!parentAttributes.IsAssemblyAttribute) - { - parentAttributes = new PlatformAttributes(); - } - + parentAttributes = new PlatformAttributes(); // The API is for guard, clear parent attributes return; } diff --git a/src/NetAnalyzers/RulesMissingDocumentation.md b/src/NetAnalyzers/RulesMissingDocumentation.md index f0385ea23d..d19d2205c7 100644 --- a/src/NetAnalyzers/RulesMissingDocumentation.md +++ b/src/NetAnalyzers/RulesMissingDocumentation.md @@ -2,7 +2,6 @@ Rule ID | Missing Help Link | Title | --------|-------------------|-------| -CA1871 | | Do not pass a nullable struct to 'ArgumentNullException.ThrowIfNull' | CA2022 | | Avoid inexact read with 'Stream.Read' | -CA2264 | | Do not pass a non-nullable value to 'ArgumentNullException.ThrowIfNull' | CA2265 | | Do not compare Span\ to 'null' or 'default' | +CA5360 | | Do Not Call Dangerous Methods In Deserialization | diff --git a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzerTests.GuardedCallsTests.cs b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzerTests.GuardedCallsTests.cs index a230b74ec1..71a11fcc72 100644 --- a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzerTests.GuardedCallsTests.cs +++ b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzerTests.GuardedCallsTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Testing; using Test.Utilities; using Xunit; @@ -1216,6 +1217,91 @@ static void M1() await VerifyAnalyzerCSAsync(csSource); } + [Fact] + public async Task CallGuardFromPlatformSpecificAssembly() + { + string csDependencyCode = @" +public class Library +{ + static bool s_isWindowsOrLinux = false; + + [System.Runtime.Versioning.SupportedOSPlatformGuard(""windows"")] + [System.Runtime.Versioning.SupportedOSPlatformGuard(""linux"")] + public static bool IsSupported => s_isWindowsOrLinux; + + [System.Runtime.Versioning.UnsupportedOSPlatformGuard(""windows"")] + [System.Runtime.Versioning.UnsupportedOSPlatformGuard(""linux"")] + public static bool IsNotSupported => false; + + public static void AMethod() { } +}"; + csDependencyCode = @$"[assembly: System.Runtime.Versioning.SupportedOSPlatform(""windows"")] + [assembly: System.Runtime.Versioning.SupportedOSPlatform(""linux"")] + {csDependencyCode}"; + + string csCurrentAssemblyCode = @" +using System; + +public class Program +{ + public void ProgramMethod() + { + [|Library.AMethod()|]; // Not guarded, warns + if (Library.IsSupported) + { + Library.AMethod(); + } + + if (Library.IsNotSupported) + { + [|Library.AMethod()|]; // warn because guarded by unsupported + } + else + { + Library.AMethod(); // guarded + } + } +}"; + var test = SetupDependencyAndTestCSWithOneSourceFile(csCurrentAssemblyCode, csDependencyCode); + test.TestState.AnalyzerConfigFiles.Add(("/.editorconfig", $@"root = true + +[*] +build_property.TargetFramework = net5 +")); + await test.RunAsync(); + } + + private static VerifyCS.Test SetupDependencyAndTestCSWithOneSourceFile(string csInput, string csDependencyCode) + { + return new VerifyCS.Test + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net80, + LanguageVersion = CodeAnalysis.CSharp.LanguageVersion.CSharp10, + MarkupOptions = MarkupOptions.UseFirstDescriptor, + TestState = + { + Sources = + { + csInput + }, + AdditionalProjects = + { + ["PreviewAssembly"] = + { + Sources = + { + ("/PreviewAssembly/AssemblyInfo.cs", csDependencyCode) + }, + }, + }, + AdditionalProjectReferences = + { + "PreviewAssembly", + }, + }, + }; + } + [Fact] public async Task GuardedWith_RuntimeInformation_IsOSPlatform_SimpleIfElseAsync() { @@ -4181,7 +4267,7 @@ public async Task GuardCallingCachedValue_CallSiteHasAssemblyAttributeAsync() [assembly: SupportedOSPlatform(""ios10.0"")] class Test { - static bool s_isiOS11OrNewer => false; + static bool s_isiOS11OrNewer = false; [SupportedOSPlatformGuard(""ios11.0"")] private bool IsIos11Supported() => s_isiOS11OrNewer; // should not warn From 25808c8458ed6bc27c52bb287b17f69a6401a239 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 18 Jun 2024 15:07:59 -0500 Subject: [PATCH 24/30] Update to Microsoft.CodeAnalysis.Testing 1.1.2-beta1.24314.1 --- eng/Versions.props | 2 +- .../ReleaseTrackingAnalyzerTests.cs | 7 +++---- ...Analysis.ResxSourceGenerator.UnitTests.csproj | 16 ++++++++-------- .../CSharpSourceGeneratorVerifier`1+Test.cs | 3 +-- .../VisualBasicSourceGeneratorVerifier`1+Test.cs | 3 +-- .../MarkAllNonSerializableFieldsTests.Fixer.cs | 10 +++++----- .../CSharpPerformanceCodeFixVerifier`2+Test.cs | 4 ++-- .../CSharpPerformanceCodeFixVerifier`2.cs | 7 +++---- ...sualBasicPerformanceCodeFixVerifier`2+Test.cs | 4 ++-- .../VisualBasicPerformanceCodeFixVerifier`2.cs | 7 +++---- .../AnnotatePublicApiAnalyzerTestsBase.cs | 9 ++++----- .../DeclarePublicAPIAnalyzerTestsBase.cs | 15 +++++++-------- .../NullableEnablePublicApiAnalyzerTests.cs | 5 ++--- src/Test.Utilities/CSharpCodeFixVerifier`2.cs | 7 +++---- .../CSharpCodeRefactoringVerifier`1+Test.cs | 3 +-- .../CSharpSecurityCodeFixVerifier`2.cs | 7 +++---- src/Test.Utilities/Test.Utilities.csproj | 8 ++++---- .../VisualBasicCodeFixVerifier`2+Test.cs | 4 ++-- .../VisualBasicCodeFixVerifier`2.cs | 7 +++---- .../VisualBasicCodeRefactoringVerifier`1+Test.cs | 4 ++-- .../VisualBasicSecurityCodeFixVerifier`2.cs | 9 ++++----- 21 files changed, 64 insertions(+), 77 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 01ba1986ee..33f9a4bcf5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -83,7 +83,7 @@ 2.0.0-pre-20160714 17.0.26-alpha - 1.1.2-beta1.24168.2 + 1.1.2-beta1.24314.1 0.13.0 2.14.1 diff --git a/src/Microsoft.CodeAnalysis.Analyzers/UnitTests/MetaAnalyzers/ReleaseTrackingAnalyzerTests.cs b/src/Microsoft.CodeAnalysis.Analyzers/UnitTests/MetaAnalyzers/ReleaseTrackingAnalyzerTests.cs index 0b7a8c45d7..eb9addb25e 100644 --- a/src/Microsoft.CodeAnalysis.Analyzers/UnitTests/MetaAnalyzers/ReleaseTrackingAnalyzerTests.cs +++ b/src/Microsoft.CodeAnalysis.Analyzers/UnitTests/MetaAnalyzers/ReleaseTrackingAnalyzerTests.cs @@ -9,7 +9,6 @@ using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.ReleaseTracking; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; using Microsoft.CodeAnalysis.VisualBasic.Testing; using Test.Utilities; using Xunit; @@ -1004,7 +1003,7 @@ private static Solution DisableNonReleaseTrackingWarnings(Solution solution, Pro private async Task VerifyCSharpAsync(string source, string shippedText, string unshippedText, params DiagnosticResult[] expected) { - var test = new CSharpCodeFixTest + var test = new CSharpCodeFixTest { TestState = { @@ -1044,8 +1043,8 @@ private async Task VerifyAdditionalFileFixAsync(string language, string source, ImmutableArray additionalExpectedDiagnosticsInInput, ImmutableArray additionalExpectedDiagnosticsInResult) { var test = language == LanguageNames.CSharp - ? new CSharpCodeFixTest() - : (CodeFixTest)new VisualBasicCodeFixTest(); + ? new CSharpCodeFixTest() + : (CodeFixTest)new VisualBasicCodeFixTest(); test.ReferenceAssemblies = AdditionalMetadataReferences.Default; test.SolutionTransforms.Add(DisableNonReleaseTrackingWarnings); diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests.csproj b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests.csproj index e7bb092aa9..51e1a94ae8 100644 --- a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests.csproj +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests.csproj @@ -10,14 +10,14 @@ - - - - - - - - + + + + + + + + diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Verifiers/CSharpSourceGeneratorVerifier`1+Test.cs b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Verifiers/CSharpSourceGeneratorVerifier`1+Test.cs index f09a5835b5..b3836af278 100644 --- a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Verifiers/CSharpSourceGeneratorVerifier`1+Test.cs +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Verifiers/CSharpSourceGeneratorVerifier`1+Test.cs @@ -23,7 +23,6 @@ using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; using Microsoft.CodeAnalysis.Text; namespace Microsoft.CodeAnalysis.ResxSourceGenerator.Test @@ -31,7 +30,7 @@ namespace Microsoft.CodeAnalysis.ResxSourceGenerator.Test public static partial class CSharpSourceGeneratorVerifier where TSourceGenerator : IIncrementalGenerator, new() { - public class Test : CSharpSourceGeneratorTest + public class Test : CSharpSourceGeneratorTest { private readonly string _identifier; private readonly string? _testFile; diff --git a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Verifiers/VisualBasicSourceGeneratorVerifier`1+Test.cs b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Verifiers/VisualBasicSourceGeneratorVerifier`1+Test.cs index 44c7e4fe02..c63807042b 100644 --- a/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Verifiers/VisualBasicSourceGeneratorVerifier`1+Test.cs +++ b/src/Microsoft.CodeAnalysis.ResxSourceGenerator/Microsoft.CodeAnalysis.ResxSourceGenerator.UnitTests/Verifiers/VisualBasicSourceGeneratorVerifier`1+Test.cs @@ -21,7 +21,6 @@ using System.Threading.Tasks; using Microsoft.CodeAnalysis.ResxSourceGenerator.VisualBasic; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; using Microsoft.CodeAnalysis.Text; using Microsoft.CodeAnalysis.VisualBasic.Testing; @@ -30,7 +29,7 @@ namespace Microsoft.CodeAnalysis.ResxSourceGenerator.Test public static partial class VisualBasicSourceGeneratorVerifier where TSourceGenerator : IIncrementalGenerator, new() { - public class Test : VisualBasicSourceGeneratorTest + public class Test : VisualBasicSourceGeneratorTest { private readonly string _identifier; private readonly string? _testFile; diff --git a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/MarkAllNonSerializableFieldsTests.Fixer.cs b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/MarkAllNonSerializableFieldsTests.Fixer.cs index f12058789a..0d7bd96274 100644 --- a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/MarkAllNonSerializableFieldsTests.Fixer.cs +++ b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/MarkAllNonSerializableFieldsTests.Fixer.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using Microsoft.CodeAnalysis.CSharp.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; +using Microsoft.CodeAnalysis.Testing; using Microsoft.CodeAnalysis.VisualBasic.Testing; using Microsoft.NetCore.CSharp.Analyzers.Runtime; using Microsoft.NetCore.VisualBasic.Analyzers.Runtime; @@ -65,7 +65,7 @@ Friend s1 As NonSerializableType [Fact] public async Task CA2235WithNonSerializableFieldsWithFix1Async() { - await new CSharpCodeFixTest + await new CSharpCodeFixTest { TestCode = @" using System; @@ -90,7 +90,7 @@ public class CA2235WithNonPublicNonSerializableFields CodeActionIndex = 1, }.RunAsync(); - await new VisualBasicCodeFixTest + await new VisualBasicCodeFixTest { TestCode = @" Imports System @@ -163,7 +163,7 @@ Public Class CA2235WithNonPublicNonSerializableFields [Fact] public async Task CA2235WithNonSerializableFieldsWithFix3Async() { - await new CSharpCodeFixTest + await new CSharpCodeFixTest { TestCode = @" using System; @@ -192,7 +192,7 @@ public class CA2235WithNonPublicNonSerializableFields CodeActionIndex = 1, }.RunAsync(); - await new VisualBasicCodeFixTest + await new VisualBasicCodeFixTest { TestCode = @" Imports System diff --git a/src/PerformanceSensitiveAnalyzers/UnitTests/CSharpPerformanceCodeFixVerifier`2+Test.cs b/src/PerformanceSensitiveAnalyzers/UnitTests/CSharpPerformanceCodeFixVerifier`2+Test.cs index 4c9f74dcf5..b40027843f 100644 --- a/src/PerformanceSensitiveAnalyzers/UnitTests/CSharpPerformanceCodeFixVerifier`2+Test.cs +++ b/src/PerformanceSensitiveAnalyzers/UnitTests/CSharpPerformanceCodeFixVerifier`2+Test.cs @@ -3,7 +3,7 @@ using Microsoft.CodeAnalysis.CodeFixes; using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Diagnostics; -using Microsoft.CodeAnalysis.Testing.Verifiers; +using Microsoft.CodeAnalysis.Testing; namespace Microsoft.CodeAnalysis.PerformanceSensitiveAnalyzers.UnitTests { @@ -11,7 +11,7 @@ public static partial class CSharpPerformanceCodeFixVerifier + internal sealed class Test : CSharpCodeFixTest { } } diff --git a/src/PerformanceSensitiveAnalyzers/UnitTests/CSharpPerformanceCodeFixVerifier`2.cs b/src/PerformanceSensitiveAnalyzers/UnitTests/CSharpPerformanceCodeFixVerifier`2.cs index fd97685c13..e31281408e 100644 --- a/src/PerformanceSensitiveAnalyzers/UnitTests/CSharpPerformanceCodeFixVerifier`2.cs +++ b/src/PerformanceSensitiveAnalyzers/UnitTests/CSharpPerformanceCodeFixVerifier`2.cs @@ -5,7 +5,6 @@ using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; #pragma warning disable CA1000 // Do not declare static members on generic types @@ -42,13 +41,13 @@ public PerformanceSensitiveAttribute(string uri) }"; public static DiagnosticResult Diagnostic() - => CSharpCodeFixVerifier.Diagnostic(); + => CSharpCodeFixVerifier.Diagnostic(); public static DiagnosticResult Diagnostic(string diagnosticId) - => CSharpCodeFixVerifier.Diagnostic(diagnosticId); + => CSharpCodeFixVerifier.Diagnostic(diagnosticId); public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) - => CSharpCodeFixVerifier.Diagnostic(descriptor); + => CSharpCodeFixVerifier.Diagnostic(descriptor); public static async Task VerifyAnalyzerAsync(string source, params DiagnosticResult[] expected) { diff --git a/src/PerformanceSensitiveAnalyzers/UnitTests/VisualBasicPerformanceCodeFixVerifier`2+Test.cs b/src/PerformanceSensitiveAnalyzers/UnitTests/VisualBasicPerformanceCodeFixVerifier`2+Test.cs index a73b10999c..0fa8964912 100644 --- a/src/PerformanceSensitiveAnalyzers/UnitTests/VisualBasicPerformanceCodeFixVerifier`2+Test.cs +++ b/src/PerformanceSensitiveAnalyzers/UnitTests/VisualBasicPerformanceCodeFixVerifier`2+Test.cs @@ -2,7 +2,7 @@ using Microsoft.CodeAnalysis.CodeFixes; using Microsoft.CodeAnalysis.Diagnostics; -using Microsoft.CodeAnalysis.Testing.Verifiers; +using Microsoft.CodeAnalysis.Testing; using Microsoft.CodeAnalysis.VisualBasic.Testing; namespace Microsoft.CodeAnalysis.PerformanceSensitiveAnalyzers.UnitTests @@ -11,7 +11,7 @@ public static partial class VisualBasicPerformanceCodeFixVerifier + internal sealed class Test : VisualBasicCodeFixTest { } } diff --git a/src/PerformanceSensitiveAnalyzers/UnitTests/VisualBasicPerformanceCodeFixVerifier`2.cs b/src/PerformanceSensitiveAnalyzers/UnitTests/VisualBasicPerformanceCodeFixVerifier`2.cs index 10759e8b88..cb8b8ca11e 100644 --- a/src/PerformanceSensitiveAnalyzers/UnitTests/VisualBasicPerformanceCodeFixVerifier`2.cs +++ b/src/PerformanceSensitiveAnalyzers/UnitTests/VisualBasicPerformanceCodeFixVerifier`2.cs @@ -4,7 +4,6 @@ using Microsoft.CodeAnalysis.CodeFixes; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; using Microsoft.CodeAnalysis.VisualBasic.Testing; namespace Microsoft.CodeAnalysis.PerformanceSensitiveAnalyzers.UnitTests @@ -40,13 +39,13 @@ End Namespace "; public static DiagnosticResult Diagnostic() - => VisualBasicCodeFixVerifier.Diagnostic(); + => VisualBasicCodeFixVerifier.Diagnostic(); public static DiagnosticResult Diagnostic(string diagnosticId) - => VisualBasicCodeFixVerifier.Diagnostic(diagnosticId); + => VisualBasicCodeFixVerifier.Diagnostic(diagnosticId); public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) - => VisualBasicCodeFixVerifier.Diagnostic(descriptor); + => VisualBasicCodeFixVerifier.Diagnostic(descriptor); public static async Task VerifyAnalyzerAsync(string source, params DiagnosticResult[] expected) { diff --git a/src/PublicApiAnalyzers/UnitTests/AnnotatePublicApiAnalyzerTestsBase.cs b/src/PublicApiAnalyzers/UnitTests/AnnotatePublicApiAnalyzerTestsBase.cs index 1998e533c3..cff801e9ad 100644 --- a/src/PublicApiAnalyzers/UnitTests/AnnotatePublicApiAnalyzerTestsBase.cs +++ b/src/PublicApiAnalyzers/UnitTests/AnnotatePublicApiAnalyzerTestsBase.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; using Test.Utilities; using Xunit; @@ -28,7 +27,7 @@ public abstract class AnnotatePublicApiAnalyzerTestsBase #region Utilities private async Task VerifyCSharpAsync(string source, string shippedApiText, string unshippedApiText, params DiagnosticResult[] expected) { - var test = new CSharpCodeFixTest + var test = new CSharpCodeFixTest { TestState = { @@ -59,7 +58,7 @@ private async Task VerifyCSharpAdditionalFileFixAsync(string source, string oldS private async Task VerifyAdditionalFileFixAsync(string source, string oldShippedApiText, string oldUnshippedApiText, string newShippedApiText, string newUnshippedApiText) { - var test = new CSharpCodeFixTest(); + var test = new CSharpCodeFixTest(); test.TestState.Sources.Add(source); test.TestState.AdditionalFiles.Add((ShippedFileName, oldShippedApiText)); @@ -130,7 +129,7 @@ public async Task NoObliviousWhenAnnotatedClassConstraintMultipleFiles() C "; - var test = new CSharpCodeFixTest + var test = new CSharpCodeFixTest { TestState = { @@ -389,7 +388,7 @@ public async Task AnnotatedMemberInAnnotatedUnshippedAPI_EnabledViaMultipleUnshi C.Field -> string? C.Field2 -> string!"; - var test = new CSharpCodeFixTest(); + var test = new CSharpCodeFixTest(); test.TestState.Sources.Add(source); diff --git a/src/PublicApiAnalyzers/UnitTests/DeclarePublicAPIAnalyzerTestsBase.cs b/src/PublicApiAnalyzers/UnitTests/DeclarePublicAPIAnalyzerTestsBase.cs index 5835dd2aac..1a58f729ee 100644 --- a/src/PublicApiAnalyzers/UnitTests/DeclarePublicAPIAnalyzerTestsBase.cs +++ b/src/PublicApiAnalyzers/UnitTests/DeclarePublicAPIAnalyzerTestsBase.cs @@ -9,7 +9,6 @@ using System.Threading.Tasks; using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; using Microsoft.CodeAnalysis.VisualBasic.Testing; using Test.Utilities; using Xunit; @@ -73,7 +72,7 @@ private static DiagnosticResult GetBasicResultAt(int line, int column, Diagnosti private async Task VerifyBasicAsync(string source, string shippedApiText, string unshippedApiText, params DiagnosticResult[] expected) { - var test = new VisualBasicCodeFixTest + var test = new VisualBasicCodeFixTest { TestState = { @@ -99,7 +98,7 @@ private async Task VerifyBasicAsync(string source, string shippedApiText, string private async Task VerifyCSharpAsync(string source, string? shippedApiText, string? unshippedApiText, params DiagnosticResult[] expected) { - var test = new CSharpCodeFixTest + var test = new CSharpCodeFixTest { TestState = { @@ -180,7 +179,7 @@ private async Task VerifyCSharpAsync(Action addSourcesAction, st private async Task VerifyCSharpAsync(string source, string shippedApiText, string unshippedApiText, string shippedApiFilePath, string unshippedApiFilePath, params DiagnosticResult[] expected) { - var test = new CSharpCodeFixTest + var test = new CSharpCodeFixTest { TestState = { @@ -223,8 +222,8 @@ private async Task VerifyAdditionalFileFixAsync(string language, string source, ReferenceAssemblies? referenceAssemblies = null) { var test = language == LanguageNames.CSharp - ? new CSharpCodeFixTest() - : (CodeFixTest)new VisualBasicCodeFixTest(); + ? new CSharpCodeFixTest() + : (CodeFixTest)new VisualBasicCodeFixTest(); if (referenceAssemblies is not null) { @@ -2271,7 +2270,7 @@ public async Task TestSimpleMissingMember_Fix_WithoutNullability_MultipleFilesAs var unshippedTextName2 = UnshippedFileNamePrefix + "test" + DeclarePublicApiAnalyzer.Extension; - var test = new CSharpCodeFixTest(); + var test = new CSharpCodeFixTest(); test.TestState.Sources.Add(source); test.TestState.AdditionalFiles.Add((ShippedFileName, shippedText)); @@ -3267,7 +3266,7 @@ public async Task TestExperimentalApiWithInvalidArgumentAsync(string invalidArgu var fixedUnshippedText = @"[???]C [???]C.C() -> void"; - var test = new CSharpCodeFixTest() + var test = new CSharpCodeFixTest() { ReferenceAssemblies = ReferenceAssemblies.Net.Net80, CompilerDiagnostics = CompilerDiagnostics.None, diff --git a/src/PublicApiAnalyzers/UnitTests/NullableEnablePublicApiAnalyzerTests.cs b/src/PublicApiAnalyzers/UnitTests/NullableEnablePublicApiAnalyzerTests.cs index 7eeee0d53b..5495a11063 100644 --- a/src/PublicApiAnalyzers/UnitTests/NullableEnablePublicApiAnalyzerTests.cs +++ b/src/PublicApiAnalyzers/UnitTests/NullableEnablePublicApiAnalyzerTests.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; using Xunit; namespace Microsoft.CodeAnalysis.PublicApiAnalyzers.UnitTests @@ -15,7 +14,7 @@ public class NullableEnablePublicApiAnalyzerTests #region Utilities private async Task VerifyCSharpAsync(string source, string shippedApiText, string unshippedApiText, params DiagnosticResult[] expected) { - var test = new CSharpCodeFixTest + var test = new CSharpCodeFixTest { TestState = { @@ -45,7 +44,7 @@ private async Task VerifyCSharpAdditionalFileFixAsync(string source, string oldS private async Task VerifyAdditionalFileFixAsync(string source, string oldShippedApiText, string oldUnshippedApiText, string newSource, string newShippedApiText, string newUnshippedApiText) { - var test = new CSharpCodeFixTest(); + var test = new CSharpCodeFixTest(); test.TestState.Sources.Add(source); test.TestState.AdditionalFiles.Add((DeclarePublicApiAnalyzer.PublicShippedFileName, oldShippedApiText)); diff --git a/src/Test.Utilities/CSharpCodeFixVerifier`2.cs b/src/Test.Utilities/CSharpCodeFixVerifier`2.cs index af659fcfd5..4453c1a217 100644 --- a/src/Test.Utilities/CSharpCodeFixVerifier`2.cs +++ b/src/Test.Utilities/CSharpCodeFixVerifier`2.cs @@ -6,7 +6,6 @@ using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; namespace Test.Utilities { @@ -15,13 +14,13 @@ public static partial class CSharpCodeFixVerifier where TCodeFix : CodeFixProvider, new() { public static DiagnosticResult Diagnostic() - => CSharpCodeFixVerifier.Diagnostic(); + => CSharpCodeFixVerifier.Diagnostic(); public static DiagnosticResult Diagnostic(string diagnosticId) - => CSharpCodeFixVerifier.Diagnostic(diagnosticId); + => CSharpCodeFixVerifier.Diagnostic(diagnosticId); public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) - => CSharpCodeFixVerifier.Diagnostic(descriptor); + => CSharpCodeFixVerifier.Diagnostic(descriptor); public static async Task VerifyAnalyzerAsync(string source, params DiagnosticResult[] expected) { diff --git a/src/Test.Utilities/CSharpCodeRefactoringVerifier`1+Test.cs b/src/Test.Utilities/CSharpCodeRefactoringVerifier`1+Test.cs index d6f1c42c95..29ec7f0ad1 100644 --- a/src/Test.Utilities/CSharpCodeRefactoringVerifier`1+Test.cs +++ b/src/Test.Utilities/CSharpCodeRefactoringVerifier`1+Test.cs @@ -6,14 +6,13 @@ using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; namespace Test.Utilities { public static partial class CSharpCodeRefactoringVerifier where TRefactoring : CodeRefactoringProvider, new() { - public class Test : CSharpCodeRefactoringTest + public class Test : CSharpCodeRefactoringTest { public Test() { diff --git a/src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs b/src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs index 5ebcdec4c1..a95af6d6d8 100644 --- a/src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs +++ b/src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs @@ -6,7 +6,6 @@ using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; namespace Test.Utilities { @@ -15,13 +14,13 @@ public static partial class CSharpSecurityCodeFixVerifier where TCodeFix : CodeFixProvider, new() { public static DiagnosticResult Diagnostic() - => CSharpCodeFixVerifier.Diagnostic(); + => CSharpCodeFixVerifier.Diagnostic(); public static DiagnosticResult Diagnostic(string diagnosticId) - => CSharpCodeFixVerifier.Diagnostic(diagnosticId); + => CSharpCodeFixVerifier.Diagnostic(diagnosticId); public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) - => CSharpCodeFixVerifier.Diagnostic(descriptor); + => CSharpCodeFixVerifier.Diagnostic(descriptor); public static async Task VerifyAnalyzerAsync(string source, params DiagnosticResult[] expected) { diff --git a/src/Test.Utilities/Test.Utilities.csproj b/src/Test.Utilities/Test.Utilities.csproj index cf418987d6..d57c29f731 100644 --- a/src/Test.Utilities/Test.Utilities.csproj +++ b/src/Test.Utilities/Test.Utilities.csproj @@ -14,10 +14,10 @@ - - - - + + + + diff --git a/src/Test.Utilities/VisualBasicCodeFixVerifier`2+Test.cs b/src/Test.Utilities/VisualBasicCodeFixVerifier`2+Test.cs index 4afdf70465..5709faf46a 100644 --- a/src/Test.Utilities/VisualBasicCodeFixVerifier`2+Test.cs +++ b/src/Test.Utilities/VisualBasicCodeFixVerifier`2+Test.cs @@ -3,7 +3,7 @@ using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CodeFixes; using Microsoft.CodeAnalysis.Diagnostics; -using Microsoft.CodeAnalysis.Testing.Verifiers; +using Microsoft.CodeAnalysis.Testing; using Microsoft.CodeAnalysis.VisualBasic; using Microsoft.CodeAnalysis.VisualBasic.Testing; @@ -13,7 +13,7 @@ public static partial class VisualBasicCodeFixVerifier where TAnalyzer : DiagnosticAnalyzer, new() where TCodeFix : CodeFixProvider, new() { - public class Test : VisualBasicCodeFixTest + public class Test : VisualBasicCodeFixTest { public Test() { diff --git a/src/Test.Utilities/VisualBasicCodeFixVerifier`2.cs b/src/Test.Utilities/VisualBasicCodeFixVerifier`2.cs index a919cb8026..605a691906 100644 --- a/src/Test.Utilities/VisualBasicCodeFixVerifier`2.cs +++ b/src/Test.Utilities/VisualBasicCodeFixVerifier`2.cs @@ -5,7 +5,6 @@ using Microsoft.CodeAnalysis.CodeFixes; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; using Microsoft.CodeAnalysis.VisualBasic.Testing; namespace Test.Utilities @@ -15,13 +14,13 @@ public static partial class VisualBasicCodeFixVerifier where TCodeFix : CodeFixProvider, new() { public static DiagnosticResult Diagnostic() - => VisualBasicCodeFixVerifier.Diagnostic(); + => VisualBasicCodeFixVerifier.Diagnostic(); public static DiagnosticResult Diagnostic(string diagnosticId) - => VisualBasicCodeFixVerifier.Diagnostic(diagnosticId); + => VisualBasicCodeFixVerifier.Diagnostic(diagnosticId); public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) - => VisualBasicCodeFixVerifier.Diagnostic(descriptor); + => VisualBasicCodeFixVerifier.Diagnostic(descriptor); public static async Task VerifyAnalyzerAsync(string source, params DiagnosticResult[] expected) { diff --git a/src/Test.Utilities/VisualBasicCodeRefactoringVerifier`1+Test.cs b/src/Test.Utilities/VisualBasicCodeRefactoringVerifier`1+Test.cs index c5cb6838fc..f9197c12bf 100644 --- a/src/Test.Utilities/VisualBasicCodeRefactoringVerifier`1+Test.cs +++ b/src/Test.Utilities/VisualBasicCodeRefactoringVerifier`1+Test.cs @@ -2,7 +2,7 @@ using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CodeRefactorings; -using Microsoft.CodeAnalysis.Testing.Verifiers; +using Microsoft.CodeAnalysis.Testing; using Microsoft.CodeAnalysis.VisualBasic; using Microsoft.CodeAnalysis.VisualBasic.Testing; @@ -11,7 +11,7 @@ namespace Test.Utilities public static partial class VisualBasicCodeRefactoringVerifier where TRefactoring : CodeRefactoringProvider, new() { - public class Test : VisualBasicCodeRefactoringTest + public class Test : VisualBasicCodeRefactoringTest { public Test() { diff --git a/src/Test.Utilities/VisualBasicSecurityCodeFixVerifier`2.cs b/src/Test.Utilities/VisualBasicSecurityCodeFixVerifier`2.cs index a260c6fb87..98579a199c 100644 --- a/src/Test.Utilities/VisualBasicSecurityCodeFixVerifier`2.cs +++ b/src/Test.Utilities/VisualBasicSecurityCodeFixVerifier`2.cs @@ -3,10 +3,9 @@ using System.Threading.Tasks; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CodeFixes; -using Microsoft.CodeAnalysis.VisualBasic.Testing; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Testing; -using Microsoft.CodeAnalysis.Testing.Verifiers; +using Microsoft.CodeAnalysis.VisualBasic.Testing; namespace Test.Utilities { @@ -15,13 +14,13 @@ public static partial class VisualBasicSecurityCodeFixVerifier VisualBasicCodeFixVerifier.Diagnostic(); + => VisualBasicCodeFixVerifier.Diagnostic(); public static DiagnosticResult Diagnostic(string diagnosticId) - => VisualBasicCodeFixVerifier.Diagnostic(diagnosticId); + => VisualBasicCodeFixVerifier.Diagnostic(diagnosticId); public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) - => VisualBasicCodeFixVerifier.Diagnostic(descriptor); + => VisualBasicCodeFixVerifier.Diagnostic(descriptor); public static async Task VerifyAnalyzerAsync(string source, params DiagnosticResult[] expected) { From 43709af7570da7140fb3e9a5237f55ffb24677e7 Mon Sep 17 00:00:00 2001 From: Collin Alpert Date: Mon, 24 Jun 2024 18:03:22 +0200 Subject: [PATCH 25/30] Don't emit CA1064 in executables (#7234) * Exclude exception types from CA1515 * Don't emit CA1064 in executables * PR comments --- .../CSharpMakeTypesInternal.cs | 42 +- .../ExceptionsShouldBePublic.cs | 13 +- .../MakeTypesInternal.Fixer.cs | 2 +- .../Maintainability/MakeTypesInternal.cs | 29 +- .../ExceptionsShouldBePublicTests.cs | 243 ++++-- .../Maintainability/MakeTypesInternalTests.cs | 782 ++++++++++-------- .../Maintainability/BasicMakeTypesInternal.vb | 41 +- 7 files changed, 638 insertions(+), 514 deletions(-) diff --git a/src/NetAnalyzers/CSharp/Microsoft.CodeQuality.Analyzers/Maintainability/CSharpMakeTypesInternal.cs b/src/NetAnalyzers/CSharp/Microsoft.CodeQuality.Analyzers/Maintainability/CSharpMakeTypesInternal.cs index 89abddcc82..081989b8ab 100644 --- a/src/NetAnalyzers/CSharp/Microsoft.CodeQuality.Analyzers/Maintainability/CSharpMakeTypesInternal.cs +++ b/src/NetAnalyzers/CSharp/Microsoft.CodeQuality.Analyzers/Maintainability/CSharpMakeTypesInternal.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. -using System.Collections.Immutable; -using Analyzer.Utilities.Extensions; using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeQuality.Analyzers.Maintainability; @@ -11,39 +8,14 @@ namespace Microsoft.CodeQuality.CSharp.Analyzers.Maintainability { [DiagnosticAnalyzer(LanguageNames.CSharp)] - public sealed class CSharpMakeTypesInternal : MakeTypesInternal + public sealed class CSharpMakeTypesInternal : MakeTypesInternal { - protected override ImmutableArray TypeKinds { get; } = - ImmutableArray.Create(SyntaxKind.ClassDeclaration, SyntaxKind.StructDeclaration, SyntaxKind.InterfaceDeclaration, SyntaxKind.RecordDeclaration); - - protected override SyntaxKind EnumKind { get; } = SyntaxKind.EnumDeclaration; - - protected override ImmutableArray DelegateKinds { get; } = ImmutableArray.Create(SyntaxKind.DelegateDeclaration); - - protected override void AnalyzeTypeDeclaration(SyntaxNodeAnalysisContext context) - { - var type = (TypeDeclarationSyntax)context.Node; - ReportIfPublic(context, type.Modifiers, type.Identifier); - } - - protected override void AnalyzeEnumDeclaration(SyntaxNodeAnalysisContext context) - { - var @enum = (EnumDeclarationSyntax)context.Node; - ReportIfPublic(context, @enum.Modifiers, @enum.Identifier); - } - - protected override void AnalyzeDelegateDeclaration(SyntaxNodeAnalysisContext context) - { - var @delegate = (DelegateDeclarationSyntax)context.Node; - ReportIfPublic(context, @delegate.Modifiers, @delegate.Identifier); - } - - private static void ReportIfPublic(SyntaxNodeAnalysisContext context, SyntaxTokenList modifiers, SyntaxToken identifier) + protected override SyntaxToken? GetIdentifier(SyntaxNode type) => type switch { - if (modifiers.Any(SyntaxKind.PublicKeyword)) - { - context.ReportDiagnostic(identifier.CreateDiagnostic(Rule)); - } - } + TypeDeclarationSyntax tds => tds.Identifier, + EnumDeclarationSyntax eds => eds.Identifier, + DelegateDeclarationSyntax dds => dds.Identifier, + _ => null + }; } } \ No newline at end of file diff --git a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ExceptionsShouldBePublic.cs b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ExceptionsShouldBePublic.cs index 95155c4dde..40bd62c5d3 100644 --- a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ExceptionsShouldBePublic.cs +++ b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ExceptionsShouldBePublic.cs @@ -39,6 +39,9 @@ public sealed class ExceptionsShouldBePublicAnalyzer : DiagnosticAnalyzer "System.ApplicationException" }; + private static readonly ImmutableHashSet s_excludedOutputKinds = + ImmutableHashSet.Create(OutputKind.ConsoleApplication, OutputKind.WindowsApplication, OutputKind.WindowsRuntimeApplication); + public override void Initialize(AnalysisContext context) { context.EnableConcurrentExecution(); @@ -49,9 +52,15 @@ public override void Initialize(AnalysisContext context) private static void AnalyzeCompilationStart(CompilationStartAnalysisContext csContext) { - // Get named type symbols for targetted exception types + if (s_excludedOutputKinds.Contains(csContext.Compilation.Options.OutputKind)) + { + return; + } + + var typeProvider = WellKnownTypeProvider.GetOrCreate(csContext.Compilation); + // Get named type symbols for targeted exception types ImmutableHashSet exceptionTypes = s_exceptionTypeNames - .Select(csContext.Compilation.GetOrCreateTypeByMetadataName) + .Select(typeProvider.GetOrCreateTypeByMetadataName) .WhereNotNull() .ToImmutableHashSet(); diff --git a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternal.Fixer.cs b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternal.Fixer.cs index acf8174fde..64feb0315b 100644 --- a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternal.Fixer.cs +++ b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternal.Fixer.cs @@ -33,6 +33,6 @@ public override async Task RegisterCodeFixesAsync(CodeFixContext context) protected abstract SyntaxNode MakeInternal(SyntaxNode node); - public override ImmutableArray FixableDiagnosticIds { get; } = ImmutableArray.Create(MakeTypesInternal.RuleId); + public override ImmutableArray FixableDiagnosticIds { get; } = ImmutableArray.Create(MakeTypesInternal.RuleId); } } \ No newline at end of file diff --git a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternal.cs b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternal.cs index 91dc865da7..e405bd81d6 100644 --- a/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternal.cs +++ b/src/NetAnalyzers/Core/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternal.cs @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. -using System; using System.Collections.Immutable; using System.Linq; using Analyzer.Utilities; +using Analyzer.Utilities.Extensions; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.Diagnostics; @@ -11,8 +11,7 @@ namespace Microsoft.CodeQuality.Analyzers.Maintainability { using static MicrosoftCodeQualityAnalyzersResources; - public abstract class MakeTypesInternal : DiagnosticAnalyzer - where TSyntaxKind : struct, Enum + public abstract class MakeTypesInternal : DiagnosticAnalyzer { internal const string RuleId = "CA1515"; @@ -42,23 +41,21 @@ public override void Initialize(AnalysisContext context) return; } - context.RegisterSyntaxNodeAction(AnalyzeTypeDeclaration, TypeKinds); - context.RegisterSyntaxNodeAction(AnalyzeEnumDeclaration, EnumKind); - context.RegisterSyntaxNodeAction(AnalyzeDelegateDeclaration, DelegateKinds); + context.RegisterSymbolAction(AnalyzeType, SymbolKind.NamedType); }); } - protected abstract ImmutableArray TypeKinds { get; } - - protected abstract TSyntaxKind EnumKind { get; } - - protected abstract ImmutableArray DelegateKinds { get; } - - protected abstract void AnalyzeTypeDeclaration(SyntaxNodeAnalysisContext context); - - protected abstract void AnalyzeEnumDeclaration(SyntaxNodeAnalysisContext context); + private void AnalyzeType(SymbolAnalysisContext context) + { + INamedTypeSymbol namedTypeSymbol = (INamedTypeSymbol)context.Symbol; + if (namedTypeSymbol.IsPublic() + && GetIdentifier(namedTypeSymbol.DeclaringSyntaxReferences[0].GetSyntax()) is SyntaxToken identifier) + { + context.ReportDiagnostic(identifier.CreateDiagnostic(Rule)); + } + } - protected abstract void AnalyzeDelegateDeclaration(SyntaxNodeAnalysisContext context); + protected abstract SyntaxToken? GetIdentifier(SyntaxNode type); public override ImmutableArray SupportedDiagnostics { get; } = ImmutableArray.Create(Rule); } diff --git a/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ExceptionsShouldBePublicTests.cs b/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ExceptionsShouldBePublicTests.cs index d09620dc0e..57b2ace7cd 100644 --- a/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ExceptionsShouldBePublicTests.cs +++ b/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/ApiDesignGuidelines/ExceptionsShouldBePublicTests.cs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; -using Microsoft.CodeAnalysis.Testing; +using Microsoft.CodeAnalysis; using Xunit; using VerifyCS = Test.Utilities.CSharpCodeFixVerifier< Microsoft.CodeQuality.Analyzers.ApiDesignGuidelines.ExceptionsShouldBePublicAnalyzer, @@ -9,110 +12,196 @@ using VerifyVB = Test.Utilities.VisualBasicCodeFixVerifier< Microsoft.CodeQuality.Analyzers.ApiDesignGuidelines.ExceptionsShouldBePublicAnalyzer, Microsoft.CodeQuality.Analyzers.ApiDesignGuidelines.ExceptionsShouldBePublicFixer>; + namespace Microsoft.CodeQuality.Analyzers.ApiDesignGuidelines.UnitTests { public class ExceptionsShouldBePublicTests { - [Fact] - public async Task TestCSharpNonPublicExceptionAsync() + private static readonly IEnumerable ExecutableOutputKinds = + [ + OutputKind.ConsoleApplication, + OutputKind.WindowsRuntimeApplication, + OutputKind.WindowsApplication + ]; + + public static readonly TheoryData DiagnosticSuppressingOutputKinds = new(ExecutableOutputKinds); + + public static readonly TheoryData DiagnosticTriggeringOutputKinds = new(Enum.GetValues().Except(ExecutableOutputKinds)); + + [Theory] + [MemberData(nameof(DiagnosticTriggeringOutputKinds))] + public async Task TestCSharpNonPublicExceptionAsync(OutputKind outputKind) { - await VerifyCS.VerifyAnalyzerAsync(@" -using System; -class InternalException : Exception -{ -}", - GetCA1064CSharpResultAt(3, 7)); + await new VerifyCS.Test + { + TestCode = """ + using System; + class [|InternalException|] : Exception + { + } + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); } - [Fact] - public async Task TestCSharpNonPublicException2Async() + [Theory] + [MemberData(nameof(DiagnosticTriggeringOutputKinds))] + public async Task TestCSharpNonPublicException2Async(OutputKind outputKind) { - await VerifyCS.VerifyAnalyzerAsync(@" -using System; -internal class Outer -{ - private class PrivateException : SystemException - { - } -}", - GetCA1064CSharpResultAt(5, 19)); + await new VerifyCS.Test + { + TestCode = """ + using System; + internal class Outer + { + private class [|PrivateException|] : SystemException + { + } + } + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); } - [Fact] - public async Task TestCSharpPublicExceptionAsync() + [Theory] + [MemberData(nameof(DiagnosticTriggeringOutputKinds))] + public async Task TestCSharpPublicExceptionAsync(OutputKind outputKind) { - await VerifyCS.VerifyAnalyzerAsync(@" -using System; -public class BasicException : Exception -{ -}"); + await new VerifyCS.Test + { + TestCode = """ + using System; + public class BasicException : Exception + { + } + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); } - [Fact] - public async Task TestCSharpNonExceptionTypeAsync() + [Theory] + [MemberData(nameof(DiagnosticTriggeringOutputKinds))] + public async Task TestCSharpNonExceptionTypeAsync(OutputKind outputKind) { - await VerifyCS.VerifyAnalyzerAsync(@" -using System.IO; -public class NonException : StringWriter -{ -}"); + await new VerifyCS.Test + { + TestCode = """ + using System.IO; + public class NonException : StringWriter + { + } + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); } - [Fact] - public async Task TestVBasicNonPublicExceptionAsync() + [Theory] + [MemberData(nameof(DiagnosticTriggeringOutputKinds))] + public async Task TestVBasicNonPublicExceptionAsync(OutputKind outputKind) { - await VerifyVB.VerifyAnalyzerAsync(@" -Imports System -Class InternalException - Inherits Exception -End Class", - GetCA1064VBasicResultAt(3, 7)); + await new VerifyVB.Test + { + TestCode = """ + Imports System + Class [|InternalException|] + Inherits Exception + End Class + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); } - [Fact] - public async Task TestVBasicNonPublicException2Async() + [Theory] + [MemberData(nameof(DiagnosticTriggeringOutputKinds))] + public async Task TestVBasicNonPublicException2Async(OutputKind outputKind) { - await VerifyVB.VerifyAnalyzerAsync(@" -Imports System -Public Class Outer - Private Class PrivateException - Inherits SystemException - End Class -End Class", - GetCA1064VBasicResultAt(4, 19)); + await new VerifyVB.Test + { + TestCode = """ + Imports System + Public Class Outer + Private Class [|PrivateException|] + Inherits SystemException + End Class + End Class + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); } - [Fact] - public async Task TestVBasicPublicExceptionAsync() + [Theory] + [MemberData(nameof(DiagnosticTriggeringOutputKinds))] + public async Task TestVBasicPublicExceptionAsync(OutputKind outputKind) { - await VerifyVB.VerifyAnalyzerAsync(@" -Imports System -Public Class BasicException - Inherits Exception -End Class"); + await new VerifyVB.Test + { + TestCode = """ + Imports System + Public Class BasicException + Inherits Exception + End Class + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); } - [Fact] - public async Task TestVBasicNonExceptionTypeAsync() + [Theory] + [MemberData(nameof(DiagnosticTriggeringOutputKinds))] + public async Task TestVBasicNonExceptionTypeAsync(OutputKind outputKind) { - await VerifyVB.VerifyAnalyzerAsync(@" -Imports System.IO -Imports System.Text -Public Class NonException - Inherits StringWriter -End Class"); + await new VerifyVB.Test + { + TestCode = """ + Imports System.IO + Imports System.Text + Public Class NonException + Inherits StringWriter + End Class + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); } - private static DiagnosticResult GetCA1064CSharpResultAt(int line, int column) -#pragma warning disable RS0030 // Do not use banned APIs - => VerifyCS.Diagnostic() - .WithLocation(line, column); -#pragma warning restore RS0030 // Do not use banned APIs + [Theory] + [MemberData(nameof(DiagnosticSuppressingOutputKinds))] + public async Task TestCSharpWithExecutableAsync(OutputKind outputKind) + { + await new VerifyCS.Test + { + TestCode = """ + using System; + class InternalException : Exception + { + } + + class Program + { + static void Main() {} + } + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); + } - private static DiagnosticResult GetCA1064VBasicResultAt(int line, int column) -#pragma warning disable RS0030 // Do not use banned APIs - => VerifyVB.Diagnostic() - .WithLocation(line, column); -#pragma warning restore RS0030 // Do not use banned APIs + [Theory] + [MemberData(nameof(DiagnosticSuppressingOutputKinds))] + public async Task TestVBasicWithExecutableAsync(OutputKind outputKind) + { + await new VerifyVB.Test + { + TestCode = """ + Imports System + Class InternalException + Inherits Exception + End Class + + Class Program + Shared Sub Main() + End Sub + End Class + """, + TestState = { OutputKind = outputKind } + }.RunAsync(); + } } } \ No newline at end of file diff --git a/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternalTests.cs b/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternalTests.cs index da107531b7..ffa3efa83b 100644 --- a/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternalTests.cs +++ b/src/NetAnalyzers/UnitTests/Microsoft.CodeQuality.Analyzers/Maintainability/MakeTypesInternalTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. +using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; @@ -16,6 +18,17 @@ namespace Microsoft.CodeQuality.Analyzers.Maintainability.UnitTests { public sealed class MakeTypesInternalTests { + private static readonly IEnumerable ExecutableOutputKinds = + [ + OutputKind.ConsoleApplication, + OutputKind.WindowsRuntimeApplication, + OutputKind.WindowsApplication + ]; + + public static readonly TheoryData DiagnosticTriggeringOutputKinds = new(ExecutableOutputKinds); + + public static readonly TheoryData NonDiagnosticTriggeringOutputKinds = new(Enum.GetValues().Except(ExecutableOutputKinds)); + [Theory] [MemberData(nameof(NonDiagnosticTriggeringOutputKinds))] public async Task LibraryCode_NoDiagnostic(OutputKind outputKind) @@ -29,201 +42,227 @@ public async Task LibraryCode_NoDiagnostic(OutputKind outputKind) [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task ApplicationCode_Diagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public class [|Program|] -{ - public static void Main() {} -}", - @" -internal class Program -{ - public static void Main() {} -}"); - - await VerifyVbAsync(outputKind, @" -Public Class [|Program|] - Public Shared Sub Main() - End Sub -End Class", - @" -Friend Class Program - Public Shared Sub Main() - End Sub -End Class"); + await VerifyCsAsync(outputKind, + """ + public class [|Program|] + { + public static void Main() {} + } + """, + """ + internal class Program + { + public static void Main() {} + } + """); + + await VerifyVbAsync(outputKind, + """ + Public Class [|Program|] + Public Shared Sub Main() + End Sub + End Class + """, + """ + Friend Class Program + Public Shared Sub Main() + End Sub + End Class + """); } [Theory] [MemberData(nameof(NonDiagnosticTriggeringOutputKinds))] public async Task MultipleTypes_LibraryCode_NoDiagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public class MyTests {} + await VerifyCsAsync(outputKind, + """ + public class MyTests {} -internal class MyService {} + internal class MyService {} -public class MyValidator {}"); + public class MyValidator {} + """); - await VerifyVbAsync(outputKind, @" -Public Class MyTests -End Class + await VerifyVbAsync(outputKind, + """ + Public Class MyTests + End Class -Friend Class MyService -End Class + Friend Class MyService + End Class -Public Class MyValidator -End Class"); + Public Class MyValidator + End Class + """); } [Theory] [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task MultipleTypes_ApplicationCode_Diagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public class [|Program|] -{ - public static void Main() {} -} + await VerifyCsAsync(outputKind, + """ + public class [|Program|] + { + public static void Main() {} + } -internal class MyService {} + internal class MyService {} -public class [|MyValidator|] {}", - @" -internal class Program -{ - public static void Main() {} -} - -internal class MyService {} - -internal class MyValidator {}"); - await VerifyVbAsync(outputKind, @" -Public Class [|Program|] - Public Shared Sub Main() - End Sub -End Class - -Friend Class MyService -End Class - -Public Class [|MyValidator|] -End Class", - @" -Friend Class Program - Public Shared Sub Main() - End Sub -End Class - -Friend Class MyService -End Class - -Friend Class MyValidator -End Class"); + public class [|MyValidator|] {} + """, + """ + internal class Program + { + public static void Main() {} + } + + internal class MyService {} + + internal class MyValidator {} + """); + await VerifyVbAsync(outputKind, + """ + Public Class [|Program|] + Public Shared Sub Main() + End Sub + End Class + + Friend Class MyService + End Class + + Public Class [|MyValidator|] + End Class + """, + """ + Friend Class Program + Public Shared Sub Main() + End Sub + End Class + + Friend Class MyService + End Class + + Friend Class MyValidator + End Class + """); } [Theory] [MemberData(nameof(NonDiagnosticTriggeringOutputKinds))] public async Task MultipleDifferentTypes_LibraryCode_NoDiagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public class MyTests {} + await VerifyCsAsync(outputKind, + """ + public class MyTests {} -public abstract class MyBaseType {} + public abstract class MyBaseType {} -public struct MyValueType {} + public struct MyValueType {} -public interface IValidator {} + public interface IValidator {} -public enum Types {}"); + public enum Types {} + """); - await VerifyVbAsync(outputKind, @" -Public Class MyTests -End Class + await VerifyVbAsync(outputKind, + """ + Public Class MyTests + End Class -Public MustInherit Class MyBaseType -End Class + Public MustInherit Class MyBaseType + End Class -Public Structure MyValueType -End Structure + Public Structure MyValueType + End Structure -Public Interface IValidator -End Interface + Public Interface IValidator + End Interface -Public Enum Types - None -End Enum"); + Public Enum Types + None + End Enum + """); } [Theory] [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task MultipleDifferentTypes_ApplicationCode_Diagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public class [|Program|] -{ - public static void Main() {} -} + await VerifyCsAsync(outputKind, + """ + public class [|Program|] + { + public static void Main() {} + } -public abstract class [|MyBaseType|] {} + public abstract class [|MyBaseType|] {} -public struct [|MyValueType|] {} + public struct [|MyValueType|] {} -public interface [|IValidator|] {} + public interface [|IValidator|] {} -public record [|Person|]; + public record [|Person|]; -public enum [|Types|] {}", - @" -internal class Program -{ - public static void Main() {} -} + public enum [|Types|] {} + """, + """ + internal class Program + { + public static void Main() {} + } -internal abstract class MyBaseType {} + internal abstract class MyBaseType {} -internal struct MyValueType {} + internal struct MyValueType {} -internal interface IValidator {} + internal interface IValidator {} -internal record Person; + internal record Person; -internal enum Types {}"); + internal enum Types {} + """); - await VerifyVbAsync(outputKind, @" -Public Class [|Program|] - Public Shared Sub Main() - End Sub -End Class + await VerifyVbAsync(outputKind, + """ + Public Class [|Program|] + Public Shared Sub Main() + End Sub + End Class -Public MustInherit Class [|MyBaseType|] -End Class + Public MustInherit Class [|MyBaseType|] + End Class -Public Structure [|MyValueType|] -End Structure + Public Structure [|MyValueType|] + End Structure -Public Interface [|IValidator|] -End Interface + Public Interface [|IValidator|] + End Interface -Public Enum [|Types|] - None -End Enum", - @" -Friend Class Program - Public Shared Sub Main() - End Sub -End Class + Public Enum [|Types|] + None + End Enum + """, + """ + Friend Class Program + Public Shared Sub Main() + End Sub + End Class -Friend MustInherit Class MyBaseType -End Class + Friend MustInherit Class MyBaseType + End Class -Friend Structure MyValueType -End Structure + Friend Structure MyValueType + End Structure -Friend Interface IValidator -End Interface + Friend Interface IValidator + End Interface -Friend Enum Types - None -End Enum"); + Friend Enum Types + None + End Enum + """); } [Theory] @@ -238,273 +277,312 @@ public Task Records_LibraryCode_NoDiagnostic(OutputKind outputKind) [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public Task Records_ApplicationCode_Diagnostic(OutputKind outputKind) { - return VerifyCsAsync(outputKind, @" -public class [|Program|] -{ - public static void Main() {} -} + return VerifyCsAsync(outputKind, + """ + public class [|Program|] + { + public static void Main() {} + } -public record [|Person|];", - @" -internal class Program -{ - public static void Main() {} -} + public record [|Person|]; + """, + """ + internal class Program + { + public static void Main() {} + } -internal record Person;"); + internal record Person; + """); } [Theory] [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task NoModifier_ApplicationCode_NoDiagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -class Program -{ - public static void Main() {} -} + await VerifyCsAsync(outputKind, + """ + class Program + { + public static void Main() {} + } -class MyService {} + class MyService {} -sealed class MyValidator {}"); - await VerifyVbAsync(outputKind, @" -Class Program - Public Shared Sub Main() - End Sub -End Class + sealed class MyValidator {} + """); + await VerifyVbAsync(outputKind, + """ + Class Program + Public Shared Sub Main() + End Sub + End Class -Class MyService -End Class + Class MyService + End Class -NotInheritable Class MyValidator -End Class"); + NotInheritable Class MyValidator + End Class + """); } [Theory] [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task MultipleModifiers_ApplicationCode_Diagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public class [|Program|] -{ - public static void Main() {} -} + await VerifyCsAsync(outputKind, + """ + public class [|Program|] + { + public static void Main() {} + } -public sealed class [|MyService|] {} + public sealed class [|MyService|] {} -public abstract partial class [|MyValidator|] {} + public abstract partial class [|MyValidator|] {} -public partial interface [|IValidator|] {}", - @" -internal class Program -{ - public static void Main() {} -} + public partial interface [|IValidator|] {} + """, + """ + internal class Program + { + public static void Main() {} + } -internal sealed class MyService {} + internal sealed class MyService {} -internal abstract partial class MyValidator {} + internal abstract partial class MyValidator {} -internal partial interface IValidator {}"); - await VerifyVbAsync(outputKind, @" -Public Class [|Program|] - Public Shared Sub Main() - End Sub -End Class + internal partial interface IValidator {} + """); + await VerifyVbAsync(outputKind, + """ + Public Class [|Program|] + Public Shared Sub Main() + End Sub + End Class -Public NotInheritable Class [|MyService|] -End Class + Public NotInheritable Class [|MyService|] + End Class -Public Partial MustInherit Class [|MyValidator|] -End Class + Public Partial MustInherit Class [|MyValidator|] + End Class -Public Partial Interface [|IValidator|] -End Interface", - @" -Friend Class Program - Public Shared Sub Main() - End Sub -End Class + Public Partial Interface [|IValidator|] + End Interface + """, + """ + Friend Class Program + Public Shared Sub Main() + End Sub + End Class -Friend NotInheritable Class MyService -End Class + Friend NotInheritable Class MyService + End Class -Friend Partial MustInherit Class MyValidator -End Class + Friend Partial MustInherit Class MyValidator + End Class -Friend Partial Interface IValidator -End Interface"); + Friend Partial Interface IValidator + End Interface + """); } [Theory] [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task MultipleUnorderedModifiers_ApplicationCode_Diagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public class [|Program|] -{ - public static void Main() {} -} + await VerifyCsAsync(outputKind, + """ + public class [|Program|] + { + public static void Main() {} + } -sealed public class [|MyService|] {}", - @" -internal class Program -{ - public static void Main() {} -} - -sealed internal class MyService {}"); - await VerifyVbAsync(outputKind, @" -Public Class [|Program|] - Public Shared Sub Main() - End Sub -End Class - -NotInheritable Public Class [|MyService|] -End Class", - @" -Friend Class Program - Public Shared Sub Main() - End Sub -End Class - -NotInheritable Friend Class MyService -End Class"); + sealed public class [|MyService|] {} + """, + """ + internal class Program + { + public static void Main() {} + } + + sealed internal class MyService {} + """); + await VerifyVbAsync(outputKind, + """ + Public Class [|Program|] + Public Shared Sub Main() + End Sub + End Class + + NotInheritable Public Class [|MyService|] + End Class + """, + """ + Friend Class Program + Public Shared Sub Main() + End Sub + End Class + + NotInheritable Friend Class MyService + End Class + """); } [Theory] [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task NestedTypes_ApplicationCode_NoDiagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -class Program -{ - public static void Main() {} - - public struct [|MyValueType|] - { - public class [|Nested|] {} - } -}", - @" -class Program -{ - public static void Main() {} - - internal struct MyValueType - { - internal class Nested {} - } -}"); - await VerifyVbAsync(outputKind, @" -Class Program - Public Shared Sub Main() - End Sub - - Public Structure [|MyValueType|] - Public Class [|Nested|] - End Class - End Structure -End Class", - @" -Class Program - Public Shared Sub Main() - End Sub - - Friend Structure MyValueType - Friend Class Nested - End Class - End Structure -End Class"); + await VerifyCsAsync(outputKind, + """ + class Program + { + public static void Main() {} + + public struct [|MyValueType|] + { + public class [|Nested|] {} + } + } + """, + """ + class Program + { + public static void Main() {} + + internal struct MyValueType + { + internal class Nested {} + } + } + """); + await VerifyVbAsync(outputKind, + """ + Class Program + Public Shared Sub Main() + End Sub + + Public Structure [|MyValueType|] + Public Class [|Nested|] + End Class + End Structure + End Class + """, + """ + Class Program + Public Shared Sub Main() + End Sub + + Friend Structure MyValueType + Friend Class Nested + End Class + End Structure + End Class + """); } [Theory] [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task ProtectedTypes_ApplicationCode_NoDiagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -class Program -{ - public static void Main() {} - - protected abstract class MyService {} -}"); - await VerifyVbAsync(outputKind, @" -Class Program - Public Shared Sub Main() - End Sub - - Protected MustInherit Class MyService - End Class -End Class"); + await VerifyCsAsync(outputKind, + """ + class Program + { + public static void Main() {} + + protected abstract class MyService {} + } + """); + await VerifyVbAsync(outputKind, + """ + Class Program + Public Shared Sub Main() + End Sub + + Protected MustInherit Class MyService + End Class + End Class + """); } [Theory] [MemberData(nameof(NonDiagnosticTriggeringOutputKinds))] public async Task Delegates_LibraryCode_NoDiagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public delegate int GetValue(string s); -public delegate void Print(object x); -public delegate bool Match(); -"); - - await VerifyVbAsync(outputKind, @" -Imports System - -Public Delegate Function GetValue(s As String) As Int32 -Public Delegate Sub Print(x As Object) -Public Delegate Function Match() As Boolean -"); + await VerifyCsAsync(outputKind, + """ + public delegate int GetValue(string s); + public delegate void Print(object x); + public delegate bool Match(); + + """); + + await VerifyVbAsync(outputKind, + """ + Imports System + + Public Delegate Function GetValue(s As String) As Int32 + Public Delegate Sub Print(x As Object) + Public Delegate Function Match() As Boolean + + """); } [Theory] [MemberData(nameof(DiagnosticTriggeringOutputKinds))] public async Task Delegates_ApplicationCode_Diagnostic(OutputKind outputKind) { - await VerifyCsAsync(outputKind, @" -public delegate int [|GetValue|](string s); -public delegate void [|Print|](object x); -public delegate bool [|Match|](); -internal delegate char GetFirst(string s); - -class Program { - public static void Main() {} -}", - @" -internal delegate int GetValue(string s); -internal delegate void Print(object x); -internal delegate bool Match(); -internal delegate char GetFirst(string s); - -class Program { - public static void Main() {} -}"); - - await VerifyVbAsync(outputKind, @" -Imports System - -Public Delegate Function [|GetValue|](s As String) As Int32 -Public Delegate Sub [|Print|](x As Object) -Public Delegate Function [|Match|]() As Boolean -Friend Delegate Function GetFirst(s As String) As Char - -Class Program - Public Shared Sub Main() - End Sub -End Class", - @" -Imports System - -Friend Delegate Function GetValue(s As String) As Int32 -Friend Delegate Sub Print(x As Object) -Friend Delegate Function Match() As Boolean -Friend Delegate Function GetFirst(s As String) As Char - -Class Program - Public Shared Sub Main() - End Sub -End Class"); + await VerifyCsAsync(outputKind, + """ + public delegate int [|GetValue|](string s); + public delegate void [|Print|](object x); + public delegate bool [|Match|](); + internal delegate char GetFirst(string s); + + class Program { + public static void Main() {} + } + """, + """ + internal delegate int GetValue(string s); + internal delegate void Print(object x); + internal delegate bool Match(); + internal delegate char GetFirst(string s); + + class Program { + public static void Main() {} + } + """); + + await VerifyVbAsync(outputKind, + """ + Imports System + + Public Delegate Function [|GetValue|](s As String) As Int32 + Public Delegate Sub [|Print|](x As Object) + Public Delegate Function [|Match|]() As Boolean + Friend Delegate Function GetFirst(s As String) As Char + + Class Program + Public Shared Sub Main() + End Sub + End Class + """, + """ + Imports System + + Friend Delegate Function GetValue(s As String) As Int32 + Friend Delegate Sub Print(x As Object) + Friend Delegate Function Match() As Boolean + Friend Delegate Function GetFirst(s As String) As Char + + Class Program + Public Shared Sub Main() + End Sub + End Class + """); } [Theory] @@ -514,7 +592,8 @@ End Sub [InlineData("WindowsApplication", OutputKind.WindowsApplication)] [InlineData("WindowsRuntimeApplication", OutputKind.WindowsRuntimeApplication)] [InlineData("ConsoleApplication,WindowsApplication", OutputKind.WindowsApplication)] - [InlineData("ConsoleApplication,WindowsApplication,WindowsRuntimeApplication", OutputKind.WindowsRuntimeApplication)] + [InlineData("ConsoleApplication,WindowsApplication,WindowsRuntimeApplication", + OutputKind.WindowsRuntimeApplication)] public async Task CompilationOptions_Diagnostic(string optionsText, OutputKind outputKind) { await new VerifyCS.Test @@ -585,7 +664,8 @@ End Class [InlineData("WindowsApplication", OutputKind.ConsoleApplication)] [InlineData("WindowsRuntimeApplication", OutputKind.ConsoleApplication)] [InlineData("ConsoleApplication,WindowsApplication", OutputKind.WindowsRuntimeApplication)] - [InlineData("ConsoleApplication,WindowsApplication,WindowsRuntimeApplication", OutputKind.DynamicallyLinkedLibrary)] + [InlineData("ConsoleApplication,WindowsApplication,WindowsRuntimeApplication", + OutputKind.DynamicallyLinkedLibrary)] public async Task CompilationOptions_NoDiagnostic(string optionsText, OutputKind outputKind) { await new VerifyCS.Test @@ -695,19 +775,5 @@ private Task VerifyVbAsync(OutputKind outputKind, string testCode, string fixedC TestState = { OutputKind = outputKind } }.RunAsync(); } - - public static IEnumerable DiagnosticTriggeringOutputKinds() - { - yield return new object[] { OutputKind.ConsoleApplication }; - yield return new object[] { OutputKind.WindowsRuntimeApplication }; - yield return new object[] { OutputKind.WindowsApplication }; - } - - public static IEnumerable NonDiagnosticTriggeringOutputKinds() - { - yield return new object[] { OutputKind.NetModule }; - yield return new object[] { OutputKind.DynamicallyLinkedLibrary }; - yield return new object[] { OutputKind.WindowsRuntimeMetadata }; - } } } \ No newline at end of file diff --git a/src/NetAnalyzers/VisualBasic/Microsoft.CodeQuality.Analyzers/Maintainability/BasicMakeTypesInternal.vb b/src/NetAnalyzers/VisualBasic/Microsoft.CodeQuality.Analyzers/Maintainability/BasicMakeTypesInternal.vb index f71e2050bd..78588de2c3 100644 --- a/src/NetAnalyzers/VisualBasic/Microsoft.CodeQuality.Analyzers/Maintainability/BasicMakeTypesInternal.vb +++ b/src/NetAnalyzers/VisualBasic/Microsoft.CodeQuality.Analyzers/Maintainability/BasicMakeTypesInternal.vb @@ -1,41 +1,32 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. -Imports System.Collections.Immutable -Imports Analyzer.Utilities.Extensions Imports Microsoft.CodeAnalysis Imports Microsoft.CodeAnalysis.Diagnostics -Imports Microsoft.CodeAnalysis.VisualBasic Imports Microsoft.CodeAnalysis.VisualBasic.Syntax Imports Microsoft.CodeQuality.Analyzers.Maintainability Namespace Microsoft.CodeQuality.VisualBasic.Analyzers.Maintainability Public NotInheritable Class BasicMakeTypesInternal - Inherits MakeTypesInternal(Of SyntaxKind) + Inherits MakeTypesInternal - Protected Overrides ReadOnly Property TypeKinds As ImmutableArray(Of SyntaxKind) = ImmutableArray.Create(SyntaxKind.ClassStatement, SyntaxKind.StructureStatement, SyntaxKind.InterfaceStatement) - Protected Overrides ReadOnly Property EnumKind As SyntaxKind = SyntaxKind.EnumStatement - Protected Overrides ReadOnly Property DelegateKinds As ImmutableArray(Of SyntaxKind) = ImmutableArray.Create(SyntaxKind.DelegateFunctionStatement, SyntaxKind.DelegateSubStatement) - - Protected Overrides Sub AnalyzeTypeDeclaration(context As SyntaxNodeAnalysisContext) - Dim type = DirectCast(context.Node, TypeStatementSyntax) - ReportIfPublic(context, type.Modifiers, type.Identifier) - End Sub - - Protected Overrides Sub AnalyzeEnumDeclaration(context As SyntaxNodeAnalysisContext) - Dim enumStatement = DirectCast(context.Node, EnumStatementSyntax) - ReportIfPublic(context, enumStatement.Modifiers, enumStatement.Identifier) - End Sub + Protected Overrides Function GetIdentifier(type As SyntaxNode) As SyntaxToken? + Dim typeStatement = TryCast(type, TypeStatementSyntax) + If typeStatement IsNot Nothing Then + Return typeStatement.Identifier + End If - Protected Overrides Sub AnalyzeDelegateDeclaration(context As SyntaxNodeAnalysisContext) - Dim delegateStatement = DirectCast(context.Node, DelegateStatementSyntax) - ReportIfPublic(context, delegateStatement.Modifiers, delegateStatement.Identifier) - End Sub + Dim enumStatement = TryCast(type, EnumStatementSyntax) + If enumStatement IsNot Nothing Then + Return enumStatement.Identifier + End If - Private Shared Sub ReportIfPublic(context As SyntaxNodeAnalysisContext, modifiers As SyntaxTokenList, identifier As SyntaxToken) - If modifiers.Any(SyntaxKind.PublicKeyword) Then - context.ReportDiagnostic(identifier.CreateDiagnostic(Rule)) + Dim delegateStatement = TryCast(type, DelegateStatementSyntax) + If delegateStatement IsNot Nothing Then + Return delegateStatement.Identifier End If - End Sub + + Return Nothing + End Function End Class End Namespace \ No newline at end of file From ccbc8e754f2e48391a9238c288812bdbcd065339 Mon Sep 17 00:00:00 2001 From: JakeYallop <30874283+JakeYallop@users.noreply.github.com> Date: Tue, 25 Jun 2024 21:34:58 +0100 Subject: [PATCH 26/30] Highlight C# code in unit tests --- src/Test.Utilities/CSharpCodeFixVerifier`2.cs | 9 +++++---- src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs | 7 ++++--- src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs | 9 +++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Test.Utilities/CSharpCodeFixVerifier`2.cs b/src/Test.Utilities/CSharpCodeFixVerifier`2.cs index 4453c1a217..68aa5692c8 100644 --- a/src/Test.Utilities/CSharpCodeFixVerifier`2.cs +++ b/src/Test.Utilities/CSharpCodeFixVerifier`2.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. +using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CodeFixes; @@ -22,7 +23,7 @@ public static DiagnosticResult Diagnostic(string diagnosticId) public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) => CSharpCodeFixVerifier.Diagnostic(descriptor); - public static async Task VerifyAnalyzerAsync(string source, params DiagnosticResult[] expected) + public static async Task VerifyAnalyzerAsync([StringSyntax("C#-test")] string source, params DiagnosticResult[] expected) { var test = new Test { @@ -33,13 +34,13 @@ public static async Task VerifyAnalyzerAsync(string source, params DiagnosticRes await test.RunAsync(); } - public static async Task VerifyCodeFixAsync(string source, string fixedSource) + public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, string fixedSource) => await VerifyCodeFixAsync(source, DiagnosticResult.EmptyDiagnosticResults, fixedSource); - public static async Task VerifyCodeFixAsync(string source, DiagnosticResult expected, string fixedSource) + public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, DiagnosticResult expected, string fixedSource) => await VerifyCodeFixAsync(source, new[] { expected }, fixedSource); - public static async Task VerifyCodeFixAsync(string source, DiagnosticResult[] expected, string fixedSource) + public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, DiagnosticResult[] expected, string fixedSource) { var test = new Test { diff --git a/src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs b/src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs index d42c9676ff..93264af3ae 100644 --- a/src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs +++ b/src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. +using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; using Microsoft.CodeAnalysis.CodeRefactorings; using Microsoft.CodeAnalysis.Testing; @@ -10,15 +11,15 @@ public static partial class CSharpCodeRefactoringVerifier where TRefactoring : CodeRefactoringProvider, new() { /// - public static async Task VerifyRefactoringAsync(string source, string fixedSource) + public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, string fixedSource) => await VerifyRefactoringAsync(source, DiagnosticResult.EmptyDiagnosticResults, fixedSource); /// - public static async Task VerifyRefactoringAsync(string source, DiagnosticResult expected, string fixedSource) + public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, DiagnosticResult expected, string fixedSource) => await VerifyRefactoringAsync(source, new[] { expected }, fixedSource); /// - public static async Task VerifyRefactoringAsync(string source, DiagnosticResult[] expected, string fixedSource) + public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, DiagnosticResult[] expected, string fixedSource) { var test = new Test { diff --git a/src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs b/src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs index a95af6d6d8..4927ce2f9d 100644 --- a/src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs +++ b/src/Test.Utilities/CSharpSecurityCodeFixVerifier`2.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information. +using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CodeFixes; @@ -22,7 +23,7 @@ public static DiagnosticResult Diagnostic(string diagnosticId) public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) => CSharpCodeFixVerifier.Diagnostic(descriptor); - public static async Task VerifyAnalyzerAsync(string source, params DiagnosticResult[] expected) + public static async Task VerifyAnalyzerAsync([StringSyntax("C#-test")] string source, params DiagnosticResult[] expected) { var test = new Test { @@ -32,13 +33,13 @@ public static async Task VerifyAnalyzerAsync(string source, params DiagnosticRes await RunTestAsync(test, expected); } - public static Task VerifyCodeFixAsync(string source, string fixedSource) + public static Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, [StringSyntax("C#-test")] string fixedSource) => VerifyCodeFixAsync(source, DiagnosticResult.EmptyDiagnosticResults, fixedSource); - public static Task VerifyCodeFixAsync(string source, DiagnosticResult expected, string fixedSource) + public static Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, DiagnosticResult expected, [StringSyntax("C#-test")] string fixedSource) => VerifyCodeFixAsync(source, new[] { expected }, fixedSource); - public static async Task VerifyCodeFixAsync(string source, DiagnosticResult[] expected, string fixedSource) + public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, DiagnosticResult[] expected, [StringSyntax("C#-test")] string fixedSource) { var test = new Test { From 08f3b314ab8a92c6c03439b1be552b0b55d5a6e9 Mon Sep 17 00:00:00 2001 From: MartyIX <203266+MartyIX@users.noreply.github.com> Date: Sun, 7 Jul 2024 21:26:39 +0200 Subject: [PATCH 27/30] Test --- .../DisposableFieldsShouldBeDisposedTests.cs | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/DisposableFieldsShouldBeDisposedTests.cs b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/DisposableFieldsShouldBeDisposedTests.cs index f7460cb30f..68cb85b41a 100644 --- a/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/DisposableFieldsShouldBeDisposedTests.cs +++ b/src/NetAnalyzers/UnitTests/Microsoft.NetCore.Analyzers/Runtime/DisposableFieldsShouldBeDisposedTests.cs @@ -3739,6 +3739,95 @@ public async ValueTask DisposeAsync() }.RunAsync(); } + [Fact] + public async Task DisposeCoreAsync_Override_NoDiagnosticAsync() + { + await new VerifyCS.Test + { + ReferenceAssemblies = AdditionalMetadataReferences.DefaultWithAsyncInterfaces, + TestCode = @" +using System; +using System.Threading.Tasks; + +class A : IAsyncDisposable +{ + private readonly object disposedValueLock = new object(); + private bool disposedValue; + private readonly Inner innerA; + + public A() + { + innerA = new Inner(); + } + + protected virtual async ValueTask DisposeCoreAsync() + { + lock (disposedValueLock) + { + if (disposedValue) + { + return; + } + + disposedValue = true; + } + + await innerA.DisposeAsync().ConfigureAwait(false); + } + + public ValueTask DisposeAsync() + { + return default(ValueTask); + } +} + +class B : A +{ + private readonly object disposedValueLock = new object(); + private bool disposedValue; + + // Newly, we want to test that no diagnostic is reported on this line. + private readonly Inner innerB; + + public B() : base() + { + innerB = new Inner(); + } + + protected override async ValueTask DisposeCoreAsync() + { + lock (disposedValueLock) + { + if (disposedValue) + { + return; + } + + disposedValue = true; + } + + await innerB.DisposeAsync().ConfigureAwait(false); + + await base.DisposeCoreAsync().ConfigureAwait(false); + } +} + +// Declares an IAsyncDisposable type. +class Inner : IAsyncDisposable +{ + public Inner() + { + } + + public ValueTask DisposeAsync() + { + return default(ValueTask); + } +} +" + }.RunAsync(); + } + [Fact, WorkItem(5099, "https://github.com/dotnet/roslyn-analyzers/issues/5099")] public async Task OwnDisposableButDoesNotOverrideDisposableMember_Dispose() { From 3c81f1ddad08647a8a1049d9c2dab62d663a14f7 Mon Sep 17 00:00:00 2001 From: MartyIX <203266+MartyIX@users.noreply.github.com> Date: Sun, 7 Jul 2024 21:31:22 +0200 Subject: [PATCH 28/30] Fix --- .../Compiler/Extensions/IMethodSymbolExtensions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Utilities/Compiler/Extensions/IMethodSymbolExtensions.cs b/src/Utilities/Compiler/Extensions/IMethodSymbolExtensions.cs index 2879a9f4d2..2755200dc8 100644 --- a/src/Utilities/Compiler/Extensions/IMethodSymbolExtensions.cs +++ b/src/Utilities/Compiler/Extensions/IMethodSymbolExtensions.cs @@ -301,13 +301,13 @@ private static bool HasOverriddenDisposeCoreAsyncMethodSignature(this IMethodSym } /// - /// Checks if the given method has the signature "virtual ValueTask DisposeCoreAsync()" or "virtual ValueTask DisposeAsyncCore()". + /// Checks if the given method has the signature "{virtual|override} ValueTask DisposeCoreAsync()" or "{virtual|override} ValueTask DisposeAsyncCore()". /// - private static bool HasVirtualDisposeCoreAsyncMethodSignature(this IMethodSymbol method, [NotNullWhen(returnValue: true)] INamedTypeSymbol? valueTask) + private static bool HasVirtualOrOverrideDisposeCoreAsyncMethodSignature(this IMethodSymbol method, [NotNullWhen(returnValue: true)] INamedTypeSymbol? valueTask) { return (method.Name == "DisposeAsyncCore" || method.Name == "DisposeCoreAsync") && method.MethodKind == MethodKind.Ordinary && - method.IsVirtual && + (method.IsVirtual || method.IsOverride) && SymbolEqualityComparer.Default.Equals(method.ReturnType, valueTask) && method.Parameters.Length == 0; } @@ -363,7 +363,7 @@ public static DisposeMethodKind GetDisposeMethodKind( { return DisposeMethodKind.DisposeCoreAsync; } - else if (method.HasVirtualDisposeCoreAsyncMethodSignature(valueTask)) + else if (method.HasVirtualOrOverrideDisposeCoreAsyncMethodSignature(valueTask)) { return DisposeMethodKind.DisposeCoreAsync; } From 81d9b5d58e3d384ed4a91b75f4a1ece6352e76d9 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 15 Jul 2024 17:26:25 -0400 Subject: [PATCH 29/30] Execute NameAndParametersValid only when needed. (#7353) The NameAndParametersValid is executed for ALL methods in a class, independenlty of the methods being static and returning a boolean. If you look carefully at the code, we have a || after a && therefore the NamedAndParametersValid is executed ALWAYS. This means that we are performing the check for EVEVERY method, rather than on those methods that are static and return a bool. --- .../InteropServices/PlatformCompatibilityAnalyzer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs index 1bf6950d50..c26cd883cd 100644 --- a/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs +++ b/src/NetAnalyzers/Core/Microsoft.NetCore.Analyzers/InteropServices/PlatformCompatibilityAnalyzer.cs @@ -223,7 +223,7 @@ static IEnumerable FilterPlatformCheckMethods(INamedTypeSymbol sy { if (m.IsStatic && m.ReturnType.SpecialType == SpecialType.System_Boolean && - (IsOSPlatform == m.Name) || NameAndParametersValid(m)) + (IsOSPlatform == m.Name || NameAndParametersValid(m))) { CheckDependentPlatforms(m, relatedPlatforms); return true; From 26e5bd1ef8d5036b90009564f4e99d0d2bc31a91 Mon Sep 17 00:00:00 2001 From: JakeYallop <30874283+JakeYallop@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:53:32 +0100 Subject: [PATCH 30/30] Add additional StringSyntax attributes --- src/Test.Utilities/CSharpCodeFixVerifier`2.cs | 6 +++--- src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Test.Utilities/CSharpCodeFixVerifier`2.cs b/src/Test.Utilities/CSharpCodeFixVerifier`2.cs index 68aa5692c8..326ae5c35e 100644 --- a/src/Test.Utilities/CSharpCodeFixVerifier`2.cs +++ b/src/Test.Utilities/CSharpCodeFixVerifier`2.cs @@ -34,13 +34,13 @@ public static async Task VerifyAnalyzerAsync([StringSyntax("C#-test")] string so await test.RunAsync(); } - public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, string fixedSource) + public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, [StringSyntax("C#-test")] string fixedSource) => await VerifyCodeFixAsync(source, DiagnosticResult.EmptyDiagnosticResults, fixedSource); - public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, DiagnosticResult expected, string fixedSource) + public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, DiagnosticResult expected, [StringSyntax("C#-test")] string fixedSource) => await VerifyCodeFixAsync(source, new[] { expected }, fixedSource); - public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, DiagnosticResult[] expected, string fixedSource) + public static async Task VerifyCodeFixAsync([StringSyntax("C#-test")] string source, DiagnosticResult[] expected, [StringSyntax("C#-test")] string fixedSource) { var test = new Test { diff --git a/src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs b/src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs index 93264af3ae..11cc4b6abd 100644 --- a/src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs +++ b/src/Test.Utilities/CSharpCodeRefactoringVerifier`1.cs @@ -11,15 +11,15 @@ public static partial class CSharpCodeRefactoringVerifier where TRefactoring : CodeRefactoringProvider, new() { /// - public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, string fixedSource) + public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, [StringSyntax("C#-test")] string fixedSource) => await VerifyRefactoringAsync(source, DiagnosticResult.EmptyDiagnosticResults, fixedSource); /// - public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, DiagnosticResult expected, string fixedSource) + public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, DiagnosticResult expected, [StringSyntax("C#-test")] string fixedSource) => await VerifyRefactoringAsync(source, new[] { expected }, fixedSource); /// - public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, DiagnosticResult[] expected, string fixedSource) + public static async Task VerifyRefactoringAsync([StringSyntax("C#-test")] string source, DiagnosticResult[] expected, [StringSyntax("C#-test")] string fixedSource) { var test = new Test {