merged from main

ReleaseHistory.md

@@ -21,6 +21,7 @@
 ## v4.5.8 UNRELEASED
 - DEP: Update SARIF SDK submodule from [7e8def7 to dd3741f(https://github.com/microsoft/sarif-sdk/compare/7e8def7..dd3741f). [Full SARIF SDK release history](https://github.com/microsoft/sarif-sdk/blob/dd3741f/ReleaseHistory.md).
 - DEP: Upgrade `Microsoft.Security.Utilities` from 6.2.1 to 6.5.0. [#788](https://github.com/microsoft/sarif-pattern-matcher/pull/788)
+- BRK: Replaced `MatchLengthToDecode` property of `MatchExpression` with new `Base64EncodingMatch` class to support detecting Base64 string from strings of specific length range. [#790](https://github.com/microsoft/sarif-pattern-matcher/pull/790)
 - FNC: Update `SEC101/041.RabbitMqCredentials` in `Security` to check loose credential combinations. [#788](https://github.com/microsoft/sarif-pattern-matcher/pull/788)
 - FPD: Removed dynamic analysis entirely for `SEC101/047.CratesApiKey` rule due to outdated validation always returning status code 200 to all tokens. No API endpoint seems to return different status codes to distinguish between valid and invalid API keys. [#786](https://github.com/microsoft/sarif-pattern-matcher/pull/786)
 - NEW: Add Linux And macOS build pipeline, Rename folder `Src` to `src`, `Scripts` to `scripts`, `Targets` to `targets`. [#787](https://github.com/microsoft/sarif-pattern-matcher/pull/787)

src/Plugins/Security/SEC101.SecurePlaintextSecrets.json

@@ -520,7 +520,7 @@
         {
           "Id": "SEC101/102",
           "Name": "AdoPat",
-          "MatchLengthToDecode": 52,
+          "Base64EncodingMatch": { "MinMatchLength": 52, "MaxMatchLength": 52 },
           "ContentsRegex": "$SEC101/102.AdoPat",
           "MessageArguments": { "secretKind": "Azure DevOps personal access token" },
           "HelpUri": "https://aka.ms/1eslivesecrets/remediation#sec101102---adopat"

src/Sarif.PatternMatcher/Base64EncodingMatch.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.CodeAnalysis.Sarif.PatternMatcher
+{
+    public class Base64EncodingMatch
+    {
+        public int MinMatchLength { get; set; }
+
+        public int MaxMatchLength { get; set; }
+
+        public bool IsValid()
+        {
+            return MinMatchLength > 0 &&
+                   MaxMatchLength > 0 &&
+                   MinMatchLength <= MaxMatchLength;
+        }
+    }
+}

src/Sarif.PatternMatcher/MatchExpression.cs

@@ -51,7 +51,7 @@ public class MatchExpression
         /// decoded string to the match expression. No base64-decoding
         /// occurs when this property is 0 or less.
         /// </summary>
-        public int MatchLengthToDecode { get; set; }
+        public Base64EncodingMatch Base64EncodingMatch { get; set; }
 
         public Dictionary<string, string> Properties { get; set; }
 

src/Sarif.PatternMatcher/SearchSkimmer.cs

@@ -4,13 +4,11 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Resources;
 using System.Text;
 using System.Text.RegularExpressions;
-using System.Xml.Schema;
 
 using Microsoft.CodeAnalysis.Sarif.Driver;
 using Microsoft.CodeAnalysis.Sarif.Driver.Sdk;
@@ -33,7 +31,9 @@ public class SearchSkimmer : Skimmer<AnalyzeContext>
         public readonly IList<MatchExpression> MatchExpressions;
 
         private const string DefaultHelpUri = "https://github.com/microsoft/sarif-pattern-matcher";
-        private const string Base64DecodingFormatString = "\\b(?i)[0-9a-z\\/+]{0}";
+        private const string Base64DecodingFixLengthFormatString = "\\b(?i)[0-9a-z\\/+]{{{0}}}={{{1}}}";
+        private const string Base64DecodingVarLengthFormatString = "\\b(?i)[0-9a-z\\/+]{{{0},{1}}}={{0,2}}";
+        private const string Base64DecodingRegex = "^(?i)(?:[0-9a-z\\/+]{4})*(?:[0-9a-z\\/+]{2}==|[0-9a-z\\/+]{3}=)?$";
 
         private static readonly Regex namedArgumentsRegex =
             new Regex(@"[^}]?{(?<index>\d+):(?i)(?<name>[a-z]+)}[\}]*", RegexDefaults.DefaultOptionsCaseSensitive);
@@ -209,25 +209,33 @@ public override void Analyze(AnalyzeContext context)
                     continue;
                 }
 
-                if (matchExpression.MatchLengthToDecode > 0)
+                if (matchExpression.Base64EncodingMatch?.IsValid() == true)
                 {
-                    decimal unencodedLength = matchExpression.MatchLengthToDecode;
-
                     // Every 3 bytes of a base64-encoded string produces 4 bytes of data.
-                    int unpaddedLength = (int)Math.Ceiling(decimal.Divide(unencodedLength * 8M, 6M));
-                    int paddedLength = 4 * (int)Math.Ceiling(decimal.Divide(unencodedLength, 3M));
-
-                    // Create proper regex for base64-encoded string which includes padding characters.
-                    string base64DecodingRegexText =
-                        string.Format(Base64DecodingFormatString, "{" + unpaddedLength + "}") +
-                        new string('=', paddedLength - unpaddedLength);
+                    int unpaddedMinLength = decimal.ToInt32(Math.Ceiling(decimal.Divide(matchExpression.Base64EncodingMatch.MinMatchLength * 8M, 6M)));
+                    int unpaddedMaxLength = decimal.ToInt32(Math.Ceiling(decimal.Divide(matchExpression.Base64EncodingMatch.MaxMatchLength * 8M, 6M)));
+                    bool isFixedLength = unpaddedMaxLength == unpaddedMinLength;
+                    int paddedLength = isFixedLength ?
+                        4 * decimal.ToInt32(Math.Ceiling(decimal.Divide(matchExpression.Base64EncodingMatch.MinMatchLength, 3M))) :
+                        0;
+
+                    // Create proper regex for all strings matches base64 charset and length requirements
+                    // But these strings may be not valid base64 strings since the padding length are vary
+                    // for different length source string.
+                    string base64DecodingRegexText = isFixedLength ?
+                        string.Format(Base64DecodingFixLengthFormatString, unpaddedMinLength, paddedLength - unpaddedMinLength) :
+                        string.Format(Base64DecodingVarLengthFormatString, unpaddedMinLength, unpaddedMaxLength);
 
                     foreach (FlexMatch flexMatch in _engine.Matches(context.CurrentTarget.Contents, base64DecodingRegexText))
                     {
-                        // This will run the match expression against the decoded content.
-                        RunMatchExpression(binary64DecodedMatch: flexMatch,
-                                           context,
-                                           matchExpression);
+                        // Check if the matched string is valid base64 string.
+                        if (isFixedLength || _engine.IsMatch(flexMatch.Value, Base64DecodingRegex))
+                        {
+                            // This will run the match expression against the decoded content.
+                            RunMatchExpression(binary64DecodedMatch: flexMatch,
+                                               context,
+                                               matchExpression);
+                        }
                     }
                 }
 

src/Test.UnitTests.Sarif.PatternMatcher/SearchSkimmerTests.cs

@@ -3,15 +3,14 @@
 
 using System;
 using System.Collections.Generic;
-using System.IO;
+using System.Linq;
 using System.Text;
 
 using FluentAssertions;
 
 using Microsoft.CodeAnalysis.Sarif.Driver;
 using Microsoft.CodeAnalysis.Sarif.PatternMatcher.Sdk;
 using Microsoft.RE2.Managed;
-using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
 using Moq;
 
@@ -26,10 +25,15 @@ private static MatchExpression CreateGuidDetectingMatchExpression(
             string allowFileExtension = null)
         {
             const string guidRegexText = "(?i)[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}";
+            const int guidStringLength = 36;
 
             return new MatchExpression
             {
-                MatchLengthToDecode = Guid.NewGuid().ToString().Length,
+                Base64EncodingMatch = new Base64EncodingMatch
+                {
+                    MinMatchLength = guidStringLength,
+                    MaxMatchLength = guidStringLength,
+                },
                 ContentsRegex = guidRegexText,
                 FileNameDenyRegex = denyFileExtension != null ? $"(?i)\\.{denyFileExtension}$" : null,
                 FileNameAllowRegex = allowFileExtension != null ? $"(?i)\\.{allowFileExtension}$" : null,
@@ -99,8 +103,8 @@ public void SearchSkimmer_DetectsBase64EncodedPattern()
             logger.Results[0].Level.Should().Be(definition.Level);
             logger.Results[0].GetMessageText(skimmer).Should().Be($"base64-encoded:{originalMessage}");
 
-            // Analyzing base64-encoded values with MatchLengthToDecode == 0 fails
-            definition.MatchExpressions[0].MatchLengthToDecode = 0;
+            // Analyzing base64-encoded values with Base64EncodingSpec == null fails
+            definition.MatchExpressions[0].Base64EncodingMatch = null;
 
             logger.Results.Clear();
             skimmer = CreateSkimmer(definition);
@@ -454,6 +458,175 @@ public void SearchSkimmer_ShouldThrowWhenRuleDoesNotHaveAnyRegularExpression()
             exception.GetType().Should().Be(typeof(InvalidOperationException));
         }
 
+        [Fact]
+        public void SearchSkimmer_DetectsBase64EncodingMatch()
+        {
+            var testCases = new[]
+            {
+                new
+                {
+                    Title = "Text matching base64 decoded length should be detected.",
+                    MinDecodedLen = 10,
+                    MaxDecodedLen = 20,
+                    DecodedPattern = "\\b[0-9A-Za-z]{12}\\b",
+                    PlainTexts = new[] { "Abcefg123456" },
+                    ScanTargetContents = "<xml>{0}</xml>",
+                    ExpectedTotalResultCount = 1,
+                    ExpectedBase64MatchCount = 1,
+                },
+                new
+                {
+                    Title = "Multiple texts match base64 decoded length should be all detected.",
+                    MinDecodedLen = 10,
+                    MaxDecodedLen = 20,
+                    DecodedPattern = "\\b[0-9]{10,20}\\b",
+                    PlainTexts = new[] { "1234567890", "098765432109876", "56481234875456213945"},
+                    ScanTargetContents = "{0}\r\n{1}\r\n{2}",
+                    ExpectedTotalResultCount = 3,
+                    ExpectedBase64MatchCount = 3,
+                },
+                new
+                {
+                    Title = "Text doesn't match decoded length shoud not be detected.",
+                    MinDecodedLen = 5,
+                    MaxDecodedLen = 10,
+                    DecodedPattern = "\\b[a-z]{5-10}\\b",
+                    PlainTexts = new[] { "abcdefghijklmnopqrstuvwxyz", },
+                    ScanTargetContents = "key : {0}",
+                    ExpectedTotalResultCount = 0,
+                    ExpectedBase64MatchCount = 0,
+                },
+                new
+                {
+                    Title = "Text matches base64 decoded length but doesn't match pattern should not be detected.",
+                    MinDecodedLen = 8,
+                    MaxDecodedLen = 10,
+                    DecodedPattern = "\\b[0-9]{8,10}\\b",
+                    PlainTexts = new[] { "abcdefghij" },
+                    ScanTargetContents = "\"value\":\"{0}\"",
+                    ExpectedTotalResultCount = 0,
+                    ExpectedBase64MatchCount = 0,
+                },
+                new
+                {
+                    Title = "Mixed texts match base64 decoded length and plain text match pattern should be all detected.",
+                    MinDecodedLen = 10,
+                    MaxDecodedLen = 10,
+                    DecodedPattern = "\\b[0-9]{10}\\b",
+                    PlainTexts = new[] { "abcdefghij", "1234567890" },
+                    ScanTargetContents = "{0}\r\n{1}\r\n1111111111",
+                    ExpectedTotalResultCount = 2,
+                    ExpectedBase64MatchCount = 1,
+                },
+                new
+                {
+                    Title = "Invalid Base64EncodingMatch: MinSourceLength is 0.",
+                    MinDecodedLen = 0,
+                    MaxDecodedLen = 10,
+                    DecodedPattern = "\\b[0-9]{10}\\b",
+                    PlainTexts = new[] { "1234567890" },
+                    ScanTargetContents = "{0}",
+                    ExpectedTotalResultCount = 0,
+                    ExpectedBase64MatchCount = 0,
+                },
+                new
+                {
+                    Title = "Invalid Base64EncodingMatch: MaxSourceLength is 0.",
+                    MinDecodedLen = 0,
+                    MaxDecodedLen = 0,
+                    DecodedPattern = "\\b[0-9]{10}\\b",
+                    PlainTexts = new[] { "1234567890" },
+                    ScanTargetContents = "{0}",
+                    ExpectedTotalResultCount = 0,
+                    ExpectedBase64MatchCount = 0,
+                },
+                new
+                {
+                    Title = "Invalid Base64EncodingMatch: MinSourceLength < MaxSourceLength.",
+                    MinDecodedLen = 10,
+                    MaxDecodedLen = 5,
+                    DecodedPattern = "\\b[0-9]{10}\\b",
+                    PlainTexts = new[] { "1234567890" },
+                    ScanTargetContents = "{0}",
+                    ExpectedTotalResultCount = 0,
+                    ExpectedBase64MatchCount = 0,
+                },
+            };
+
+            foreach (var testCase in testCases)
+            {
+                string[] encodedTexts = testCase.PlainTexts.Select(text => Convert.ToBase64String(Encoding.UTF8.GetBytes(text))).ToArray();
+                string scanTargetContents = string.Format(testCase.ScanTargetContents, encodedTexts);
+
+                MatchExpression expr = CreateMatchExpressionWithBase64EncodingMatch(
+                                       testCase.MinDecodedLen,
+                                       testCase.MaxDecodedLen,
+                                       testCase.DecodedPattern);
+                SearchDefinition definition = CreateDefaultSearchDefinition(expr);
+
+                // We inject the well-known encoding name that reports with
+                // 'plaintext' or 'base64-encoded' depending on how a match
+                // was made.
+                definition.Message = $"{{0:encoding}}";
+
+                var mockFileSystem = new Mock<IFileSystem>();
+                mockFileSystem.Setup(x => x.FileInfoLength(It.IsAny<string>())).Returns(10);
+
+                var logger = new TestLogger();
+
+                var target = new EnumeratedArtifact(FileSystem.Instance)
+                {
+                    Uri = new Uri($"file:///c:/{definition.Name}.{definition.FileNameAllowRegex}"),
+                    Contents = scanTargetContents,
+                };
+
+                var context = new AnalyzeContext
+                {
+                    CurrentTarget = target,
+                    FileSystem = mockFileSystem.Object,
+                    Logger = logger,
+                };
+
+                SearchSkimmer skimmer = CreateSkimmer(definition);
+                skimmer.Analyze(context);
+
+                // assert
+                if (testCase.ExpectedTotalResultCount == 0)
+                {
+                    logger.Results.Should().BeNull();
+                }
+                else
+                {
+                    logger.Results.Count.Should().Be(testCase.ExpectedTotalResultCount);
+                }
+
+                for (int i = 0; i < testCase.ExpectedBase64MatchCount; i++)
+                {
+                    logger.Results[0].RuleId.Should().Be(definition.Id);
+                    logger.Results[0].Level.Should().Be(definition.Level);
+                    logger.Results[0].GetMessageText(skimmer).Should().StartWith("base64-encoded");
+                }
+            }
+        }
+
+        private static MatchExpression CreateMatchExpressionWithBase64EncodingMatch(
+            int minDecodedLength,
+            int maxDecodedLength,
+            string decodedPattern)
+        {
+            return new MatchExpression
+            {
+                Base64EncodingMatch = new Base64EncodingMatch
+                {
+                    MinMatchLength = minDecodedLength,
+                    MaxMatchLength = maxDecodedLength
+                },
+                ContentsRegex = decodedPattern,
+                FileNameDenyRegex = null,
+                FileNameAllowRegex = null,
+            };
+        }
+
         private AnalyzeContext CreateGuidMatchingSkimmer(
             string scanTargetExtension,
             ref SearchDefinition definition,