Give system key store providers precedence over instance-level providers

doc/snippets/Microsoft.Data.SqlClient/SqlCommand.xml

@@ -2798,7 +2798,9 @@ Custom master key store providers can be registered with the driver at three dif
 
 Once any key store provider is found at a registration level, the driver will **NOT** fall back to the other registrations to search for a provider. If providers are registered but the proper provider is not found at a level, an exception will be thrown containing only the registered providers in the registration that was checked.
 
-The built-in column master key store providers that are available for the Windows Certificate Store, CNG Store and CSP are pre-registered. No providers should be registered on the connection or command instances if one of the built-in column master key store providers is needed.
+The built-in column master key store providers that are available for the Windows Certificate Store, CNG Store and CSP are pre-registered.
+
+This does shallow copying of the dictionary so that the app cannot alter the custom provider list once it has been set.
 ]]></format>
 			</remarks>
 		</RegisterColumnEncryptionKeyStoreProvidersOnCommand>

doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml

@@ -1026,7 +1026,7 @@ GO
             <summary>
             Registers the column encryption key store providers. This function should only be called once in an app. This does shallow copying of the dictionary so that the app cannot alter the custom provider list once it has been set.
 
-            The built-in column master key store providers that are available for the Windows Certificate Store, CNG Store and CSP are pre-registered. No providers should be registered on the connection or command instances if one of the built-in column master key store providers is needed.
+            The built-in column master key store providers that are available for the Windows Certificate Store, CNG Store and CSP are pre-registered.
             </summary>
             <remarks>
             <format type="text/markdown"><![CDATA[  
@@ -1085,7 +1085,9 @@ Custom master key store providers can be registered with the driver at three dif
 
 Once any key store provider is found at a registration level, the driver will **NOT** fall back to the other registrations to search for a provider. If providers are registered but the proper provider is not found at a level, an exception will be thrown containing only the registered providers in the registration that was checked.
 
-The built-in column master key store providers that are available for the Windows Certificate Store, CNG Store and CSP are pre-registered. No providers should be registered on the connection or command instances if one of the built-in column master key store providers is needed.
+The built-in column master key store providers that are available for the Windows Certificate Store, CNG Store and CSP are pre-registered.
+
+This does shallow copying of the dictionary so that the app cannot alter the custom provider list once it has been set.
 ]]></format>
 			</remarks>
         </RegisterColumnEncryptionKeyStoreProvidersOnConnection>

src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlConnection.cs

@@ -225,10 +225,15 @@ private SqlConnection(SqlConnection connection)
             CacheConnectionStringProperties();
         }
 
+        internal static bool TryGetSystemColumnEncryptionKeyStoreProvider(string keyStoreName, out SqlColumnEncryptionKeyStoreProvider provider)
+        {
+            return s_systemColumnEncryptionKeyStoreProviders.TryGetValue(keyStoreName, out provider);
+        }
+
         /// <summary>
-        /// This function walks through both system and custom column encryption key store providers and returns an object if found.
+        /// This function walks through both instance-level and global custom column encryption key store providers and returns an object if found.
         /// </summary>
-        /// <param name="providerName">Provider Name to be searched in System Provider dictionary and Custom provider dictionary.</param>
+        /// <param name="providerName">Provider Name to be searched for.</param>
         /// <param name="columnKeyStoreProvider">If the provider is found, initializes the corresponding SqlColumnEncryptionKeyStoreProvider instance.</param>
         /// <returns>true if the provider is found, else returns false</returns>
         internal bool TryGetColumnEncryptionKeyStoreProvider(string providerName, out SqlColumnEncryptionKeyStoreProvider columnKeyStoreProvider)
@@ -240,17 +245,12 @@ internal bool TryGetColumnEncryptionKeyStoreProvider(string providerName, out Sq
                 return _customColumnEncryptionKeyStoreProviders.TryGetValue(providerName, out columnKeyStoreProvider);
             }
 
-            // Search in the sytem provider list.
-            if (s_systemColumnEncryptionKeyStoreProviders.TryGetValue(providerName, out columnKeyStoreProvider))
-            {
-                return true;
-            }
-
             lock (s_globalCustomColumnEncryptionKeyProvidersLock)
             {
                 // If custom provider is not set, then return false
                 if (s_globalCustomColumnEncryptionKeyStoreProviders is null)
                 {
+                    columnKeyStoreProvider = null;
                     return false;
                 }
 

src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlConnection.cs

@@ -213,11 +213,16 @@ private static void ValidateCustomProviders(IDictionary<string, SqlColumnEncrypt
             }
         }
 
+        internal static bool TryGetSystemColumnEncryptionKeyStoreProvider(string keyStoreName, out SqlColumnEncryptionKeyStoreProvider provider)
+        {
+            return s_systemColumnEncryptionKeyStoreProviders.TryGetValue(keyStoreName, out provider);
+        }
+
         /// <summary>
-        /// This function walks through both system and custom column encryption key store providers and returns an object if found.
+        /// This function walks through both instance-level and global custom column encryption key store providers and returns an object if found.
         /// </summary>
-        /// <param name="providerName">Provider Name to be searched in System Provider diction and Custom provider dictionary.</param>
-        /// <param name="columnKeyStoreProvider">If the provider is found, returns the corresponding SqlColumnEncryptionKeyStoreProvider instance.</param>
+        /// <param name="providerName">Provider Name to be searched for.</param>
+        /// <param name="columnKeyStoreProvider">If the provider is found, initializes the corresponding SqlColumnEncryptionKeyStoreProvider instance.</param>
         /// <returns>true if the provider is found, else returns false</returns>
         internal bool TryGetColumnEncryptionKeyStoreProvider(string providerName, out SqlColumnEncryptionKeyStoreProvider columnKeyStoreProvider)
         {
@@ -228,17 +233,12 @@ internal bool TryGetColumnEncryptionKeyStoreProvider(string providerName, out Sq
                 return _customColumnEncryptionKeyStoreProviders.TryGetValue(providerName, out columnKeyStoreProvider);
             }
 
-            // Search in the system provider list.
-            if (s_systemColumnEncryptionKeyStoreProviders.TryGetValue(providerName, out columnKeyStoreProvider))
-            {
-                return true;
-            }
-
             lock (s_globalCustomColumnEncryptionKeyProvidersLock)
             {
                 // If custom provider is not set, then return false
                 if (s_globalCustomColumnEncryptionKeyStoreProviders is null)
                 {
+                    columnKeyStoreProvider = null;
                     return false;
                 }
 

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlSecurityUtility.cs

@@ -9,6 +9,7 @@
 using System.Reflection;
 using System.Security.Cryptography;
 using System.Text;
+using Microsoft.Data.Common;
 
 namespace Microsoft.Data.SqlClient
 {
@@ -274,7 +275,7 @@ internal static void DecryptSymmetricKey(SqlTceCipherInfoEntry sqlTceCipherInfoE
             {
                 try
                 {
-                    sqlClientSymmetricKey = InstanceLevelProvidersAreRegistered(connection, command) ?
+                    sqlClientSymmetricKey = ShouldUseInstanceLevelProviderFlow(keyInfo.keyStoreName, connection, command) ?
                         GetKeyFromLocalProviders(keyInfo, connection, command) :
                         globalCekCache.GetKey(keyInfo, connection, command);
                     encryptionkeyInfoChosen = keyInfo;
@@ -367,7 +368,7 @@ internal static void VerifyColumnMasterKeySignature(string keyStoreName, string
                         GetListOfProviderNamesThatWereSearched(connection, command));
                 }
 
-                if (InstanceLevelProvidersAreRegistered(connection, command))
+                if (ShouldUseInstanceLevelProviderFlow(keyStoreName,connection, command))
                 {
                     isValidSignature = provider.VerifyColumnMasterKeyMetadata(keyPath, isEnclaveEnabled, CMKSignature);
                 }
@@ -399,6 +400,15 @@ internal static void VerifyColumnMasterKeySignature(string keyStoreName, string
             }
         }
 
+        // Instance-level providers will be used if at least one is registered on a connection or command and
+        // the required provider is not a system provider. System providers are pre-registered globally and
+        // must use the global provider flow
+        private static bool ShouldUseInstanceLevelProviderFlow(string keyStoreName, SqlConnection connection, SqlCommand command)
+        {
+            return InstanceLevelProvidersAreRegistered(connection, command) &&
+                !keyStoreName.StartsWith(ADP.ColumnEncryptionSystemProviderNamePrefix);
+        }
+
         private static bool InstanceLevelProvidersAreRegistered(SqlConnection connection, SqlCommand command) =>
             connection.HasColumnEncryptionKeyStoreProvidersRegistered ||
             (command is not null && command.HasColumnEncryptionKeyStoreProvidersRegistered);
@@ -423,6 +433,11 @@ internal static bool TryGetColumnEncryptionKeyStoreProvider(string keyStoreName,
         {
             Debug.Assert(!string.IsNullOrWhiteSpace(keyStoreName), "Provider name is invalid");
 
+            if (SqlConnection.TryGetSystemColumnEncryptionKeyStoreProvider(keyStoreName, out provider))
+            {
+                return true;
+            }
+
             // command may be null because some callers do not have a command object, eg SqlBulkCopy
             if (command is not null && command.HasColumnEncryptionKeyStoreProvidersRegistered)
             {

src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs

@@ -2242,6 +2242,38 @@ public void TestCommandCustomKeyStoreProviderDuringAeQuery(string connectionStri
             }
         }
 
+        // On Windows, "_fixture" will be type SQLSetupStrategyCertStoreProvider
+        // On non-Windows, "_fixture" will be type SQLSetupStrategyAzureKeyVault
+        // Test will pass on both but only SQLSetupStrategyCertStoreProvider is a system provider
+        [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringSetupForAE))]
+        [ClassData(typeof(AEConnectionStringProvider))]
+        public void TestSystemProvidersHavePrecedenceOverInstanceLevelProviders(string connectionString)
+        {
+            Dictionary<string, SqlColumnEncryptionKeyStoreProvider> customKeyStoreProviders = new()
+            {
+                {
+                    SqlColumnEncryptionAzureKeyVaultProvider.ProviderName,
+                    new SqlColumnEncryptionAzureKeyVaultProvider(new SqlClientCustomTokenCredential())
+                }
+            };
+
+            using (SqlConnection connection = new(connectionString))
+            {
+                connection.Open();
+                using SqlCommand command = CreateCommandThatRequiresSystemKeyStoreProvider(connection);
+                connection.RegisterColumnEncryptionKeyStoreProvidersOnConnection(customKeyStoreProviders);
+                command.ExecuteReader();
+            }
+
+            using (SqlConnection connection = new(connectionString))
+            {
+                connection.Open();
+                using SqlCommand command = CreateCommandThatRequiresSystemKeyStoreProvider(connection);
+                command.RegisterColumnEncryptionKeyStoreProvidersOnCommand(customKeyStoreProviders);
+                command.ExecuteReader();
+            }
+        }
+
         [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringSetupForAE), nameof(DataTestUtility.EnclaveEnabled))]
         [ClassData(typeof(AEConnectionStringProvider))]
         public void TestRetryWhenAEParameterMetadataCacheIsStale(string connectionString)
@@ -2318,6 +2350,15 @@ private SqlCommand CreateCommandThatRequiresCustomKeyStoreProvider(SqlConnection
             return command;
         }
 
+        private SqlCommand CreateCommandThatRequiresSystemKeyStoreProvider(SqlConnection connection)
+        {
+            SqlCommand command = new(
+                    $"SELECT * FROM [{_fixture.CustomKeyStoreProviderTestTable.Name}] WHERE FirstName = @firstName",
+                    connection, null, SqlCommandColumnEncryptionSetting.Enabled);
+            command.Parameters.AddWithValue("firstName", "abc");
+            return command;
+        }
+
         private void AssertExceptionCausedByFailureToDecrypt(Exception ex)
         {
             Assert.Contains(_failedToDecryptMessage, ex.Message);