Add Async API Tests for Timeout

src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs

@@ -125,8 +125,14 @@ internal enum SnapshottedStateFlags : byte
         internal volatile bool _attentionSent;              // true if we sent an Attention to the server
         internal volatile bool _attentionSending;
 
-        private readonly LastIOTimer _lastSuccessfulIOTimer;                     
-
+        // Below 2 properties are used to enforce timeout delays in code to 
+        // reproduce issues related to theadpool starvation and timeout delay.
+        // It should always be set to false by default, and only be enabled during testing.
+        internal bool _enforceTimeoutDelay = false;
+        internal int _enforcedTimeoutDelayInMilliSeconds = 5000;
+
+        private readonly LastIOTimer _lastSuccessfulIOTimer;
+
         // secure password information to be stored
         //  At maximum number of secure string that need to be stored is two; one for login password and the other for new change password
         private SecureString[] _securePasswords = new SecureString[2] { null, null };
@@ -1455,7 +1461,7 @@ internal bool TryReadInt16(out short value)
             {
                 // The entire int16 is in the packet and in the buffer, so just return it
                 // and take care of the counters.
-                buffer = _inBuff.AsSpan(_inBytesUsed,2);
+                buffer = _inBuff.AsSpan(_inBytesUsed, 2);
                 _inBytesUsed += 2;
                 _inBytesPacket -= 2;
             }
@@ -1489,7 +1495,7 @@ internal bool TryReadInt32(out int value)
             }
 
             AssertValidState();
-            value = (buffer[3] << 24) + (buffer[2] <<16) + (buffer[1] << 8) + buffer[0];
+            value = (buffer[3] << 24) + (buffer[2] << 16) + (buffer[1] << 8) + buffer[0];
             return true;
 
         }
@@ -2277,9 +2283,11 @@ private sealed class TimeoutState
 
         private void OnTimeoutAsync(object state)
         {
-#if DEBUG
-            Thread.Sleep(13000);
-#endif
+            if (_enforceTimeoutDelay)
+            {
+                Thread.Sleep(_enforcedTimeoutDelayInMilliSeconds);
+            }
+
             int currentIdentityValue = _timeoutIdentityValue;
             TimeoutState timeoutState = (TimeoutState)state;
             if (timeoutState.IdentityValue == _timeoutIdentityValue)
@@ -2467,7 +2475,7 @@ internal void ReadSni(TaskCompletionSource<object> completion)
                     Timeout.Infinite,
                     Timeout.Infinite
                 );
-                
+
 
                 // -1 == Infinite
                 //  0 == Already timed out (NOTE: To simulate the same behavior as sync we will only timeout on 0 if we receive an IO Pending from SNI)
@@ -3555,11 +3563,10 @@ internal void SendAttention(bool mustTakeWriteLock = false)
                     // Set _attentionSending to true before sending attention and reset after setting _attentionSent
                     // This prevents a race condition between receiving the attention ACK and setting _attentionSent
                     _attentionSending = true;
-
 #if DEBUG
                     if (!_skipSendAttention)
-                    {
 #endif
+                    {
                         // Take lock and send attention
                         bool releaseLock = false;
                         if ((mustTakeWriteLock) && (!_parser.Connection.ThreadHasParserLockForClose))
@@ -3589,9 +3596,7 @@ internal void SendAttention(bool mustTakeWriteLock = false)
                                 _parser.Connection._parserLock.Release();
                             }
                         }
-#if DEBUG
                     }
-#endif
 
                     SetTimeoutSeconds(AttentionTimeoutSeconds); // Initialize new attention timeout of 5 seconds.
                     _attentionSent = true;

src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj

@@ -66,6 +66,7 @@
     <Compile Include="SQL\DataClassificationTest\DataClassificationTest.cs" />
     <Compile Include="TracingTests\EventSourceTest.cs" />
     <Compile Include="SQL\AdapterTest\AdapterTest.cs" />
+    <Compile Include="SQL\AsyncTest\AsyncTimeoutTest.cs" />
     <Compile Include="SQL\AsyncTest\BeginExecAsyncTest.cs" />
     <Compile Include="SQL\AsyncTest\BeginExecReaderAsyncTest.cs" />
     <Compile Include="SQL\AsyncTest\XmlReaderAsyncTest.cs" />

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AsyncTest/AsyncTimeoutTest.cs

@@ -0,0 +1,209 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Data;
+using System.Threading.Tasks;
+using System.Xml;
+using Microsoft.Data.SqlClient.ManualTesting.Tests.SystemDataInternals;
+using Xunit;
+
+namespace Microsoft.Data.SqlClient.ManualTesting.Tests
+{
+    public static class AsyncTimeoutTest
+    {
+        static string delayQuery2s = "WAITFOR DELAY '00:00:02'";
+        static string delayQuery10s = "WAITFOR DELAY '00:00:10'";
+
+        public enum AsyncAPI
+        {
+            ExecuteReaderAsync,
+            ExecuteScalarAsync,
+            ExecuteXmlReaderAsync
+        }
+
+        [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))]
+        [ClassData(typeof(AsyncTimeoutTestVariations))]
+        public static void TestDelayedAsyncTimeout(AsyncAPI api, string commonObj, int delayPeriod, bool marsEnabled) =>
+            RunTest(api, commonObj, delayPeriod, marsEnabled);
+
+        public class AsyncTimeoutTestVariations : IEnumerable<object[]>
+        {
+            public IEnumerator<object[]> GetEnumerator()
+            {
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 8000, true };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 5000, true };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 0, true };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 8000, false };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 5000, false };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 0, false };
+
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 8000, true };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 5000, true };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 0, true };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 8000, false };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 5000, false };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 0, false };
+
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 8000, true };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 5000, true };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 0, true };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 8000, false };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 5000, false };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 0, false };
+
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 8000, true };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 5000, true };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 0, true };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 8000, false };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 5000, false };
+                yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 0, false };
+
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 8000, true };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 5000, true };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 0, true };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 8000, false };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 5000, false };
+                yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 0, false };
+
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 8000, true };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 5000, true };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 0, true };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 8000, false };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 5000, false };
+                yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 0, false };
+            }
+
+            IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+        }
+
+        private static void RunTest(AsyncAPI api, string commonObj, int timeoutDelay, bool marsEnabled)
+        {
+            string connString = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString)
+            {
+                MultipleActiveResultSets = marsEnabled
+            }.ConnectionString;
+
+            using (SqlConnection sqlConnection = new SqlConnection(connString))
+            {
+                sqlConnection.Open();
+                if (timeoutDelay != 0)
+                {
+                    ConnectionHelper.SetEnforcedTimeout(sqlConnection, true, timeoutDelay);
+                }
+                switch (commonObj)
+                {
+                    case "Connection":
+                        QueryAndValidate(api, 1, delayQuery2s, 1, true, true, sqlConnection).Wait();
+                        QueryAndValidate(api, 2, delayQuery2s, 5, false, true, sqlConnection).Wait();
+                        QueryAndValidate(api, 3, delayQuery10s, 1, true, true, sqlConnection).Wait();
+                        QueryAndValidate(api, 4, delayQuery2s, 10, false, true, sqlConnection).Wait();
+                        break;
+                    case "Command":
+                        using (SqlCommand cmd = sqlConnection.CreateCommand())
+                        {
+                            QueryAndValidate(api, 1, delayQuery2s, 1, true, false, sqlConnection, cmd).Wait();
+                            QueryAndValidate(api, 2, delayQuery2s, 5, false, false, sqlConnection, cmd).Wait();
+                            QueryAndValidate(api, 3, delayQuery10s, 1, true, false, sqlConnection, cmd).Wait();
+                            QueryAndValidate(api, 4, delayQuery2s, 10, false, false, sqlConnection, cmd).Wait();
+                        }
+                        break;
+                }
+            }
+        }
+
+        private static async Task QueryAndValidate(AsyncAPI api, int index, string delayQuery, int timeout,
+            bool timeoutExExpected = false, bool useTransaction = false, SqlConnection cn = null, SqlCommand cmd = null)
+        {
+            SqlTransaction tx = null;
+            try
+            {
+                if (cn != null)
+                {
+                    if (cn.State != ConnectionState.Open)
+                    {
+                        await cn.OpenAsync();
+                    }
+                    cmd = cn.CreateCommand();
+                    if (useTransaction)
+                    {
+                        tx = cn.BeginTransaction(IsolationLevel.ReadCommitted);
+                        cmd.Transaction = tx;
+                    }
+                }
+
+                cmd.CommandTimeout = timeout;
+                if (api != AsyncAPI.ExecuteXmlReaderAsync)
+                {
+                    cmd.CommandText = delayQuery + $";select {index} as Id;";
+                }
+                else
+                {
+                    cmd.CommandText = delayQuery + $";select {index} as Id FOR XML PATH;";
+                }
+
+                var result = -1;
+                switch (api)
+                {
+                    case AsyncAPI.ExecuteReaderAsync:
+                        using (SqlDataReader reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false))
+                        {
+                            while (await reader.ReadAsync().ConfigureAwait(false))
+                            {
+                                var columnIndex = reader.GetOrdinal("Id");
+                                result = reader.GetInt32(columnIndex);
+                                break;
+                            }
+                        }
+                        break;
+                    case AsyncAPI.ExecuteScalarAsync:
+                        result = (int)await cmd.ExecuteScalarAsync().ConfigureAwait(false);
+                        break;
+                    case AsyncAPI.ExecuteXmlReaderAsync:
+                        using (XmlReader reader = await cmd.ExecuteXmlReaderAsync().ConfigureAwait(false))
+                        {
+                            try
+                            {
+                                Assert.True(reader.Settings.Async);
+                                reader.ReadToDescendant("Id");
+                                result = reader.ReadElementContentAsInt();
+                            }
+                            catch (Exception ex)
+                            {
+                                Assert.False(true, "Exception occurred: " + ex.Message);
+                            }
+                        }
+                        break;
+                }
+
+                if (result != index)
+                {
+                    throw new Exception("High Alert! Wrong data received for index: " + index);
+                }
+                else
+                {
+                    Assert.True(!timeoutExExpected && result == index);
+                }
+            }
+            catch (SqlException e)
+            {
+                if (!timeoutExExpected)
+                    throw new Exception("Index " + index + " failed with: " + e.Message);
+                else
+                    Assert.True(timeoutExExpected && e.Class == 11 && e.Number == -2);
+            }
+            finally
+            {
+                if (cn != null)
+                {
+                    if (useTransaction)
+                        tx.Commit();
+                    cn.Close();
+                }
+            }
+        }
+    }
+}