[ENHANCEMENT] Improve JSON deserialize performance for large strings #7

[ENHANCEMENT] Allow large objects over 2GB in memory #6
[ENHANCEMENT] Add database selection for Performance monitor #5

Author: lorenzobruni

rls/API.Library.dll

@@ -65,7 +65,7 @@ public static RESTful_API JSONRPC2RESTful_API(JSONRPC_API JsonRpcApi)
         /// <param name="mimeType"></param>
         /// <param name="statusCode4NoContent"></param>
         /// <returns></returns>
-        public static RESTful_Output JSONRPC2RESTful_Output(JSONRPC_Output JsonRpcOutput, string mimeType = null, HttpStatusCode statusCode4NoContent = HttpStatusCode.NoContent)
+        public static RESTful_Output JSONRPC2RESTful_Output(JSONRPC_Output JsonRpcOutput, string mimeType = null, HttpStatusCode statusCode4NoContent = HttpStatusCode.NoContent, HttpStatusCode statusCode4Error = HttpStatusCode.InternalServerError)
         {
             Log.Instance.Info("Map JSON-RPC Output to RESTful Output");
 
@@ -78,7 +78,7 @@ public static RESTful_Output JSONRPC2RESTful_Output(JSONRPC_Output JsonRpcOutput
                 return new RESTful_Output
                 {
                     mimeType = null,
-                    statusCode = HttpStatusCode.InternalServerError,
+                    statusCode = statusCode4Error,
                     response = JsonRpcOutput.error
                 };
             }

rls/API.Library.dll.config

@@ -1,5 +1,6 @@
 using Enyim.Caching;
 using Enyim.Caching.Memcached;
+using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
 using System.Configuration;
@@ -18,6 +19,11 @@ public static class MemCacheD
         /// </summary>
         internal static bool API_MEMCACHED_ENABLED = Convert.ToBoolean(ConfigurationManager.AppSettings["API_MEMCACHED_ENABLED"]);
 
+        /// <summary>
+        /// Max size in MB before splitting a string record in sub-cache entries 
+        /// </summary>
+        internal static uint API_MEMCACHED_MAX_SIZE = Convert.ToUInt32(ConfigurationManager.AppSettings["API_MEMCACHED_MAX_SIZE"]);
+
         /// <summary>
         /// Maximum validity in number of seconds that MemCacheD can handle (30 days = 2592000)
         /// </summary>
@@ -38,6 +44,11 @@ public static class MemCacheD
         /// </summary>
         internal static MemcachedClient MemcachedClient = API_MEMCACHED_ENABLED ? new MemcachedClient() : null;
 
+        /// <summary>
+        /// SubKey prefix
+        /// </summary>
+        internal static String SubKeyPrefix = "subKey_";
+
         #endregion
 
         #region Methods
@@ -99,7 +110,7 @@ private static string GenerateKey_ADO(string nameSpace, string procedureName, Li
         }
 
         /// <summary>
-        /// Generate the Key for ADO
+        /// Generate the Key for BSO
         /// </summary>
         /// <typeparam name="T"></typeparam>
         /// <param name="nameSpace"></param>
@@ -142,6 +153,35 @@ private static string GenerateKey_BSO<T>(string nameSpace, string className, str
             }
         }
 
+        /// <summary>
+        /// Get a SubKey for a Key
+        /// </summary>
+        /// <param name="key"></param>
+        /// <returns></returns>
+        private static string GetSubKey(string key)
+        {
+            return SubKeyPrefix + key;
+        }
+
+        /// <summary>
+        /// Check if a subKey exists
+        /// </summary>
+        /// <param name="key"></param>
+        /// <returns></returns>
+        private static bool IsSubKey(dynamic data, string key)
+        {
+            // A key must be either a String or a JValue returned from deserialisation
+            if (data.GetType() == typeof(String) || data.GetType() == typeof(JValue))
+            {
+                // Check the explicit casting to String against the subKey 
+                if ((String)data == GetSubKey(key))
+                {
+                    return true;
+                }
+            }
+            return false;
+        }
+
         /// <summary>
         /// Generate the Hash
         /// </summary>
@@ -326,9 +366,33 @@ public static bool Store_BSO<T>(string nameSpace, string className, string metho
         /// <param name="key"></param>
         /// <param name="value"></param>
         /// <param name="validFor"></param>
+        /// <param name="repository"></param>
         /// <returns></returns>
         private static bool Store(string key, MemCachedD_Value value, TimeSpan validFor, string repository)
         {
+            // Check if data is of type String or JValue 
+            if (value.data.GetType() == typeof(String) || value.data.GetType() == typeof(JValue))
+            {
+                // Cast data to String and check if oversized
+                string sData = (String)value.data;
+                if (sData.Length * sizeof(Char) > API_MEMCACHED_MAX_SIZE * 1024 * 1024)
+                {
+                    // Get a subKey
+                    string subKey = GetSubKey(key);
+
+                    // SubStore the data by subKey
+                    if (SubStore(subKey, sData, validFor, repository))
+                    {
+                        // Override data with the subKey to fish it out later
+                        value.data = subKey;
+                    }
+                    else
+                    {
+                        return false;
+                    }
+                }
+            }
+
             try
             {
                 // The value must be serialised
@@ -371,6 +435,57 @@ private static bool Store(string key, MemCachedD_Value value, TimeSpan validFor,
             }
         }
 
+        /// <summary>
+        /// SubStore the data string by subKey
+        /// </summary>
+        /// <param name="subKey"></param>
+        /// <param name="data"></param>
+        /// <param name="validFor"></param>
+        /// <param name="repository"></param>
+        /// <returns></returns>
+        private static bool SubStore(string subKey, string data, TimeSpan validFor, string repository)
+        {
+            try
+            {
+                // The data is a string, no need to serialize
+                Log.Instance.Info("SubCache Size Uncompressed (Byte): " + data.Length * sizeof(Char));
+
+                // The data must be compressed
+                string subCacheCompressed = Utility.GZipCompress(data);
+                Log.Instance.Info("SubCache Size Compressed (Byte): " + subCacheCompressed.Length * sizeof(Char));
+
+                bool isStored = false;
+
+                // Fix the MemCacheD issue with bad Windows' compiled version: use validFor instead of expiresAt
+                // validFor and expiresAt match each other
+                isStored = MemcachedClient.Store(StoreMode.Set, subKey, subCacheCompressed, validFor);
+
+                // Store Value by Key
+                if (isStored)
+                {
+                    Log.Instance.Info("SubStore succesfull: " + subKey);
+
+                    // Add the cached record into a Repository
+                    if (!String.IsNullOrEmpty(repository))
+                    {
+                        CasRepositoryStore(subKey, repository);
+                    }
+
+                    return true;
+                }
+                else
+                {
+                    Log.Instance.Fatal("SubStore failed: " + subKey);
+                    return false;
+                }
+            }
+            catch (Exception e)
+            {
+                Log.Instance.Fatal(e);
+                return false;
+            }
+        }
+
         /// <summary>
         /// Add a Key into a Cas Repository
         /// N.B. Cas records DO NOT exipire, see Enyim inline documentation
@@ -578,12 +693,10 @@ private static MemCachedD_Value Get(string key)
                     Log.Instance.Info("Cache Valid For (s): " + cacheValidFor.TotalSeconds.ToString());
                     Log.Instance.Info("Cache Has Data: " + cache.hasData.ToString());
 
-                    if (!Convert.ToBoolean(cache.hasData))
-                    {
-                        Log.Instance.Info("Force removal of cache without data");
-                        // Remove the record with no data
-                        Remove(key);
-                    }
+                    // Init subKey
+                    string subKey = GetSubKey(key);
+                    // Init isSubKey
+                    bool isSubKey = IsSubKey(cache.data, key);
 
                     // double check the cache record is still valid if not cleared by the garbage collector
                     if (cacheExpiresAt > DateTime.Now
@@ -594,22 +707,45 @@ private static MemCachedD_Value Get(string key)
                         value.expiresAt = cacheExpiresAt;
                         value.validFor = cacheValidFor;
                         value.hasData = Convert.ToBoolean(cache.hasData);
-                        value.data = cache.data;
+                        value.data = isSubKey ? null : cache.data;
+
+                        // Check for data in the subKey
+                        if (isSubKey)
+                        {
+                            // Get subCacheCompressed from the subKey
+                            string subCacheCompressed = MemcachedClient.Get<string>(subKey);
+
+                            if (!String.IsNullOrEmpty(subCacheCompressed))
+                            {
+                                Log.Instance.Info("SubCache found: " + subKey);
+                                Log.Instance.Info("SubCache Size Compressed (Byte): " + subCacheCompressed.Length * sizeof(Char));
+
+                                // Decompress the cache
+                                string subCache = Utility.GZipDecompress(subCacheCompressed);
+                                Log.Instance.Info("SubCache Size Decompressed (Byte): " + subCache.Length * sizeof(Char));
+
+                                value.data = subCache;
+                            }
+                            else
+                            {
+                                Log.Instance.Info("SubCache not found: " + key);
+                            }
+                        }
 
                         return value;
                     }
                     else
                     {
-                        Log.Instance.Info("Force removal of expired cache");
+                        Log.Instance.Info("Forced removal of expired cache");
                         // Remove the expired record
                         Remove(key);
+
                     }
                 }
                 else
                 {
                     Log.Instance.Info("Cache not found: " + key);
                 }
-
             }
             catch (Exception e)
             {
@@ -666,7 +802,15 @@ private static bool Remove(string key)
 
             try
             {
-                // Remove the record by the Key
+                string subKey = GetSubKey(key);
+
+                // Remove the (optional) subKey
+                if (MemcachedClient.Remove(subKey))
+                {
+                    Log.Instance.Info("SubRemoval succesfull: " + subKey);
+                }
+
+                // Remove the Key
                 if (MemcachedClient.Remove(key))
                 {
                     Log.Instance.Info("Removal succesfull: " + key);

rls/API.Library.pdb

@@ -13,6 +13,7 @@ internal static class Performance
         /// Flag to indicate if Performance is enabled 
         /// </summary>
         internal static bool API_PERFORMANCE_ENABLED = Convert.ToBoolean(ConfigurationManager.AppSettings["API_PERFORMANCE_ENABLED"]);
+        internal static string API_PERFORMANCE_DATABASE = ConfigurationManager.AppSettings["API_PERFORMANCE_DATABASE"];
 
         internal static PerformanceCounter ProcessorPercentage = API_PERFORMANCE_ENABLED ? new PerformanceCounter("Processor", "% Processor Time", "_Total") : null;
         internal static PerformanceCounter MemoryAvailableMBytes = API_PERFORMANCE_ENABLED ? new PerformanceCounter("Memory", "Available MBytes") : null;
@@ -107,7 +108,7 @@ protected virtual void Dispose(bool disposing)
             if (disposing)
             {
                 // Store data
-                Performance_ADO.Create(new ADO(), items);
+                Performance_ADO.Create(String.IsNullOrEmpty(Performance.API_PERFORMANCE_DATABASE) ? new ADO() : new ADO(Performance.API_PERFORMANCE_DATABASE), items);
             }
         }
     }

src/API.Library/App.config

@@ -31,8 +31,8 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("4.4.0")]
-[assembly: AssemblyFileVersion("4.4.0")]
+[assembly: AssemblyVersion("4.4.1")]
+[assembly: AssemblyFileVersion("4.4.1")]
 
 // Configure log4net using the Web.config file by default
 [assembly: log4net.Config.XmlConfigurator(Watch = true)]

src/API.Library/Entities/API.Map.cs

@@ -72,9 +72,9 @@
     <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="API.Library, Version=4.4.0, Culture=neutral, processorArchitecture=MSIL">
+    <Reference Include="API.Library, Version=4.4.1, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\API.Library.4.4.0\API.Library.dll</HintPath>
+      <HintPath>..\packages\API.Library.4.4.1\API.Library.dll</HintPath>
     </Reference>
     <Reference Include="Enyim.Caching, Version=2.16.0.0, Culture=neutral, PublicKeyToken=cec98615db04012e, processorArchitecture=MSIL">
       <HintPath>..\packages\EnyimMemcached.2.16.0\lib\net35\Enyim.Caching.dll</HintPath>

src/API.Library/Entities/MemCacheD.cs

@@ -148,7 +148,7 @@
 		<!-- ADO - Execution timeout in seconds -->
 		<add key="API_ADO_EXECUTION_TIMEOUT" value="600" />
 		<!-- ADO - Bulk Copy timeout in seconds -->
-		<add key="API_ADO_BULKCOPY_TIMEOUT" value="600" />		  
+		<add key="API_ADO_BULKCOPY_TIMEOUT" value="600" />
 		<!-- ADO - Bulk Copy BatchSize in rows (below 5000 to avoid Table locking) -->
 		<add key="API_ADO_BULKCOPY_BATCHSIZE" value="4999" />
 
@@ -159,6 +159,8 @@
 	-->
 		<!-- MemCacheD - Switch on [TRUE] or off [FALSE] the MemCacheD -->
 		<add key="API_MEMCACHED_ENABLED" value="TRUE" />
+		<!-- MemCacheD - Set the max size in MB before splitting a string record in sub-cache entries -->
+		<add key="API_MEMCACHED_MAX_SIZE" value="128" />
 		<!-- MemCacheD - Maximum validity in number of seconds that MemCacheD can handle (30 days = 2592000) -->
 		<add  key="API_MEMCACHED_MAX_VALIDITY" value="2592000" />
 		<!-- MemCacheD - Salsa code to isolate the cache records form other applications or environments -->
@@ -171,6 +173,8 @@
 	-->
 		<!-- Performance - Switch on [TRUE] or off [FALSE] the Performance -->
 		<add key="API_PERFORMANCE_ENABLED" value="FALSE" />
+		<!-- Performance - Choose the Database connection string where to store the records -->
+		<add key="API_PERFORMANCE_DATABASE" value="defaultConnection" />
 
 	</appSettings>