Add Parallel option to improve database construct performance, Add DISABLE symbols to generated code for reduce code size

.gitignore

@@ -122,4 +122,15 @@ nuget/tools/*
 
 # Unity
 Library/
-Temp/
+Temp/
+src/MasterMemory.Unity/Assembly-CSharp-Editor.csproj
+
+src/MasterMemory.Unity/MasterMemory.csproj
+
+src/MasterMemory.Unity/MasterMemory.Tests.csproj
+
+src/MasterMemory.Unity/MasterMemory.Unity.sln
+
+src/MasterMemory.Unity/MessagePack.Annotations.csproj
+
+src/MasterMemory.Unity/MessagePack.csproj

README.md

@@ -24,6 +24,7 @@ Embedded Typed Readonly In-Memory Document Database for .NET Core and Unity.
 - [Validator](#validator)
 - [Metadata](#metadata)
 - [Inheritance](#inheritance)
+- [Optimization](#optimization)
 - [Code Generator](#code-generator)
 - [License](#license)
 
@@ -369,7 +370,7 @@ In default, `MemoryDatabase` do all string data automatically interning(see: [Wi
 
 Use intern or not is selected in constructor. If you want to disable automatically interning, use `internString:false`.
 
-`MemoryDatabase(byte[] databaseBinary, bool internString = true, MessagePack.IFormatterResolver formatterResolver = null)`.
+`MemoryDatabase(byte[] databaseBinary, bool internString = true, MessagePack.IFormatterResolver formatterResolver = null, int maxDegreeOfParallelism = 1)`.
 
 MemoryDatabase has three(or four) query methods.
 
@@ -883,6 +884,18 @@ public class BarTable : FooAndBarBase
 }
 ```
 
+Optimization
+---
+When invoking `new MemoryDatabase(byte[] databaseBinary...)`, read and construct database from binary. If binary size is large then construct performance will slow down. `MemoryDatabase` has `ctor(..., int maxDegreeOfParallelism = 1)` option in constructor to construct in parallel.
+
+```csharp
+var database = new MemoryDatabase(bin, maxDegreeOfParallelism: Environment.ProcessorCount);
+```
+
+The use of Parallel can greatly improve the construct performance. Recommend to use `Environment.ProcessorCount`.
+
+If you want to reduce code size of generated code, Validator and MetaDatabase info can omit in runtime. Generated code has two symbols `DISABLE_MASTERMEMORY_VALIDATOR` and `DISABLE_MASTERMEMORY_METADATABASE`.  By defining them, can be erased from the build code.
+
 Code Generator
 ---
 MasterMemory has two kinds of code-generator. `MSBuild Task`, `.NET Core Global/Local Tools`.

sandbox/Benchmark/Generated/MemoryDatabase.cs

@@ -25,12 +25,12 @@ TestDocTable TestDocTable
             this.TestDocTable = TestDocTable;
         }
 
-        public MemoryDatabase(byte[] databaseBinary, bool internString = true, MessagePack.IFormatterResolver formatterResolver = null)
-            : base(databaseBinary, internString, formatterResolver)
+        public MemoryDatabase(byte[] databaseBinary, bool internString = true, MessagePack.IFormatterResolver formatterResolver = null, int maxDegreeOfParallelism = 1)
+            : base(databaseBinary, internString, formatterResolver, maxDegreeOfParallelism)
         {
         }
 
-        protected override void Init(Dictionary<string, (int offset, int count)> header, System.ReadOnlyMemory<byte> databaseBinary, MessagePack.MessagePackSerializerOptions options)
+        protected override void Init(Dictionary<string, (int offset, int count)> header, System.ReadOnlyMemory<byte> databaseBinary, MessagePack.MessagePackSerializerOptions options, int maxDegreeOfParallelism)
         {
             this.TestDocTable = ExtractTableData<TestDoc, TestDocTable>(header, databaseBinary, options, xs => new TestDocTable(xs));
         }

sandbox/ConsoleApp/ConsoleApp.csproj

@@ -5,6 +5,14 @@
     <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <DefineConstants>$(DefineConstants)TRACE;</DefineConstants>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <DefineConstants>$(DefineConstants)TRACE;</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="MessagePack" Version="2.1.90" />
   </ItemGroup>

sandbox/ConsoleApp/Generated/MemoryDatabase.cs

@@ -48,12 +48,24 @@ Test2Table Test2Table
             this.Test2Table = Test2Table;
         }
 
-        public MemoryDatabase(byte[] databaseBinary, bool internString = true, MessagePack.IFormatterResolver formatterResolver = null)
-            : base(databaseBinary, internString, formatterResolver)
+        public MemoryDatabase(byte[] databaseBinary, bool internString = true, MessagePack.IFormatterResolver formatterResolver = null, int maxDegreeOfParallelism = 1)
+            : base(databaseBinary, internString, formatterResolver, maxDegreeOfParallelism)
         {
         }
 
-        protected override void Init(Dictionary<string, (int offset, int count)> header, System.ReadOnlyMemory<byte> databaseBinary, MessagePack.MessagePackSerializerOptions options)
+        protected override void Init(Dictionary<string, (int offset, int count)> header, System.ReadOnlyMemory<byte> databaseBinary, MessagePack.MessagePackSerializerOptions options, int maxDegreeOfParallelism)
+        {
+            if(maxDegreeOfParallelism == 1)
+            {
+                InitSequential(header, databaseBinary, options, maxDegreeOfParallelism);
+            }
+            else
+            {
+                InitParallel(header, databaseBinary, options, maxDegreeOfParallelism);
+            }
+        }
+
+        void InitSequential(Dictionary<string, (int offset, int count)> header, System.ReadOnlyMemory<byte> databaseBinary, MessagePack.MessagePackSerializerOptions options, int maxDegreeOfParallelism)
         {
             this.EnumKeyTableTable = ExtractTableData<EnumKeyTable, EnumKeyTableTable>(header, databaseBinary, options, xs => new EnumKeyTableTable(xs));
             this.ItemTable = ExtractTableData<Item, ItemTable>(header, databaseBinary, options, xs => new ItemTable(xs));
@@ -64,6 +76,25 @@ protected override void Init(Dictionary<string, (int offset, int count)> header,
             this.Test2Table = ExtractTableData<Test2, Test2Table>(header, databaseBinary, options, xs => new Test2Table(xs));
         }
 
+        void InitParallel(Dictionary<string, (int offset, int count)> header, System.ReadOnlyMemory<byte> databaseBinary, MessagePack.MessagePackSerializerOptions options, int maxDegreeOfParallelism)
+        {
+            var extracts = new Action[]
+            {
+                () => this.EnumKeyTableTable = ExtractTableData<EnumKeyTable, EnumKeyTableTable>(header, databaseBinary, options, xs => new EnumKeyTableTable(xs)),
+                () => this.ItemTable = ExtractTableData<Item, ItemTable>(header, databaseBinary, options, xs => new ItemTable(xs)),
+                () => this.MonsterTable = ExtractTableData<Monster, MonsterTable>(header, databaseBinary, options, xs => new MonsterTable(xs)),
+                () => this.PersonTable = ExtractTableData<Person, PersonTable>(header, databaseBinary, options, xs => new PersonTable(xs)),
+                () => this.QuestTable = ExtractTableData<Quest, QuestTable>(header, databaseBinary, options, xs => new QuestTable(xs)),
+                () => this.Test1Table = ExtractTableData<Test1, Test1Table>(header, databaseBinary, options, xs => new Test1Table(xs)),
+                () => this.Test2Table = ExtractTableData<Test2, Test2Table>(header, databaseBinary, options, xs => new Test2Table(xs)),
+            };
+
+            System.Threading.Tasks.Parallel.ForEach(extracts, new System.Threading.Tasks.ParallelOptions
+            {
+                MaxDegreeOfParallelism = maxDegreeOfParallelism
+            }, action => action.Invoke());
+        }
+
         public ImmutableBuilder ToImmutableBuilder()
         {
             return new ImmutableBuilder(this);
@@ -95,6 +126,8 @@ public DatabaseBuilder ToDatabaseBuilder(MessagePack.IFormatterResolver resolver
             return builder;
         }
 
+#if !DISABLE_MASTERMEMORY_VALIDATOR
+
         public ValidateResult Validate()
         {
             var result = new ValidateResult();
@@ -127,6 +160,8 @@ public ValidateResult Validate()
             return result;
         }
 
+#endif
+
         static MasterMemory.Meta.MetaDatabase metaTable;
 
         public static object GetTable(MemoryDatabase db, string tableName)
@@ -153,6 +188,8 @@ public static object GetTable(MemoryDatabase db, string tableName)
             }
         }
 
+#if !DISABLE_MASTERMEMORY_METADATABASE
+
         public static MasterMemory.Meta.MetaDatabase GetMetaDatabase()
         {
             if (metaTable != null) return metaTable;
@@ -169,5 +206,7 @@ public static MasterMemory.Meta.MetaDatabase GetMetaDatabase()
             metaTable = new MasterMemory.Meta.MetaDatabase(dict);
             return metaTable;
         }
+
+#endif
     }
 }

sandbox/ConsoleApp/Generated/Tables/EnumKeyTableTable.cs

@@ -57,9 +57,15 @@ public RangeView<EnumKeyTable> FindRangeByGender(Gender min, Gender max, bool as
 
         void ITableUniqueValidate.ValidateUnique(ValidateResult resultSet)
         {
+#if !DISABLE_MASTERMEMORY_VALIDATOR
+
             ValidateUniqueCore(data, primaryIndexSelector, "Gender", resultSet);       
+
+#endif
         }
 
+#if !DISABLE_MASTERMEMORY_METADATABASE
+
         public static MasterMemory.Meta.MetaTable CreateMetaTable()
         {
             return new MasterMemory.Meta.MetaTable(typeof(EnumKeyTable), typeof(EnumKeyTableTable), "enumkeytable",
@@ -74,5 +80,6 @@ public static MasterMemory.Meta.MetaTable CreateMetaTable()
                 });
         }
 
+#endif
     }
 }

sandbox/ConsoleApp/Generated/Tables/ItemTable.cs

@@ -82,9 +82,15 @@ public RangeView<Item> FindRangeByItemId(int min, int max, bool ascendant = true
 
         void ITableUniqueValidate.ValidateUnique(ValidateResult resultSet)
         {
+#if !DISABLE_MASTERMEMORY_VALIDATOR
+
             ValidateUniqueCore(data, primaryIndexSelector, "ItemId", resultSet);       
+
+#endif
         }
 
+#if !DISABLE_MASTERMEMORY_METADATABASE
+
         public static MasterMemory.Meta.MetaTable CreateMetaTable()
         {
             return new MasterMemory.Meta.MetaTable(typeof(Item), typeof(ItemTable), "item",
@@ -99,5 +105,6 @@ public static MasterMemory.Meta.MetaTable CreateMetaTable()
                 });
         }
 
+#endif
     }
 }

sandbox/ConsoleApp/Generated/Tables/MonsterTable.cs

@@ -82,9 +82,15 @@ public RangeView<Monster> FindRangeByMonsterId(int min, int max, bool ascendant
 
         void ITableUniqueValidate.ValidateUnique(ValidateResult resultSet)
         {
+#if !DISABLE_MASTERMEMORY_VALIDATOR
+
             ValidateUniqueCore(data, primaryIndexSelector, "MonsterId", resultSet);       
+
+#endif
         }
 
+#if !DISABLE_MASTERMEMORY_METADATABASE
+
         public static MasterMemory.Meta.MetaTable CreateMetaTable()
         {
             return new MasterMemory.Meta.MetaTable(typeof(Monster), typeof(MonsterTable), "monster",
@@ -101,5 +107,6 @@ public static MasterMemory.Meta.MetaTable CreateMetaTable()
                 });
         }
 
+#endif
     }
 }

sandbox/ConsoleApp/Generated/Tables/PersonTable.cs

@@ -142,9 +142,15 @@ public RangeView<Person> FindRangeByGender(Gender min, Gender max, bool ascendan
 
         void ITableUniqueValidate.ValidateUnique(ValidateResult resultSet)
         {
+#if !DISABLE_MASTERMEMORY_VALIDATOR
+
             ValidateUniqueCore(data, primaryIndexSelector, "PersonId", resultSet);       
+
+#endif
         }
 
+#if !DISABLE_MASTERMEMORY_METADATABASE
+
         public static MasterMemory.Meta.MetaTable CreateMetaTable()
         {
             return new MasterMemory.Meta.MetaTable(typeof(Person), typeof(PersonTable), "person",
@@ -172,5 +178,6 @@ public static MasterMemory.Meta.MetaTable CreateMetaTable()
                 });
         }
 
+#endif
     }
 }

sandbox/ConsoleApp/Generated/Tables/QuestTable.cs

@@ -82,9 +82,15 @@ public RangeView<Quest> FindRangeById(int min, int max, bool ascendant = true)
 
         void ITableUniqueValidate.ValidateUnique(ValidateResult resultSet)
         {
+#if !DISABLE_MASTERMEMORY_VALIDATOR
+
             ValidateUniqueCore(data, primaryIndexSelector, "Id", resultSet);       
+
+#endif
         }
 
+#if !DISABLE_MASTERMEMORY_METADATABASE
+
         public static MasterMemory.Meta.MetaTable CreateMetaTable()
         {
             return new MasterMemory.Meta.MetaTable(typeof(Quest), typeof(QuestTable), "quest_master",
@@ -103,5 +109,6 @@ public static MasterMemory.Meta.MetaTable CreateMetaTable()
                 });
         }
 
+#endif
     }
 }

sandbox/ConsoleApp/Generated/Tables/Test1Table.cs

@@ -82,9 +82,15 @@ public RangeView<Test1> FindRangeById(int min, int max, bool ascendant = true)
 
         void ITableUniqueValidate.ValidateUnique(ValidateResult resultSet)
         {
+#if !DISABLE_MASTERMEMORY_VALIDATOR
+
             ValidateUniqueCore(data, primaryIndexSelector, "Id", resultSet);       
+
+#endif
         }
 
+#if !DISABLE_MASTERMEMORY_METADATABASE
+
         public static MasterMemory.Meta.MetaTable CreateMetaTable()
         {
             return new MasterMemory.Meta.MetaTable(typeof(Test1), typeof(Test1Table), "Test1",
@@ -99,5 +105,6 @@ public static MasterMemory.Meta.MetaTable CreateMetaTable()
                 });
         }
 
+#endif
     }
 }

sandbox/ConsoleApp/Generated/Tables/Test2Table.cs

@@ -82,9 +82,15 @@ public RangeView<Test2> FindRangeById(int min, int max, bool ascendant = true)
 
         void ITableUniqueValidate.ValidateUnique(ValidateResult resultSet)
         {
+#if !DISABLE_MASTERMEMORY_VALIDATOR
+
             ValidateUniqueCore(data, primaryIndexSelector, "Id", resultSet);       
+
+#endif
         }
 
+#if !DISABLE_MASTERMEMORY_METADATABASE
+
         public static MasterMemory.Meta.MetaTable CreateMetaTable()
         {
             return new MasterMemory.Meta.MetaTable(typeof(Test2), typeof(Test2Table), "Test2",
@@ -99,5 +105,6 @@ public static MasterMemory.Meta.MetaTable CreateMetaTable()
                 });
         }
 
+#endif
     }
 }

sandbox/ConsoleApp/Program.cs

@@ -162,7 +162,7 @@ public void Advance(int count)
 
         public Memory<byte> GetMemory(int sizeHint = 0)
         {
-            AGAIN:
+        AGAIN:
             var nextSize = index + sizeHint;
             if (buffer.Length < nextSize)
             {
@@ -258,7 +258,7 @@ static void Main(string[] args)
             builder.AppendDynamic(table.DataType, tableData);
 
             var bin = builder.Build();
-            var database = new MemoryDatabase(bin);
+            var database = new MemoryDatabase(bin, maxDegreeOfParallelism: Environment.ProcessorCount);
         }
 
         static object ParseValue(Type type, string rawValue)

sandbox/PerfTest2/Generated/MemoryDatabase.cs

@@ -25,12 +25,12 @@ TestDocTable TestDocTable
             this.TestDocTable = TestDocTable;
         }
 
-        public MemoryDatabase(byte[] databaseBinary, bool internString = true, MessagePack.IFormatterResolver formatterResolver = null)
-            : base(databaseBinary, internString, formatterResolver)
+        public MemoryDatabase(byte[] databaseBinary, bool internString = true, MessagePack.IFormatterResolver formatterResolver = null, int maxDegreeOfParallelism = 1)
+            : base(databaseBinary, internString, formatterResolver, maxDegreeOfParallelism)
         {
         }
 
-        protected override void Init(Dictionary<string, (int offset, int count)> header, System.ReadOnlyMemory<byte> databaseBinary, MessagePack.MessagePackSerializerOptions options)
+        protected override void Init(Dictionary<string, (int offset, int count)> header, System.ReadOnlyMemory<byte> databaseBinary, MessagePack.MessagePackSerializerOptions options, int maxDegreeOfParallelism)
         {
             this.TestDocTable = ExtractTableData<TestDoc, TestDocTable>(header, databaseBinary, options, xs => new TestDocTable(xs));
         }