Skip to content

Speed up preloads using unity-scene-repacker #157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 20 commits into
base: master
Choose a base branch
from
Open

Speed up preloads using unity-scene-repacker #157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
4f9ff05
build: copy files top <CopyDir> if set
jakobhellermann Jun 25, 2025
6052319
preloader: implement optimization using unity-scene-repacker
jakobhellermann Jun 25, 2025
d03305f
preloader: support preloading using non-scene asset bundles
jakobhellermann Jun 27, 2025
e2b0714
fix progress bar not being able to catch up
jakobhellermann Jun 25, 2025
3908ac2
refactor: move ProgressBar to separate file
jakobhellermann Jun 30, 2025
91f1840
build: remove SolutionDir to support `dotnet build Assembly-CSharp ...`
jakobhellermann Jun 30, 2025
df17ebd
disable <CopyDir> by default
jakobhellermann Jun 30, 2025
d9dc78c
actions: build assembly-csharp with specified RID
jakobhellermann Jun 30, 2025
cfa8860
preloader: log total duration for preloading
jakobhellermann Jun 30, 2025
b9bb6ba
Merge branch 'master' into repack-preloads
jakobhellermann Jul 1, 2025
8ef02d0
switch `PreloadMode` setting to enum
jakobhellermann Jul 1, 2025
24db6bd
fall back to `RepackScene` when scene hooks are present
jakobhellermann Jul 1, 2025
f6e258f
set default preload mode to `RepackAssets`
jakobhellermann Jul 1, 2025
08bc9d4
don't use new c# languge features
jakobhellermann Jul 1, 2025
afd2498
fix typo
jakobhellermann Jul 2, 2025
0ed58bf
refactor: make preloading method names clearer
jakobhellermann Jul 2, 2025
cc80fec
replace string with `nameof()` reference
jakobhellermann Jul 2, 2025
b57bdf8
pass embedded dump of monobehaviour typetrees to unity-scene-repacker
jakobhellermann Jul 2, 2025
6f4808d
rename preload bundles for consistency
jakobhellermann Jul 2, 2025
e662c9b
remove leftover python script for timing analysis
jakobhellermann Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion .github/workflows/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,14 @@ jobs:
mkdir Vanilla
cd ./Vanilla
tar -xzf ../hk-binary-archives/${{ env.HK_VERSION }}/managed.${{ matrix.platform }}.tar.gz

- name: Set RID
run: |
case "${{ matrix.platform }}" in
windows) echo "RID=win-x64" >> $GITHUB_ENV ;;
linux) echo "RID=linux-x64" >> $GITHUB_ENV ;;
macos) echo "RID=osx-x64" >> $GITHUB_ENV ;;
esac

- uses: actions/setup-dotnet@v4
with:
Expand All @@ -67,7 +75,7 @@ jobs:
dotnet build PrePatcher -o PrePatcher/Output -p:Configuration=Release
- name: Build Assembly-CSharp
run: |
dotnet build Assembly-CSharp -p:SolutionDir=$PWD -p:Configuration=Release
dotnet build Assembly-CSharp --runtime $RID -p:Configuration=Release
- name: Upload Binary
if: inputs.upload-artifact
uses: actions/upload-artifact@v4
Expand Down
56 changes: 44 additions & 12 deletions Assembly-CSharp/Assembly-CSharp.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
</PropertyGroup>

<ItemGroup>
<Files Include="$(SolutionDir)Vanilla/*" />
<Files Include="../Vanilla/*" />

<BuildDir Include="$(TargetDir)" />

Expand All @@ -26,7 +26,12 @@
</ItemGroup>

<PropertyGroup>
<OutputDir>$(SolutionDir)/OutputFinal</OutputDir>
<OutputDir>../OutputFinal</OutputDir>
<!-- todo macOS -->
<GamePath Condition="'$(OS)' == 'Windows_NT'">C:/Program Files (x86)/Steam/steamapps/common/Hollow Knight</GamePath>
<GamePath Condition="'$(OS)' != 'Windows_NT'">$(HOME)/.local/share/Steam/steamapps/common/Hollow Knight</GamePath>
<CopyDir>$(GamePath)/hollow_knight_Data/Managed</CopyDir>

<Mono Condition="$(OS) == WINDOWS_NT" />
<Mono Condition="$(OS) != WINDOWS_NT">mono</Mono>
</PropertyGroup>
Expand All @@ -36,7 +41,7 @@

<Copy SkipUnchangedFiles="true" SourceFiles="@(Files)" DestinationFolder="@(BuildDir)" />

<Exec WorkingDirectory="@(BuildDir)" Command="$(Mono) &quot;$(SolutionDir)/PrePatcher/Output/PrePatcher.exe&quot; Assembly-CSharp.dll Assembly-CSharp-patched.dll" />
<Exec WorkingDirectory="@(BuildDir)" Command="$(Mono) &quot;$(MSBuildProjectDirectory)/../PrePatcher/Output/PrePatcher.exe&quot; Assembly-CSharp.dll Assembly-CSharp-patched.dll" />
<Copy SkipUnchangedFiles="true" SourceFiles="$(TargetDir)Assembly-CSharp-patched.dll" DestinationFiles="$(TargetDir)Assembly-CSharp.dll" />

<Delete Condition="Exists('MONOMODDED_Assembly-CSharp.dll')" Files="MONOMODDED_Assembly-CSharp.dll" />
Expand All @@ -48,16 +53,16 @@
</Target>

<Target Name="SetupExamples" AfterTargets="PostBuildEvent" Condition="'$(SetupExamples)' == 'true'">
<RemoveDir Directories="$(SolutionDir)/Examples/HollowKnightManaged/" />
<MakeDir Directories="$(SolutionDir)/Examples/HollowKnightManaged/" />
<RemoveDir Directories="../Examples/HollowKnightManaged/" />
<MakeDir Directories="../Examples/HollowKnightManaged/" />
<!-- Copy the vanilla files. -->
<Copy SourceFiles="@(Files)" DestinationFolder="$(SolutionDir)/Examples/HollowKnightManaged/" />
<Copy SourceFiles="@(Files)" DestinationFolder="../Examples/HollowKnightManaged/" />
<!-- Copy the API and the documentation -->
<Copy SourceFiles="$(TargetDir)MONOMODDED_Assembly-CSharp.dll" DestinationFiles="$(SolutionDir)/Examples/HollowKnightManaged/Assembly-CSharp.dll" />
<Copy SourceFiles="$(TargetDir)Assembly-CSharp.mm.xml" DestinationFiles="$(SolutionDir)/Examples/HollowKnightManaged/Assembly-CSharp.xml" />
<Copy SourceFiles="$(TargetDir)MONOMODDED_Assembly-CSharp.dll" DestinationFiles="../Examples/HollowKnightManaged/Assembly-CSharp.dll" />
<Copy SourceFiles="$(TargetDir)Assembly-CSharp.mm.xml" DestinationFiles="../Examples/HollowKnightManaged/Assembly-CSharp.xml" />

<!-- Copy the dependencies for the API and hooks for mods -->
<Copy SourceFiles="@(Dependencies)" DestinationFolder="$(SolutionDir)/Examples/HollowKnightManaged/" />
<Copy SourceFiles="@(Dependencies)" DestinationFolder="../Examples/HollowKnightManaged/" />
</Target>

<Target Name="OutputFinal" AfterTargets="PostBuild">
Expand All @@ -68,14 +73,34 @@
<MakeDir Directories="$(OutputDir)/" />

<!-- Copy the API, the documentation, the overridden mscorlib, and the README. -->
<Copy SourceFiles="$(SolutionDir)/README.ModdingApi.md" DestinationFiles="$(OutputDir)/README.md" />
<Copy SourceFiles="$(SolutionDir)/override/mscorlib.dll" DestinationFiles="$(OutputDir)/mscorlib.dll" />
<Copy SourceFiles="$(SolutionDir)/override/mscorlib.xml" DestinationFiles="$(OutputDir)/mscorlib.xml" />
<Copy SourceFiles="../README.ModdingApi.md" DestinationFiles="$(OutputDir)/README.md" />
<Copy SourceFiles="../override/mscorlib.dll" DestinationFiles="$(OutputDir)/mscorlib.dll" />
<Copy SourceFiles="../override/mscorlib.xml" DestinationFiles="$(OutputDir)/mscorlib.xml" />
<Copy SourceFiles="$(TargetDir)MONOMODDED_Assembly-CSharp.dll" DestinationFiles="$(OutputDir)/Assembly-CSharp.dll" />
<Copy SourceFiles="$(TargetDir)Assembly-CSharp.mm.xml" DestinationFiles="$(OutputDir)/Assembly-CSharp.xml" />

<!-- Copy the dependencies for the API and hooks for mods -->
<Copy SourceFiles="@(Dependencies)" DestinationFolder="$(OutputDir)" />

<PropertyGroup>
<NativeLibPrefix Condition="'$(RuntimeIdentifier)' == 'win-x64'" />
<NativeLibExtension Condition="'$(RuntimeIdentifier)' == 'win-x64'">.dll</NativeLibExtension>
<NativeLibPrefix Condition=" '$(RuntimeIdentifier)' == 'linux-x64'">lib</NativeLibPrefix>
<NativeLibExtension Condition=" '$(RuntimeIdentifier)' == 'linux-x64'">.so</NativeLibExtension>
<NativeLibPrefix Condition=" '$(RuntimeIdentifier)' == 'osx-x64'">lib</NativeLibPrefix>
<NativeLibExtension Condition=" '$(RuntimeIdentifier)' == 'osx-x64'">.dylib</NativeLibExtension>
</PropertyGroup>

<Warning Condition="'$(RuntimeIdentifier)' == ''" Text="Runtime identifier unknown, pass `--runtime win-x64` to include native dependencies" />
<!-- on .NET 5 and up, msbuild can automatically copy the relevant files from /runtimes/$RID/native/library.{dll,so,dylib}. Unfortunately, we're on net472 -->
<Copy Condition="'$(RuntimeIdentifier)' != ''" SourceFiles="$(Pkgunityscenerepacker)/runtimes/$(RuntimeIdentifier)/native/$(NativeLibPrefix)unityscenerepacker$(NativeLibExtension)" DestinationFolder="$(OutputDir)" />
</Target>

<Target Name="CopyOutputAfterFinal" AfterTargets="OutputFinal" Condition="'$(CopyDir)' != ''">
<ItemGroup>
<OutputFiles Include="$(OutputDir)\**\*" />
</ItemGroup>
<Copy SourceFiles="@(OutputFiles)" DestinationFolder="$(CopyDir)" />
</Target>

<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
Expand All @@ -99,6 +124,8 @@
<PackageReference Include="MonoMod.RuntimeDetour.HookGen" Version="21.4.29.1">
<IncludeAssets>all</IncludeAssets>
</PackageReference>

<PackageReference Include="unityscenerepacker" Version="0.2.0" GeneratePathProperty="true" />
</ItemGroup>

<ItemGroup>
Expand All @@ -109,6 +136,8 @@

<ItemGroup>
<EmbeddedResource Include="logo.png"/>
<!-- generated using https://github.com/jakobhellermann/unity-scene-repacker/blob/main/examples/export_monobehaviour_typetrees.rs -->
<EmbeddedResource Include="monobehaviour-typetree-dump.lz4" />
Comment on lines +139 to +140
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When packing the scenes into an assetbundle, we need to have information about the layout of the serialized data of the game's MonoBehaviours, so that we can patch up some layout changes.

Here I'm just bundling them into the dll (they're 200KiB big).

But the question is, does the modding API support earlier game versions as well?
In that case, this would have to be done differently:

  1. include a dump for each game version
  2. generate the dumps at runtime:

Copy link
Collaborator

@Yurihaia Yurihaia Jul 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But the question is, does the modding API support earlier game versions as well?

we only support the current version really. there are builds for some earlier patches but they don't really need these types of new features (and the API itself is wildly different)

</ItemGroup>

<ItemGroup>
Expand All @@ -135,6 +164,9 @@
<Reference Include="UnityEngine.AnimationModule, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null">
<HintPath>../Vanilla/UnityEngine.AnimationModule.dll</HintPath>
</Reference>
<Reference Include="UnityEngine.AssetBundleModule, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null">
<HintPath>../Vanilla/UnityEngine.AssetBundleModule.dll</HintPath>
</Reference>
<Reference Include="UnityEngine.AudioModule, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null">
<HintPath>../Vanilla/UnityEngine.AudioModule.dll</HintPath>
</Reference>
Expand Down
27 changes: 27 additions & 0 deletions Assembly-CSharp/ModHooksGlobalSettings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,31 @@
using System.Collections.Generic;
using JetBrains.Annotations;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;

namespace Modding
{

/// <summary>
/// Strategy preloading game objects
/// </summary>
[PublicAPI]
public enum PreloadMode
{
/// <summary>
/// Load the entire scene unmodified into memory
/// </summary>
FullScene,
/// <summary>
/// Preprocess the scenes into an assetbundle, containing filtered versions of the originals
/// </summary>
RepackScene,
/// <summary>
/// Preprocess the scenes into an assetbundle that contains individual game object assets
/// </summary>
RepackAssets,
}

/// <summary>
/// Class to hold GlobalSettings for the Modding API
/// </summary>
Expand Down Expand Up @@ -44,6 +66,11 @@ public class ModHooksGlobalSettings
/// </summary>
public int PreloadBatchSize = 5;

/// <summary>
/// Determines the strategy used for preloading game objects.
/// </summary>
[JsonConverter(typeof(StringEnumConverter))]
public PreloadMode PreloadMode = PreloadMode.RepackAssets;

/// <summary>
/// Maximum number of days to preserve modlogs for.
Expand Down
3 changes: 1 addition & 2 deletions Assembly-CSharp/ModLoader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,7 @@ Assembly Resolve(object sender, ResolveEventArgs args)
}
}

foreach (Assembly asm in asms)
{
foreach (Assembly asm in asms) {
Logger.APILogger.LogDebug($"Loading mods in assembly `{asm.FullName}`");

bool foundMod = false;
Expand Down
Loading