gh-66: gh-73: rework shader cache management & unify mount api

EgorOrachyov · Aug 11, 2024 · 005ed8b · 005ed8b
1 parent b7bcdb6
commit 005ed8b
Show file tree

Hide file tree

Showing 72 changed files with 978 additions and 982 deletions.
diff --git a/engine/config/engine.cfg b/engine/config/engine.cfg
@@ -9,51 +9,43 @@ log_to_file_level = "Info"
 log_to_console = 1
 log_to_console_level = "Info"
 
-[task_manager]
-workers = 4
-
-[file_system]
-engine_path = "engine/"
-asset_path = "assets/"
-local_path = ".wgengine/"
-cache_path = "local://cache/"
-debug_path = "local://debug/"
-logs_path = "local://logs/"
-
-[window]
-title = "Wmoge game"
+[engine.window]
+title = "Wmoge Game"
 width = 1280
 height = 720
 vsync = 1
 exit = 1
-icon_default = "engine://icons/wmoge-128.png"
-icon_small = "engine://icons/wmoge-64.png"
+icon_default = "engine/icons/wmoge-128.png"
+icon_small = "engine/icons/wmoge-64.png"
+
+[task_manager]
+workers = 4
 
-[grc.shader]
+[engine.shader]
 compiler.workers = 4
 library.load_cache = true
 library.save_cache = true
-library.path = "cache://"
+library.path = "cache/"
 library.prefix = "shader_library"
 library.suffix = "slf"
 cache.load_cache = true
 cache.save_cache = true
-cache.path = "cache://"
+cache.path = "cache/"
 cache.prefix = "shader_cache"
 cache.suffix = "scf"
 compilation.enable = true
 hot_reload.enable = true
 hot_reload.on_change = true
 hot_reload.on_trigger = true
 hot_reload.interval_sec = 5.0
-shaders_folder = "engine://shaders"
+shaders_folder = "engine/shaders"
 
 [gfx]
 vsync = true
 driver = vulkan
 
 [gfx.vulkan]
-pipeline_cache = "cache://driver.pso_cache.vulkan.pcf"
+pipeline_cache = "cache/driver.pso_cache.vulkan.pcf"
 validation_layer = true
 desc_pool_max_images = 8192
 desc_pool_max_ub = 8192

diff --git a/engine/config/game.cfg b/engine/config/game.cfg
@@ -1,11 +1,4 @@
 ; particular game config (specifics for a project)
 ; use this to override common values for the game only
 
-[window]
-title = "Wmoge game"
-width = 1280
-height = 720
-vsync = 1
-exit = 1
-icon_default = "icons/wmoge-128.png"
-icon_small = "icons/wmoge-64.png"
+[game]
diff --git a/engine/plugins/runtime/asset/shader_asset_loader.cpp b/engine/plugins/runtime/asset/shader_asset_loader.cpp
@@ -28,7 +28,10 @@
 #include "shader_asset_loader.hpp"
 
 #include "grc/shader.hpp"
+#include "grc/shader_file.hpp"
+#include "grc/shader_manager.hpp"
 #include "profiler/profiler.hpp"
+#include "system/ioc_container.hpp"
 
 namespace wmoge {
 
@@ -51,23 +54,25 @@ namespace wmoge {
             return StatusCode::InvalidData;
         }
 
-        Ref<Shader> shader = meta.rtti->instantiate().cast<Shader>();
-        if (!shader) {
-            WG_LOG_ERROR("failed to instantiate shader " << name);
-            return StatusCode::FailedInstantiate;
-        }
-
         ShaderFile shader_file;
         if (!yaml_read_file(path_on_disk, shader_file)) {
             WG_LOG_ERROR("failed to read parse shader file " << path_on_disk);
             return StatusCode::FailedParse;
         }
 
+        auto* shader_manager = IocContainer::iresolve_v<ShaderManager>();
+
+        ShaderReflection shader_reflection;
+        WG_CHECKED(shader_manager->load_shader_reflection(shader_file, shader_reflection));
+
+        Ref<Shader> shader = make_ref<Shader>(std::move(shader_reflection));
+        shader_manager->add_shader(shader);
+
         asset = shader;
         asset->set_name(name);
         asset->set_import_data(meta.import_data);
 
-        return shader->load(shader_file);
+        return WG_OK;
     }
 
 }// namespace wmoge
diff --git a/engine/plugins/runtime/asset/texture_asset_loader.cpp b/engine/plugins/runtime/asset/texture_asset_loader.cpp
@@ -35,7 +35,6 @@
 #include "grc/texture_manager.hpp"
 #include "grc/texture_resize.hpp"
 #include "profiler/profiler.hpp"
-#include "system/engine.hpp"
 #include "system/ioc_container.hpp"
 
 namespace wmoge {

diff --git a/engine/runtime/asset/asset.cpp b/engine/runtime/asset/asset.cpp
@@ -28,7 +28,6 @@
 #include "asset.hpp"
 
 #include "core/class.hpp"
-#include "system/engine.hpp"
 
 #include <algorithm>
 #include <cassert>

diff --git a/engine/runtime/asset/asset_pak_fs.cpp b/engine/runtime/asset/asset_pak_fs.cpp
@@ -28,6 +28,7 @@
 #include "asset_pak_fs.hpp"
 
 #include "asset/asset_manager.hpp"
+#include "core/string_utils.hpp"
 #include "io/yaml.hpp"
 #include "platform/file_system.hpp"
 #include "profiler/profiler.hpp"
@@ -46,7 +47,12 @@ namespace wmoge {
     Status AssetPakFileSystem::get_meta(const AssetId& name, AssetMeta& meta) {
         WG_AUTO_PROFILE_ASSET("AssetPakFileSystem::meta");
 
-        const std::string meta_file_path = name.str() + AssetMetaFile::FILE_EXTENSION;
+        std::string meta_file_path = name.str();
+        if (StringUtils::is_starts_with(meta_file_path, "asset://")) {
+            meta_file_path = StringUtils::find_replace_first(meta_file_path, "asset://", "assets/");
+        }
+
+        meta_file_path += AssetMetaFile::FILE_EXTENSION;
 
         AssetMetaFile asset_file;
 

diff --git a/engine/runtime/asset/asset_ref.hpp b/engine/runtime/asset/asset_ref.hpp
@@ -31,7 +31,6 @@
 #include "asset/asset_manager.hpp"
 #include "io/archive.hpp"
 #include "io/yaml.hpp"
-#include "system/engine.hpp"
 
 #include <cassert>
 #include <optional>

diff --git a/engine/runtime/audio/audio_stream_wav.cpp b/engine/runtime/audio/audio_stream_wav.cpp
@@ -29,7 +29,7 @@
 
 #include "platform/file_system.hpp"
 #include "profiler/profiler.hpp"
-#include "system/engine.hpp"
+#include "system/ioc_container.hpp"
 
 #include <AudioFile.h>
 
@@ -42,7 +42,7 @@ namespace wmoge {
 
         std::vector<std::uint8_t> file_data;
 
-        if (!Engine::instance()->file_system()->read_file(file_path, file_data)) {
+        if (!IocContainer::iresolve_v<FileSystem>()->read_file(file_path, file_data)) {
             WG_LOG_ERROR("field to read wav file " << file_path);
             return StatusCode::FailedRead;
         }

diff --git a/engine/runtime/core/log.cpp b/engine/runtime/core/log.cpp
@@ -98,7 +98,7 @@ namespace wmoge {
         std::time_t       time = engine_time->get_time();
         std::stringstream log_file_name;
 
-        log_file_name << "logs://log_"
+        log_file_name << "logs/log_"
                       << m_name << " "
                       << engine_time->get_time_formatted("%Y-%m-%d %H-%M-%S", time)
                       << ".log";

diff --git a/engine/runtime/gfx/gfx_vector.hpp b/engine/runtime/gfx/gfx_vector.hpp
@@ -32,7 +32,6 @@
 #include "gfx/gfx_cmd_list.hpp"
 #include "gfx/gfx_driver.hpp"
 #include "math/math_utils.hpp"
-#include "system/engine.hpp"
 
 #include <cassert>
 #include <cstring>
@@ -155,16 +154,16 @@ namespace wmoge {
             const Strid buffer_name = SID(m_name.str() + " cap=" + StringUtils::from_int(int(new_capacity)));
 
             if constexpr (std::is_same_v<Storage, GfxVertBuffer>) {
-                m_buffer = Engine::instance()->gfx_driver()->make_vert_buffer(new_size, GfxMemUsage::GpuLocal, buffer_name);
+                // m_buffer = Engine::instance()->gfx_driver()->make_vert_buffer(new_size, GfxMemUsage::GpuLocal, buffer_name);
             }
             if constexpr (std::is_same_v<Storage, GfxIndexBuffer>) {
-                m_buffer = Engine::instance()->gfx_driver()->make_index_buffer(new_size, GfxMemUsage::GpuLocal, buffer_name);
+                // m_buffer = Engine::instance()->gfx_driver()->make_index_buffer(new_size, GfxMemUsage::GpuLocal, buffer_name);
             }
             if constexpr (std::is_same_v<Storage, GfxStorageBuffer>) {
-                m_buffer = Engine::instance()->gfx_driver()->make_storage_buffer(new_size, GfxMemUsage::GpuLocal, buffer_name);
+                // m_buffer = Engine::instance()->gfx_driver()->make_storage_buffer(new_size, GfxMemUsage::GpuLocal, buffer_name);
             }
 
-            assert(m_buffer);
+            // assert(m_buffer);
         }
     }
 

diff --git a/engine/runtime/gfx/vulkan/vk_driver.cpp b/engine/runtime/gfx/vulkan/vk_driver.cpp
@@ -96,7 +96,7 @@ namespace wmoge {
         m_required_device_extensions.emplace_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME);
         WG_LOG_INFO("request " << VK_KHR_SWAPCHAIN_EXTENSION_NAME);
 
-        m_pipeline_cache_path = config->get_string_or_default(SID("gfx.vulkan.pipeline_cache"), "cache://pipelines_vk.cache");
+        m_pipeline_cache_path = config->get_string_or_default(SID("gfx.vulkan.pipeline_cache"), "cache/pipelines_vk.cache");
 
         // load vulkan functions from volk
         init_functions();

diff --git a/engine/runtime/gfx/vulkan/vk_pipeline.cpp b/engine/runtime/gfx/vulkan/vk_pipeline.cpp
@@ -31,7 +31,6 @@
 #include "core/timer.hpp"
 #include "gfx/vulkan/vk_driver.hpp"
 #include "profiler/profiler.hpp"
-#include "system/engine.hpp"
 
 namespace wmoge {
 

diff --git a/engine/runtime/glsl/glsl_builder.hpp b/engine/runtime/glsl/glsl_builder.hpp
@@ -28,6 +28,7 @@
 #pragma once
 
 #include "core/string_id.hpp"
+#include "grc/shader_compiler.hpp"
 #include "grc/shader_reflection.hpp"
 
 #include <sstream>
@@ -39,30 +40,31 @@ namespace wmoge {
      * @class GlslBuilder
      * @brief Builder to construct glsl code file
     */
-    class GlslBuilder {
+    class GlslBuilder : public ShaderCodeBuilder {
     public:
-        GlslBuilder() = default;
+        GlslBuilder()           = default;
+        ~GlslBuilder() override = default;
 
-        void set_version(int version, bool core_profile);
-        void set_module(GfxShaderModule module);
-        void add_define(Strid define);
-        void add_define(Strid define, const std::string& value);
-        void add_vertex_input(int location, const std::string& type, const std::string& name);
-        void add_sampler2d_binding(int space, int slot, Strid name);
-        void add_sampler2dArray_binding(int space, int slot, Strid name);
-        void add_samplerCube_binding(int space, int slot, Strid name);
-        void add_image_binding(int space, int slot, Strid name, ShaderQualifiers qualifiers);
-        void begin_storage_binding(int space, int slot, Strid name, ShaderQualifiers qualifiers);
-        void end_storage_binding();
-        void begin_uniform_binding(int space, int slot, Strid name, ShaderQualifiers qualifiers);
-        void end_uniform_binding();
-        void begin_struct(Strid name);
-        void end_struct();
-        void add_field(Strid type_name, Strid field_name);
-        void add_field(Strid type_name, Strid field_name, std::optional<int> num_elements);
-        void add_source(const std::string& source);
+        void set_version(int version, bool core_profile) override;
+        void set_module(GfxShaderModule module) override;
+        void add_define(Strid define) override;
+        void add_define(Strid define, const std::string& value) override;
+        void add_vertex_input(int location, const std::string& type, const std::string& name) override;
+        void add_sampler2d_binding(int space, int slot, Strid name) override;
+        void add_sampler2dArray_binding(int space, int slot, Strid name) override;
+        void add_samplerCube_binding(int space, int slot, Strid name) override;
+        void add_image_binding(int space, int slot, Strid name, ShaderQualifiers qualifiers) override;
+        void begin_storage_binding(int space, int slot, Strid name, ShaderQualifiers qualifiers) override;
+        void end_storage_binding() override;
+        void begin_uniform_binding(int space, int slot, Strid name, ShaderQualifiers qualifiers) override;
+        void end_uniform_binding() override;
+        void begin_struct(Strid name) override;
+        void end_struct() override;
+        void add_field(Strid type_name, Strid field_name) override;
+        void add_field(Strid type_name, Strid field_name, std::optional<int> num_elements) override;
+        void add_source(const std::string& source) override;
 
-        [[nodiscard]] std::string emit() const;
+        [[nodiscard]] std::string emit() const override;
 
     private:
         GfxShaderModule   m_module;

diff --git a/engine/runtime/glsl/glsl_shader_compiler.cpp b/engine/runtime/glsl/glsl_shader_compiler.cpp
@@ -85,7 +85,7 @@ namespace wmoge {
 
             if (input.options.dump_on_failure) {
                 std::stringstream file_content;
-                std::string       file_name = "debug://" + StringUtils::to_sha256(code).to_string() + ".txt";
+                std::string       file_name = "debug/" + StringUtils::to_sha256(code).to_string() + ".txt";
 
                 file_content << code << "\n";
                 file_content << "\n\n";
@@ -383,6 +383,10 @@ namespace wmoge {
         return task.schedule(depends_on).as_async();
     }
 
+    std::shared_ptr<ShaderCodeBuilder> GlslShaderCompilerAdapter::make_builder() {
+        return std::make_shared<GlslBuilder>();
+    }
+
     GfxShaderPlatform GlslShaderCompilerAdapter::get_platform() {
         return m_platform;
     }

diff --git a/engine/runtime/glsl/glsl_shader_compiler.hpp b/engine/runtime/glsl/glsl_shader_compiler.hpp
@@ -31,6 +31,7 @@
 #include "core/data.hpp"
 #include "core/string_id.hpp"
 #include "gfx/gfx_defs.hpp"
+#include "glsl/glsl_builder.hpp"
 #include "grc/shader_compiler.hpp"
 
 #include <vector>
@@ -65,9 +66,10 @@ namespace wmoge {
 
         GlslShaderCompilerAdapter(GfxShaderPlatform platform);
 
-        Async             compile(const Ref<ShaderCompilerRequest>& request, const Async& depends_on) override;
-        GfxShaderPlatform get_platform() override;
-        GfxShaderLang     get_lang() override;
+        Async                              compile(const Ref<ShaderCompilerRequest>& request, const Async& depends_on) override;
+        std::shared_ptr<ShaderCodeBuilder> make_builder() override;
+        GfxShaderPlatform                  get_platform() override;
+        GfxShaderLang                      get_lang() override;
 
     private:
         class GlslShaderCompiler* m_glsl_compiler = nullptr;

diff --git a/engine/runtime/grc/font.cpp b/engine/runtime/grc/font.cpp
@@ -31,7 +31,6 @@
 #include "grc/image.hpp"
 #include "platform/file_system.hpp"
 #include "profiler/profiler.hpp"
-#include "system/engine.hpp"
 
 #include <sstream>
 #include <string>