gh-73: add shader library bytecode file cache (save/load)

engine/core/date_time.cpp

@@ -55,14 +55,7 @@ namespace wmoge {
     }
 
     DateTimeTm DateTime::to_tm() const {
-        std::tm     s_tm;
-        std::time_t s_time_t = to_time_t();
-
-#if defined(TARGET_WINDOWS)
-        localtime_s(&s_tm, &s_time_t);
-#else
-        localtime_r(&s_time_t, &s_tm);
-#endif
+        std::tm s_tm = to_tm_t();
 
         DateTimeTm tm{};
         tm.year   = s_tm.tm_year;
@@ -78,6 +71,19 @@ namespace wmoge {
         return Clock::to_time_t(m_value);
     }
 
+    std::tm DateTime::to_tm_t() const {
+        std::tm     s_tm;
+        std::time_t s_time_t = to_time_t();
+
+#if defined(TARGET_WINDOWS)
+        localtime_s(&s_tm, &s_time_t);
+#else
+        localtime_r(&s_time_t, &s_tm);
+#endif
+
+        return s_tm;
+    }
+
     std::string DateTime::to_string() const {
         std::stringstream str;
         DateTimeTm        tm = to_tm();
@@ -92,6 +98,18 @@ namespace wmoge {
         return str.str();
     }
 
+    std::string DateTime::to_formatted(const std::string& format) const {
+        char    buffer[128];
+        std::tm tm = to_tm_t();
+        strftime(buffer, sizeof(buffer), format.c_str(), &tm);
+        return buffer;
+    }
+
+    std::string DateTime::to_pretty_string() const {
+        static const std::string format = "%Y.%m.%d %H:%M:%S";
+        return to_formatted(format);
+    }
+
     DateTime DateTime::now() {
         DateTime t;
         t.m_value = Clock::now();

engine/core/date_time.hpp

@@ -68,7 +68,10 @@ namespace wmoge {
 
         [[nodiscard]] DateTimeTm  to_tm() const;
         [[nodiscard]] std::time_t to_time_t() const;
+        [[nodiscard]] std::tm     to_tm_t() const;
         [[nodiscard]] std::string to_string() const;
+        [[nodiscard]] std::string to_formatted(const std::string& format) const;
+        [[nodiscard]] std::string to_pretty_string() const;
 
         [[nodiscard]] TimePoint get_time_point() const { return m_value; }
 
@@ -84,7 +87,7 @@ namespace wmoge {
     };
 
     inline std::ostream& operator<<(std::ostream& stream, const DateTime& dt) {
-        stream << dt.to_string();
+        stream << dt.to_pretty_string();
         return stream;
     }
 

engine/gfx/gfx_defs.hpp

@@ -75,6 +75,15 @@ namespace wmoge {
             "PLATFORM_METAL_MACOS",
             "PLATFORM_MAX"};
 
+    static constexpr const char* GfxShaderPlatformFileName[] = {
+            "none",
+            "vk_linux",
+            "vk_windows",
+            "vk_macos",
+            "dx12_windows",
+            "metal_macos",
+            "max"};
+
     /** @brief Gfx common device limits */
     struct GfxLimits {
         /** Vertex shader max input elements */

engine/gfx/gfx_pipeline.hpp

@@ -52,6 +52,8 @@ namespace wmoge {
         GfxDescSetLayouts m_layouts;
     };
 
+    using GfxPsoLayoutRef = Ref<GfxPsoLayout>;
+
     /**
      * @class GfxPsoStateGraphics
      * @brief Gfx pipeline state description
@@ -105,6 +107,8 @@ namespace wmoge {
         ~GfxPso() override = default;
     };
 
+    using GfxPsoRef = Ref<GfxPso>;
+
     /**
      * @class GfxPsoGraphics
      * @brief Represents created and compiled graphics pipeline state object
@@ -116,6 +120,8 @@ namespace wmoge {
         ~GfxPsoGraphics() override = default;
     };
 
+    using GfxPsoGraphicsRef = Ref<GfxPsoGraphics>;
+
     /**
      * @class GfxPsoCompute
      * @brief Represents created and compiled compute pipeline state object
@@ -127,6 +133,8 @@ namespace wmoge {
         ~GfxPsoCompute() override = default;
     };
 
+    using GfxPsoComputeRef = Ref<GfxPsoCompute>;
+
     /**
      * @class GfxAsyncPsoRequestGraphics
      * @brief Request for async pso compilation
@@ -140,6 +148,8 @@ namespace wmoge {
         buffered_vector<Ref<GfxPsoGraphics>, 1> pso;
     };
 
+    using GfxAsyncPsoRequestGraphicsRef = Ref<GfxAsyncPsoRequestGraphics>;
+
     /**
      * @class GfxAsyncPsoRequestCompute
      * @brief Request for async pso compilation
@@ -153,6 +163,8 @@ namespace wmoge {
         buffered_vector<Ref<GfxPsoCompute>, 1> pso;
     };
 
+    using GfxAsyncPsoRequestComputeRef = Ref<GfxAsyncPsoRequestCompute>;
+
 }// namespace wmoge
 
 namespace std {

engine/gfx/vulkan/vk_driver.cpp

@@ -69,6 +69,16 @@ namespace wmoge {
         m_engine_name         = info.engine_name;
         m_required_extensions = info.required_ext;
 
+#ifdef TARGET_WINDOWS
+        m_shader_patform = GfxShaderPlatform::VulkanWindows;
+#endif
+#ifdef TARGET_LINUX
+        m_shader_patform = GfxShaderPlatform::VulkanLinux;
+#endif
+#ifdef TARGET_MACOS
+        m_shader_patform = GfxShaderPlatform::VulkanMacOS;
+#endif
+
 #ifndef WMOGE_RELEASE
         m_use_validation = m_config->get_bool(SID("gfx.vulkan.validation_layer"), true);
 #endif
@@ -144,16 +154,6 @@ namespace wmoge {
         // finally init wrapper for ctx immediate
         m_ctx_immediate_wrapper = std::make_unique<GfxCtxWrapper>(m_ctx_immediate.get());
 
-#ifdef TARGET_WINDOWS
-        m_shader_patform = GfxShaderPlatform::VulkanWindows;
-#endif
-#ifdef TARGET_LINUX
-        m_shader_patform = GfxShaderPlatform::VulkanLinux;
-#endif
-#ifdef TARGET_MACOS
-        m_shader_patform = GfxShaderPlatform::VulkanMacOS;
-#endif
-
         WG_LOG_INFO("init vulkan gfx driver");
     }
     VKDriver::~VKDriver() {

engine/gfx/vulkan/vk_shader.cpp

@@ -47,7 +47,7 @@ namespace wmoge {
     void VKShader::create(GfxShaderDesc desc) {
         WG_AUTO_PROFILE_VULKAN("VKShader::create");
 
-        m_desc = std::move(m_desc);
+        m_desc = std::move(desc);
 
         VkShaderModuleCreateInfo create_info{};
         create_info.sType    = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO;

engine/grc/shader_interface.hpp

@@ -95,12 +95,12 @@ namespace wmoge {
         [[nodiscard]] const Ref<GfxPsoLayout>&     get_pso_layout() const { return m_pso_layout; }
 
     protected:
-        ShaderReflection                       m_reflection;
-        ShaderCache                            m_cache;
-        ShaderCompilerEnv                      m_env;
-        buffered_vector<Ref<GfxDescSetLayout>> m_layouts;
-        Ref<GfxPsoLayout>                      m_pso_layout;
-        flat_map<GfxShaderLang, std::string>   m_cached_declarations;
+        ShaderReflection                     m_reflection;
+        ShaderCache                          m_cache;
+        ShaderCompilerEnv                    m_env;
+        GfxDescSetLayouts                    m_layouts;
+        GfxPsoLayoutRef                      m_pso_layout;
+        flat_map<GfxShaderLang, std::string> m_cached_declarations;
 
         mutable RwMutexReadPrefer m_mutex;
     };

engine/grc/shader_library.cpp

@@ -1,3 +1,4 @@
+#include "shader_library.hpp"
 /**********************************************************************************/
 /* Wmoge game engine                                                              */
 /* Available at github https://github.com/EgorOrachyov/wmoge                      */
@@ -27,20 +28,60 @@
 
 #include "shader_library.hpp"
 
+#include "core/date_time.hpp"
 #include "core/log.hpp"
 #include "gfx/gfx_driver.hpp"
+#include "io/archive_file.hpp"
 #include "platform/file_system.hpp"
 #include "profiler/profiler.hpp"
+#include "rtti/traits.hpp"
 #include "system/ioc_container.hpp"
 
 #include <algorithm>
 #include <cassert>
 #include <functional>
 #include <mutex>
 #include <shared_mutex>
+#include <sstream>
 
 namespace wmoge {
 
+    struct FileShaderModule {
+        WG_RTTI_STRUCT(FileShaderModule);
+
+        Ref<Data>       bytecode;
+        GfxShaderModule module_type;
+        Sha256          source_hash;
+        Sha256          bytecode_hash;
+        Strid           name;
+    };
+
+    WG_RTTI_STRUCT_BEGIN(FileShaderModule) {
+        WG_RTTI_FIELD(bytecode, {});
+        WG_RTTI_FIELD(module_type, {});
+        WG_RTTI_FIELD(source_hash, {});
+        WG_RTTI_FIELD(bytecode_hash, {});
+        WG_RTTI_FIELD(name, {});
+    }
+    WG_RTTI_END;
+
+    struct FileShaderLibrary {
+        WG_RTTI_STRUCT(FileShaderLibrary);
+
+        GfxShaderPlatform             platform;
+        DateTime                      timestamp;
+        std::size_t                   total_size;
+        std::vector<FileShaderModule> modules;
+    };
+
+    WG_RTTI_STRUCT_BEGIN(FileShaderLibrary) {
+        WG_RTTI_FIELD(platform, {});
+        WG_RTTI_FIELD(timestamp, {});
+        WG_RTTI_FIELD(total_size, {});
+        WG_RTTI_FIELD(modules, {});
+    }
+    WG_RTTI_END;
+
     ShaderModuleMap::ShaderModuleMap() {
         m_driver = IocContainer::iresolve_v<GfxDriver>();
     }
@@ -90,9 +131,9 @@ namespace wmoge {
         return query->second.shader;
     }
 
-    void ShaderModuleMap::fit_module(const ShaderModule& module) {
+    void ShaderModuleMap::fit_module(ShaderModule& module) {
         ShaderModule& new_entry = m_modules[module.bytecode_hash];
-        new_entry               = module;
+        new_entry               = std::move(module);
     }
 
     void ShaderModuleMap::dump_modules(std::vector<ShaderModule>& out_modules) {
@@ -101,6 +142,11 @@ namespace wmoge {
         }
     }
 
+    ShaderLibrary::ShaderLibrary() {
+        rtti_type<FileShaderModule>();
+        rtti_type<FileShaderLibrary>();
+    }
+
     Ref<GfxShader> ShaderLibrary::get_or_create_shader(GfxShaderPlatform platform, GfxShaderModule module_type, const Sha256& bytecode_hash) {
         WG_AUTO_PROFILE_GRC("ShaderLibrary::get_or_create_shader");
 
@@ -125,7 +171,7 @@ namespace wmoge {
         return m_libraries[int(platform)].find_shader(module_type, bytecode_hash);
     }
 
-    void ShaderLibrary::fit_module(GfxShaderPlatform platform, const ShaderModule& module) {
+    void ShaderLibrary::fit_module(GfxShaderPlatform platform, ShaderModule& module) {
         WG_AUTO_PROFILE_GRC("ShaderLibrary::fit_module");
 
         std::unique_lock lock(m_mutex);
@@ -143,4 +189,85 @@ namespace wmoge {
         m_libraries[int(platform)].dump_modules(out_modules);
     }
 
-}// namespace wmoge
+    std::string ShaderLibrary::get_cache_file_name(const std::string& folder, const GfxShaderPlatform platform) {
+        std::stringstream file_name;
+        file_name << folder;
+        file_name << "/shader_library." << GfxShaderPlatformFileName[int(platform)] << ".slf";
+        return file_name.str();
+    }
+
+    Status ShaderLibrary::load_cache(const std::string& folder, const GfxShaderPlatform platform) {
+        WG_AUTO_PROFILE_GRC("ShaderLibrary::load_cache");
+
+        const std::string file_path = get_cache_file_name(folder, platform);
+        ArchiveReaderFile archive;
+        IoContext         context;
+
+        if (!archive.open(file_path)) {
+            WG_LOG_ERROR("failed to open shader library " << file_path << " for platform " << Enum::to_str(platform));
+            return StatusCode::FailedOpenFile;
+        }
+
+        FileShaderLibrary library;
+        WG_ARCHIVE_READ(context, archive, library);
+
+        std::unique_lock lock(m_mutex);
+
+        if (library.platform != platform) {
+            WG_LOG_ERROR("mismatched platfrom in file " << file_path);
+            return StatusCode::InvalidState;
+        }
+
+        for (FileShaderModule& file_module : library.modules) {
+            ShaderModule module;
+            module.bytecode      = std::move(file_module.bytecode);
+            module.bytecode_hash = std::move(file_module.bytecode_hash);
+            module.source_hash   = std::move(file_module.source_hash);
+            module.module_type   = std::move(file_module.module_type);
+            module.name          = std::move(file_module.name);
+            m_libraries[int(platform)].fit_module(module);
+        }
+
+        WG_LOG_INFO("load " << file_path << " created=" << library.timestamp << " size=" << StringUtils::from_mem_size(library.total_size));
+
+        return WG_OK;
+    }
+
+    Status ShaderLibrary::save_cache(const std::string& folder, const GfxShaderPlatform platform) {
+        WG_AUTO_PROFILE_GRC("ShaderLibrary::save_cache");
+
+        std::vector<ShaderModule> modules;
+        dump_modules(platform, modules);
+
+        FileShaderLibrary library;
+        library.platform   = platform;
+        library.timestamp  = DateTime::now();
+        library.total_size = 0;
+        library.modules.reserve(modules.size());
+
+        for (ShaderModule& module : modules) {
+            FileShaderModule& file_module = library.modules.emplace_back();
+            file_module.bytecode          = std::move(module.bytecode);
+            file_module.bytecode_hash     = std::move(module.bytecode_hash);
+            file_module.source_hash       = std::move(module.source_hash);
+            file_module.module_type       = std::move(module.module_type);
+            file_module.name              = std::move(module.name);
+            library.total_size += file_module.bytecode->size();
+        }
+
+        const std::string file_path = get_cache_file_name(folder, platform);
+        ArchiveWriterFile archive;
+        IoContext         context;
+
+        if (!archive.open(file_path)) {
+            WG_LOG_ERROR("failed to open shader library " << file_path << " for platform " << Enum::to_str(platform));
+            return StatusCode::FailedOpenFile;
+        }
+
+        WG_ARCHIVE_WRITE(context, archive, library);
+        WG_LOG_INFO("save " << file_path << " at=" << library.timestamp << " size=" << StringUtils::from_mem_size(library.total_size));
+
+        return WG_OK;
+    }
+
+}// namespace wmoge