From 615eb4af74e396ff90e59b2eca24820abbf8d2c9 Mon Sep 17 00:00:00 2001 From: Mitchell Hwang <16830051+mdh1418@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:42:42 -0400 Subject: [PATCH] [mono][tasks] Extract EmitWasmBundle into generic EmitBundle task and enable bundling in mono self-contained library (#84191) * [tasks] Move EmitWasmBundle out of wasm specific task * [tasks] Make EmitBundle more generic * [tasks] Generate EmitBundle task and replace invocations of old task * Extend preloading assemblies with mono_assembly_open * [LibraryBuilder] Integrate bundling * Refactor LibraryBuilder targets to isolate Bundling * [mono] Add mono_get_bundled_assembly in preparation for bundled runtimeconfig data * [mono][librarymode] Bundle runtimeconfig.bin * Address feedback Redesign bundling apis to handle various types of resources Move bundle registering into generated function and handle all forms of bundle registering there Move bundling api into assembly.c and prototypes into assembly-internals.h Rename assemblies to resources in bundling context Include EmitBundle task in wasm nuget Declare macro for case invariant string comparison Add documentation for bundling apis * Consolidate runtimeconfig functions * [LibraryBuilder] Enable bundling by default * Handle pdb bundling * Default EmitBundle callback function to mono_add_bundled_resource * Combine bundled resource types into generic bundled data * Add api to free memory dynamically allocated for the bundled resources hash table * Prepend enum and entries with mono * Move bundling api into separate source and header * Fix source file name * Prefix struct with Mono * Integrate new bundling api into mono debug * Fix types in autoinit * [mono] Base bundling apis around preallocation of MonoBundled*Resource structs * [mono] Clean up bundled assembly and satellite assembly search * [mono] Incorporate new bundling APIs into bundled resource retrievals * [EmitBundle][task] Add templates for bundled source files Templates added: - header mirroring bundled-resources-internals.h and MonoBundledSatelliteAssembly - source file template for each resource's byte data and size - individual templates to build preallocated MonoBundled*Resource structs - source file template to hold preallocated MonoBundled*Resource structs * [EmitBundle] Enable consolidation of bundled resource symbols * [EmitBundle] Generate bundled source file to preallocate MonoBundled*Resources * [EmitBundle] Output symbol to resource type map for populating function to add bundled resources * [LibraryBuilder] Generate source file to add preallocated MonoBundled*Resources * [LibraryBuilder] Incorporate EmitBundle templates into targets flow * [EmitBundle] Move bundle registration back into EmitBundle [EmitBundle] Leverage Emit for file generation * Update invocations of EmitBundle task * Address feedback Migrate logic to check hash table for assemblies or satellite assemblies back to bundled-resources.c Prepend apis with consistent mono_bundled_resources prefix * [mono] Prefer C99 compliant types for partially exposed APIs * Address various fixups Remove unused function Remove unnecessary string duplication Add length checks for comparisons Fix spacing in MonoBundledResource structs Avoid g_hash_table_lookup_extended precondition macro assertion fail * Combine preallocation registration and rename to templates * Prefer symbol data over symfile * Differentiate resource name and symbol * Update invocations of EmitBundle * Add inlined resource getters * Add unique id field for each MonoBundledResource * Include culture information for satellite resources * Generate preallocation and registration source file iff BundleFile provided * Generate output itemgroup detailing resource data * Leverage new bundling api from old bundling api and deprecate * [mono][debug] Migrate symbol data registration to new bundling api * Remove symbol data from MonoBundleSatelliteAssemblyResource * Directly determine resource type * Update bundled header and switch to non internal types Update bundled header to declare resource symbols and registration function Switch types from mono internal types to C99 compliant as bundling apis are partially exposed to public * Update Autoinitialization logic to leverage bundled runtimeconfig symbols * Integrate header into bundled source * Add functions to free allocated MonoBundledResources * Fixup Leverage new bundling api from old bundling api and deprecate * Fixup Address various fixups * Remove unnecessary includes * Fix trailing commas in structs and extra lines in generated files * Move specific bundled resource getters into source file * Various cleanups Instantiating boolean variables More direct insertion into hash table Removing unnecessary assertions Proper type casting Preferring symbol_data over symfile Adding void to prototype functions arg Fix indentation and spacing Prefer id over name * Cleanup logic surrounding EmitBundle task invocations * Rename output metadata to DataLenSymbol and DataLenSymbolValue * Simplify bundled resource hashtable insertion * Cleanup variable instantiation * Fixup mono-debug * Cleanup header * fixup wasi * fixup template format * Add flexibility to auto initialize runtime without bundled runtimeconfig * Various fixups * Use g_free for memory allocated through glib apis * Move condition end tag into LibraryBuilder to avoid empty line and cluttered line * Add bundled resource getters to access data values without MonoBundledResource structs * Emit one object file for single resource data source file mode * Rename resource size symbol * Use public friendly API in file linking towards runtime * Cleanup types populated through bundled resource getters * Move extern block completely to LibraryBuilder side * Update EmitBundle task invocations to use BundleRegistrationFunctionName * Refactor EmitBundle to account for timezone duplicate resource contents * wip * wip * WIP * [mono] Alias assemblies with known extension in bundled resources hashtable * [tasks] Calculate destination files within EmitBundle task itself * Update invocations of EmitBundle to grab DestinationFiles from output * Guard webcil specific logic * Include OutputDirectory path into output destinationFiles * Fix debugger tests and memory leak * Various fixups and Documentation * Try adding resources with known assembly extensions as dll * Revert to custom hash and equal without WEBCIL guard This reverts commit 0e3344214e306709c68ef0aa453863b9f79563ed. * Consolidate file and bundledResource variables * Add culture metadata to SatelliteAssembly resources * Truncate Encoding for readability * Enable custom hash for bundling resources on wasm * Leverage new bundling api for wasm wasi drivers * Add helper to dynamically add bundled assembly and satellite_assembly resources * Limit when bundled resources can be added and freed * Restrict MonoBundledResource getters * Restrict exposure of underlying MonoBundledAssemblyResource modification * WIP * Free function chaining. * Fix crash when output parameters are NULL. * Fix WASM/WASI build errors. * Align bundled resources registration struct * [task] Check PE metadata to determine resource type * Cleanup Satellite Assembly bundling * Address Feedback * Split MonoBundledAssemblyResource getter * Addressing more feedback * Validate OutputDirectory --------- Co-authored-by: Larry Ewing Co-authored-by: lateralusX --- src/mono/mono/component/mini-wasm-debugger.c | 18 +- src/mono/mono/metadata/CMakeLists.txt | 2 + src/mono/mono/metadata/assembly.c | 111 ++-- .../metadata/bundled-resources-internals.h | 103 ++++ src/mono/mono/metadata/bundled-resources.c | 538 ++++++++++++++++++ src/mono/mono/metadata/mono-debug.c | 81 +-- src/mono/mono/metadata/webcil-loader.h | 2 + src/mono/mono/mini/mini-wasm.h | 3 - src/mono/mono/utils/mono-dl-wasm.c | 42 -- src/mono/mono/utils/mono-dl.h | 5 - .../android/build/AndroidBuild.targets | 3 +- .../msbuild/apple/build/AppleBuild.targets | 3 +- .../msbuild/common/LibraryBuilder.targets | 37 +- src/mono/wasi/build/WasiApp.Native.targets | 57 +- src/mono/wasi/runtime/driver.c | 127 ++--- src/mono/wasi/wasi.proj | 34 +- src/mono/wasm/runtime/driver.c | 131 ++--- src/mono/wasm/wasm.proj | 45 +- src/native/libs/System.Native/pal_datetime.c | 16 +- src/tasks/Common/Utils.cs | 83 ++- src/tasks/LibraryBuilder/LibraryBuilder.cs | 57 +- src/tasks/LibraryBuilder/Templates/autoinit.c | 47 +- .../Templates/library-builder.h | 11 +- .../Templates/preloaded-assemblies.c | 16 +- .../EmitBundleTask/EmitBundleBase.cs | 463 +++++++++++++++ .../EmitBundleTask/EmitBundleObjectFiles.cs} | 14 +- .../EmitBundleTask/EmitBundleSourceFiles.cs} | 17 +- .../MonoTargetsTasks/MonoTargetsTasks.csproj | 5 + .../Templates/mono-bundled-assembly.template | 7 + .../Templates/mono-bundled-data.template | 7 + ...ce-preallocation-and-registration.template | 73 +++ .../mono-bundled-satellite-assembly.template | 8 + .../Templates/mono-bundled-symbol.template | 2 + .../WasmAppBuilder/EmitWasmBundleBase.cs | 240 -------- 34 files changed, 1699 insertions(+), 709 deletions(-) create mode 100644 src/mono/mono/metadata/bundled-resources-internals.h create mode 100644 src/mono/mono/metadata/bundled-resources.c create mode 100644 src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs rename src/tasks/{WasmAppBuilder/EmitWasmBundleObjectFiles.cs => MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs} (79%) rename src/tasks/{WasmAppBuilder/EmitWasmBundleSourceFiles.cs => MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs} (54%) create mode 100644 src/tasks/MonoTargetsTasks/Templates/mono-bundled-assembly.template create mode 100644 src/tasks/MonoTargetsTasks/Templates/mono-bundled-data.template create mode 100644 src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template create mode 100644 src/tasks/MonoTargetsTasks/Templates/mono-bundled-satellite-assembly.template create mode 100644 src/tasks/MonoTargetsTasks/Templates/mono-bundled-symbol.template delete mode 100644 src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs diff --git a/src/mono/mono/component/mini-wasm-debugger.c b/src/mono/mono/component/mini-wasm-debugger.c index 2b38e5c17bbbb..b73db552ebc62 100644 --- a/src/mono/mono/component/mini-wasm-debugger.c +++ b/src/mono/mono/component/mini-wasm-debugger.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -198,7 +199,14 @@ assembly_loaded (MonoProfiler *prof, MonoAssembly *assembly) return; } - if (mono_wasm_assembly_already_added(assembly->aname.name)) + gboolean already_loaded = mono_bundled_resources_get_assembly_resource_values (assembly->aname.name, NULL, NULL); + if (!already_loaded && !g_str_has_suffix (assembly->aname.name, ".dll")) { + char *assembly_name_with_extension = g_strdup_printf ("%s.dll", assembly->aname.name); + already_loaded = mono_bundled_resources_get_assembly_resource_values (assembly_name_with_extension, NULL, NULL); + g_free (assembly_name_with_extension); + } + + if (already_loaded) return; if (mono_has_pdb_checksum ((char *) assembly_image->raw_data, assembly_image->raw_data_len)) { //if it's a release assembly we don't need to send to DebuggerProxy @@ -435,10 +443,12 @@ mono_wasm_send_dbg_command (int id, MdbgProtCommandSet command_set, int command, } else { + const unsigned char* assembly_bytes = NULL; unsigned int assembly_size = 0; - int symfile_size = 0; - const unsigned char* assembly_bytes = mono_wasm_get_assembly_bytes (assembly_name, &assembly_size); - const unsigned char* pdb_bytes = mono_get_symfile_bytes_from_bundle (assembly_name, &symfile_size); + mono_bundled_resources_get_assembly_resource_values (assembly_name, &assembly_bytes, &assembly_size); + const unsigned char* pdb_bytes = NULL; + unsigned int symfile_size = 0; + mono_bundled_resources_get_assembly_resource_symbol_values (assembly_name, &pdb_bytes, &symfile_size); m_dbgprot_buffer_init (&buf, assembly_size + symfile_size); m_dbgprot_buffer_add_byte_array (&buf, (uint8_t *) assembly_bytes, assembly_size); m_dbgprot_buffer_add_byte_array (&buf, (uint8_t *) pdb_bytes, symfile_size); diff --git a/src/mono/mono/metadata/CMakeLists.txt b/src/mono/mono/metadata/CMakeLists.txt index 1c692c9f6f0e1..dac92188d4f68 100644 --- a/src/mono/mono/metadata/CMakeLists.txt +++ b/src/mono/mono/metadata/CMakeLists.txt @@ -48,6 +48,8 @@ set(metadata_common_sources appdomain-icalls.h assembly.c assembly-internals.h + bundled-resources.c + bundled-resources-internals.h cil-coff.h class.c class-getters.h diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 6758877040c01..4ba8463c68518 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -18,6 +18,7 @@ #include #include #include "assembly-internals.h" +#include #include #include "image-internals.h" #include "object-internals.h" @@ -81,11 +82,6 @@ mono_assemblies_unlock (void) mono_os_mutex_unlock (&assemblies_mutex); } -/* If defined, points to the bundled assembly information */ -static const MonoBundledAssembly **bundles; - -static const MonoBundledSatelliteAssembly **satellite_bundles; - /* Class lazy loading functions */ static GENERATE_TRY_GET_CLASS_WITH_CACHE (debuggable_attribute, "System.Diagnostics", "DebuggableAttribute") @@ -711,7 +707,8 @@ mono_assembly_get_assemblyref (MonoImage *image, int index, MonoAssemblyName *an static MonoAssembly * search_bundle_for_assembly (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname) { - if (bundles == NULL && satellite_bundles == NULL) + if (!mono_bundled_resources_contains_assemblies () && + !mono_bundled_resources_contains_satellite_assemblies ()) return NULL; MonoImageOpenStatus status; @@ -794,7 +791,7 @@ netcore_load_reference (MonoAssemblyName *aname, MonoAssemblyLoadContext *alc, M } } - if (bundles != NULL && !is_satellite) { + if (mono_bundled_resources_contains_assemblies () && !is_satellite) { reference = search_bundle_for_assembly (mono_alc_get_default (), aname); if (reference) { mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Assembly found in the bundle: '%s'.", aname->name); @@ -802,7 +799,7 @@ netcore_load_reference (MonoAssemblyName *aname, MonoAssemblyLoadContext *alc, M } } - if (satellite_bundles != NULL && is_satellite) { + if (mono_bundled_resources_contains_satellite_assemblies () && is_satellite) { // Satellite assembly byname requests should be loaded in the same ALC as their parent assembly size_t name_len = strlen (aname->name); char *parent_name = NULL; @@ -1451,47 +1448,19 @@ absolute_dir (const gchar *filename) return res; } -static gboolean -bundled_assembly_match (const char *bundled_name, const char *name) -{ -#ifndef ENABLE_WEBCIL - return strcmp (bundled_name, name) == 0; -#else - if (strcmp (bundled_name, name) == 0) - return TRUE; - /* if they want a .dll and we have the matching .webcil, return it */ - if (g_str_has_suffix (bundled_name, ".webcil") && g_str_has_suffix (name, ".dll")) { - size_t bprefix = strlen (bundled_name) - strlen (".webcil"); - size_t nprefix = strlen (name) - strlen (".dll"); - if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) - return TRUE; - } - /* if they want a .dll and we have the matching .wasm webcil-in-wasm, return it */ - if (g_str_has_suffix (bundled_name, MONO_WEBCIL_IN_WASM_EXTENSION) && g_str_has_suffix (name, ".dll")) { - size_t bprefix = strlen (bundled_name) - strlen (MONO_WEBCIL_IN_WASM_EXTENSION); - size_t nprefix = strlen (name) - strlen (".dll"); - if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) - return TRUE; - } - return FALSE; -#endif -} - static MonoImage * -open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, gboolean is_satellite) +open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status) { - if (!bundles) + if (!mono_bundled_resources_contains_assemblies ()) return NULL; MonoImage *image = NULL; - char *name = is_satellite ? g_strdup (filename) : g_path_get_basename (filename); - for (int i = 0; !image && bundles [i]; ++i) { - if (bundled_assembly_match (bundles[i]->name, name)) { - // Since bundled images don't exist on disk, don't give them a legit filename - image = mono_image_open_from_data_internal (alc, (char*)bundles [i]->data, bundles [i]->size, FALSE, status, FALSE, name, NULL); - break; - } - } + char *name = g_path_get_basename (filename); + + const uint8_t *data = NULL; + uint32_t size = 0; + if (mono_bundled_resources_get_assembly_resource_values (name, &data, &size)) + image = mono_image_open_from_data_internal (alc, (char *)data, size, FALSE, status, FALSE, name, NULL); g_free (name); return image; @@ -1500,22 +1469,18 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M static MonoImage * open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, const char *culture) { - if (!satellite_bundles) + if (!mono_bundled_resources_contains_satellite_assemblies ()) return NULL; MonoImage *image = NULL; - char *name = g_strdup (filename); + char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); - for (int i = 0; !image && satellite_bundles [i]; ++i) { - if (bundled_assembly_match (satellite_bundles[i]->name, name) && strcmp (satellite_bundles [i]->culture, culture) == 0) { - char *bundle_name = g_strconcat (culture, "/", name, (const char *)NULL); - image = mono_image_open_from_data_internal (alc, (char *)satellite_bundles [i]->data, satellite_bundles [i]->size, FALSE, status, FALSE, bundle_name, NULL); - g_free (bundle_name); - break; - } - } + const uint8_t *data = NULL; + uint32_t size = 0; + if (mono_bundled_resources_get_satellite_assembly_resource_values (bundle_name, &data, &size)) + image = mono_image_open_from_data_internal (alc, (char *)data, size, FALSE, status, FALSE, bundle_name, NULL); - g_free (name); + g_free (bundle_name); return image; } @@ -1525,7 +1490,7 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, * \param status return status code * * This routine tries to open the assembly specified by \p filename from the - * defined bundles, if found, returns the MonoImage for it, if not found + * bundles hashtable in bundled-resources.c, if found, returns the MonoImage for it, if not found * returns NULL */ MonoImage * @@ -1537,12 +1502,10 @@ mono_assembly_open_from_bundle (MonoAssemblyLoadContext *alc, const char *filena */ MonoImage *image = NULL; MONO_ENTER_GC_UNSAFE; - gboolean is_satellite = culture && culture [0] != 0; - - if (is_satellite) + if (culture && culture [0] != 0) image = open_from_satellite_bundle (alc, filename, status, culture); else - image = open_from_bundle_internal (alc, filename, status, FALSE); + image = open_from_bundle_internal (alc, filename, status); if (image) { mono_image_addref (image); @@ -1620,7 +1583,7 @@ mono_assembly_request_open (const char *filename, const MonoAssemblyOpenRequest // If VM built with mkbundle loaded_from_bundle = FALSE; - if (bundles != NULL || satellite_bundles != NULL) { + if (mono_bundled_resources_contains_assemblies ()) { /* We don't know the culture of the filename we're loading here, so this call is not culture aware. */ image = mono_assembly_open_from_bundle (load_req.alc, fname, status, NULL); loaded_from_bundle = image != NULL; @@ -3172,11 +3135,16 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) /** * mono_register_bundled_assemblies: + * Dynamically allocates MonoBundledAssemblyResources to leverage + * preferred bundling api mono_bundled_resources_add. */ void mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) { - bundles = assemblies; + for (int i = 0; assemblies [i]; ++i) { + const MonoBundledAssembly *assembly = assemblies [i]; + mono_bundled_resources_add_assembly_resource (assembly->name, assembly->name, (const uint8_t *)assembly->data, (uint32_t)assembly->size, NULL, NULL); + } } /** @@ -3187,19 +3155,34 @@ mono_create_new_bundled_satellite_assembly (const char *name, const char *cultur { MonoBundledSatelliteAssembly *satellite_assembly = g_new0 (MonoBundledSatelliteAssembly, 1); satellite_assembly->name = strdup (name); + g_assert (satellite_assembly->name); satellite_assembly->culture = strdup (culture); + g_assert (satellite_assembly->culture); satellite_assembly->data = data; satellite_assembly->size = size; return satellite_assembly; } +static void +mono_free_bundled_satellite_assembly_func (void *resource, void *free_data) +{ + g_free (free_data); +} + /** * mono_register_bundled_satellite_assemblies: + * Dynamically allocates MonoBundledSatelliteAssemblyResources to leverage + * preferred bundling api mono_bundled_resources_add. */ void -mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly **assemblies) +mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly **satellite_assemblies) { - satellite_bundles = assemblies; + for (int i = 0; satellite_assemblies [i]; ++i) { + const MonoBundledSatelliteAssembly *satellite_assembly = satellite_assemblies [i]; + char *id = g_strconcat (satellite_assembly->culture, "/", satellite_assembly->name, (const char*)NULL); + g_assert (id); + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly->name, satellite_assembly->culture, (const uint8_t *)satellite_assembly->data, (uint32_t)satellite_assembly->size, mono_free_bundled_satellite_assembly_func, id); + } } /** diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h new file mode 100644 index 0000000000000..1a8c03f4fb4e7 --- /dev/null +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -0,0 +1,103 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#ifndef __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ +#define __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ + +#include +#include + +typedef enum { + MONO_BUNDLED_DATA, + MONO_BUNDLED_ASSEMBLY, + MONO_BUNDLED_SATELLITE_ASSEMBLY, + MONO_BUNDLED_RESOURCE_COUNT +} MonoBundledResourceType; + +typedef void (*free_bundled_resource_func)(void *, void*); + +typedef struct _MonoBundledResource { + MonoBundledResourceType type; + const char *id; + free_bundled_resource_func free_func; + void *free_data; +} MonoBundledResource; + +typedef struct _MonoBundledData { + const char *name; + const uint8_t *data; + uint32_t size; +} MonoBundledData; + +typedef struct _MonoBundledDataResource { + MonoBundledResource resource; + MonoBundledData data; +} MonoBundledDataResource; + +typedef struct _MonoBundledSymbolData { + const uint8_t *data; + uint32_t size; +} MonoBundledSymbolData; + +typedef struct _MonoBundledAssemblyData { + const char *name; + const uint8_t *data; + uint32_t size; +} MonoBundledAssemblyData; + +typedef struct _MonoBundledAssemblyResource { + MonoBundledResource resource; + MonoBundledAssemblyData assembly; + MonoBundledSymbolData symbol_data; +} MonoBundledAssemblyResource; + +typedef struct _MonoBundledSatelliteAssemblyData { + const char *name; + const char *culture; + const uint8_t *data; + uint32_t size; +} MonoBundledSatelliteAssemblyData; + +typedef struct _MonoBundledSatelliteAssemblyResource { + MonoBundledResource resource; + MonoBundledSatelliteAssemblyData satellite_assembly; +} MonoBundledSatelliteAssemblyResource; + +void +mono_bundled_resources_free (void); + +void +mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); + +bool +mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); + +bool +mono_bundled_resources_get_assembly_resource_symbol_values (const char *id, const uint8_t **symbol_data_out, uint32_t *symbol_size_out); + +bool +mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); + +bool +mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); + +void +mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data); + +void +mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data); + +void +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data); + +void +mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data); + +bool +mono_bundled_resources_contains_assemblies (void); + +bool +mono_bundled_resources_contains_satellite_assemblies (void); + +#endif /* __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ */ diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c new file mode 100644 index 0000000000000..9eae4f5db8fa8 --- /dev/null +++ b/src/mono/mono/metadata/bundled-resources.c @@ -0,0 +1,538 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#include +#include + +#include +#include +#include + +static GHashTable *bundled_resources = NULL; +static bool bundled_resources_contains_assemblies = false; +static bool bundled_resources_contains_satellite_assemblies = false; + +typedef struct _BundledResourcesChainedFreeFunc { + free_bundled_resource_func free_func; + void *free_data; + void *next; +} BundledResourcesChainedFreeFunc; + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_free frees all memory allocated for bundled resources. +// It should only be called when the runtime no longer needs access to the data, +// most likely to happen during runtime shutdown. +// + +void +mono_bundled_resources_free (void) +{ + g_assert (mono_runtime_is_shutting_down ()); + + g_hash_table_destroy (bundled_resources); + bundled_resources = NULL; + + bundled_resources_contains_assemblies = false; + bundled_resources_contains_satellite_assemblies = false; +} + +//--------------------------------------------------------------------------------------- +// +// bundled_resources_value_destroy_func frees the memory allocated by the hashtable's +// MonoBundled*Resource by invoking its underlying free_bundled_resource_func when possible. +// + +static void +bundled_resources_value_destroy_func (void *resource) +{ + MonoBundledResource *value = (MonoBundledResource *)resource; + if (value->free_func) + value->free_func (resource, value->free_data); +} + +static bool +bundled_resources_is_known_assembly_extension (const char *ext) +{ +#ifdef ENABLE_WEBCIL + return !strcmp (ext, ".dll") || !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); +#else + return !strcmp (ext, ".dll") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); +#endif +} + +static gboolean +bundled_resources_resource_id_equal (const char *id_one, const char *id_two) +{ + const char *extension_one = strrchr (id_one, '.'); + const char *extension_two = strrchr (id_two, '.'); + if (extension_one && extension_two && bundled_resources_is_known_assembly_extension (extension_one) && bundled_resources_is_known_assembly_extension (extension_two)) { + size_t len_one = extension_one - id_one; + size_t len_two = extension_two - id_two; + return (len_one == len_two) && !strncmp (id_one, id_two, len_one); + } + + return !strcmp (id_one, id_two); +} + +static guint +bundled_resources_resource_id_hash (const char *id) +{ + const char *current = id; + const char *extension = NULL; + guint previous_hash = 0; + guint hash = 0; + + while (*current) { + hash = (hash << 5) - (hash + *current); + if (*current == '.') { + extension = current; + previous_hash = hash; + } + current++; + } + + // alias all extensions to .dll + if (extension && bundled_resources_is_known_assembly_extension (extension)) { + hash = previous_hash; + hash = (hash << 5) - (hash + 'd'); + hash = (hash << 5) - (hash + 'l'); + hash = (hash << 5) - (hash + 'l'); + } + + return hash; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_add handles bundling of many types of resources to circumvent +// needing to find or have those resources on disk. The MonoBundledResource struct models +// the union of information carried by all supported types of resources which are +// enumerated in MonoBundledResourceType. +// +// bundled_resources: +// A single hash table will hold all resources being bundled with the understanding +// that all resources being bundled are uniquely named. The bundled resource is tagged +// with the type of resource it represents, and the pointer added to this hash table +// should fully represent a MonoBundled*Resource struct defined in `bundled-resources-internals.h +// +// Arguments: +// ** resources_to_bundle - An array of pointers to `MonoBundledResource`, which details +// the type of MonoBundled*Resource information follows this pointer in memory. +// len - The number of resources being added to the hash table +// + +void +mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len) +{ + MonoDomain *domain = mono_get_root_domain (); + g_assert (!domain); + + if (!bundled_resources) + bundled_resources = g_hash_table_new_full ((GHashFunc)bundled_resources_resource_id_hash, (GEqualFunc)bundled_resources_resource_id_equal, NULL, bundled_resources_value_destroy_func); + + bool assemblyAdded = false; + bool satelliteAssemblyAdded = false; + + for (uint32_t i = 0; i < len; ++i) { + MonoBundledResource *resource_to_bundle = (MonoBundledResource *)resources_to_bundle[i]; + if (resource_to_bundle->type == MONO_BUNDLED_ASSEMBLY) + assemblyAdded = true; + + if (resource_to_bundle->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) + satelliteAssemblyAdded = true; + + g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); + } + + if (assemblyAdded) + bundled_resources_contains_assemblies = true; + + if (satelliteAssemblyAdded) + bundled_resources_contains_satellite_assemblies = true; +} + + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get retrieves the pointer of the MonoBundledResource associated +// with a key equivalent to the requested resource id. +// +// Arguments: +// * id - Unique name of the resource +// +// Returns: +// MonoBundledResource * - Pointer to the resource in the hashmap with the key `id` +// + +static MonoBundledResource * +bundled_resources_get (const char *id) +{ + if (!bundled_resources) + return NULL; + + return g_hash_table_lookup (bundled_resources, id); +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_assembly_resource retrieves MonoBundledAssemblyResource* associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// +// Returns: +// MonoBundledAssemblyResource * - Pointer to the bundled assembly resource in the hashmap with the key `id` +// +// Note: As MonoBundled*Resource types are not public, prefer `mono_bundled_resources_get_assembly_resource_values` +// in external contexts to grab assembly and symbol data. +// + +static MonoBundledAssemblyResource * +bundled_resources_get_assembly_resource (const char *id) +{ + MonoBundledAssemblyResource *assembly = + (MonoBundledAssemblyResource*)bundled_resources_get (id); + if (!assembly) + return NULL; + g_assert (assembly->resource.type == MONO_BUNDLED_ASSEMBLY); + return assembly; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_satellite_assembly_resource retrieves MonoBundledSatelliteAssemblyResource* associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// +// Returns: +// MonoBundledSatelliteAssemblyResource * - Pointer to the bundled assembly resource in the hashmap with the key `id` +// +// Note: As MonoBundled*Resource types are not public, prefer `mono_bundled_resources_get_satellite_assembly_resource_values` +// in external contexts to grab satellite assembly data. +// + +static MonoBundledSatelliteAssemblyResource * +bundled_resources_get_satellite_assembly_resource (const char *id) +{ + MonoBundledSatelliteAssemblyResource *satellite_assembly = + (MonoBundledSatelliteAssemblyResource*)bundled_resources_get (id); + if (!satellite_assembly) + return NULL; + g_assert (satellite_assembly->resource.type == MONO_BUNDLED_SATELLITE_ASSEMBLY); + return satellite_assembly; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_data_resource retrieves MonoBundledDataResource* associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// +// Returns: +// MonoBundledDataResource * - Pointer to the bundled assembly resource in the hashmap with the key `id` +// +// Note: As MonoBundled*Resource types are not public, prefer `mono_bundled_resources_get_data_resource_values` +// in external contexts to grab data. +// + +static MonoBundledDataResource * +bundled_resources_get_data_resource (const char *id) +{ + MonoBundledDataResource *data = + (MonoBundledDataResource*)bundled_resources_get (id); + if (!data) + return NULL; + g_assert (data->resource.type == MONO_BUNDLED_DATA); + return data; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_assembly_resource_values retrieves assembly data associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// ** data_out - address to point at assembly byte data +// ** size_out - address to point at assembly byte data size +// +// Returns: +// bool - whether or not a valid MonoBundledAssemblyResource->assembly was found with key 'id' +// + +bool +mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) +{ + MonoBundledAssemblyResource *bundled_assembly_resource = bundled_resources_get_assembly_resource (id); + if (!bundled_assembly_resource || + !bundled_assembly_resource->assembly.data || + bundled_assembly_resource->assembly.size == 0) + return false; + + if (data_out) + *data_out = bundled_assembly_resource->assembly.data; + if (size_out) + *size_out = bundled_assembly_resource->assembly.size; + + return true; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_assembly_resource_symbol_values retrieves assembly symbol data associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// ** symbol_data_out - address to point at assembly symbol byte data +// ** symbol_size_out - address to point at assembly symbol byte data size +// +// Returns: +// bool - whether or not a valid MonoBundledAssemblyResource->symbol_data was found with key 'id' +// + +bool +mono_bundled_resources_get_assembly_resource_symbol_values (const char *id, const uint8_t **symbol_data_out, uint32_t *symbol_size_out) +{ + MonoBundledAssemblyResource *bundled_assembly_resource = bundled_resources_get_assembly_resource (id); + if (!bundled_assembly_resource || + !bundled_assembly_resource->symbol_data.data || + bundled_assembly_resource->symbol_data.size == 0) + return false; + + if (symbol_data_out) + *symbol_data_out = bundled_assembly_resource->symbol_data.data; + if (symbol_size_out) + *symbol_size_out = bundled_assembly_resource->symbol_data.size; + + return true; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_satellite_assembly_resource_values retrieves satellite assembly data associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// ** data_out - address to point at satellite assembly byte data +// ** size_out - address to point at satellite assembly byte data size +// +// Returns: +// bool - whether or not a valid MonoBundledSatelliteAssemblyResource was found with key 'id' +// + +bool +mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) +{ + MonoBundledSatelliteAssemblyResource *bundled_satellite_assembly_resource = bundled_resources_get_satellite_assembly_resource (id); + if (!bundled_satellite_assembly_resource || + !bundled_satellite_assembly_resource->satellite_assembly.data || + bundled_satellite_assembly_resource->satellite_assembly.size == 0) + return false; + + if (data_out) + *data_out = bundled_satellite_assembly_resource->satellite_assembly.data; + if (size_out) + *size_out = bundled_satellite_assembly_resource->satellite_assembly.size; + + return true; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_data_resource_values retrieves data associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// ** data_out - address to point at resource byte data +// ** size_out - address to point at resource byte data size +// +// Returns: +// bool - whether or not a valid MonoBundledDataResource was found with key 'id' +// + +bool +mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) +{ + MonoBundledDataResource *bundled_data_resource = bundled_resources_get_data_resource (id); + if (!bundled_data_resource || + !bundled_data_resource->data.data || + bundled_data_resource->data.size == 0) + return false; + + if (data_out) + *data_out = bundled_data_resource->data.data; + if (size_out) + *size_out = bundled_data_resource->data.size; + + return true; +} + +static void +bundled_resources_chained_free_func (void *resource, void *free_data) +{ + BundledResourcesChainedFreeFunc *node = (BundledResourcesChainedFreeFunc *)free_data; + if (node && node->free_func) + node->free_func (resource, node->free_data); + if (node && node->next) + bundled_resources_chained_free_func (resource, node->next); + + g_free (free_data); +} + +static void +bundled_resources_free_func (void *resource, void *free_data) +{ + bundled_resources_chained_free_func (resource, free_data); + g_free (resource); +} + +static void +bundled_resource_add_free_func (MonoBundledResource *resource, free_bundled_resource_func free_func, void *free_data) +{ + if (!free_func) + return; + + if (!resource->free_func) { + resource->free_func = free_func; + resource->free_data = free_data; + } else if (resource->free_func == bundled_resources_chained_free_func || resource->free_func == bundled_resources_free_func) { + BundledResourcesChainedFreeFunc *node = g_new0 (BundledResourcesChainedFreeFunc, 1); + node->free_func = free_func; + node->free_data = free_data; + node->next = resource->free_data; + resource->free_data = node; + } else { + BundledResourcesChainedFreeFunc *node1 = g_new0 (BundledResourcesChainedFreeFunc, 1); + BundledResourcesChainedFreeFunc *node2 = g_new0 (BundledResourcesChainedFreeFunc, 2); + + node2->free_func = resource->free_func; + node2->free_data = resource->free_data; + + node1->free_func = free_func; + node1->free_data = free_data; + node1->next = node2; + + resource->free_func = bundled_resources_chained_free_func; + resource->free_data = node1; + } +} + +void +mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data) +{ + // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly + MonoBundledAssemblyResource *assembly_resource = bundled_resources_get_assembly_resource (name); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = id; + assembly_resource->resource.free_func = bundled_resources_free_func; + bundled_resource_add_free_func ((MonoBundledResource *)assembly_resource, free_func, free_data); + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); + } else { + // Ensure the MonoBundledAssemblyData has not been initialized + g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); + bundled_resource_add_free_func ((MonoBundledResource *)assembly_resource, free_func, free_data); + } + assembly_resource->assembly.name = name; + assembly_resource->assembly.data = data; + assembly_resource->assembly.size = size; +} + +void +mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data) +{ + // Check if assembly dll counterpart had been added via mono_register_bundled_assemblies + MonoBundledAssemblyResource *assembly_resource = bundled_resources_get_assembly_resource (id); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = id; + assembly_resource->resource.free_func = bundled_resources_free_func; + bundled_resource_add_free_func ((MonoBundledResource *)assembly_resource, free_func, free_data); + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); + } else { + // Ensure the MonoBundledSymbolData has not been initialized + g_assert (!assembly_resource->symbol_data.data && assembly_resource->symbol_data.size == 0); + bundled_resource_add_free_func ((MonoBundledResource *)assembly_resource, free_func, free_data); + } + assembly_resource->symbol_data.data = (const uint8_t *)data; + assembly_resource->symbol_data.size = (uint32_t)size; +} + +void +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data) +{ + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = bundled_resources_get_satellite_assembly_resource (id); + g_assert (!satellite_assembly_resource); + + satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); + satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; + satellite_assembly_resource->resource.id = id; + satellite_assembly_resource->resource.free_func = bundled_resources_free_func; + satellite_assembly_resource->satellite_assembly.name = name; + satellite_assembly_resource->satellite_assembly.culture = culture; + satellite_assembly_resource->satellite_assembly.data = data; + satellite_assembly_resource->satellite_assembly.size = size; + + bundled_resource_add_free_func ((MonoBundledResource *)satellite_assembly_resource, free_func, free_data); + mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); +} + +void +mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data) +{ + MonoBundledDataResource *data_resource = bundled_resources_get_data_resource (id); + g_assert (!data_resource); + + data_resource = g_new0 (MonoBundledDataResource, 1); + data_resource->resource.type = MONO_BUNDLED_DATA; + data_resource->resource.id = id; + data_resource->resource.free_func = bundled_resources_free_func; + data_resource->data.name = name; + data_resource->data.data = data; + data_resource->data.size = size; + + bundled_resource_add_free_func ((MonoBundledResource *)data_resource, free_func, free_data); + mono_bundled_resources_add ((MonoBundledResource **)&data_resource, 1); +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_contains_assemblies returns whether or not assemblies +// have been added to the bundled resource hash table via mono_bundled_resources_add. +// +// Returns: +// bool - bool value indicating whether or not a bundled assembly resource had been added. +// + +bool +mono_bundled_resources_contains_assemblies (void) +{ + return bundled_resources_contains_assemblies; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_contains_satellite_assemblies returns whether or not satellite assemblies +// have been added to the bundled resource hash table via mono_bundled_resources_add. +// +// Returns: +// bool - bool value indicating whether or not a bundled satellite assembly resource had been added. +// + +bool +mono_bundled_resources_contains_satellite_assemblies (void) +{ + return bundled_resources_contains_satellite_assemblies; +} diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index c415e3b4ce640..2472c62e62d33 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1067,88 +1068,38 @@ mono_is_debugger_attached (void) return is_attached; } -/* - * Bundles - */ - -typedef struct _BundledSymfile BundledSymfile; - -struct _BundledSymfile { - BundledSymfile *next; - const char *aname; - const mono_byte *raw_contents; - int size; -}; - -static BundledSymfile *bundled_symfiles = NULL; - /** * mono_register_symfile_for_assembly: + * Dynamically allocates MonoBundledAssemblyResource to leverage + * preferred bundling api mono_bundled_resources_add. */ void mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte *raw_contents, int size) { - BundledSymfile *bsymfile; - - bsymfile = g_new0 (BundledSymfile, 1); - bsymfile->aname = assembly_name; - bsymfile->raw_contents = raw_contents; - bsymfile->size = size; - bsymfile->next = bundled_symfiles; - bundled_symfiles = bsymfile; -} - -static gboolean -bsymfile_match (BundledSymfile *bsymfile, const char *assembly_name) -{ - if (!strcmp (bsymfile->aname, assembly_name)) - return TRUE; -#ifdef ENABLE_WEBCIL - const char *p = strstr (assembly_name, ".webcil"); - /* if assembly_name ends with .webcil, check if aname matches, with a .dll extension instead */ - if (p && *(p + strlen(".webcil")) == 0) { - size_t n = p - assembly_name; - if (!strncmp (bsymfile->aname, assembly_name, n) - && !strcmp (bsymfile->aname + n, ".dll")) - return TRUE; - } - p = strstr (assembly_name, MONO_WEBCIL_IN_WASM_EXTENSION); - if (p && *(p + strlen(MONO_WEBCIL_IN_WASM_EXTENSION)) == 0) { - size_t n = p - assembly_name; - if (!strncmp (bsymfile->aname, assembly_name, n) - && !strcmp (bsymfile->aname + n, ".dll")) - return TRUE; - } -#endif - return FALSE; + mono_bundled_resources_add_assembly_symbol_resource (assembly_name, raw_contents, size, NULL, NULL); } static MonoDebugHandle * open_symfile_from_bundle (MonoImage *image) { - BundledSymfile *bsymfile; - - for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (!bsymfile_match (bsymfile, image->module_name)) - continue; - - return mono_debug_open_image (image, bsymfile->raw_contents, bsymfile->size); - } + const uint8_t *data = NULL; + uint32_t size = 0; + if (!mono_bundled_resources_get_assembly_resource_symbol_values (image->module_name, &data, &size)) + return NULL; - return NULL; + return mono_debug_open_image (image, data, size); } const mono_byte * mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { - BundledSymfile *bsymfile; - for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (!bsymfile_match (bsymfile, assembly_name)) - continue; - *size = bsymfile->size; - return bsymfile->raw_contents; - } - return NULL; + const uint8_t *symbol_data = NULL; + uint32_t symbol_size = 0; + if (!mono_bundled_resources_get_assembly_resource_symbol_values (assembly_name, &symbol_data, &symbol_size)) + return NULL; + + *size = symbol_size; + return (mono_byte *)symbol_data; } void diff --git a/src/mono/mono/metadata/webcil-loader.h b/src/mono/mono/metadata/webcil-loader.h index daf6217150483..a3466356c9cb8 100644 --- a/src/mono/mono/metadata/webcil-loader.h +++ b/src/mono/mono/metadata/webcil-loader.h @@ -5,6 +5,8 @@ #ifndef _MONO_METADATA_WEBCIL_LOADER_H #define _MONO_METADATA_WEBCIL_LOADER_H +#include + #define MONO_WEBCIL_IN_WASM_EXTENSION ".wasm" void diff --git a/src/mono/mono/mini/mini-wasm.h b/src/mono/mono/mini/mini-wasm.h index 9de72fdd840e8..22c998018b59a 100644 --- a/src/mono/mono/mini/mini-wasm.h +++ b/src/mono/mono/mini/mini-wasm.h @@ -98,9 +98,6 @@ G_EXTERN_C void mono_wasm_enable_debugging (int log_level); void mono_wasm_main_thread_schedule_timer (void *timerHandler, int shortestDueTimeMs); -int mono_wasm_assembly_already_added (const char *assembly_name); -const unsigned char *mono_wasm_get_assembly_bytes (const char *name, unsigned int *size); - void mono_wasm_print_stack_trace (void); gboolean diff --git a/src/mono/mono/utils/mono-dl-wasm.c b/src/mono/mono/utils/mono-dl-wasm.c index 774d1ba6dd2b1..1490b086ccfc8 100644 --- a/src/mono/mono/utils/mono-dl-wasm.c +++ b/src/mono/mono/utils/mono-dl-wasm.c @@ -94,45 +94,3 @@ mono_dl_close_handle (MonoDl *module, MonoError *error) MONO_EMPTY_SOURCE_FILE (mono_dl_wasm); #endif - -#if defined (HOST_WASM) - -static GHashTable *name_to_blob = NULL; - -typedef struct { - const unsigned char *data; - unsigned int size; -} FileBlob; - -int -mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size) -{ - // printf("mono_wasm_add_bundled_file: %s %p %d\n", name, data, size); - if(name_to_blob == NULL) - { - name_to_blob = g_hash_table_new (g_str_hash, g_str_equal); - } - FileBlob *blob = g_new0 (FileBlob, 1); - blob->data = data; - blob->size = size; - g_hash_table_insert (name_to_blob, (gpointer) name, blob); - return 0; -} - -const unsigned char* -mono_wasm_get_bundled_file (const char *name, int* out_length) -{ - FileBlob *blob = (FileBlob *)g_hash_table_lookup (name_to_blob, name); - if (blob != NULL) - { - // printf("mono_wasm_get_bundled_file: %s %p %d \n", name, blob->data, blob->size); - *out_length = blob->size; - return blob->data; - } - - // printf("mono_wasm_get_bundled_file: %s not found \n", name); - *out_length = 0; - return NULL; -} - -#endif /* HOST_WASM */ diff --git a/src/mono/mono/utils/mono-dl.h b/src/mono/mono/utils/mono-dl.h index 7ed4fe089bb53..7a65b6fd1a97e 100644 --- a/src/mono/mono/utils/mono-dl.h +++ b/src/mono/mono/utils/mono-dl.h @@ -56,10 +56,5 @@ int mono_dl_convert_flags (int mono_flags, int native_flags); char* mono_dl_current_error_string (void); const char* mono_dl_get_system_dir (void); -#if defined (HOST_WASM) -int mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size); -const unsigned char* mono_wasm_get_bundled_file (const char *name, int* out_length); -#endif /* HOST_WASM */ - #endif /* __MONO_UTILS_DL_H__ */ diff --git a/src/mono/msbuild/android/build/AndroidBuild.targets b/src/mono/msbuild/android/build/AndroidBuild.targets index 0db739451372c..26cc272768a37 100644 --- a/src/mono/msbuild/android/build/AndroidBuild.targets +++ b/src/mono/msbuild/android/build/AndroidBuild.targets @@ -247,7 +247,8 @@ + Condition="Exists('$(_AndroidRuntimeConfigFilePath)')" + BeforeTargets="_GenerateBundle"> <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/> <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/> diff --git a/src/mono/msbuild/apple/build/AppleBuild.targets b/src/mono/msbuild/apple/build/AppleBuild.targets index d5c141358d816..f54f37b7e0778 100644 --- a/src/mono/msbuild/apple/build/AppleBuild.targets +++ b/src/mono/msbuild/apple/build/AppleBuild.targets @@ -279,7 +279,8 @@ + Condition="Exists('$(_AppleRuntimeConfigFilePath)')" + BeforeTargets="_GenerateBundle"> <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/> <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/> diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index 3edfb5c1a6242..9ea4a4562ee1e 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -1,25 +1,35 @@ + + Condition="'$(_IsLibraryMode)' == 'true' and '$(RunAOTCompilation)' == 'true'" + DependsOnTargets="_GenerateBundle"> <_IsSharedLibrary>false <_IsSharedLibrary Condition="'$(NativeLib)' == 'shared'">true <_UsesCustomRuntimeInitCallback>false <_UsesCustomRuntimeInitCallback Condition="$(CustomRuntimeInitCallback) != ''">true + <_BundlesResources>$(BundlesResources) + <_BundlesResources Condition="'$(_BundlesResources)' == ''">true <_ExtraLibrarySources Include="$(_AotModuleTablePath)" /> + <_ExtraLibrarySources Include="@(BundledSources)" /> + <_ExtraLibrarySources Include="%(_BundledResources.DestinationFile)" /> <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" /> + <_BundledRuntimeConfig Include="@(_BundledResources)" Condition="$([System.String]::Copy('%(Identity)').EndsWith('runtimeconfig.bin'))" /> + + + mono-bundled-source.c + $(BundleDir) + + + + <_ResourcesToBundle Remove="@(_ResourcesToBundle)" /> + <_ResourcesToBundle Include="@(_AssembliesToBundleInternal)" /> + <_ResourcesToBundle Include="$(_ParsedRuntimeConfigFilePath)" /> + + + + + + + + + + + \ No newline at end of file diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index daeb8515c1388..e0e43c0edffc4 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -341,6 +341,7 @@ + - <_WasmAssembliesBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_assemblies.o - <_WasmIcuBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_icu.o + <_WasmAssembliesBundleObjectFile>wasi_bundled_assemblies.o + <_WasmIcuBundleObjectFile>wasi_bundled_icu.o - - - - - - - - - $(_WasmIntermediateOutputPath)%(WasmBundleAssembliesWithHashes.Filename)%(WasmBundleAssembliesWithHashes.Extension).$([System.String]::Copy(%(WasmBundleAssembliesWithHashes.FileHash)).Substring(0, 8)).o - - - $(_WasmIntermediateOutputPath)%(WasmBundleIcuWithHashes.Filename)%(WasmBundleIcuWithHashes.Extension).$([System.String]::Copy(%(WasmBundleIcuWithHashes.FileHash)).Substring(0, 8)).o - - - + OutputDirectory="$(_WasmIntermediateOutputPath)"> + + - <_WasiObjectFilesForBundle Include="$(_WasmAssembliesBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="%(WasmBundleAssembliesWithHashes.DestinationFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmAssembliesBundleObjectFile)" /> + <_WasiObjectFilesForBundle Include="%(_WasmBundledAssemblies.DestinationFile)" /> - - + + - + OutputDirectory="$(_WasmIntermediateOutputPath)"> + + - <_WasiObjectFilesForBundle Include="$(_WasmIcuBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="%(WasmBundleIcuWithHashes.DestinationFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmIcuBundleObjectFile)" /> + <_WasiObjectFilesForBundle Include="%(BundledWasmIcu.DestinationFile)" /> - - + + diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 52457c5069277..d6a51f9be51b0 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#include #include #include #include @@ -62,15 +63,19 @@ int32_t monoeg_g_hasenv(const char *variable); void mono_free (void*); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); -extern void mono_wasm_register_timezones_bundle(); +extern void mono_register_timezones_bundle (void); #ifdef WASM_SINGLE_FILE -extern void mono_wasm_register_assemblies_bundle(); +extern void mono_register_assemblies_bundle (void); #ifndef INVARIANT_GLOBALIZATION -extern void mono_wasm_register_icu_bundle(); +extern bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); +extern void mono_register_icu_bundle (void); #endif /* INVARIANT_GLOBALIZATION */ -extern const unsigned char* mono_wasm_get_bundled_file (const char *name, int* out_length); #endif /* WASM_SINGLE_FILE */ +extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_func)(void *, void *), void *free_data); + extern const char* dotnet_wasi_getentrypointassemblyname(); int32_t mono_wasi_load_icu_data(const void* pData); void load_icu_data (void); @@ -112,15 +117,11 @@ typedef SgenDescriptor MonoGCDescriptor; #include "driver-gen.c" #endif -typedef struct WasmAssembly_ WasmAssembly; - -struct WasmAssembly_ { - MonoBundledAssembly assembly; - WasmAssembly *next; -}; - -static WasmAssembly *assemblies; -static int assembly_count; +static void +bundled_resources_free_func (void *resource, void *free_data) +{ + free (free_data); +} int mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned int size) @@ -131,43 +132,55 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in char *new_name = strdup (name); //FIXME handle debugging assemblies with .exe extension strcpy (&new_name [len - 3], "dll"); - mono_register_symfile_for_assembly (new_name, data, size); + mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, bundled_resources_free_func, new_name); return 1; } - WasmAssembly *entry = g_new0 (WasmAssembly, 1); - entry->assembly.name = strdup (name); - entry->assembly.data = data; - entry->assembly.size = size; - entry->next = assemblies; - assemblies = entry; - ++assembly_count; + char *assembly_name = strdup (name); + assert (assembly_name); + mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, bundled_resources_free_func, assembly_name); return mono_has_pdb_checksum ((char*)data, size); } -typedef struct WasmSatelliteAssembly_ WasmSatelliteAssembly; - -struct WasmSatelliteAssembly_ { - MonoBundledSatelliteAssembly *assembly; - WasmSatelliteAssembly *next; -}; - -static WasmSatelliteAssembly *satellite_assemblies; -static int satellite_assembly_count; - char* gai_strerror(int code) { char* result = malloc(256); sprintf(result, "Error code %i", code); return result; } +static void +bundled_resources_free_slots_func (void *resource, void *free_data) +{ + if (free_data) { + void **slots = (void **)free_data; + for (int i = 0; slots [i]; i++) + free (slots [i]); + } +} + void mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) { - WasmSatelliteAssembly *entry = g_new0 (WasmSatelliteAssembly, 1); - entry->assembly = mono_create_new_bundled_satellite_assembly (name, culture, data, size); - entry->next = satellite_assemblies; - satellite_assemblies = entry; - ++satellite_assembly_count; + int id_len = strlen (culture) + 1 + strlen (name); // +1 is for the "/" + char *id = (char *)malloc (sizeof (char) * (id_len + 1)); // +1 is for the terminating null character + assert (id); + + int num_char = snprintf (id, (id_len + 1), "%s/%s", culture, name); + assert (num_char > 0 && num_char == id_len); + + char *satellite_assembly_name = strdup (name); + assert (satellite_assembly_name); + + char *satellite_assembly_culture = strdup (culture); + assert (satellite_assembly_culture); + + void **slots = malloc (sizeof (void *) * 4); + assert (slots); + slots [0] = id; + slots [1] = satellite_assembly_name; + slots [2] = satellite_assembly_culture; + slots [3] = NULL; + + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, bundled_resources_free_slots_func, slots); } static void *sysglobal_native_handle; @@ -323,32 +336,15 @@ get_native_to_interp (MonoMethod *method, void *extra_arg) return addr; } -void -mono_wasm_register_bundled_satellite_assemblies (void) -{ - /* In legacy satellite_assembly_count is always false */ - if (satellite_assembly_count) { - MonoBundledSatelliteAssembly **satellite_bundle_array = g_new0 (MonoBundledSatelliteAssembly *, satellite_assembly_count + 1); - WasmSatelliteAssembly *cur = satellite_assemblies; - int i = 0; - while (cur) { - satellite_bundle_array [i] = cur->assembly; - cur = cur->next; - ++i; - } - mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_bundle_array); - } -} - #ifndef INVARIANT_GLOBALIZATION void load_icu_data (void) { #ifdef WASM_SINGLE_FILE - mono_wasm_register_icu_bundle(); + mono_register_icu_bundle (); - int length = -1; - const unsigned char* buffer = mono_wasm_get_bundled_file("icudt.dat", &length); - if (!buffer) { + const uint8_t *buffer = NULL; + uint32_t data_len = 0; + if (!mono_bundled_resources_get_data_resource_values ("icudt.dat", &buffer, &data_len)) { printf("Could not load icudt.dat from the bundle"); assert(buffer); } @@ -443,9 +439,9 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); - mono_wasm_register_timezones_bundle(); + mono_register_timezones_bundle (); #ifdef WASM_SINGLE_FILE - mono_wasm_register_assemblies_bundle(); + mono_register_assemblies_bundle (); #endif mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); @@ -501,19 +497,6 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mono_method_builder_ilgen_init (); mono_sgen_mono_ilgen_init (); - if (assembly_count) { - MonoBundledAssembly **bundle_array = g_new0 (MonoBundledAssembly*, assembly_count + 1); - WasmAssembly *cur = assemblies; - int i = 0; - while (cur) { - bundle_array [i] = &cur->assembly; - cur = cur->next; - ++i; - } - mono_register_bundled_assemblies ((const MonoBundledAssembly **)bundle_array); - } - - mono_wasm_register_bundled_satellite_assemblies (); mono_trace_init (); mono_trace_set_log_handler (wasi_trace_logger, NULL); diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index e8742d36f8345..fbf434ce4964a 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -65,38 +65,34 @@ - + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) - <_WasmTimezonesBundleObjectFile>$(WasiObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundleObjectFile>wasm-bundled-timezones.o <_WasmTimezonesBundleArchive>$(WasiObjDir)\wasm-bundled-timezones.a <_WasmTimezonesArchiveRsp>$(WasiObjDir)\wasm-bundled-timezones-archive.rsp <_WasmTimezonesInternal Include="$(_WasmTimezonesPath)\**\*.*" WasmRole="Timezone"/> - - - - <_WasmBundleTimezonesWithHashes Update="@(_WasmBundleTimezonesWithHashes)"> - $(WasiObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).o - /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) - + <_WasmTimezonesInternal Update="@(_WasmTimezonesInternal)"> + /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmTimezonesInternal.Identity)).Replace('\','/')) + - - + OutputDirectory="$(WasiObjDir)"> + + - - + + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> - <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)\$(_WasmTimezonesBundleObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="%(BundledWasiTimezones.DestinationFile)" /> diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index 6f2bee81e07d8..51cd2497a1b9d 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -54,10 +54,14 @@ int monoeg_g_setenv(const char *variable, const char *value, int overwrite); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); -extern void mono_wasm_register_timezones_bundle(); +extern void mono_register_timezones_bundle (void); extern void mono_wasm_set_entrypoint_breakpoint (const char* assembly_name, int method_token); static void mono_wasm_init_finalizer_thread (void); +extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_func)(void *, void*), void *free_data); +extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_func)(void *, void*), void *free_data); + #ifndef DISABLE_LEGACY_JS_INTEROP #define MARSHAL_TYPE_NULL 0 @@ -181,15 +185,11 @@ mono_wasm_deregister_root (char *addr) #include "driver-gen.c" #endif -typedef struct WasmAssembly_ WasmAssembly; - -struct WasmAssembly_ { - MonoBundledAssembly assembly; - WasmAssembly *next; -}; - -static WasmAssembly *assemblies; -static int assembly_count; +static void +bundled_resources_free_func (void *resource, void *free_data) +{ + free (free_data); +} EMSCRIPTEN_KEEPALIVE int mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned int size) @@ -199,72 +199,49 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in char *new_name = strdup (name); //FIXME handle debugging assemblies with .exe extension strcpy (&new_name [len - 3], "dll"); - mono_register_symfile_for_assembly (new_name, data, size); + mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, bundled_resources_free_func, new_name); return 1; } - WasmAssembly *entry = g_new0 (WasmAssembly, 1); - entry->assembly.name = strdup (name); - entry->assembly.data = data; - entry->assembly.size = size; - entry->next = assemblies; - assemblies = entry; - ++assembly_count; + char *assembly_name = strdup (name); + assert (assembly_name); + mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, bundled_resources_free_func, assembly_name); return mono_has_pdb_checksum ((char*)data, size); } -int -mono_wasm_assembly_already_added (const char *assembly_name) +static void +bundled_resources_free_slots_func (void *resource, void *free_data) { - if (assembly_count == 0) - return 0; - - WasmAssembly *entry = assemblies; - while (entry != NULL) { - int entry_name_minus_extn_len = strrchr (entry->assembly.name, '.') - entry->assembly.name; - if (entry_name_minus_extn_len == strlen(assembly_name) && strncmp (entry->assembly.name, assembly_name, entry_name_minus_extn_len) == 0) - return 1; - entry = entry->next; + if (free_data) { + void **slots = (void **)free_data; + for (int i = 0; slots [i]; i++) + free (slots [i]); } - - return 0; } -const unsigned char * -mono_wasm_get_assembly_bytes (const char *assembly_name, unsigned int *size) +EMSCRIPTEN_KEEPALIVE void +mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) { - if (assembly_count == 0) - return 0; + int id_len = strlen (culture) + 1 + strlen (name); // +1 is for the "/" + char *id = (char *)malloc (sizeof (char) * (id_len + 1)); // +1 is for the terminating null character + assert (id); - WasmAssembly *entry = assemblies; - while (entry != NULL) { - if (strcmp (entry->assembly.name, assembly_name) == 0) - { - *size = entry->assembly.size; - return entry->assembly.data; - } - entry = entry->next; - } - return NULL; -} + int num_char = snprintf (id, (id_len + 1), "%s/%s", culture, name); + assert (num_char > 0 && num_char == id_len); -typedef struct WasmSatelliteAssembly_ WasmSatelliteAssembly; + char *satellite_assembly_name = strdup (name); + assert (satellite_assembly_name); -struct WasmSatelliteAssembly_ { - MonoBundledSatelliteAssembly *assembly; - WasmSatelliteAssembly *next; -}; + char *satellite_assembly_culture = strdup (culture); + assert (satellite_assembly_culture); -static WasmSatelliteAssembly *satellite_assemblies; -static int satellite_assembly_count; + void **slots = malloc (sizeof (void *) * 4); + assert (slots); + slots [0] = id; + slots [1] = satellite_assembly_name; + slots [2] = satellite_assembly_culture; + slots [3] = NULL; -EMSCRIPTEN_KEEPALIVE void -mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) -{ - WasmSatelliteAssembly *entry = g_new0 (WasmSatelliteAssembly, 1); - entry->assembly = mono_create_new_bundled_satellite_assembly (name, culture, data, size); - entry->next = satellite_assemblies; - satellite_assemblies = entry; - ++satellite_assembly_count; + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, bundled_resources_free_slots_func, slots); } EMSCRIPTEN_KEEPALIVE void @@ -432,23 +409,6 @@ get_native_to_interp (MonoMethod *method, void *extra_arg) return addr; } -void -mono_wasm_register_bundled_satellite_assemblies (void) -{ - /* In legacy satellite_assembly_count is always false */ - if (satellite_assembly_count) { - MonoBundledSatelliteAssembly **satellite_bundle_array = g_new0 (MonoBundledSatelliteAssembly *, satellite_assembly_count + 1); - WasmSatelliteAssembly *cur = satellite_assemblies; - int i = 0; - while (cur) { - satellite_bundle_array [i] = cur->assembly; - cur = cur->next; - ++i; - } - mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_bundle_array); - } -} - void mono_wasm_link_icu_shim (void); void @@ -513,7 +473,7 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); - mono_wasm_register_timezones_bundle(); + mono_register_timezones_bundle (); mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); @@ -570,19 +530,6 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mono_sgen_mono_ilgen_init (); #endif - if (assembly_count) { - MonoBundledAssembly **bundle_array = g_new0 (MonoBundledAssembly*, assembly_count + 1); - WasmAssembly *cur = assemblies; - int i = 0; - while (cur) { - bundle_array [i] = &cur->assembly; - cur = cur->next; - ++i; - } - mono_register_bundled_assemblies ((const MonoBundledAssembly **)bundle_array); - } - - mono_wasm_register_bundled_satellite_assemblies (); mono_trace_init (); mono_trace_set_log_handler (wasm_trace_logger, NULL); root_domain = mono_jit_init_version ("mono", NULL); diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 55f4c6c0e7399..c150a1fe09d31 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -79,12 +79,11 @@ - + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) - <_WasmTimezonesBundleSourceFile>$(WasmObjDir)\wasm-bundled-timezones.c - <_WasmTimezonesBundleObjectFile>$(WasmObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundleSourceFile>wasm-bundled-timezones.c <_WasmTimezonesBundleArchive>$(WasmObjDir)\wasm-bundled-timezones.a <_WasmTimezonesSourcesRsp>$(WasmObjDir)\wasm-bundled-timezones-sources.rsp <_WasmTimezonesArchiveRsp>$(WasmObjDir)\wasm-bundled-timezones-archive.rsp @@ -92,27 +91,23 @@ <_WasmTimezonesInternal Include="$(_WasmTimezonesPath)\**\*.*" WasmRole="Timezone"/> - - - - <_WasmBundleTimezonesWithHashes Update="@(_WasmBundleTimezonesWithHashes)"> - $(WasmObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).c - $(WasmObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).o - /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) - + <_WasmTimezonesInternal Update="@(_WasmTimezonesInternal)"> + /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmTimezonesInternal.Identity)).Replace('\','/')) + - - + + + + - <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" /> - <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundleSourceFile)).Replace('\','/'))" /> + <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(BundledWasmTimezones.DestinationFile)).Replace('\','/'))" /> + <_WasmBundleTimezonesSources Include="$(_WasmTimezonesBundleSourceFile)" /> - - + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.c" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleSourceFile)" /> - <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> - <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.ObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\$(_WasmTimezonesBundleSourceFile)" /> + <_WasmBundleTimezonesToDelete Remove="%(BundledWasmTimezones.DestinationFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\%(WasmBundleTimezonesObjects)" /> diff --git a/src/native/libs/System.Native/pal_datetime.c b/src/native/libs/System.Native/pal_datetime.c index 61b0dba23a919..94a05ebb63e69 100644 --- a/src/native/libs/System.Native/pal_datetime.c +++ b/src/native/libs/System.Native/pal_datetime.c @@ -4,6 +4,7 @@ #include "pal_config.h" #include "pal_datetime.h" #include "pal_utilities.h" +#include #include #include #include @@ -21,7 +22,7 @@ static const int64_t TICKS_PER_MICROSECOND = 10; /* 1000 / 100 */ #endif #if defined(TARGET_WASI) || defined(TARGET_BROWSER) -extern const unsigned char* mono_wasm_get_bundled_file(const char* name, int* out_length); +extern bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); #endif // @@ -67,7 +68,18 @@ const char* SystemNative_GetTimeZoneData(const char* name, int* length) assert(name != NULL); assert(length != NULL); #if defined(TARGET_WASI) || defined(TARGET_BROWSER) - return (const char*) mono_wasm_get_bundled_file(name, length); + const uint8_t *data = NULL; + uint32_t data_len = 0; + + mono_bundled_resources_get_data_resource_values (name, &data, &data_len); + assert (data_len <= INT_MAX); + if (data_len > INT_MAX) { + data_len = 0; + data = NULL; + } + + *length = (int)data_len; + return (const char *)data; #else assert_msg(false, "Not supported on this platform", 0); (void)name; // unused diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 2f1cbd98fb3b6..b4a63116ed81e 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -16,6 +16,19 @@ internal static class Utils { + public enum HashAlgorithmType + { + SHA256, + SHA384, + SHA512 + }; + + public enum HashEncodingType + { + Base64, + Base64Safe + }; + public static string WebcilInWasmExtension = ".wasm"; private static readonly object s_SyncObj = new object(); @@ -213,13 +226,75 @@ public static bool CopyIfDifferent(string src, string dst, bool useHash) return areDifferent; } + private static string ToBase64SafeString(byte[] data) + { + if (data.Length == 0) + return string.Empty; + + int outputLength = ((4 * data.Length / 3) + 3) & ~3; + char[] base64Safe = new char[outputLength]; + int base64SafeLength = Convert.ToBase64CharArray(data, 0, data.Length, base64Safe, 0, Base64FormattingOptions.None); + + //RFC3548, URL and Filename Safe Alphabet. + for (int i = 0; i < base64SafeLength; i++) + { + if (base64Safe[i] == '+') + base64Safe[i] = '-'; + else if (base64Safe[i] == '/') + base64Safe[i] = '_'; + } + + return new string(base64Safe); + } + + private static byte[] ComputeHashFromStream(Stream stream, HashAlgorithmType algorithm) + { + if (algorithm == HashAlgorithmType.SHA512) + { + using HashAlgorithm hashAlgorithm = SHA512.Create(); + return hashAlgorithm.ComputeHash(stream); + } + else if (algorithm == HashAlgorithmType.SHA384) + { + using HashAlgorithm hashAlgorithm = SHA384.Create(); + return hashAlgorithm.ComputeHash(stream); + } + else if (algorithm == HashAlgorithmType.SHA256) + { + using HashAlgorithm hashAlgorithm = SHA256.Create(); + return hashAlgorithm.ComputeHash(stream); + } + else + { + throw new ArgumentException($"Unsupported hash algorithm: {algorithm}"); + } + } + + private static string EncodeHash(byte[] data, HashEncodingType encoding) + { + if (encoding == HashEncodingType.Base64) + { + return Convert.ToBase64String(data); + } + else if (encoding == HashEncodingType.Base64Safe) + { + return ToBase64SafeString(data); + } + else + { + throw new ArgumentException($"Unsupported hash encoding: {encoding}"); + } + } + public static string ComputeHash(string filepath) { - using var stream = File.OpenRead(filepath); - using HashAlgorithm hashAlgorithm = SHA512.Create(); + return ComputeHashEx(filepath); + } - byte[] hash = hashAlgorithm.ComputeHash(stream); - return Convert.ToBase64String(hash); + public static string ComputeHashEx(string filepath, HashAlgorithmType algorithm = HashAlgorithmType.SHA512, HashEncodingType encoding = HashEncodingType.Base64) + { + using var stream = File.OpenRead(filepath); + return EncodeHash(ComputeHashFromStream(stream, algorithm), encoding); } public static string ComputeIntegrity(string filepath) diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index e3d11c6efdc72..e53a7f7c36647 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -85,6 +85,19 @@ public bool IsSharedLibrary /// public string? AssembliesLocation { get; set; } + /// + /// Determines whether or not assemblies are bundled into the library + /// + public bool BundlesResources { get; set; } + + /// + /// An Item containing the bundled runtimeconfig.bin metadata detailing + /// DataSymbol - Symbol corresponding to the runtimeconfig.bin byte array data + /// DataLenSymbol - Symbol corresponding to the runtimeconfig.bin byte array size + /// DataLenSymbolValue - Literal size of the runtimeconfig.bin byte array data + /// + public ITaskItem? BundledRuntimeConfig { get; set; } + public bool StripDebugSymbols { get; set; } /// @@ -182,7 +195,7 @@ private void GatherAotSourcesObjects(StringBuilder aotSources, StringBuilder aot if (symbolsAdded > 0) { - exportedAssemblies.Add(Path.GetFileNameWithoutExtension(compiledAssembly.Path)); + exportedAssemblies.Add(Path.GetFileName(compiledAssembly.Path)); } } } @@ -273,10 +286,41 @@ private static void WriteLinkerScriptFile(string exportsFile, List expor private void WriteAutoInitializationFromTemplate() { - File.WriteAllText(Path.Combine(OutputDirectory, "autoinit.c"), - Utils.GetEmbeddedResource("autoinit.c") + string autoInitialization = Utils.GetEmbeddedResource("autoinit.c") .Replace("%ASSEMBLIES_LOCATION%", !string.IsNullOrEmpty(AssembliesLocation) ? AssembliesLocation : "DOTNET_LIBRARY_ASSEMBLY_PATH") - .Replace("%RUNTIME_IDENTIFIER%", RuntimeIdentifier)); + .Replace("%RUNTIME_IDENTIFIER%", RuntimeIdentifier); + + if (BundlesResources) + { + string dataSymbol = "NULL"; + string dataLenSymbol = "0"; + StringBuilder externBundledResourcesSymbols = new ("#if defined(BUNDLED_RESOURCES)\nextern void mono_register_resources_bundle (void);"); + if (BundledRuntimeConfig?.ItemSpec != null) + { + dataSymbol = BundledRuntimeConfig.GetMetadata("DataSymbol"); + if (string.IsNullOrEmpty(dataSymbol)) + { + throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataSymbol' metadata."); + } + dataLenSymbol = BundledRuntimeConfig.GetMetadata("DataLenSymbol"); + if (string.IsNullOrEmpty(dataLenSymbol)) + { + throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataLenSymbol' metadata."); + } + externBundledResourcesSymbols.AppendLine(); + externBundledResourcesSymbols.AppendLine($"extern uint8_t {dataSymbol}[];"); + externBundledResourcesSymbols.AppendLine($"extern const uint32_t {dataLenSymbol};"); + } + + externBundledResourcesSymbols.AppendLine("#endif"); + + autoInitialization = autoInitialization + .Replace("%EXTERN_BUNDLED_RESOURCES_SYMBOLS%", externBundledResourcesSymbols.ToString()) + .Replace("%RUNTIME_CONFIG_DATA%", dataSymbol) + .Replace("%RUNTIME_CONFIG_DATA_LEN%", dataLenSymbol); + } + + File.WriteAllText(Path.Combine(OutputDirectory, "autoinit.c"), autoInitialization); } private void GenerateAssembliesLoader() @@ -318,6 +362,11 @@ private string GenerateExtraDefinitions() extraDefinitions.AppendLine("add_definitions(-DUSES_AOT_DATA=1)"); } + if (BundlesResources) + { + extraDefinitions.AppendLine("add_definitions(-DBUNDLED_RESOURCES=1)"); + } + return extraDefinitions.ToString(); } diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 6fd37510d00a1..e104345a43408 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -18,10 +18,14 @@ #include "library-builder.h" +%EXTERN_BUNDLED_RESOURCES_SYMBOLS% + static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) { - free ((void *)args->runtimeconfig.name.path); + if (args->kind == 0) + free ((void *)args->runtimeconfig.name.path); + free (args); free (user_data); } @@ -29,7 +33,18 @@ cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) static void initialize_runtimeconfig (const char *bundle_path) { - char *file_name = "runtimeconfig.bin"; + MonovmRuntimeConfigArguments *arg = (MonovmRuntimeConfigArguments *)malloc (sizeof (MonovmRuntimeConfigArguments)); + if (!arg) + LOG_ERROR ("Out of memory.\n"); + +#if defined(BUNDLED_RESOURCES) + arg->kind = 1; + arg->runtimeconfig.data.data = %RUNTIME_CONFIG_DATA%; + arg->runtimeconfig.data.data_len = %RUNTIME_CONFIG_DATA_LEN%; + + if (!arg->runtimeconfig.data.data) + return; +#else size_t str_len = sizeof (char) * (strlen (bundle_path) + strlen (file_name) + 2); // +1 "/", +1 null-terminating char char *file_path = (char *)malloc (str_len); if (!file_path) @@ -40,18 +55,16 @@ initialize_runtimeconfig (const char *bundle_path) LOG_ERROR ("Encoding error while formatting '%s' and '%s' into \"%%s/%%s\".\n", bundle_path, file_name); struct stat buffer; - - if (stat (file_path, &buffer) == 0) { - MonovmRuntimeConfigArguments *arg = (MonovmRuntimeConfigArguments *)malloc (sizeof (MonovmRuntimeConfigArguments)); - if (!arg) - LOG_ERROR ("Out of memory.\n"); - - arg->kind = 0; - arg->runtimeconfig.name.path = file_path; - monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL); - } else { + if (stat (file_path, &buffer) != 0) { free (file_path); + return; } + + arg->kind = 0; + arg->runtimeconfig.name.path = file_path; +#endif // BUNDLED_RESOURCES + + monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL); } static void @@ -105,11 +118,17 @@ free_aot_data (MonoAssembly *assembly, int size, void *user_data, void *handle) { munmap (handle, size); } -#endif +#endif // USES_AOT_DATA static void runtime_init_callback () { + register_aot_modules (); + +#if defined(BUNDLED_RESOURCES) + mono_register_resources_bundle (); +#endif + const char *assemblies_location = getenv ("%ASSEMBLIES_LOCATION%"); if (!assemblies_location || assemblies_location[0] == '\0') assemblies_location = "./"; @@ -123,8 +142,6 @@ runtime_init_callback () initialize_appctx_env_variables (bundle_path); - register_aot_modules (); - mono_set_assemblies_path (bundle_path); mono_jit_set_aot_only (true); diff --git a/src/tasks/LibraryBuilder/Templates/library-builder.h b/src/tasks/LibraryBuilder/Templates/library-builder.h index c0dee67be95ad..eae270c004891 100644 --- a/src/tasks/LibraryBuilder/Templates/library-builder.h +++ b/src/tasks/LibraryBuilder/Templates/library-builder.h @@ -5,7 +5,9 @@ #define __MONO_LIBRARY_BUILDER_H__ #include +#include +// Logging #if defined(HOST_ANDROID) #include @@ -32,7 +34,14 @@ #error Unsupported Host Platform. Ensure the hosting platform is supported by the LibraryBuilder and the appropriate logging functions are added. -#endif +#endif // Logging + +// Platform specific native functions +#if defined(HOST_WINDOWS) +#define STR_CASE_CMP _stricmp +#else +#define STR_CASE_CMP strcasecmp +#endif // Platform specific native functions void register_aot_modules (void); void preload_assemblies_with_exported_symbols (); diff --git a/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c b/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c index 71be26d870dc0..f650ad78650c4 100644 --- a/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c +++ b/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c @@ -9,7 +9,21 @@ static void preload_assembly (const char* filename) { - MonoAssembly *assembly = mono_assembly_load_with_partial_name (filename, NULL); + MonoAssembly *assembly = mono_assembly_open (filename, NULL); + if (assembly) + return; + + int len = strlen (filename); + char *filename_without_extension = strdup (filename); + if (!filename_without_extension) + LOG_ERROR ("Out of memory.\n"); + + if (len >= 4 && !STR_CASE_CMP (".dll", &filename [len - 4])) + *(filename_without_extension + len - 4) = '\0'; + + assembly = mono_assembly_load_with_partial_name (filename_without_extension, NULL); + + free (filename_without_extension); if (!assembly) LOG_ERROR ("Could not open assembly '%s'.\n", filename); } diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs new file mode 100644 index 0000000000000..4378dc4046140 --- /dev/null +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -0,0 +1,463 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource BuildTaskCancelled { get; } = new(); + + private readonly Dictionary _resourceDataSymbolDictionary = new(); + + private readonly Dictionary _resourcesForDataSymbolDictionary = new(); + + private const string RegisteredName = "RegisteredName"; + + /// Truncate encoded hashes used in file names and symbols to 24 characters. + /// Represents a 128-bit hash output encoded in base64 format. + private const int MaxEncodedHashLength = 24; + + /// Must have DestinationFile metadata, which is the output filename + /// Could have RegisteredName, otherwise it would be the filename. + /// RegisteredName should be prefixed with namespace in form of unix like path. For example: "/usr/share/zoneinfo/" + [Required] + public ITaskItem[] FilesToBundle { get; set; } = Array.Empty(); + + /// + /// The function to call before mono runtime initialization + /// in order to register the bundled resources in FilesToBundle + /// + public string BundleRegistrationFunctionName { get; set; } = "mono_register_resources_bundle"; + + /// + /// The filename for the generated source file that registers + /// the bundled resources. + /// + public string? BundleFile { get; set; } + + /// + /// Path to store build artifacts + /// + [Required] + public string? OutputDirectory { get; set; } + + /// + /// Resources that were bundled + /// + /// Successful bundling will set the following metadata on the items: + /// - DataSymbol + /// - LenSymbol + /// + [Output] + public ITaskItem[] BundledResources { get; set; } = default!; + + public override bool Execute() + { + if (!Directory.Exists(OutputDirectory)) + { + Log.LogError($"OutputDirectory={OutputDirectory} doesn't exist."); + return false; + } + + List bundledResources = new(FilesToBundle.Length); + foreach (ITaskItem bundledResource in FilesToBundle) + { + var resourcePath = bundledResource.ItemSpec; + + bundledResource.SetMetadata("ResourceType", "DataResource"); + try + { + using FileStream resourceContents = File.OpenRead(resourcePath); + using PEReader resourcePEReader = new(resourceContents); + if (resourcePEReader.HasMetadata) + { + string? managedAssemblyCulture = null; + + var resourceMetadataReader = PEReaderExtensions.GetMetadataReader(resourcePEReader); + if (resourceMetadataReader.IsAssembly) + { + bundledResource.SetMetadata("ResourceType", "AssemblyResource"); + managedAssemblyCulture = resourceMetadataReader.GetString(resourceMetadataReader.GetAssemblyDefinition().Culture); + } + + bool isSatelliteAssembly = !string.IsNullOrEmpty(managedAssemblyCulture) && !managedAssemblyCulture!.Equals("neutral", StringComparison.OrdinalIgnoreCase); + if (resourcePath.EndsWith(".resources.dll", StringComparison.InvariantCultureIgnoreCase) || isSatelliteAssembly) + { + bundledResource.SetMetadata("ResourceType", "SatelliteAssemblyResource"); + if (isSatelliteAssembly) + bundledResource.SetMetadata("Culture", managedAssemblyCulture); + } + } + } + catch (BadImageFormatException e) + { + if (resourcePath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) + Log.LogMessage(MessageImportance.High, $"Resource '{resourcePath}' was interpreted with ResourceType 'DataResource' but has a '.dll' extension. Error: {e}"); + } + + var registeredName = bundledResource.GetMetadata(RegisteredName); + if (string.IsNullOrEmpty(registeredName)) + { + string culture = bundledResource.GetMetadata("Culture"); + registeredName = !string.IsNullOrEmpty(culture) ? culture + "/" + Path.GetFileName(resourcePath) : Path.GetFileName(resourcePath); + bundledResource.SetMetadata(RegisteredName, registeredName); + } + + string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(TruncateEncodedHash(Utils.ComputeHashEx(resourcePath, Utils.HashAlgorithmType.SHA256, Utils.HashEncodingType.Base64Safe), MaxEncodedHashLength))}"; + if (_resourceDataSymbolDictionary.ContainsKey(registeredName)) + { + throw new LogAsErrorException($"Multiple resources have the same {RegisteredName} '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); + } + _resourceDataSymbolDictionary.Add(registeredName, resourceDataSymbol); + + string destinationFile = Path.Combine(OutputDirectory, resourceDataSymbol + GetDestinationFileExtension()); + bundledResource.SetMetadata("DestinationFile", destinationFile); + + string[] resourcesWithDataSymbol; + if (_resourcesForDataSymbolDictionary.TryGetValue(resourceDataSymbol, out string[]? resourcesAlreadyWithDataSymbol)) + { + _resourcesForDataSymbolDictionary.Remove(resourceDataSymbol); + Log.LogMessage(MessageImportance.Low, $"Resource '{registeredName}' has the same output destination file '{destinationFile}' as '{string.Join("', '", resourcesAlreadyWithDataSymbol)}'"); + resourcesWithDataSymbol = resourcesAlreadyWithDataSymbol.Append(registeredName).ToArray(); + } + else + { + resourcesWithDataSymbol = new[] { registeredName }; + Log.LogMessage(MessageImportance.Low, $"Resource '{registeredName}' is associated with output destination file '{destinationFile}'"); + } + _resourcesForDataSymbolDictionary.Add(resourceDataSymbol, resourcesWithDataSymbol); + + bundledResources.Add(bundledResource); + } + + // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore + // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. + var remainingDestinationFilesToBundle = bundledResources.GroupBy(file => file.GetMetadata("DestinationFile")).ToArray(); + + var verboseCount = 0; + + // Generate source file(s) containing each resource's byte data and size + int allowedParallelism = Math.Max(Math.Min(bundledResources.Count, Environment.ProcessorCount), 1); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + + Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => + { + var group = remainingDestinationFilesToBundle[i]; + + var contentSourceFile = group.First(); + + var inputFile = contentSourceFile.ItemSpec; + var destinationFile = contentSourceFile.GetMetadata("DestinationFile"); + var registeredName = contentSourceFile.GetMetadata(RegisteredName); + + var count = Interlocked.Increment(ref verboseCount); + Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); + + Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, destinationFile); + var symbolName = _resourceDataSymbolDictionary[registeredName]; + if (!EmitBundleFile(destinationFile, (codeStream) => + { + using var inputStream = File.OpenRead(inputFile); + using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); + BundleFileToCSource(symbolName, inputStream, outputUtf8Writer); + })) + { + state.Stop(); + } + }); + + foreach (ITaskItem bundledResource in bundledResources) + { + string registeredName = bundledResource.GetMetadata(RegisteredName); + string resourceDataSymbol = _resourceDataSymbolDictionary[registeredName]; + bundledResource.SetMetadata("DataSymbol", $"{resourceDataSymbol}_data"); + bundledResource.SetMetadata("DataLenSymbol", $"{resourceDataSymbol}_data_len"); + bundledResource.SetMetadata("DataLenSymbolValue", symbolDataLen[resourceDataSymbol].ToString()); + } + + BundledResources = bundledResources.ToArray(); + + if (!string.IsNullOrEmpty(BundleFile)) + { + string resourceSymbols = GatherUniqueExportedResourceDataSymbols(bundledResources); + + var files = bundledResources.Select(bundledResource => { + var resourceType = bundledResource.GetMetadata("ResourceType"); + var registeredName = bundledResource.GetMetadata(RegisteredName); + var resourceName = ToSafeSymbolName(registeredName, false); + // Different timezone resources may have the same contents, use registered name to differentiate preallocated resources + var resourceDataSymbol = _resourceDataSymbolDictionary[registeredName]; + + string culture = bundledResource.GetMetadata("Culture"); + string? resourceSymbolName = null; + if (File.Exists(bundledResource.GetMetadata("SymbolFile"))) + resourceSymbolName = ToSafeSymbolName(bundledResource.GetMetadata("SymbolFile")); + + return (resourceType, registeredName, resourceName, resourceDataSymbol, culture, resourceSymbolName); + }).ToList(); + + Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); + + // Generate source file to preallocate resources and register bundled resources + EmitBundleFile(Path.Combine(OutputDirectory, BundleFile), (outputStream) => + { + using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); + GenerateBundledResourcePreallocationAndRegistration(resourceSymbols, BundleRegistrationFunctionName, files, outputUtf8Writer); + }); + } + + return !Log.HasLoggedErrors; + } + + public void Cancel() + { + BuildTaskCancelled.Cancel(); + } + + #region Helpers + + private static readonly Encoding Utf8NoBom = new UTF8Encoding(false); + private static readonly byte[] HexToUtf8Lookup = InitLookupTable(); + private static readonly byte[] NewLineAndIndentation = new[] { (byte)0x0a, (byte)0x20, (byte)0x20 }; + + private static byte[] InitLookupTable() + { + // Every 6 bytes in this array represents the output for a different input byte value. + // For example, the input byte 0x1a (26 decimal) corresponds to bytes 156-161 (26*6=156), + // whose values will be ['0', 'x', '1', 'a', ',', ' '], which is the UTF-8 representation + // for "0x1a, ". This is just a faster alternative to calling .ToString("x2") on every + // byte of the input file and then pushing that string through UTF8Encoding. + var lookup = new byte[256 * 6]; + for (int i = 0; i < 256; i++) + { + string byteAsHex = i.ToString("x2"); + char highOrderChar = BitConverter.IsLittleEndian ? byteAsHex[0] : byteAsHex[1]; + char lowOrderChar = BitConverter.IsLittleEndian ? byteAsHex[1] : byteAsHex[0]; + lookup[i * 6 + 0] = (byte)'0'; + lookup[i * 6 + 1] = (byte)'x'; + lookup[i * 6 + 2] = (byte)highOrderChar; + lookup[i * 6 + 3] = (byte)lowOrderChar; + lookup[i * 6 + 4] = (byte)','; + lookup[i * 6 + 5] = (byte)' '; + } + + return lookup; + } + + public abstract bool EmitBundleFile(string destinationFile, Action writeToOutputStream); + + public abstract string GetDestinationFileExtension(); + + private static Dictionary symbolDataLen = new(); + + private string GatherUniqueExportedResourceDataSymbols(List uniqueDestinationFiles) + { + StringBuilder resourceSymbols = new (); + HashSet resourcesAdded = new (); // Different Timezone resources may have the same contents + foreach (var uniqueDestinationFile in uniqueDestinationFiles) + { + string registeredName = uniqueDestinationFile.GetMetadata(RegisteredName); + string resourceDataSymbol = _resourceDataSymbolDictionary[registeredName]; + if (!resourcesAdded.Contains(resourceDataSymbol)) + { + resourceSymbols.AppendLine($"extern uint8_t {resourceDataSymbol}_data[];"); + resourceSymbols.AppendLine($"extern const uint32_t {resourceDataSymbol}_data_len;"); + resourceSymbols.AppendLine($"#define {resourceDataSymbol}_data_len_val {symbolDataLen[resourceDataSymbol]}"); + resourcesAdded.Add(resourceDataSymbol); + } + } + return resourceSymbols.ToString(); + } + + private static void GenerateBundledResourcePreallocationAndRegistration(string resourceSymbols, string bundleRegistrationFunctionName, ICollection<(string resourceType, string registeredName, string resourceName, string resourceDataSymbol, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) + { + List preallocatedSource = new (); + + string assemblyTemplate = Utils.GetEmbeddedResource("mono-bundled-assembly.template"); + string satelliteAssemblyTemplate = Utils.GetEmbeddedResource("mono-bundled-satellite-assembly.template"); + string symbolDataTemplate = Utils.GetEmbeddedResource("mono-bundled-data.template"); + + var preallocatedResources = new StringBuilder(); + List preallocatedAssemblies = new (); + List preallocatedSatelliteAssemblies = new (); + List preallocatedData = new (); + int assembliesCount = 0; + int satelliteAssembliesCount = 0; + int dataCount = 0; + foreach (var tuple in files) + { + string resourceId = tuple.registeredName; + + // Generate Preloaded MonoBundled*Resource structs + string preloadedStruct; + switch (tuple.resourceType) + { + case "SatelliteAssemblyResource": + { + preloadedStruct = satelliteAssemblyTemplate; + preloadedStruct = preloadedStruct.Replace("%Culture%", tuple.culture); + resourceId = tuple.registeredName; + preallocatedSatelliteAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); + satelliteAssembliesCount += 1; + break; + } + case "AssemblyResource": + { + preloadedStruct = assemblyTemplate; + // Add associated symfile information to MonoBundledAssemblyResource structs + string preloadedSymbolData = ""; + if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) + { + preloadedSymbolData = $",\n{Utils.GetEmbeddedResource("mono-bundled-symbol.template") + .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) + .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName!].ToString())}"; + } + preloadedStruct = preloadedStruct.Replace("%MonoBundledSymbolData%", preloadedSymbolData); + preallocatedAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); + assembliesCount += 1; + break; + } + case "DataResource": + { + preloadedStruct = symbolDataTemplate; + preallocatedData.Add($" (MonoBundledResource *)&{tuple.resourceName}"); + dataCount += 1; + break; + } + default: + { + throw new Exception($"Unsupported ResourceType '{tuple.resourceType}' for Resource '{tuple.resourceName}' with registered name '{tuple.registeredName}'. Ensure that the resource's ResourceType metadata is populated."); + } + } + + var resourceDataSymbol = tuple.resourceDataSymbol; + + preallocatedSource.Add(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) + .Replace("%ResourceDataSymbol%", resourceDataSymbol) + .Replace("%ResourceID%", resourceId) + .Replace("%RegisteredFilename%", tuple.registeredName) + .Replace("%Len%", $"{resourceDataSymbol}_data_len_val")); + } + + List addPreallocatedResources = new (); + if (assembliesCount != 0) { + preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_assembly_resources[] = {{\n{string.Join(",\n", preallocatedAssemblies)}\n}};"); + addPreallocatedResources.Add($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_assembly_resources, {assembliesCount});"); + } + if (satelliteAssembliesCount != 0) { + preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_satellite_assembly_resources[] = {{\n{string.Join(",\n", preallocatedSatelliteAssemblies)}\n}};"); + addPreallocatedResources.Add($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_satellite_assembly_resources, {satelliteAssembliesCount});"); + } + if (dataCount != 0) { + preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_data_resources[] = {{\n{string.Join(",\n", preallocatedData)}\n}};"); + addPreallocatedResources.Add($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_data_resources, {dataCount});"); + } + + outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-preallocation-and-registration.template") + .Replace("%ResourceSymbols%", resourceSymbols) + .Replace("%PreallocatedStructs%", string.Join("\n", preallocatedSource)) + .Replace("%PreallocatedResources%", preallocatedResources.ToString()) + .Replace("%BundleRegistrationFunctionName%", bundleRegistrationFunctionName) + .Replace("%AddPreallocatedResources%", string.Join("\n", addPreallocatedResources))); + } + + private void BundleFileToCSource(string symbolName, FileStream inputStream, StreamWriter outputUtf8Writer) + { + // Emits a C source file in the same format as "xxd --include". Example: + // + // unsigned char Some_File_dll[] = { + // 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x0a + // }; + // unsigned int Some_File_dll_len = 6; + + var buf = new byte[4096]; + int bytesRead; + var generatedArrayLength = 0; + var bytesEmitted = 0; + + outputUtf8Writer.WriteLine("#include "); + + string[] resourcesForDataSymbol = _resourcesForDataSymbolDictionary[symbolName]; + outputUtf8Writer.WriteLine($"// Resource Registered Names: {string.Join(", ", resourcesForDataSymbol)}"); + outputUtf8Writer.Write($"uint8_t {symbolName}_data[] = {{"); + outputUtf8Writer.Flush(); + while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) + { + for (var i = 0; i < bytesRead; i++) + { + if (bytesEmitted++ % 12 == 0) + { + outputUtf8Writer.BaseStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); + } + + var byteValue = buf[i]; + outputUtf8Writer.BaseStream.Write(HexToUtf8Lookup, byteValue * 6, 6); + } + + generatedArrayLength += bytesRead; + } + + outputUtf8Writer.WriteLine("0\n};"); + outputUtf8Writer.WriteLine($"const uint32_t {symbolName}_data_len = {generatedArrayLength};"); + outputUtf8Writer.Flush(); + outputUtf8Writer.BaseStream.Flush(); + + lock (symbolDataLen) + { + int len = 0; + if (symbolDataLen.TryGetValue(symbolName, out len)) + { + if (len != generatedArrayLength) + Log.LogMessage(MessageImportance.High, $"There are duplicate resources with the same output symbol '{symbolName}' but have differing content sizes '{symbolDataLen[symbolName]}' != '{generatedArrayLength}'."); + } + else + { + symbolDataLen.Add(symbolName, generatedArrayLength); + } + } + } + + private static string ToSafeSymbolName(string destinationFileName, bool filenameOnly = true) + { + var filename = destinationFileName; + if (filenameOnly) + filename = Path.GetFileName(destinationFileName); + + // Equivalent to the logic from "xxd --include" + var sb = new StringBuilder(); + foreach (var c in filename) + { + sb.Append(IsAlphanumeric(c) ? c : + (c == '+') ? "plus" : '_'); // To help differentiate timezones differing by a symbol (i.e. GMT+0 GMT-0) + } + + return sb.ToString(); + } + + private static string TruncateEncodedHash(string encodedHash, int maxEncodedHashLength) + => string.IsNullOrEmpty(encodedHash) + ? string.Empty + : encodedHash.Substring(0, Math.Min(encodedHash.Length, maxEncodedHashLength)); + + // Equivalent to "isalnum" + private static bool IsAlphanumeric(char c) => c + is (>= 'a' and <= 'z') + or (>= 'A' and <= 'Z') + or (>= '0' and <= '9'); + + #endregion + +} diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs similarity index 79% rename from src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs rename to src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs index 48c1f509768cd..3e0d8e0e4407d 100644 --- a/src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs @@ -6,9 +6,7 @@ using System.IO; using Microsoft.Build.Framework; -namespace Microsoft.WebAssembly.Build.Tasks; - -public class EmitWasmBundleObjectFiles : EmitWasmBundleBase +public class EmitBundleObjectFiles : EmitBundleBase { [Required] public string ClangExecutable { get; set; } = default!; @@ -24,7 +22,7 @@ public override bool Execute() return base.Execute(); } - public override bool Emit(string destinationFile, Action inputProvider) + public override bool EmitBundleFile(string destinationFile, Action EmitBundleFile) { if (Path.GetDirectoryName(destinationFile) is string destDir && !string.IsNullOrEmpty(destDir)) Directory.CreateDirectory(destDir); @@ -33,8 +31,8 @@ public override bool Emit(string destinationFile, Action inputProvider) ClangExecutable!, args: $"-xc -o \"{destinationFile}\" -c -", envVars: null, workingDir: null, silent: true, logStdErrAsMessage: false, - debugMessageImportance: MessageImportance.Low, label: null, - inputProvider); + debugMessageImportance: MessageImportance.Low, label: Path.GetFileName(destinationFile), + EmitBundleFile); if (exitCode != 0) { Log.LogError($"Failed to compile with exit code {exitCode}{Environment.NewLine}Output: {output}"); @@ -42,4 +40,8 @@ public override bool Emit(string destinationFile, Action inputProvider) return exitCode == 0; } + public override string GetDestinationFileExtension() + { + return ".o"; + } } diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs similarity index 54% rename from src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs rename to src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs index b57a8f4627a82..fb2793d4f7ef0 100644 --- a/src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs @@ -5,20 +5,23 @@ using System.IO; using Microsoft.Build.Framework; -namespace Microsoft.WebAssembly.Build.Tasks; - -// It would be ideal that this Task would always produce object files as EmitWasmBundleObjectFiles does. -// EmitWasmBundleObjectFiles could do it with clang by streaming code directly to clang input stream. +// It would be ideal that this Task would always produce object files as EmitBundleObjectFiles does. +// EmitBundleObjectFiles could do it with clang by streaming code directly to clang input stream. // For emcc it's not possible, so we need to write the code to disk first and then compile it in MSBuild. -public class EmitWasmBundleSourceFiles : EmitWasmBundleBase +public class EmitBundleSourceFiles : EmitBundleBase { - public override bool Emit(string destinationFile, Action inputProvider) + public override bool EmitBundleFile(string destinationFile, Action EmitBundleFile) { using (var fileStream = File.Create(destinationFile)) { - inputProvider(fileStream); + EmitBundleFile(fileStream); } return true; } + + public override string GetDestinationFileExtension() + { + return ".c"; + } } diff --git a/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj b/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj index 340cc4e5463b2..21861ede49281 100644 --- a/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj +++ b/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj @@ -25,6 +25,11 @@ + + + + + diff --git a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-assembly.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-assembly.template new file mode 100644 index 0000000000000..a2ff6570a5011 --- /dev/null +++ b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-assembly.template @@ -0,0 +1,7 @@ +const MonoBundledAssemblyResource %ResourceName% = { + .resource = { .type = MONO_BUNDLED_ASSEMBLY, + .id = "%ResourceID%" }, + .assembly = { .name = "%RegisteredFilename%", + .data = %ResourceDataSymbol%_data, + .size = %Len% }%MonoBundledSymbolData% +}; \ No newline at end of file diff --git a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-data.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-data.template new file mode 100644 index 0000000000000..03a9b43823be5 --- /dev/null +++ b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-data.template @@ -0,0 +1,7 @@ +const MonoBundledDataResource %ResourceName% = { + .resource = { .type = MONO_BUNDLED_DATA, + .id = "%ResourceID%" }, + .data = { .name = "%RegisteredFilename%", + .data = %ResourceDataSymbol%_data, + .size = %Len% } +}; \ No newline at end of file diff --git a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template new file mode 100644 index 0000000000000..40192fc002b2b --- /dev/null +++ b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include + +%ResourceSymbols% +typedef enum { + MONO_BUNDLED_DATA, + MONO_BUNDLED_ASSEMBLY, + MONO_BUNDLED_SATELLITE_ASSEMBLY, + MONO_BUNDLED_RESOURCE_COUNT +} MonoBundledResourceType; + +typedef void (*free_bundled_resource_func)(void *, void*); + +typedef struct _MonoBundledResource { + MonoBundledResourceType type; + const char *id; + free_bundled_resource_func free_func; + void *free_data; +} MonoBundledResource; + +typedef struct _MonoBundledData { + const char *name; + const uint8_t *data; + uint32_t size; +} MonoBundledData; + +typedef struct _MonoBundledDataResource { + MonoBundledResource resource; + MonoBundledData data; +} MonoBundledDataResource; + +typedef struct _MonoBundledSymbolData { + const uint8_t *data; + uint32_t size; +} MonoBundledSymbolData; + +typedef struct _MonoBundledAssemblyData { + const char *name; + const uint8_t *data; + uint32_t size; +} MonoBundledAssemblyData; + +typedef struct _MonoBundledAssemblyResource { + MonoBundledResource resource; + MonoBundledAssemblyData assembly; + MonoBundledSymbolData symbol_data; +} MonoBundledAssemblyResource; + +typedef struct _MonoBundledSatelliteAssemblyData { + const char *name; + const char *culture; + const uint8_t *data; + uint32_t size; +} MonoBundledSatelliteAssemblyData; + +typedef struct _MonoBundledSatelliteAssemblyResource { + MonoBundledResource resource; + MonoBundledSatelliteAssemblyData satellite_assembly; +} MonoBundledSatelliteAssemblyResource; + +extern void +mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); + +%PreallocatedStructs% + +%PreallocatedResources% +void +%BundleRegistrationFunctionName% (void) +{ +%AddPreallocatedResources% +} diff --git a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-satellite-assembly.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-satellite-assembly.template new file mode 100644 index 0000000000000..1fbee8bbf2bf1 --- /dev/null +++ b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-satellite-assembly.template @@ -0,0 +1,8 @@ +const MonoBundledSatelliteAssemblyResource %ResourceName% = { + .resource = { .type = MONO_BUNDLED_SATELLITE_ASSEMBLY, + .id = "%ResourceID%" }, + .satellite_assembly = { .name = "%RegisteredFilename%", + .culture = "%Culture%", + .data = %ResourceDataSymbol%_data, + .size = %Len% } +}; \ No newline at end of file diff --git a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-symbol.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-symbol.template new file mode 100644 index 0000000000000..a7677ec6e270b --- /dev/null +++ b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-symbol.template @@ -0,0 +1,2 @@ + .symbol_data = { .data = &%ResourceSymbolName%_data, + .size = %SymbolLen% } diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs b/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs deleted file mode 100644 index fb6e897d7ff3d..0000000000000 --- a/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs +++ /dev/null @@ -1,240 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Build.Framework; - -namespace Microsoft.WebAssembly.Build.Tasks; - -public abstract class EmitWasmBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask -{ - private CancellationTokenSource BuildTaskCancelled { get; } = new(); - - /// Must have DestinationFile metadata, which is the output filename - /// Could have RegisteredName, otherwise it would be the filename. - /// RegisteredName should be prefixed with namespace in form of unix like path. For example: "/usr/share/zoneinfo/" - [Required] - public ITaskItem[] FilesToBundle { get; set; } = default!; - - [Required] - public string BundleName { get; set; } = default!; - - [Required] - public string BundleFile { get; set; } = default!; - - [Required] - public string RegistrationCallbackFunctionName { get; set; } = default!; - - public override bool Execute() - { - // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore - // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. - var filesToBundleByDestinationFileName = FilesToBundle.GroupBy(f => f.GetMetadata("DestinationFile")).ToList(); - - // We're handling the incrementalism within this task, because it needs to be based on file content hashes - // and not on timetamps. The output filenames contain a content hash, so if any such file already exists on - // disk with that name, we know it must be up-to-date. - var remainingDestinationFilesToBundle = filesToBundleByDestinationFileName.Where(g => !File.Exists(g.Key)).ToArray(); - - // If you're only touching the leaf project, we don't really need to tell you that. - // But if there's more work to do it's valuable to show progress. - var verbose = remainingDestinationFilesToBundle.Length > 1; - var verboseCount = 0; - - var filesToBundleByRegisteredName = FilesToBundle.GroupBy(file => { - var registeredName = file.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(file.ItemSpec); - } - return registeredName; - }).ToList(); - - var files = filesToBundleByRegisteredName.Select(group => { - var registeredFile = group.First(); - var outputFile = registeredFile.GetMetadata("DestinationFile"); - var registeredName = group.Key; - var symbolName = ToSafeSymbolName(outputFile); - return (registeredName, symbolName); - }).ToList(); - - Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleName}"); - - if (remainingDestinationFilesToBundle.Length > 0) - { - int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); - - Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => - { - var group = remainingDestinationFilesToBundle[i]; - - // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, - // since we know each group's ITaskItems all contain the same binary data - var contentSourceFile = group.First(); - - var outputFile = group.Key; - var inputFile = contentSourceFile.ItemSpec; - if (verbose) - { - var registeredName = contentSourceFile.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(inputFile); - } - var count = Interlocked.Increment(ref verboseCount); - Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); - } - - Log.LogMessage(MessageImportance.Low, "Bundling {0} as {1}", inputFile, outputFile); - var symbolName = ToSafeSymbolName(outputFile); - if (!Emit(outputFile, (codeStream) => { - using var inputStream = File.OpenRead(inputFile); - BundleFileToCSource(symbolName, inputStream, codeStream); - })) - { - state.Stop(); - } - }); - } - - return Emit(BundleFile, (inputStream) => - { - using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateRegisteredBundledObjects($"mono_wasm_register_{BundleName}_bundle", RegistrationCallbackFunctionName, files, outputUtf8Writer); - }) && !Log.HasLoggedErrors; - } - - public void Cancel() - { - BuildTaskCancelled.Cancel(); - } - - #region Helpers - - private static readonly Encoding Utf8NoBom = new UTF8Encoding(false); - private static readonly byte[] HexToUtf8Lookup = InitLookupTable(); - private static readonly byte[] NewLineAndIndentation = new[] { (byte)0x0a, (byte)0x20, (byte)0x20 }; - - private static byte[] InitLookupTable() - { - // Every 6 bytes in this array represents the output for a different input byte value. - // For example, the input byte 0x1a (26 decimal) corresponds to bytes 156-161 (26*6=156), - // whose values will be ['0', 'x', '1', 'a', ',', ' '], which is the UTF-8 representation - // for "0x1a, ". This is just a faster alternative to calling .ToString("x2") on every - // byte of the input file and then pushing that string through UTF8Encoding. - var lookup = new byte[256 * 6]; - for (int i = 0; i < 256; i++) - { - string byteAsHex = i.ToString("x2"); - char highOrderChar = BitConverter.IsLittleEndian ? byteAsHex[0] : byteAsHex[1]; - char lowOrderChar = BitConverter.IsLittleEndian ? byteAsHex[1] : byteAsHex[0]; - lookup[i * 6 + 0] = (byte)'0'; - lookup[i * 6 + 1] = (byte)'x'; - lookup[i * 6 + 2] = (byte)highOrderChar; - lookup[i * 6 + 3] = (byte)lowOrderChar; - lookup[i * 6 + 4] = (byte)','; - lookup[i * 6 + 5] = (byte)' '; - } - - return lookup; - } - - public abstract bool Emit(string destinationFile, Action inputProvider); - - public static void GenerateRegisteredBundledObjects(string newFunctionName, string callbackFunctionName, ICollection<(string registeredName, string symbol)> files, StreamWriter outputUtf8Writer) - { - outputUtf8Writer.WriteLine($"int {callbackFunctionName}(const char* name, const unsigned char* data, unsigned int size);"); - outputUtf8Writer.WriteLine(); - - foreach (var tuple in files) - { - outputUtf8Writer.WriteLine($"extern const unsigned char {tuple.symbol}[];"); - outputUtf8Writer.WriteLine($"extern const int {tuple.symbol}_len;"); - } - - outputUtf8Writer.WriteLine(); - outputUtf8Writer.WriteLine($"void {newFunctionName}() {{"); - - foreach (var tuple in files) - { - outputUtf8Writer.WriteLine($" {callbackFunctionName} (\"{tuple.registeredName}\", {tuple.symbol}, {tuple.symbol}_len);"); - } - - outputUtf8Writer.WriteLine("}"); - } - - private static void BundleFileToCSource(string symbolName, FileStream inputStream, Stream outputStream) - { - // Emits a C source file in the same format as "xxd --include". Example: - // - // unsigned char Some_File_dll[] = { - // 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x0a - // }; - // unsigned int Some_File_dll_len = 6; - - var buf = new byte[4096]; - int bytesRead; - var generatedArrayLength = 0; - var bytesEmitted = 0; - - using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); - - outputUtf8Writer.Write($"unsigned char {symbolName}[] = {{"); - outputUtf8Writer.Flush(); - while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) - { - for (var i = 0; i < bytesRead; i++) - { - if (bytesEmitted++ % 12 == 0) - { - outputStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); - } - - var byteValue = buf[i]; - outputStream.Write(HexToUtf8Lookup, byteValue * 6, 6); - } - - generatedArrayLength += bytesRead; - } - - outputUtf8Writer.WriteLine("0\n};"); - outputUtf8Writer.WriteLine($"unsigned int {symbolName}_len = {generatedArrayLength};"); - outputUtf8Writer.Flush(); - outputStream.Flush(); - } - - private static string ToSafeSymbolName(string destinationFileName) - { - // Since destinationFileName includes a content hash, we can safely strip off the directory name - // as the filename is always unique enough. This avoid disclosing information about the build - // file structure in the resulting symbols. - var filename = Path.GetFileName(destinationFileName); - - // Equivalent to the logic from "xxd --include" - var sb = new StringBuilder(); - foreach (var c in filename) - { - sb.Append(IsAlphanumeric(c) ? c : '_'); - } - - return sb.ToString(); - } - - // Equivalent to "isalnum" - private static bool IsAlphanumeric(char c) => c - is (>= 'a' and <= 'z') - or (>= 'A' and <= 'Z') - or (>= '0' and <= '9'); - - #endregion - -}