From 86a28febbfbd640ce9b02905b8d9479e0b5d6aab Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 8 Jan 2024 16:28:52 -0800 Subject: [PATCH 1/7] fix --- src/worker.js | 10 +++++++++- test/test_other.py | 15 +++++++++++++-- tools/link.py | 27 ++++++++++++++++----------- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/worker.js b/src/worker.js index 425d084e9508..67421838915b 100644 --- a/src/worker.js +++ b/src/worker.js @@ -18,6 +18,13 @@ var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions if (ENVIRONMENT_IS_NODE) { // Create as web-worker-like an environment as we can. + // See the parallel code in shell.js. +#if EXPORT_ES6 && ENVIRONMENT_MAY_BE_WEB + const { createRequire } = await import('module'); + /** @suppress{duplicate} */ + var require = createRequire(import.meta.url); +#endif + var nodeWorkerThreads = require('worker_threads'); var parentPort = nodeWorkerThreads.parentPort; @@ -32,7 +39,8 @@ if (ENVIRONMENT_IS_NODE) { require, Module, location: { - href: __filename + // __filename is undefined in ES6 modules + href: typeof __filename !== 'undefined' ? __filename : undefined }, Worker: nodeWorkerThreads.Worker, importScripts: (f) => vm.runInThisContext(fs.readFileSync(f, 'utf8'), {filename: f}), diff --git a/test/test_other.py b/test/test_other.py index 82ad41e5835a..35ba2c2dc6b5 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -400,11 +400,16 @@ def test_export_es6_allows_export_in_post_js(self): src = read_file('a.out.js') self.assertContained('export{doNothing};', src) + @parameterized({ + '': (False,), + 'package_json': (True,), + }) @parameterized({ '': ([],), - 'pthreads': (['-pthread'],), + # load a worker before startup to check ES6 modules there as well + 'pthreads': (['-pthread', '-sPTHREAD_POOL_SIZE=1'],), }) - def test_export_es6(self, args): + def test_export_es6(self, args, package_json): self.run_process([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6', '-o', 'hello.mjs'] + args) # In ES6 mode we use MODULARIZE, so we must instantiate an instance of the @@ -413,6 +418,12 @@ def test_export_es6(self, args): import Hello from "./hello.mjs"; Hello(); ''') + + if package_json: + # This makes node load all files in the directory as ES6 modules, + # including the worker.js file. + create_file('package.json', '{"type":"module"}') + self.assertContained('hello, world!', self.run_js('runner.mjs')) def test_emcc_out_file(self): diff --git a/tools/link.py b/tools/link.py index 45b0aa1e78c0..e26d92de0926 100644 --- a/tools/link.py +++ b/tools/link.py @@ -1999,12 +1999,26 @@ def phase_memory_initializer(memfile): final_js += '.mem.js' +# Unmangle previously mangled `import.meta` and `await import` references in +# both main code and libraries. +# See also: `preprocess` in parseTools.js. +def fix_es6_import_statements(js_file): + if not settings.EXPORT_ES6 or not settings.USE_ES6_IMPORT_META: + return + + src = read_file(js_file) + write_file(js_file, src + .replace('EMSCRIPTEN$IMPORT$META', 'import.meta') + .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import')) + def create_worker_file(input_file, target_dir, output_file): output_file = os.path.join(target_dir, output_file) input_file = utils.path_from_root(input_file) contents = shared.read_and_preprocess(input_file, expand_macros=True) write_file(output_file, contents) + fix_es6_import_statements(output_file) + # Minify the worker JS file, if JS minification is enabled. if settings.MINIFY_WHITESPACE: contents = building.acorn_optimizer(output_file, ['minifyWhitespace'], return_output=True) @@ -2045,17 +2059,8 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): # mode) final_js = building.closure_compiler(final_js, advanced=False, extra_closure_args=options.closure_args) - # Unmangle previously mangled `import.meta` and `await import` references in - # both main code and libraries. - # See also: `preprocess` in parseTools.js. - if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META: - src = read_file(final_js) - final_js += '.esmeta.js' - write_file(final_js, src - .replace('EMSCRIPTEN$IMPORT$META', 'import.meta') - .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import')) - shared.get_temp_files().note(final_js) - save_intermediate('es6-module') + fix_es6_import_statements(final_js) + save_intermediate('es6-module') # Apply pre and postjs files if options.extern_pre_js or options.extern_post_js: From dc7c00f750bd3061abd749edcc14c8dca5a1c07e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 8 Jan 2024 16:59:33 -0800 Subject: [PATCH 2/7] feedback: import.meta.url --- src/worker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.js b/src/worker.js index 67421838915b..12675419f52b 100644 --- a/src/worker.js +++ b/src/worker.js @@ -40,7 +40,7 @@ if (ENVIRONMENT_IS_NODE) { Module, location: { // __filename is undefined in ES6 modules - href: typeof __filename !== 'undefined' ? __filename : undefined + href: typeof __filename !== 'undefined' ? __filename : import.meta.url }, Worker: nodeWorkerThreads.Worker, importScripts: (f) => vm.runInThisContext(fs.readFileSync(f, 'utf8'), {filename: f}), From 5435bf4d68e047b7111d2010b8895a7436119414 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 9 Jan 2024 14:05:42 -0800 Subject: [PATCH 3/7] python lint --- tools/link.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/link.py b/tools/link.py index e26d92de0926..70248f5db28c 100644 --- a/tools/link.py +++ b/tools/link.py @@ -2011,6 +2011,7 @@ def fix_es6_import_statements(js_file): .replace('EMSCRIPTEN$IMPORT$META', 'import.meta') .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import')) + def create_worker_file(input_file, target_dir, output_file): output_file = os.path.join(target_dir, output_file) input_file = utils.path_from_root(input_file) From 42cf278b8fbacd47c4489322d2ad44bb46b10dff Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 9 Jan 2024 14:09:33 -0800 Subject: [PATCH 4/7] fix --- src/worker.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/worker.js b/src/worker.js index 12675419f52b..40a926d06d90 100644 --- a/src/worker.js +++ b/src/worker.js @@ -39,8 +39,13 @@ if (ENVIRONMENT_IS_NODE) { require, Module, location: { - // __filename is undefined in ES6 modules + // __filename is undefined in ES6 modules, and import.meta.url only in ES6 + // modules. +#if EXPORT_ES6 href: typeof __filename !== 'undefined' ? __filename : import.meta.url +#else + href: typeof __filename +#endif }, Worker: nodeWorkerThreads.Worker, importScripts: (f) => vm.runInThisContext(fs.readFileSync(f, 'utf8'), {filename: f}), From 578e3facacb805ff244b11d3f4593ac1e21b4f14 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 9 Jan 2024 14:22:04 -0800 Subject: [PATCH 5/7] use .worker.mjs when EXPORT_ES6 --- ChangeLog.md | 4 ++++ tools/link.py | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 757355b940f2..7bc619cc29ba 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works. 3.1.52 (in development) ----------------------- +- Building with `pthreads+EXPORT_ES6` will now emit the worker file as + `NAME.worker.mjs` rather than `.js`. This is a necessary breaking change to + resolve other `pthreads+EXPORT_ES6` issues in Node.js (because Node.js is + affected by the suffix in some cases). (#21041) - Include paths added by ports (e.g. `-sUSE_SDL=2`) now use `-isystem` rather then `-I`. This means that files in user-specified include directories will now take precedence over port includes. (#21014) diff --git a/tools/link.py b/tools/link.py index 70248f5db28c..7763c0f9d9a4 100644 --- a/tools/link.py +++ b/tools/link.py @@ -513,6 +513,10 @@ def do_split_module(wasm_file, options): building.run_binaryen_command('wasm-split', wasm_file + '.orig', outfile=wasm_file, args=args) +def get_worker_js_suffix(): + return '.worker.mjs' if settings.EXPORT_ES6 else '.worker.js' + + def setup_pthreads(target): if settings.RELOCATABLE: # phtreads + dyanmic linking has certain limitations @@ -569,7 +573,7 @@ def setup_pthreads(target): building.user_requested_exports.update(worker_imports) # set location of worker.js - settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js' + settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + get_worker_js_suffix() if settings.MINIMAL_RUNTIME: building.user_requested_exports.add('exit') @@ -2606,7 +2610,7 @@ def generate_worker_js(target, js_target, target_basename): proxy_worker_filename = get_subresource_location(js_target) else: # compiler output goes in .worker.js file - move_file(js_target, shared.replace_suffix(js_target, '.worker.js')) + move_file(js_target, shared.replace_suffix(js_target, get_worker_js_suffix())) worker_target_basename = target_basename + '.worker' proxy_worker_filename = (settings.PROXY_TO_WORKER_FILENAME or worker_target_basename) + '.js' From 49ad5083020377b49cf54a42a0da137ce80be701 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 9 Jan 2024 15:38:00 -0800 Subject: [PATCH 6/7] fix tests --- src/worker.js | 7 +++++-- test/test_other.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/worker.js b/src/worker.js index 40a926d06d90..c0dcc77793ff 100644 --- a/src/worker.js +++ b/src/worker.js @@ -18,8 +18,11 @@ var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions if (ENVIRONMENT_IS_NODE) { // Create as web-worker-like an environment as we can. - // See the parallel code in shell.js. -#if EXPORT_ES6 && ENVIRONMENT_MAY_BE_WEB + // See the parallel code in shell.js, but here we don't need the condition on + // multi-environment builds, as we do not have the need to interact with the + // modularization logic as shell.js must (see link.py:node_es6_imports and + // how that is used in link.py). +#if EXPORT_ES6 const { createRequire } = await import('module'); /** @suppress{duplicate} */ var require = createRequire(import.meta.url); diff --git a/test/test_other.py b/test/test_other.py index 35ba2c2dc6b5..4504eefd71e9 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -343,10 +343,10 @@ def test_emcc_output_worker_mjs(self, args): test_file('hello_world.c')] + args) src = read_file('subdir/hello_world.mjs') self.assertContained("new URL('hello_world.wasm', import.meta.url)", src) - self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src) + self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src) self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src) self.assertContained('export default Module;', src) - src = read_file('subdir/hello_world.worker.js') + src = read_file('subdir/hello_world.worker.mjs') self.assertContained("import('./hello_world.mjs')", src) self.assertContained('hello, world!', self.run_js('subdir/hello_world.mjs')) @@ -358,7 +358,7 @@ def test_emcc_output_worker_mjs_single_file(self): test_file('hello_world.c'), '-sSINGLE_FILE']) src = read_file('hello_world.mjs') self.assertNotContained("new URL('data:", src) - self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src) + self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src) self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src) self.assertContained('hello, world!', self.run_js('hello_world.mjs')) From 48dc5efcbcbc3ddaf4f3d15f03788d5fcd70d55a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 10 Jan 2024 08:40:43 -0800 Subject: [PATCH 7/7] fix --- src/worker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.js b/src/worker.js index c0dcc77793ff..0b04ceedc227 100644 --- a/src/worker.js +++ b/src/worker.js @@ -47,7 +47,7 @@ if (ENVIRONMENT_IS_NODE) { #if EXPORT_ES6 href: typeof __filename !== 'undefined' ? __filename : import.meta.url #else - href: typeof __filename + href: __filename #endif }, Worker: nodeWorkerThreads.Worker,