feat: I guess it's esm (electron#37535)

* fix: allow ESM loads from within ASAR files

* fix: ensure that ESM entry points finish loading before app ready

* fix: allow loading ESM entrypoints via default_app

* fix: allow ESM loading for renderer preloads

* docs: document current known limitations of esm

* chore: add patches to support blending esm handlers

* refactor: use SetDefersLoading instead of JoinAppCode in renderers

Blink has it's own event loop so pumping the uv loop in the renderer is not enough, luckily in blink we can suspend the loading of the frame while we do additional work.

* chore: add patch to expose SetDefersLoading

* fix: use fileURLToPath instead of pathname

* chore: update per PR feedback

* fix: fs.exists/existsSync should never throw

* fix: convert path to file url before importing

* fix: oops

* fix: oops

* Update docs/tutorial/esm-limitations.md

Co-authored-by: Jeremy Rose <jeremya@chromium.org>

* windows...

* windows...

* chore: update patches

* spec: fix tests and document empty body edge case

* Apply suggestions from code review

Co-authored-by: Daniel Scalzi <d_scalzi@yahoo.com>
Co-authored-by: Jeremy Rose <jeremya@chromium.org>

* spec: add tests for esm

* spec: windows

* chore: update per PR feedback

* chore: update patches

* Update shell/common/node_bindings.h

Co-authored-by: Jeremy Rose <jeremya@chromium.org>

* chore: update patches

* rebase

* use cjs loader by default for preload scripts

* chore: fix lint

* chore: update patches

* chore: update patches

* chore: fix patches

* build: debug depshash

* ?

* Revert "build: debug depshash"

This reverts commit 0de8252.

* chore: allow electron as builtin protocol in esm loader

* Revert "Revert "build: debug depshash""

This reverts commit ff86b12.

* chore: fix esm doc

* chore: update node patches

---------

Co-authored-by: Jeremy Rose <jeremya@chromium.org>
Co-authored-by: electron-patch-conflict-fixer[bot] <83340002+electron-patch-conflict-fixer[bot]@users.noreply.github.com>
Co-authored-by: PatchUp <73610968+patchup[bot]@users.noreply.github.com>
Co-authored-by: Daniel Scalzi <d_scalzi@yahoo.com>

default_app/default_app.ts

@@ -51,16 +51,17 @@ async function createWindow (backgroundColor?: string) {
     autoHideMenuBar: true,
     backgroundColor,
     webPreferences: {
-      preload: path.resolve(__dirname, 'preload.js'),
+      preload: url.fileURLToPath(new URL('preload.js', import.meta.url)),
       contextIsolation: true,
-      sandbox: true
+      sandbox: true,
+      nodeIntegration: false
     },
     useContentSize: true,
     show: false
   };
 
   if (process.platform === 'linux') {
-    options.icon = path.join(__dirname, 'icon.png');
+    options.icon = url.fileURLToPath(new URL('icon.png', import.meta.url));
   }
 
   mainWindow = new BrowserWindow(options);

default_app/main.ts

@@ -1,6 +1,7 @@
 import * as electron from 'electron/main';
 
 import * as fs from 'node:fs';
+import { Module } from 'node:module';
 import * as path from 'node:path';
 import * as url from 'node:url';
 const { app, dialog } = electron;
@@ -15,8 +16,6 @@ type DefaultAppOptions = {
   modules: string[];
 }
 
-const Module = require('node:module');
-
 // Parse command line options.
 const argv = process.argv.slice(1);
 
@@ -71,10 +70,10 @@ if (nextArgIsRequire) {
 
 // Set up preload modules
 if (option.modules.length > 0) {
-  Module._preloadModules(option.modules);
+  (Module as any)._preloadModules(option.modules);
 }
 
-function loadApplicationPackage (packagePath: string) {
+async function loadApplicationPackage (packagePath: string) {
   // Add a flag indicating app is started from default app.
   Object.defineProperty(process, 'defaultApp', {
     configurable: false,
@@ -89,11 +88,19 @@ function loadApplicationPackage (packagePath: string) {
     let appPath;
     if (fs.existsSync(packageJsonPath)) {
       let packageJson;
+      const emitWarning = process.emitWarning;
       try {
-        packageJson = require(packageJsonPath);
+        process.emitWarning = () => {};
+        packageJson = (await import(url.pathToFileURL(packageJsonPath).toString(), {
+          assert: {
+            type: 'json'
+          }
+        })).default;
       } catch (e) {
         showErrorMessage(`Unable to parse ${packageJsonPath}\n\n${(e as Error).message}`);
         return;
+      } finally {
+        process.emitWarning = emitWarning;
       }
 
       if (packageJson.version) {
@@ -112,21 +119,23 @@ function loadApplicationPackage (packagePath: string) {
       // Set v8 flags, deliberately lazy load so that apps that do not use this
       // feature do not pay the price
       if (packageJson.v8Flags) {
-        require('node:v8').setFlagsFromString(packageJson.v8Flags);
+        (await import('node:v8')).setFlagsFromString(packageJson.v8Flags);
       }
       appPath = packagePath;
     }
 
+    let filePath: string;
+
     try {
-      const filePath = Module._resolveFilename(packagePath, module, true);
+      filePath = (Module as any)._resolveFilename(packagePath, null, true);
       app.setAppPath(appPath || path.dirname(filePath));
     } catch (e) {
       showErrorMessage(`Unable to find Electron app at ${packagePath}\n\n${(e as Error).message}`);
       return;
     }
 
     // Run the app.
-    Module._load(packagePath, module, true);
+    await import(url.pathToFileURL(filePath).toString());
   } catch (e) {
     console.error('App threw an error during load');
     console.error((e as Error).stack || e);
@@ -141,16 +150,16 @@ function showErrorMessage (message: string) {
 }
 
 async function loadApplicationByURL (appUrl: string) {
-  const { loadURL } = await import('./default_app');
+  const { loadURL } = await import('./default_app.js');
   loadURL(appUrl);
 }
 
 async function loadApplicationByFile (appPath: string) {
-  const { loadFile } = await import('./default_app');
+  const { loadFile } = await import('./default_app.js');
   loadFile(appPath);
 }
 
-function startRepl () {
+async function startRepl () {
   if (process.platform === 'win32') {
     console.error('Electron REPL not currently supported on Windows');
     process.exit(1);
@@ -171,8 +180,8 @@ function startRepl () {
     Using: Node.js ${nodeVersion} and Electron.js ${electronVersion}
   `);
 
-  const { REPLServer } = require('node:repl');
-  const repl = new REPLServer({
+  const { start } = await import('node:repl');
+  const repl = start({
     prompt: '> '
   }).on('exit', () => {
     process.exit(0);
@@ -225,8 +234,8 @@ function startRepl () {
 
   const electronBuiltins = [...Object.keys(electron), 'original-fs', 'electron'];
 
-  const defaultComplete = repl.completer;
-  repl.completer = (line: string, callback: Function) => {
+  const defaultComplete: Function = repl.completer;
+  (repl as any).completer = (line: string, callback: Function) => {
     const lastSpace = line.lastIndexOf(' ');
     const currentSymbol = line.substring(lastSpace + 1, repl.cursor);
 
@@ -249,11 +258,11 @@ if (option.file && !option.webdriver) {
   const protocol = url.parse(file).protocol;
   const extension = path.extname(file);
   if (protocol === 'http:' || protocol === 'https:' || protocol === 'file:' || protocol === 'chrome:') {
-    loadApplicationByURL(file);
+    await loadApplicationByURL(file);
   } else if (extension === '.html' || extension === '.htm') {
-    loadApplicationByFile(path.resolve(file));
+    await loadApplicationByFile(path.resolve(file));
   } else {
-    loadApplicationPackage(file);
+    await loadApplicationPackage(file);
   }
 } else if (option.version) {
   console.log('v' + process.versions.electron);
@@ -262,7 +271,7 @@ if (option.file && !option.webdriver) {
   console.log(process.versions.modules);
   process.exit(0);
 } else if (option.interactive) {
-  startRepl();
+  await startRepl();
 } else {
   if (!option.noHelp) {
     const welcomeMessage = `
@@ -285,5 +294,5 @@ Options:
     console.log(welcomeMessage);
   }
 
-  loadApplicationByFile('index.html');
+  await loadApplicationByFile('index.html');
 }

default_app/package.json

@@ -1,5 +1,6 @@
 {
   "name": "electron",
   "productName": "Electron",
-  "main": "main.js"
+  "main": "main.js",
+  "type": "module"
 }

default_app/preload.ts

@@ -1,4 +1,4 @@
-import { ipcRenderer, contextBridge } from 'electron/renderer';
+const { ipcRenderer, contextBridge } = require('electron/renderer');
 
 const policy = window.trustedTypes.createPolicy('electron-default-app', {
   // we trust the SVG contents

docs/tutorial/esm-limitations.md

@@ -0,0 +1,38 @@
+# ESM Limitations
+
+This document serves to outline the limitations / differences between ESM in Electron and ESM in Node.js and Chromium.
+
+## ESM Support Matrix
+
+This table gives a general overview of where ESM is supported and most importantly which ESM loader is used.
+
+| | Supported | Loader | Supported in Preload | Loader in Preload | Applicable Requirements |
+|-|-|-|-|-|-|
+| Main Process | Yes | Node.js | N/A | N/A | <ul><li> [You must `await` generously in the main process to avoid race conditions](#you-must-use-await-generously-in-the-main-process-to-avoid-race-conditions) </li></ul> |
+| Sandboxed Renderer | Yes | Chromium | No | | <ul><li> [Sandboxed preload scripts can't use ESM imports](#sandboxed-preload-scripts-cant-use-esm-imports) </li></ul> |
+| Node.js Renderer + Context Isolation | Yes | Chromium | Yes | Node.js | <ul><li> [Node.js ESM Preload Scripts will run after page load on pages with no content](#nodejs-esm-preload-scripts-will-run-after-page-load-on-pages-with-no-content) </li> <li>[ESM Preload Scripts must have the `.mjs` extension](#esm-preload-scripts-must-have-the-mjs-extension)</li></ul> |
+| Node.js Renderer + No Context Isolation | Yes | Chromium | Yes | Node.js | <ul><li> [Non-context-isolated renderers can't use dynamic Node.js ESM imports](#non-context-isolated-renderers-cant-use-dynamic-nodejs-esm-imports) </li> <li>[ESM Preload Scripts must have the `.mjs` extension](#esm-preload-scripts-must-have-the-mjs-extension)</li></ul> |
+
+## Requirements
+
+### You must use `await` generously in the main process to avoid race conditions
+
+Certain APIs in Electron (`app.setPath` for instance) are documented as needing to be called **before** the `app.on('ready')` event is emitted.  When using ESM in the main process it is only guaranteed that the `ready` event hasn't been emitted while executing the side-effects of the primary import.  i.e. if `index.mjs` calls `import('./set-up-paths.mjs')` at the top level the app will likely already be "ready" by the time that dynamic import resolves.  To avoid this you should `await import('./set-up-paths.mjs')` at the top level of `index.mjs`.  It's not just import calls you should await, if you are reading files asynchronously or performing other asynchronous actions you must await those at the top-level as well to ensure the app does not resume initialization and become ready too early.
+
+### Sandboxed preload scripts can't use ESM imports
+
+Sandboxed preload scripts are run as plain javascript without an ESM context.  It is recommended that preload scripts are bundled via something like `webpack` or `vite` for performance reasons regardless, so your preload script should just be a single file that doesn't need to use ESM imports.  Loading the `electron` API is still done via `require('electron')`.
+
+### Node.js ESM Preload Scripts will run after page load on pages with no content
+
+If the response body for the page is **completely** empty, i.e. `Content-Length: 0`, the preload script will not block the page load, which may result in race conditions. If this impacts you, change your response body to have _something_ in it, for example an empty `html` tag (`<html></html>`) or swap back to using a CommonJS preload script (`.js` or `.cjs`) which will block the page load.
+
+### ESM Preload Scripts must have the `.mjs` extension
+
+In order to load an ESM preload script it must have a `.mjs` file extension.  Using `type: module` in a nearby package.json is not sufficient.  Please also note the limitation above around not blocking page load if the page is empty.
+
+### Non-context-isolated renderers can't use dynamic Node.js ESM imports
+
+If your renderer process does not have `contextIsolation` enabled you can not `import()` ESM files via the Node.js module loader.  This means that you can't `import('fs')` or `import('./foo')`.  If you want to be able to do so you must enable context isolation.  This is because in the renderer Chromium's `import()` function takes precedence and without context isolation there is no way for Electron to know which loader to route the request to.
+
+If you enable context isolation `import()` from the isolated preload context will use the Node.js loader and `import()` from the main context will continue using Chromium's loader.

lib/asar/fs-wrapper.ts

@@ -41,8 +41,13 @@ const getOrCreateArchive = (archivePath: string) => {
 
 const asarRe = /\.asar/i;
 
+const { getValidatedPath } = __non_webpack_require__('internal/fs/utils');
+// In the renderer node internals use the node global URL but we do not set that to be
+// the global URL instance.  We need to do instanceof checks against the internal URL impl
+const { URL: NodeURL } = __non_webpack_require__('internal/url');
+
 // Separate asar package's path from full path.
-const splitPath = (archivePathOrBuffer: string | Buffer) => {
+const splitPath = (archivePathOrBuffer: string | Buffer | URL) => {
   // Shortcut for disabled asar.
   if (isAsarDisabled()) return { isAsar: <const>false };
 
@@ -51,6 +56,9 @@ const splitPath = (archivePathOrBuffer: string | Buffer) => {
   if (Buffer.isBuffer(archivePathOrBuffer)) {
     archivePath = archivePathOrBuffer.toString();
   }
+  if (archivePath instanceof NodeURL) {
+    archivePath = getValidatedPath(archivePath);
+  }
   if (typeof archivePath !== 'string') return { isAsar: <const>false };
   if (!asarRe.test(archivePath)) return { isAsar: <const>false };
 
@@ -384,7 +392,13 @@ export const wrapFsWithAsar = (fs: Record<string, any>) => {
 
   const { exists: nativeExists } = fs;
   fs.exists = function exists (pathArgument: string, callback: any) {
-    const pathInfo = splitPath(pathArgument);
+    let pathInfo: ReturnType<typeof splitPath>;
+    try {
+      pathInfo = splitPath(pathArgument);
+    } catch {
+      nextTick(callback, [false]);
+      return;
+    }
     if (!pathInfo.isAsar) return nativeExists(pathArgument, callback);
     const { asarPath, filePath } = pathInfo;
 
@@ -415,7 +429,12 @@ export const wrapFsWithAsar = (fs: Record<string, any>) => {
 
   const { existsSync } = fs;
   fs.existsSync = (pathArgument: string) => {
-    const pathInfo = splitPath(pathArgument);
+    let pathInfo: ReturnType<typeof splitPath>;
+    try {
+      pathInfo = splitPath(pathArgument);
+    } catch {
+      return false;
+    }
     if (!pathInfo.isAsar) return existsSync(pathArgument);
     const { asarPath, filePath } = pathInfo;
 

lib/browser/init.ts

@@ -182,11 +182,31 @@ const { setDefaultApplicationMenu } = require('@electron/internal/browser/defaul
 // menu is set before any user window is created.
 app.once('will-finish-launching', setDefaultApplicationMenu);
 
+const { appCodeLoaded } = process;
+delete process.appCodeLoaded;
+
 if (packagePath) {
   // Finally load app's main.js and transfer control to C++.
-  process._firstFileName = Module._resolveFilename(path.join(packagePath, mainStartupScript), null, false);
-  Module._load(path.join(packagePath, mainStartupScript), Module, true);
+  if ((packageJson.type === 'module' && !mainStartupScript.endsWith('.cjs')) || mainStartupScript.endsWith('.mjs')) {
+    const { loadESM } = __non_webpack_require__('internal/process/esm_loader');
+    const main = require('url').pathToFileURL(path.join(packagePath, mainStartupScript));
+    loadESM(async (esmLoader: any) => {
+      try {
+        await esmLoader.import(main.toString(), undefined, Object.create(null));
+        appCodeLoaded!();
+      } catch (err) {
+        appCodeLoaded!();
+        process.emit('uncaughtException', err as Error);
+      }
+    });
+  } else {
+    // Call appCodeLoaded before just for safety, it doesn't matter here as _load is syncronous
+    appCodeLoaded!();
+    process._firstFileName = Module._resolveFilename(path.join(packagePath, mainStartupScript), null, false);
+    Module._load(path.join(packagePath, mainStartupScript), Module, true);
+  }
 } else {
   console.error('Failed to locate a valid package to load (app, app.asar or default_app.asar)');
   console.error('This normally means you\'ve damaged the Electron package somehow');
+  appCodeLoaded!();
 }

lib/renderer/init.ts

@@ -1,4 +1,5 @@
 import * as path from 'path';
+import { pathToFileURL } from 'url';
 import { IPC_MESSAGES } from '@electron/internal/common/ipc-messages';
 
 import type * as ipcRendererInternalModule from '@electron/internal/renderer/ipc-renderer-internal';
@@ -122,18 +123,39 @@ if (nodeIntegration) {
   }
 }
 
-const { preloadPaths } = ipcRendererUtils.invokeSync<{
-  preloadPaths: string[]
-}>(IPC_MESSAGES.BROWSER_NONSANDBOX_LOAD);
-
-// Load the preload scripts.
-for (const preloadScript of preloadPaths) {
-  try {
-    Module._load(preloadScript);
-  } catch (error) {
-    console.error(`Unable to load preload script: ${preloadScript}`);
-    console.error(error);
-
-    ipcRendererInternal.send(IPC_MESSAGES.BROWSER_PRELOAD_ERROR, preloadScript, error);
+const { appCodeLoaded } = process;
+delete process.appCodeLoaded;
+
+const { preloadPaths } = ipcRendererUtils.invokeSync<{ preloadPaths: string[] }>(IPC_MESSAGES.BROWSER_NONSANDBOX_LOAD);
+const cjsPreloads = preloadPaths.filter(p => path.extname(p) !== '.mjs');
+const esmPreloads = preloadPaths.filter(p => path.extname(p) === '.mjs');
+if (cjsPreloads.length) {
+  // Load the preload scripts.
+  for (const preloadScript of cjsPreloads) {
+    try {
+      Module._load(preloadScript);
+    } catch (error) {
+      console.error(`Unable to load preload script: ${preloadScript}`);
+      console.error(error);
+
+      ipcRendererInternal.send(IPC_MESSAGES.BROWSER_PRELOAD_ERROR, preloadScript, error);
+    }
   }
 }
+if (esmPreloads.length) {
+  const { loadESM } = __non_webpack_require__('internal/process/esm_loader');
+
+  loadESM(async (esmLoader: any) => {
+    // Load the preload scripts.
+    for (const preloadScript of esmPreloads) {
+      await esmLoader.import(pathToFileURL(preloadScript).toString(), undefined, Object.create(null)).catch((err: Error) => {
+        console.error(`Unable to load preload script: ${preloadScript}`);
+        console.error(err);
+
+        ipcRendererInternal.send(IPC_MESSAGES.BROWSER_PRELOAD_ERROR, preloadScript, err);
+      });
+    }
+  }).finally(() => appCodeLoaded!());
+} else {
+  appCodeLoaded!();
+}

patches/chromium/.patches

@@ -119,6 +119,8 @@ expose_v8initializer_codegenerationcheckcallbackinmainthread.patch
 chore_patch_out_profile_methods_in_profile_selections_cc.patch
 add_gin_converter_support_for_arraybufferview.patch
 chore_defer_usb_service_getdevices_request_until_usb_service_is.patch
+refactor_expose_hostimportmoduledynamically_and.patch
+feat_expose_documentloader_setdefersloading_on_webdocumentloader.patch
 fix_remove_profiles_from_spellcheck_service.patch
 chore_patch_out_profile_methods_in_chrome_browser_pdf.patch
 chore_patch_out_profile_methods_in_titlebar_config.patch