module: Add support for fetch-style ESM hooks

lib/internal/modules/esm/loader.js

@@ -62,6 +62,13 @@ class Loader {
     this._dynamicInstantiate = undefined;
     // The index for assigning unique URLs to anonymous module evaluation
     this.evalIndex = 0;
+
+    this._fetchListener = undefined;
+  }
+
+  setFetchListener(listener) {
+    console.log('fetch listener added', listener);
+    this._fetchListener = listener;
   }
 
   async resolve(specifier, parentURL) {
@@ -152,7 +159,73 @@ class Loader {
     }
   }
 
+  async resolveToURLOnly(specifier, parentURL) {
+    try {
+      const { url } = await this.resolve(specifier, parentURL);
+      return url;
+    } catch (err) {
+      const UNKNOWN_EXTENSION_PATTERN = /^Unknown file extension "\.(?:[^"]*)" for (.+?) imported from /;
+      const MODULE_NOT_FOUND_PATTERN = /^Cannot find module (.+?) imported from /;
+
+      if (err.code === 'ERR_UNKNOWN_FILE_EXTENSION') {
+        const m = err.message.match(UNKNOWN_EXTENSION_PATTERN);
+        if (m) {
+          return pathToFileURL(m[1]).href;
+        }
+      } else if (err.code === 'ERR_MODULE_NOT_FOUND') {
+        const m = err.message.match(MODULE_NOT_FOUND_PATTERN);
+        if (m) {
+          return pathToFileURL(m[1]).href;
+        }
+      }
+      throw err;
+    }
+  }
+
+  async getModuleJobFromFetch(specifier, parentURL) {
+    const url = await this.resolveToURLOnly(specifier, parentURL);
+    let job = this.moduleMap.get(url);
+    // CommonJS will set functions for lazy job evaluation.
+    if (typeof job === 'function') {
+      this.moduleMap.set(url, job = job());
+    }
+    if (job !== undefined) {
+      return job;
+    }
+
+    const request = new Request(url);
+    const event = new FetchEvent('fetch', { request });
+    console.log('calling fetch listener', event);
+    this._fetchListener(event);
+
+    const loaderInstance = async (url) => {
+      const response = await (event.responsePromise || fetch(request));
+
+      // TODO: Add last-minute transforms
+
+      // TODO: Check for content-type
+
+      const source = await response.text();
+      const createModule = translators.get('module:hack');
+      return createModule(url, source);
+    };
+
+    // TODO: inspectBrk checks for this
+    const format = 'module';
+    const inspectBrk = parentURL === undefined &&
+        format === 'module' && getOptionValue('--inspect-brk');
+    job = new ModuleJob(this, url, loaderInstance, parentURL === undefined,
+                        inspectBrk);
+    this.moduleMap.set(url, job);
+    return job;
+  }
+
   async getModuleJob(specifier, parentURL) {
+    console.log('getModuleJob', this._fetchListener);
+    if (this._fetchListener) {
+      return this.getModuleJobFromFetch(specifier, parentURL);
+    }
+
     const { url, format } = await this.resolve(specifier, parentURL);
     let job = this.moduleMap.get(url);
     // CommonJS will set functions for lazy job evaluation.

lib/internal/modules/esm/translators.js

@@ -75,6 +75,17 @@ async function importModuleDynamically(specifier, { url }) {
   return esmLoader.ESMLoader.import(specifier, url);
 }
 
+translators.set('module:hack', function hackyStrategy(url, source) {
+  maybeCacheSourceMap(url, source);
+  debug(`Translating StandardModule ${url}`);
+  const module = new ModuleWrap(url, undefined, source, 0, 0);
+  moduleWrap.callbackMap.set(module, {
+    initializeImportMeta,
+    importModuleDynamically,
+  });
+  return module;
+});
+
 // Strategy for loading a standard JavaScript module
 translators.set('module', async function moduleStrategy(url) {
   const source = `${await getSource(url)}`;

lib/internal/process/esm_loader.js

@@ -44,6 +44,85 @@ exports.importModuleDynamicallyCallback = async function(wrap, specifier) {
 let ESMLoader = new Loader();
 exports.ESMLoader = ESMLoader;
 
+function addLoaderWorkerGlobals(loader) {
+  globalThis.self = globalThis;
+
+  class FetchEvent {
+    constructor(type, init) {
+      this._type = type;
+      this._init = init;
+      this.responsePromise = null;
+    }
+
+    get request() {
+      return this._init.request;
+    }
+
+    respondWith(responsePromise) {
+      this.responsePromise = responsePromise;
+    }
+  }
+  globalThis.FetchEvent = FetchEvent;
+
+  // TODO: Use full Headers API
+  class Headers {
+    constructor(values = []) {
+      this.values = new Map(values);
+    }
+
+    set(key, value) {
+      this.values.set(key, value);
+    }
+  }
+  globalThis.Headers = Headers;
+
+  // TODO: Use full Request API
+  class Request {
+    constructor(url) {
+      this.url = url;
+      this.method = 'GET';
+    }
+  }
+  globalThis.Request = Request;
+
+  // TODO: Use full Response API
+  class Response {
+    constructor(body, init = {}) {
+      this.url = null;
+      this.body = body;
+      this.status = init.status || 200;
+      this.headers = new Map();
+    }
+
+    evilAddURL(url) {
+      this.url = url;
+      return this;
+    }
+
+    async text() {
+      return this.body;
+    }
+  }
+  globalThis.Response = Response;
+
+  async function fetch(request) {
+    // TODO: Setting the URL shouldn't be exposed like this but *shrug*
+    const url = new URL(request.url);
+
+    if (url.protocol === 'file:') {
+      return new Response(require('fs').readFileSync(url, 'utf8')).evilAddURL(request.url);
+    }
+    throw new TypeError('Failed to fetch');
+  }
+  globalThis.fetch = fetch;
+
+  globalThis.addEventListener = (eventName, handler) => {
+    if (eventName === 'fetch') {
+      loader.setFetchListener(handler);
+    }
+  };
+}
+
 let calledInitialize = false;
 exports.initializeLoader = initializeLoader;
 async function initializeLoader() {
@@ -65,9 +144,12 @@ async function initializeLoader() {
   const { emitExperimentalWarning } = require('internal/util');
   emitExperimentalWarning('--experimental-loader');
   return (async () => {
+    // TODO: In a perfect world the loader wouldn't run in the same realm
+    const newLoader = new Loader();
+    addLoaderWorkerGlobals(newLoader);
     const hooks =
         await ESMLoader.import(userLoader, pathToFileURL(cwd).href);
-    ESMLoader = new Loader();
+    ESMLoader = newLoader;
     ESMLoader.hook(hooks);
     return exports.ESMLoader = ESMLoader;
   })();

test/fixtures/es-module-loaders/fetch-style-transform.mjs

@@ -0,0 +1,50 @@
+/**
+ * @param {string} urlString
+ * @param {string} fileExtension
+ */
+function isFileExtensionURL(urlString, fileExtension) {
+  const url = new URL(urlString);
+  return url.protocol === 'file:' && url.pathname.endsWith(fileExtension);
+}
+
+/**
+ * @param {Response} res
+ * @param {string} mimeType
+ * @param {string} fileExtension
+ */
+function isType(res, mimeType, fileExtension) {
+  const contentType = (res.headers.get('content-type') || '').toLocaleLowerCase(
+    'en'
+  );
+  if (contentType === mimeType) {
+    return true;
+  }
+  return !contentType && isFileExtensionURL(res.url, fileExtension);
+}
+
+function compile(source) {
+  return `\
+const data = ${JSON.stringify(source)};
+console.log(import.meta.url, data);
+export default data;
+`;
+}
+
+function isCustomScript(res) {
+  return isType(res, 'application/vnd.customscript', '.custom');
+}
+
+self.addEventListener('fetch', event => {
+  event.respondWith(
+    fetch(event.request).then(async res => {
+      if (res.status !== 200 || !isCustomScript(res)) {
+        return res;
+      }
+      const source = await res.text();
+      const body = compile(source);
+      const headers = new Headers(res.headers);
+      headers.set('content-type', 'text/javascript');
+      return new Response(body, { headers });
+    })
+  );
+});

test/fixtures/es-modules/custom.custom

@@ -0,0 +1 @@
+Custom file format