core(lhr): add top-level runtimeError (#6014)

lighthouse-core/lib/lh-error.js

@@ -12,18 +12,20 @@ const strings = require('./strings');
  * @property {string} code
  * @property {string} message
  * @property {RegExp} [pattern]
+ * @property {boolean} [lhrRuntimeError] True if it should appear in the top-level LHR.runtimeError property.
  */
 
 class LighthouseError extends Error {
   /**
    * @param {LighthouseErrorDefinition} errorDefinition
-   * @param {Record<string, string|undefined>=} properties
+   * @param {Record<string, string|boolean|undefined>=} properties
    */
   constructor(errorDefinition, properties) {
     super(errorDefinition.code);
     this.name = 'LHError';
     this.code = errorDefinition.code;
     this.friendlyMessage = errorDefinition.message;
+    this.lhrRuntimeError = !!errorDefinition.lhrRuntimeError;
     if (properties) Object.assign(this, properties);
 
     Error.captureStackTrace(this, LighthouseError);
@@ -75,17 +77,52 @@ class LighthouseError extends Error {
 
 const ERRORS = {
   // Screenshot/speedline errors
-  NO_SPEEDLINE_FRAMES: {code: 'NO_SPEEDLINE_FRAMES', message: strings.didntCollectScreenshots},
-  SPEEDINDEX_OF_ZERO: {code: 'SPEEDINDEX_OF_ZERO', message: strings.didntCollectScreenshots},
-  NO_SCREENSHOTS: {code: 'NO_SCREENSHOTS', message: strings.didntCollectScreenshots},
-  INVALID_SPEEDLINE: {code: 'INVALID_SPEEDLINE', message: strings.didntCollectScreenshots},
+  NO_SPEEDLINE_FRAMES: {
+    code: 'NO_SPEEDLINE_FRAMES',
+    message: strings.didntCollectScreenshots,
+    lhrRuntimeError: true,
+  },
+  SPEEDINDEX_OF_ZERO: {
+    code: 'SPEEDINDEX_OF_ZERO',
+    message: strings.didntCollectScreenshots,
+    lhrRuntimeError: true,
+  },
+  NO_SCREENSHOTS: {
+    code: 'NO_SCREENSHOTS',
+    message: strings.didntCollectScreenshots,
+    lhrRuntimeError: true,
+  },
+  INVALID_SPEEDLINE: {
+    code: 'INVALID_SPEEDLINE',
+    message: strings.didntCollectScreenshots,
+    lhrRuntimeError: true,
+  },
 
   // Trace parsing errors
-  NO_TRACING_STARTED: {code: 'NO_TRACING_STARTED', message: strings.badTraceRecording},
-  NO_NAVSTART: {code: 'NO_NAVSTART', message: strings.badTraceRecording},
-  NO_FCP: {code: 'NO_FCP', message: strings.badTraceRecording},
-  NO_FMP: {code: 'NO_FMP', message: strings.badTraceRecording},
-  NO_DCL: {code: 'NO_DCL', message: strings.badTraceRecording},
+  NO_TRACING_STARTED: {
+    code: 'NO_TRACING_STARTED',
+    message: strings.badTraceRecording,
+    lhrRuntimeError: true,
+  },
+  NO_NAVSTART: {
+    code: 'NO_NAVSTART',
+    message: strings.badTraceRecording,
+    lhrRuntimeError: true,
+  },
+  NO_FCP: {
+    code: 'NO_FCP',
+    message: strings.badTraceRecording,
+    lhrRuntimeError: true,
+  },
+  NO_DCL: {
+    code: 'NO_DCL',
+    message: strings.badTraceRecording,
+    lhrRuntimeError: true,
+  },
+  NO_FMP: {
+    code: 'NO_FMP',
+    message: strings.badTraceRecording,
+  },
 
   // TTI calculation failures
   FMP_TOO_LATE_FOR_FCPUI: {code: 'FMP_TOO_LATE_FOR_FCPUI', message: strings.pageLoadTookTooLong},
@@ -97,21 +134,36 @@ const ERRORS = {
   },
 
   // Page load failures
-  NO_DOCUMENT_REQUEST: {code: 'NO_DOCUMENT_REQUEST', message: strings.pageLoadFailed},
-  FAILED_DOCUMENT_REQUEST: {code: 'FAILED_DOCUMENT_REQUEST', message: strings.pageLoadFailed},
+  NO_DOCUMENT_REQUEST: {
+    code: 'NO_DOCUMENT_REQUEST',
+    message: strings.pageLoadFailed,
+    lhrRuntimeError: true,
+  },
+  FAILED_DOCUMENT_REQUEST: {
+    code: 'FAILED_DOCUMENT_REQUEST',
+    message: strings.pageLoadFailed,
+    lhrRuntimeError: true,
+  },
 
   // Protocol internal failures
   TRACING_ALREADY_STARTED: {
     code: 'TRACING_ALREADY_STARTED',
     message: strings.internalChromeError,
     pattern: /Tracing.*started/,
+    lhrRuntimeError: true,
   },
   PARSING_PROBLEM: {
     code: 'PARSING_PROBLEM',
     message: strings.internalChromeError,
     pattern: /Parsing problem/,
+    lhrRuntimeError: true,
+  },
+  READ_FAILED: {
+    code: 'READ_FAILED',
+    message: strings.internalChromeError,
+    pattern: /Read failed/,
+    lhrRuntimeError: true,
   },
-  READ_FAILED: {code: 'READ_FAILED', message: strings.internalChromeError, pattern: /Read failed/},
 
   // Protocol timeout failures
   REQUEST_CONTENT_TIMEOUT: {
@@ -128,5 +180,7 @@ const ERRORS = {
 
 /** @type {Record<keyof typeof ERRORS, LighthouseErrorDefinition>} */
 LighthouseError.errors = ERRORS;
+LighthouseError.NO_ERROR = 'NO_ERROR';
+LighthouseError.UNKNOWN_ERROR = 'UNKNOWN_ERROR';
 module.exports = LighthouseError;
 

lighthouse-core/runner.js

@@ -18,6 +18,7 @@ const path = require('path');
 const URL = require('./lib/url-shim');
 const Sentry = require('./lib/sentry');
 const generateReport = require('./report/report-generator').generateReport;
+const LHError = require('./lib/lh-error.js');
 
 /** @typedef {import('./gather/connections/connection.js')} Connection */
 /** @typedef {import('./config/config.js')} Config */
@@ -131,6 +132,7 @@ class Runner {
         requestedUrl: requestedUrl,
         finalUrl: artifacts.URL.finalUrl,
         runWarnings: lighthouseRunWarnings,
+        runtimeError: Runner.getArtifactRuntimeError(artifacts),
         audits: resultsById,
         configSettings: settings,
         categories,
@@ -291,6 +293,31 @@ class Runner {
     return auditResult;
   }
 
+  /**
+   * Returns first runtimeError found in artifacts.
+   * @param {LH.Artifacts} artifacts
+   * @return {LH.Result['runtimeError']}
+   */
+  static getArtifactRuntimeError(artifacts) {
+    for (const possibleErrorArtifact of Object.values(artifacts)) {
+      if (possibleErrorArtifact instanceof LHError && possibleErrorArtifact.lhrRuntimeError) {
+        const errorMessage = possibleErrorArtifact.friendlyMessage ?
+            `${possibleErrorArtifact.friendlyMessage} (${possibleErrorArtifact.message})` :
+            possibleErrorArtifact.message;
+
+        return {
+          code: possibleErrorArtifact.code,
+          message: errorMessage,
+        };
+      }
+    }
+
+    return {
+      code: LHError.NO_ERROR,
+      message: '',
+    };
+  }
+
   /**
    * Returns list of audit names for external querying.
    * @return {Array<string>}

lighthouse-core/test/results/sample_v2.json

@@ -10,6 +10,10 @@
   "requestedUrl": "http://localhost:10200/dobetterweb/dbw_tester.html",
   "finalUrl": "http://localhost:10200/dobetterweb/dbw_tester.html",
   "runWarnings": [],
+  "runtimeError": {
+    "code": "NO_ERROR",
+    "message": ""
+  },
   "audits": {
     "is-on-https": {
       "id": "is-on-https",

lighthouse-core/test/runner-test.js

@@ -10,12 +10,14 @@ const GatherRunner = require('../gather/gather-runner');
 const driverMock = require('./gather/fake-driver');
 const Config = require('../config/config');
 const Audit = require('../audits/audit');
+const Gatherer = require('../gather/gatherers/gatherer.js');
 const assetSaver = require('../lib/asset-saver');
 const fs = require('fs');
 const assert = require('assert');
 const path = require('path');
 const sinon = require('sinon');
 const rimraf = require('rimraf');
+const LHError = require('../lib/lh-error.js');
 
 /* eslint-env jest */
 
@@ -605,6 +607,41 @@ describe('Runner', () => {
     });
   });
 
+  it('includes a top-level runtimeError when a gatherer throws one', async () => {
+    const NO_FCP = LHError.errors.NO_FCP;
+    class RuntimeErrorGatherer extends Gatherer {
+      afterPass() {
+        throw new LHError(NO_FCP);
+      }
+    }
+    class WarningAudit extends Audit {
+      static get meta() {
+        return {
+          id: 'test-audit',
+          title: 'A test audit',
+          description: 'An audit for testing',
+          requiredArtifacts: ['RuntimeErrorGatherer'],
+        };
+      }
+      static audit() {
+        throw new Error('Should not get here');
+      }
+    }
+
+    const config = new Config({
+      passes: [{gatherers: [RuntimeErrorGatherer]}],
+      audits: [WarningAudit],
+    });
+    const {lhr} = await Runner.run(null, {url: 'https://example.com/', config, driverMock});
+
+    // Audit error included the runtimeError
+    assert.strictEqual(lhr.audits['test-audit'].scoreDisplayMode, 'error');
+    assert.ok(lhr.audits['test-audit'].errorMessage.includes(NO_FCP.code));
+    // And it bubbled up to the runtimeError.
+    assert.strictEqual(lhr.runtimeError.code, NO_FCP.code);
+    assert.ok(lhr.runtimeError.message.includes(NO_FCP.code));
+  });
+
   it('can handle array of outputs', async () => {
     const url = 'https://example.com';
     const config = new Config({

lighthouse-extension/app/src/lighthouse-ext-background.js

@@ -11,6 +11,7 @@ const background = require('./lighthouse-background');
 const ExtensionProtocol = require('../../../lighthouse-core/gather/connections/extension');
 const log = require('lighthouse-logger');
 const assetSaver = require('../../../lighthouse-core/lib/asset-saver.js');
+const LHError = require('../../../lighthouse-core/lib/lh-error.js');
 
 /** @type {Record<'mobile'|'desktop', LH.Config.Json>} */
 const LR_PRESETS = {
@@ -111,14 +112,33 @@ async function runLighthouseInLR(connection, url, flags, {lrDevice, categoryIDs,
     config.settings.onlyCategories = categoryIDs;
   }
 
-  const results = await lighthouse(url, flags, config, connection);
-  if (!results) return;
+  try {
+    const results = await lighthouse(url, flags, config, connection);
+    if (!results) return;
 
-  if (logAssets) {
-    await assetSaver.logAssets(results.artifacts, results.lhr.audits);
-  }
+    if (logAssets) {
+      await assetSaver.logAssets(results.artifacts, results.lhr.audits);
+    }
+    return results.report;
+  } catch (err) {
+    // If an error ruined the entire lighthouse run, attempt to return a meaningful error.
+    let runtimeError;
+    if (!(err instanceof LHError) || !err.lhrRuntimeError) {
+      runtimeError = {
+        code: LHError.UNKNOWN_ERROR,
+        message: `Unknown error encountered with message '${err.message}'`,
+      };
+    } else {
+      runtimeError = {
+        code: err.code,
+        message: err.friendlyMessage ?
+            `${err.friendlyMessage} (${err.message})` :
+            err.message,
+      };
+    }
 
-  return results.report;
+    return JSON.stringify({runtimeError}, null, 2);
+  }
 }
 
 /**
@@ -222,28 +242,32 @@ if ('chrome' in window && chrome.runtime) {
 }
 
 if (typeof module !== 'undefined' && module.exports) {
-  // Export for popup.js to import types. We don't want tsc to infer an index
-  // type, so use exports instead of module.exports.
-  exports.runLighthouseInExtension = runLighthouseInExtension;
-  exports.getDefaultCategories = background.getDefaultCategories;
-  exports.isRunning = isRunning;
-  exports.listenForStatus = listenForStatus;
-  exports.saveSettings = saveSettings;
-  exports.loadSettings = loadSettings;
+  // Export for importing types into popup.js, require()ing into unit tests.
+  module.exports = {
+    runLighthouseInExtension,
+    runLighthouseInLR,
+    getDefaultCategories: background.getDefaultCategories,
+    isRunning,
+    listenForStatus,
+    saveSettings,
+    loadSettings,
+  };
 }
 
-// Expose on window for extension, other consumers of file.
-// @ts-ignore
-window.runLighthouseInExtension = runLighthouseInExtension;
-// @ts-ignore
-window.runLighthouseInLR = runLighthouseInLR;
-// @ts-ignore
-window.getDefaultCategories = background.getDefaultCategories;
-// @ts-ignore
-window.isRunning = isRunning;
-// @ts-ignore
-window.listenForStatus = listenForStatus;
-// @ts-ignore
-window.loadSettings = loadSettings;
-// @ts-ignore
-window.saveSettings = saveSettings;
+// Expose on window for extension, other browser-residing consumers of file.
+if (typeof window !== 'undefined') {
+  // @ts-ignore
+  window.runLighthouseInExtension = runLighthouseInExtension;
+  // @ts-ignore
+  window.runLighthouseInLR = runLighthouseInLR;
+  // @ts-ignore
+  window.getDefaultCategories = background.getDefaultCategories;
+  // @ts-ignore
+  window.isRunning = isRunning;
+  // @ts-ignore
+  window.listenForStatus = listenForStatus;
+  // @ts-ignore
+  window.loadSettings = loadSettings;
+  // @ts-ignore
+  window.saveSettings = saveSettings;
+}