core(config): switch to lantern by default, add presets (#5041)

lighthouse-cli/bin.js

@@ -14,8 +14,6 @@ const getFlags = require('./cli-flags.js').getFlags;
 const runLighthouse = require('./run').runLighthouse;
 
 const log = require('lighthouse-logger');
-const perfOnlyConfig = require('../lighthouse-core/config/perf-config.js');
-const mixedContentConfig = require('../lighthouse-core/config/mixed-content-config.js');
 // @ts-ignore
 const pkg = require('../package.json');
 const Sentry = require('../lighthouse-core/lib/sentry');
@@ -54,12 +52,13 @@ if (cliFlags.configPath) {
   // Resolve the config file path relative to where cli was called.
   cliFlags.configPath = path.resolve(process.cwd(), cliFlags.configPath);
   config = /** @type {LH.Config.Json} */ (require(cliFlags.configPath));
-} else if (cliFlags.perf) {
-  config = perfOnlyConfig;
-} else if (cliFlags.mixedContent) {
-  config = mixedContentConfig;
-  // The mixed-content audits require headless Chrome (https://crbug.com/764505).
-  cliFlags.chromeFlags = `${cliFlags.chromeFlags} --headless`;
+} else if (cliFlags.preset) {
+  if (cliFlags.preset === 'mixed-content') {
+    // The mixed-content audits require headless Chrome (https://crbug.com/764505).
+    cliFlags.chromeFlags = `${cliFlags.chromeFlags} --headless`;
+  }
+
+  config = require(`../lighthouse-core/config/${cliFlags.preset}-config.js`);
 }
 
 // set logging preferences

lighthouse-cli/cli-flags.js

@@ -58,7 +58,7 @@ function getFlags(manualArgv) {
       .group(
         [
           'save-assets', 'list-all-audits', 'list-trace-categories', 'additional-trace-categories',
-          'config-path', 'chrome-flags', 'perf', 'mixed-content', 'port', 'hostname',
+          'config-path', 'preset', 'chrome-flags', 'port', 'hostname',
           'max-wait-for-load', 'enable-error-reporting', 'gather-mode', 'audit-mode',
           'only-audits', 'only-categories', 'skip-audits',
         ],
@@ -86,11 +86,10 @@ function getFlags(manualArgv) {
         'additional-trace-categories':
             'Additional categories to capture with the trace (comma-delimited).',
         'config-path': 'The path to the config JSON.',
-        'mixed-content': 'Use the mixed-content auditing configuration.',
+        'preset': 'Use a built-in configuration.',
         'chrome-flags':
             `Custom flags to pass to Chrome (space-delimited). For a full list of flags, see http://bit.ly/chrome-flags
             Additionally, use the CHROME_PATH environment variable to use a specific Chrome binary. Requires Chromium version 54.0 or later. If omitted, any detected Chrome Canary or Chrome stable will be used.`,
-        'perf': 'Use a performance-test-only configuration',
         'hostname': 'The hostname to use for the debugging protocol.',
         'port': 'The port to use for the debugging protocol. Use 0 for a random port',
         'max-wait-for-load':
@@ -117,11 +116,11 @@ function getFlags(manualArgv) {
       // boolean values
       .boolean([
         'disable-storage-reset', 'disable-device-emulation', 'save-assets', 'list-all-audits',
-        'list-trace-categories', 'perf', 'view', 'verbose', 'quiet', 'help',
-        'mixed-content',
+        'list-trace-categories', 'view', 'verbose', 'quiet', 'help',
       ])
       .choices('output', printer.getValidOutputOptions())
       .choices('throttling-method', ['devtools', 'provided', 'simulate'])
+      .choices('preset', ['full', 'perf', 'mixed-content'])
       // force as an array
       // note MUST use camelcase versions or only the kebab-case version will be forced
       .array('blockedUrlPatterns')

lighthouse-cli/test/smokehouse/byte-config.js

@@ -6,7 +6,7 @@
 'use strict';
 
 /**
- * Config file for running PWA smokehouse audits.
+ * Config file for running byte efficiency smokehouse audits.
  */
 module.exports = {
   extends: 'lighthouse:full',
@@ -22,5 +22,8 @@ module.exports = {
       'unused-css-rules',
       'unused-javascript',
     ],
+
+    // TODO(phulce): re-write testers to work with faster lantern loading
+    throttlingMethod: 'devtools',
   },
 };

lighthouse-cli/test/smokehouse/perf/expectations.js

@@ -6,7 +6,7 @@
 'use strict';
 
 /**
- * Expected Lighthouse audit values for --perf tests
+ * Expected Lighthouse audit values for --preset=perf tests
  */
 module.exports = [
   {

lighthouse-cli/test/smokehouse/run-smoke.js

@@ -63,7 +63,7 @@ const SMOKETESTS = [{
   id: 'offline',
   expectations: 'offline-local/offline-expectations.js',
   config: smokehouseDir + 'offline-config.js',
-  batch: 'parallel-second',
+  batch: 'offline',
 }, {
   id: 'byte',
   expectations: 'byte-efficiency/expectations.js',
@@ -77,7 +77,7 @@ const SMOKETESTS = [{
 }, {
   id: 'tti',
   expectations: 'tricky-tti/expectations.js',
-  config: 'lighthouse-core/config/default-config.js',
+  config: 'lighthouse-core/config/perf-config.js',
   batch: 'parallel-second',
 }];
 

lighthouse-cli/test/smokehouse/tricky-tti/expectations.js

@@ -6,7 +6,7 @@
 'use strict';
 
 /**
- * Expected Lighthouse audit values for --perf tests
+ * Expected Lighthouse audit values for tricky TTI tests
  */
 module.exports = [
   {

lighthouse-core/config/config.js

@@ -187,7 +187,7 @@ function deepClone(json) {
   // injection of plugins.
   if (Array.isArray(json.passes)) {
     cloned.passes.forEach((pass, i) => {
-      pass.gatherers = cloneArrayWithPluginSafety(json.passes[i].gatherers);
+      pass.gatherers = cloneArrayWithPluginSafety(json.passes[i].gatherers || []);
     });
   }
 
@@ -249,6 +249,8 @@ class Config {
           skipAuditIds);
     }
 
+    Config.adjustDefaultPassForThrottling(configJSON);
+
     // Store the directory of the config path, if one was provided.
     this._configDir = configPath ? path.dirname(configPath) : undefined;
 
@@ -381,6 +383,30 @@ class Config {
     return mergedItems;
   }
 
+  /**
+   * Observed throttling methods (devtools/provided) require at least 5s of quiet for the metrics to
+   * be computed. This method adjusts the quiet thresholds to the required minimums if necessary.
+   *
+   * @param {LH.Config.Json} config
+   */
+  static adjustDefaultPassForThrottling(config) {
+    if (config.settings.throttlingMethod !== 'devtools' &&
+        config.settings.throttlingMethod !== 'provided') {
+      return;
+    }
+
+    const defaultPass = config.passes.find(pass => pass.passName === 'defaultPass');
+    if (!defaultPass) return;
+
+    const overrides = constants.nonSimulatedPassConfigOverrides;
+    defaultPass.pauseAfterLoadMs =
+      Math.max(overrides.pauseAfterLoadMs, defaultPass.pauseAfterLoadMs);
+    defaultPass.cpuQuietThresholdMs =
+      Math.max(overrides.cpuQuietThresholdMs, defaultPass.cpuQuietThresholdMs);
+    defaultPass.networkQuietThresholdMs =
+      Math.max(overrides.networkQuietThresholdMs, defaultPass.networkQuietThresholdMs);
+  }
+
   /**
    * Filter out any unrequested items from the config, based on requested top-level categories.
    * @param {!Object} oldConfig Lighthouse config object

lighthouse-core/config/constants.js

@@ -30,7 +30,7 @@ const throttling = {
 const defaultSettings = {
   output: 'json',
   maxWaitForLoad: 45 * 1000,
-  throttlingMethod: 'devtools',
+  throttlingMethod: 'simulate',
   throttling: throttling.mobile3G,
   auditMode: false,
   gatherMode: false,
@@ -60,8 +60,15 @@ const defaultPassConfig = {
   gatherers: [],
 };
 
+const nonSimulatedPassConfigOverrides = {
+  pauseAfterLoadMs: 5250,
+  networkQuietThresholdMs: 5250,
+  cpuQuietThresholdMs: 5250,
+};
+
 module.exports = {
   throttling,
   defaultSettings,
   defaultPassConfig,
+  nonSimulatedPassConfigOverrides,
 };

lighthouse-core/config/default-config.js

@@ -15,9 +15,9 @@ module.exports = {
     passName: 'defaultPass',
     recordTrace: true,
     useThrottling: true,
-    pauseAfterLoadMs: 5250,
-    networkQuietThresholdMs: 5250,
-    cpuQuietThresholdMs: 5250,
+    pauseAfterLoadMs: 1000,
+    networkQuietThresholdMs: 1000,
+    cpuQuietThresholdMs: 1000,
     gatherers: [
       'url',
       'scripts',

lighthouse-core/config/mixed-content-config.js

@@ -11,10 +11,6 @@ module.exports = {
   // (2) Re-load page but attempt to upgrade each request to HTTPS.
   passes: [{
     passName: 'defaultPass',
-    // overwrite the throttling and load wait parameters to regular pass defaults
-    pauseAfterLoadMs: 0,
-    networkQuietThresholdMs: 0,
-    cpuQuietThresholdMs: 0,
     gatherers: ['url'],
   }, {
     passName: 'mixedContentPass',

lighthouse-core/config/perf-config.js

@@ -8,6 +8,7 @@
 module.exports = {
   extends: 'lighthouse:default',
   settings: {
+    throttlingMethod: 'devtools',
     onlyCategories: ['performance'],
   },
 };

lighthouse-core/scripts/assert-golden-lhr-unchanged.sh

@@ -28,7 +28,8 @@ trap teardown EXIT
 
 colorText "Generating a fresh LHR..." "$purple"
 set -x
-node "$lhroot_path/lighthouse-cli" -A="$lhroot_path/lighthouse-core/test/results/artifacts" --quiet --output=json --output-path="$freshLHRPath" http://localhost/dobetterweb/dbw_tester.html
+# TODO(phulce): add a lantern LHR-differ
+node "$lhroot_path/lighthouse-cli" -A="$lhroot_path/lighthouse-core/test/results/artifacts" --throttling-method=devtools --quiet --output=json --output-path="$freshLHRPath" http://localhost/dobetterweb/dbw_tester.html
 set +x
 
 # remove timing from both

lighthouse-core/test/config/config-test.js

@@ -454,6 +454,44 @@ describe('Config', () => {
     assert.ok(auditNames.has('first-meaningful-paint'), 'did not include default audits');
   });
 
+  it('ensures quiet thresholds are sufficient when using devtools', () => {
+    const config = new Config({
+      extends: 'lighthouse:default',
+      settings: {
+        throttlingMethod: 'devtools',
+      },
+    });
+
+    assert.equal(config.settings.throttlingMethod, 'devtools');
+    assert.equal(config.passes[0].passName, 'defaultPass');
+    assert.ok(config.passes[0].pauseAfterLoadMs >= 5000, 'did not adjust load quiet ms');
+    assert.ok(config.passes[0].cpuQuietThresholdMs >= 5000, 'did not adjust cpu quiet ms');
+    assert.ok(config.passes[0].networkQuietThresholdMs >= 5000, 'did not adjust network quiet ms');
+    assert.equal(config.passes[1].pauseAfterLoadMs, 0, 'should not have touched non-defaultPass');
+  });
+
+  it('does nothing when thresholds for devtools are already sufficient', () => {
+    const config = new Config({
+      extends: 'lighthouse:default',
+      settings: {
+        throttlingMethod: 'devtools',
+        onlyCategories: ['performance'],
+      },
+      passes: [
+        {
+          pauseAfterLoadMs: 10001,
+          cpuQuietThresholdMs: 10002,
+          networkQuietThresholdMs: 10003,
+        },
+      ],
+    });
+
+    assert.equal(config.settings.throttlingMethod, 'devtools');
+    assert.equal(config.passes[0].pauseAfterLoadMs, 10001);
+    assert.equal(config.passes[0].cpuQuietThresholdMs, 10002);
+    assert.equal(config.passes[0].networkQuietThresholdMs, 10003);
+  });
+
   it('merges settings with correct priority', () => {
     const config = new Config(
       {

lighthouse-extension/app/src/lighthouse-background.js

@@ -9,7 +9,6 @@ const RawProtocol = require('../../../lighthouse-core/gather/connections/raw');
 const Runner = require('../../../lighthouse-core/runner');
 const Config = require('../../../lighthouse-core/config/config');
 const defaultConfig = require('../../../lighthouse-core/config/default-config.js');
-const fastConfig = require('../../../lighthouse-core/config/fast-config.js');
 const log = require('lighthouse-logger');
 
 /**
@@ -22,7 +21,7 @@ const log = require('lighthouse-logger');
 window.runLighthouseForConnection = function(
     connection, url, options, categoryIDs,
     updateBadgeFn = function() { }) {
-  const config = options && options.fastMode ? new Config(fastConfig, options.flags) : new Config({
+  const config = new Config({
     extends: 'lighthouse:default',
     settings: {onlyCategories: categoryIDs},
   }, options.flags);

package.json

@@ -60,7 +60,7 @@
     "changelog": "conventional-changelog --config ./build/changelog-generator/index.js --infile changelog.md --same-file",
     "type-check": "tsc -p .",
     "update:sample-artifacts": "node lighthouse-core/scripts/update-report-fixtures.js -G",
-    "update:sample-json": "node ./lighthouse-cli -A=./lighthouse-core/test/results/artifacts --output=json --output-path=./lighthouse-core/test/results/sample_v2.json http://localhost/dobetterweb/dbw_tester.html && node lighthouse-core/scripts/cleanup-LHR-for-diff.js ./lighthouse-core/test/results/sample_v2.json --only-remove-timing",
+    "update:sample-json": "node ./lighthouse-cli -A=./lighthouse-core/test/results/artifacts --throttling-method=devtools --output=json --output-path=./lighthouse-core/test/results/sample_v2.json http://localhost/dobetterweb/dbw_tester.html && node lighthouse-core/scripts/cleanup-LHR-for-diff.js ./lighthouse-core/test/results/sample_v2.json --only-remove-timing",
     "diff:sample-json": "bash lighthouse-core/scripts/assert-golden-lhr-unchanged.sh",
     "update:crdp-typings": "node lighthouse-core/scripts/extract-crdp-mapping.js",
     "mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js"

readme.md

@@ -58,8 +58,8 @@ Configuration:
                                  CHROME_PATH: Explicit path of intended Chrome binary. If set must point to an executable of a build of
                                  Chromium version 54.0 or later. By default, any detected Chrome Canary or Chrome (stable) will be launched.
                                                                                                                                        [default: ""]
-  --perf                         Use a performance-test-only configuration                                                                 [boolean]
   --port                         The port to use for the debugging protocol. Use 0 for a random port                                    [default: 0]
+  --preset                       Use a built-in configuration.                                            [choices: "full", "perf", "mixed-content"]
   --hostname                     The hostname to use for the debugging protocol.                                              [default: "localhost"]
   --max-wait-for-load            The timeout (in milliseconds) to wait before the page is considered done loading and the run should continue.
                                  WARNING: Very high values can lead to large traces and instability                                 [default: 45000]

typings/config.d.ts

@@ -45,7 +45,7 @@ declare global {
         blockedUrlPatterns?: string[];
         blankPage?: string;
         blankDuration?: number;
-        gatherers: GathererJson[];
+        gatherers?: GathererJson[];
       }
 
       export type GathererJson = {

typings/externs.d.ts

@@ -74,6 +74,7 @@ declare global {
       listAllAudits: boolean;
       listTraceCategories: boolean;
       configPath?: string;
+      preset?: 'full'|'mixed-content'|'perf';
       perf: boolean;
       mixedContent: boolean;
       verbose: boolean;