diff --git a/lighthouse-cli/bin.js b/lighthouse-cli/bin.js index a8bd557c2161..0a476317467e 100644 --- a/lighthouse-cli/bin.js +++ b/lighthouse-cli/bin.js @@ -14,8 +14,6 @@ const getFlags = require('./cli-flags.js').getFlags; const runLighthouse = require('./run').runLighthouse; const log = require('lighthouse-logger'); -const perfOnlyConfig = require('../lighthouse-core/config/perf-config.js'); -const mixedContentConfig = require('../lighthouse-core/config/mixed-content-config.js'); // @ts-ignore const pkg = require('../package.json'); const Sentry = require('../lighthouse-core/lib/sentry'); @@ -54,12 +52,13 @@ if (cliFlags.configPath) { // Resolve the config file path relative to where cli was called. cliFlags.configPath = path.resolve(process.cwd(), cliFlags.configPath); config = /** @type {LH.Config.Json} */ (require(cliFlags.configPath)); -} else if (cliFlags.perf) { - config = perfOnlyConfig; -} else if (cliFlags.mixedContent) { - config = mixedContentConfig; - // The mixed-content audits require headless Chrome (https://crbug.com/764505). - cliFlags.chromeFlags = `${cliFlags.chromeFlags} --headless`; +} else if (cliFlags.preset) { + if (cliFlags.preset === 'mixed-content') { + // The mixed-content audits require headless Chrome (https://crbug.com/764505). + cliFlags.chromeFlags = `${cliFlags.chromeFlags} --headless`; + } + + config = require(`../lighthouse-core/config/${cliFlags.preset}-config.js`); } // set logging preferences diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js index 63f2bae30ced..0f63451451c2 100644 --- a/lighthouse-cli/cli-flags.js +++ b/lighthouse-cli/cli-flags.js @@ -58,7 +58,7 @@ function getFlags(manualArgv) { .group( [ 'save-assets', 'list-all-audits', 'list-trace-categories', 'additional-trace-categories', - 'config-path', 'chrome-flags', 'perf', 'mixed-content', 'port', 'hostname', + 'config-path', 'preset', 'chrome-flags', 'port', 'hostname', 'max-wait-for-load', 'enable-error-reporting', 'gather-mode', 'audit-mode', 'only-audits', 'only-categories', 'skip-audits', ], @@ -86,11 +86,10 @@ function getFlags(manualArgv) { 'additional-trace-categories': 'Additional categories to capture with the trace (comma-delimited).', 'config-path': 'The path to the config JSON.', - 'mixed-content': 'Use the mixed-content auditing configuration.', + 'preset': 'Use a built-in configuration.', 'chrome-flags': `Custom flags to pass to Chrome (space-delimited). For a full list of flags, see http://bit.ly/chrome-flags Additionally, use the CHROME_PATH environment variable to use a specific Chrome binary. Requires Chromium version 54.0 or later. If omitted, any detected Chrome Canary or Chrome stable will be used.`, - 'perf': 'Use a performance-test-only configuration', 'hostname': 'The hostname to use for the debugging protocol.', 'port': 'The port to use for the debugging protocol. Use 0 for a random port', 'max-wait-for-load': @@ -117,11 +116,11 @@ function getFlags(manualArgv) { // boolean values .boolean([ 'disable-storage-reset', 'disable-device-emulation', 'save-assets', 'list-all-audits', - 'list-trace-categories', 'perf', 'view', 'verbose', 'quiet', 'help', - 'mixed-content', + 'list-trace-categories', 'view', 'verbose', 'quiet', 'help', ]) .choices('output', printer.getValidOutputOptions()) .choices('throttling-method', ['devtools', 'provided', 'simulate']) + .choices('preset', ['full', 'perf', 'mixed-content']) // force as an array // note MUST use camelcase versions or only the kebab-case version will be forced .array('blockedUrlPatterns') diff --git a/lighthouse-cli/test/smokehouse/byte-config.js b/lighthouse-cli/test/smokehouse/byte-config.js index d41aa7a18d6e..78c58b7fbab2 100644 --- a/lighthouse-cli/test/smokehouse/byte-config.js +++ b/lighthouse-cli/test/smokehouse/byte-config.js @@ -6,7 +6,7 @@ 'use strict'; /** - * Config file for running PWA smokehouse audits. + * Config file for running byte efficiency smokehouse audits. */ module.exports = { extends: 'lighthouse:full', @@ -22,5 +22,8 @@ module.exports = { 'unused-css-rules', 'unused-javascript', ], + + // TODO(phulce): re-write testers to work with faster lantern loading + throttlingMethod: 'devtools', }, }; diff --git a/lighthouse-cli/test/smokehouse/perf/expectations.js b/lighthouse-cli/test/smokehouse/perf/expectations.js index c5b30793d89d..50ff11f68bf5 100644 --- a/lighthouse-cli/test/smokehouse/perf/expectations.js +++ b/lighthouse-cli/test/smokehouse/perf/expectations.js @@ -6,7 +6,7 @@ 'use strict'; /** - * Expected Lighthouse audit values for --perf tests + * Expected Lighthouse audit values for --preset=perf tests */ module.exports = [ { diff --git a/lighthouse-cli/test/smokehouse/run-smoke.js b/lighthouse-cli/test/smokehouse/run-smoke.js index 7ea899137317..96c185130fca 100644 --- a/lighthouse-cli/test/smokehouse/run-smoke.js +++ b/lighthouse-cli/test/smokehouse/run-smoke.js @@ -63,7 +63,7 @@ const SMOKETESTS = [{ id: 'offline', expectations: 'offline-local/offline-expectations.js', config: smokehouseDir + 'offline-config.js', - batch: 'parallel-second', + batch: 'offline', }, { id: 'byte', expectations: 'byte-efficiency/expectations.js', @@ -77,7 +77,7 @@ const SMOKETESTS = [{ }, { id: 'tti', expectations: 'tricky-tti/expectations.js', - config: 'lighthouse-core/config/default-config.js', + config: 'lighthouse-core/config/perf-config.js', batch: 'parallel-second', }]; diff --git a/lighthouse-cli/test/smokehouse/tricky-tti/expectations.js b/lighthouse-cli/test/smokehouse/tricky-tti/expectations.js index aba59f13caa9..dd4c539b2853 100644 --- a/lighthouse-cli/test/smokehouse/tricky-tti/expectations.js +++ b/lighthouse-cli/test/smokehouse/tricky-tti/expectations.js @@ -6,7 +6,7 @@ 'use strict'; /** - * Expected Lighthouse audit values for --perf tests + * Expected Lighthouse audit values for tricky TTI tests */ module.exports = [ { diff --git a/lighthouse-core/config/config.js b/lighthouse-core/config/config.js index 4f2a891d7b0b..b25233db8254 100644 --- a/lighthouse-core/config/config.js +++ b/lighthouse-core/config/config.js @@ -187,7 +187,7 @@ function deepClone(json) { // injection of plugins. if (Array.isArray(json.passes)) { cloned.passes.forEach((pass, i) => { - pass.gatherers = cloneArrayWithPluginSafety(json.passes[i].gatherers); + pass.gatherers = cloneArrayWithPluginSafety(json.passes[i].gatherers || []); }); } @@ -249,6 +249,8 @@ class Config { skipAuditIds); } + Config.adjustDefaultPassForThrottling(configJSON); + // Store the directory of the config path, if one was provided. this._configDir = configPath ? path.dirname(configPath) : undefined; @@ -381,6 +383,30 @@ class Config { return mergedItems; } + /** + * Observed throttling methods (devtools/provided) require at least 5s of quiet for the metrics to + * be computed. This method adjusts the quiet thresholds to the required minimums if necessary. + * + * @param {LH.Config.Json} config + */ + static adjustDefaultPassForThrottling(config) { + if (config.settings.throttlingMethod !== 'devtools' && + config.settings.throttlingMethod !== 'provided') { + return; + } + + const defaultPass = config.passes.find(pass => pass.passName === 'defaultPass'); + if (!defaultPass) return; + + const overrides = constants.nonSimulatedPassConfigOverrides; + defaultPass.pauseAfterLoadMs = + Math.max(overrides.pauseAfterLoadMs, defaultPass.pauseAfterLoadMs); + defaultPass.cpuQuietThresholdMs = + Math.max(overrides.cpuQuietThresholdMs, defaultPass.cpuQuietThresholdMs); + defaultPass.networkQuietThresholdMs = + Math.max(overrides.networkQuietThresholdMs, defaultPass.networkQuietThresholdMs); + } + /** * Filter out any unrequested items from the config, based on requested top-level categories. * @param {!Object} oldConfig Lighthouse config object diff --git a/lighthouse-core/config/constants.js b/lighthouse-core/config/constants.js index a667d3efc983..a8995d633264 100644 --- a/lighthouse-core/config/constants.js +++ b/lighthouse-core/config/constants.js @@ -30,7 +30,7 @@ const throttling = { const defaultSettings = { output: 'json', maxWaitForLoad: 45 * 1000, - throttlingMethod: 'devtools', + throttlingMethod: 'simulate', throttling: throttling.mobile3G, auditMode: false, gatherMode: false, @@ -60,8 +60,15 @@ const defaultPassConfig = { gatherers: [], }; +const nonSimulatedPassConfigOverrides = { + pauseAfterLoadMs: 5250, + networkQuietThresholdMs: 5250, + cpuQuietThresholdMs: 5250, +}; + module.exports = { throttling, defaultSettings, defaultPassConfig, + nonSimulatedPassConfigOverrides, }; diff --git a/lighthouse-core/config/default-config.js b/lighthouse-core/config/default-config.js index 3123e00672f3..649920c65542 100644 --- a/lighthouse-core/config/default-config.js +++ b/lighthouse-core/config/default-config.js @@ -15,9 +15,9 @@ module.exports = { passName: 'defaultPass', recordTrace: true, useThrottling: true, - pauseAfterLoadMs: 5250, - networkQuietThresholdMs: 5250, - cpuQuietThresholdMs: 5250, + pauseAfterLoadMs: 1000, + networkQuietThresholdMs: 1000, + cpuQuietThresholdMs: 1000, gatherers: [ 'url', 'scripts', diff --git a/lighthouse-core/config/fast-config.js b/lighthouse-core/config/fast-config.js deleted file mode 100644 index ae3d68d31c1f..000000000000 --- a/lighthouse-core/config/fast-config.js +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @license Copyright 2017 Google Inc. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - */ -'use strict'; - -module.exports = { - extends: 'lighthouse:default', - settings: { - skipAudits: [ - // disabled for now because they are too slow - 'no-mutation-events', - 'screenshot-thumbnails', - - // disabled for now because their results are not meaningful/cannot be computed anymore - 'first-meaningful-paint', - 'first-cpu-idle', - 'interactive', - 'estimated-input-latency', - 'speed-index', - 'offscreen-images', - 'load-fast-enough-for-pwa', - ], - }, - passes: [ - { - passName: 'defaultPass', - // overwrite the throttling and load wait parameters - useThrottling: false, - pauseAfterLoadMs: 0, - networkQuietThresholdMs: 500, - cpuQuietThresholdMs: 500, - // no need to add any gatherers yet, but this property is required - gatherers: [], - }, - ], - audits: [ - 'predictive-perf', - ], - categories: { - performance: { - audits: [ - {id: 'predictive-perf', weight: 5, group: 'perf-metric'}, - ], - }, - }, -}; diff --git a/lighthouse-core/config/mixed-content-config.js b/lighthouse-core/config/mixed-content-config.js index 1afe3de322a8..29ba715239e1 100644 --- a/lighthouse-core/config/mixed-content-config.js +++ b/lighthouse-core/config/mixed-content-config.js @@ -11,10 +11,6 @@ module.exports = { // (2) Re-load page but attempt to upgrade each request to HTTPS. passes: [{ passName: 'defaultPass', - // overwrite the throttling and load wait parameters to regular pass defaults - pauseAfterLoadMs: 0, - networkQuietThresholdMs: 0, - cpuQuietThresholdMs: 0, gatherers: ['url'], }, { passName: 'mixedContentPass', diff --git a/lighthouse-core/config/perf-config.js b/lighthouse-core/config/perf-config.js index 916dd5968efc..a00e1445912e 100644 --- a/lighthouse-core/config/perf-config.js +++ b/lighthouse-core/config/perf-config.js @@ -8,6 +8,7 @@ module.exports = { extends: 'lighthouse:default', settings: { + throttlingMethod: 'devtools', onlyCategories: ['performance'], }, }; diff --git a/lighthouse-core/scripts/assert-golden-lhr-unchanged.sh b/lighthouse-core/scripts/assert-golden-lhr-unchanged.sh index 9b1ccae2b8aa..efe6a3311bb7 100644 --- a/lighthouse-core/scripts/assert-golden-lhr-unchanged.sh +++ b/lighthouse-core/scripts/assert-golden-lhr-unchanged.sh @@ -28,7 +28,8 @@ trap teardown EXIT colorText "Generating a fresh LHR..." "$purple" set -x -node "$lhroot_path/lighthouse-cli" -A="$lhroot_path/lighthouse-core/test/results/artifacts" --quiet --output=json --output-path="$freshLHRPath" http://localhost/dobetterweb/dbw_tester.html +# TODO(phulce): add a lantern LHR-differ +node "$lhroot_path/lighthouse-cli" -A="$lhroot_path/lighthouse-core/test/results/artifacts" --throttling-method=devtools --quiet --output=json --output-path="$freshLHRPath" http://localhost/dobetterweb/dbw_tester.html set +x # remove timing from both diff --git a/lighthouse-core/test/config/config-test.js b/lighthouse-core/test/config/config-test.js index e47d1c81b269..156509a144f0 100644 --- a/lighthouse-core/test/config/config-test.js +++ b/lighthouse-core/test/config/config-test.js @@ -454,6 +454,44 @@ describe('Config', () => { assert.ok(auditNames.has('first-meaningful-paint'), 'did not include default audits'); }); + it('ensures quiet thresholds are sufficient when using devtools', () => { + const config = new Config({ + extends: 'lighthouse:default', + settings: { + throttlingMethod: 'devtools', + }, + }); + + assert.equal(config.settings.throttlingMethod, 'devtools'); + assert.equal(config.passes[0].passName, 'defaultPass'); + assert.ok(config.passes[0].pauseAfterLoadMs >= 5000, 'did not adjust load quiet ms'); + assert.ok(config.passes[0].cpuQuietThresholdMs >= 5000, 'did not adjust cpu quiet ms'); + assert.ok(config.passes[0].networkQuietThresholdMs >= 5000, 'did not adjust network quiet ms'); + assert.equal(config.passes[1].pauseAfterLoadMs, 0, 'should not have touched non-defaultPass'); + }); + + it('does nothing when thresholds for devtools are already sufficient', () => { + const config = new Config({ + extends: 'lighthouse:default', + settings: { + throttlingMethod: 'devtools', + onlyCategories: ['performance'], + }, + passes: [ + { + pauseAfterLoadMs: 10001, + cpuQuietThresholdMs: 10002, + networkQuietThresholdMs: 10003, + }, + ], + }); + + assert.equal(config.settings.throttlingMethod, 'devtools'); + assert.equal(config.passes[0].pauseAfterLoadMs, 10001); + assert.equal(config.passes[0].cpuQuietThresholdMs, 10002); + assert.equal(config.passes[0].networkQuietThresholdMs, 10003); + }); + it('merges settings with correct priority', () => { const config = new Config( { diff --git a/lighthouse-extension/app/src/lighthouse-background.js b/lighthouse-extension/app/src/lighthouse-background.js index 7ac1fc2f4cf1..3e33a6d288ca 100644 --- a/lighthouse-extension/app/src/lighthouse-background.js +++ b/lighthouse-extension/app/src/lighthouse-background.js @@ -9,7 +9,6 @@ const RawProtocol = require('../../../lighthouse-core/gather/connections/raw'); const Runner = require('../../../lighthouse-core/runner'); const Config = require('../../../lighthouse-core/config/config'); const defaultConfig = require('../../../lighthouse-core/config/default-config.js'); -const fastConfig = require('../../../lighthouse-core/config/fast-config.js'); const log = require('lighthouse-logger'); /** @@ -22,7 +21,7 @@ const log = require('lighthouse-logger'); window.runLighthouseForConnection = function( connection, url, options, categoryIDs, updateBadgeFn = function() { }) { - const config = options && options.fastMode ? new Config(fastConfig, options.flags) : new Config({ + const config = new Config({ extends: 'lighthouse:default', settings: {onlyCategories: categoryIDs}, }, options.flags); diff --git a/package.json b/package.json index c6a201369cd1..999d2c22f842 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "changelog": "conventional-changelog --config ./build/changelog-generator/index.js --infile changelog.md --same-file", "type-check": "tsc -p .", "update:sample-artifacts": "node lighthouse-core/scripts/update-report-fixtures.js -G", - "update:sample-json": "node ./lighthouse-cli -A=./lighthouse-core/test/results/artifacts --output=json --output-path=./lighthouse-core/test/results/sample_v2.json http://localhost/dobetterweb/dbw_tester.html && node lighthouse-core/scripts/cleanup-LHR-for-diff.js ./lighthouse-core/test/results/sample_v2.json --only-remove-timing", + "update:sample-json": "node ./lighthouse-cli -A=./lighthouse-core/test/results/artifacts --throttling-method=devtools --output=json --output-path=./lighthouse-core/test/results/sample_v2.json http://localhost/dobetterweb/dbw_tester.html && node lighthouse-core/scripts/cleanup-LHR-for-diff.js ./lighthouse-core/test/results/sample_v2.json --only-remove-timing", "diff:sample-json": "bash lighthouse-core/scripts/assert-golden-lhr-unchanged.sh", "update:crdp-typings": "node lighthouse-core/scripts/extract-crdp-mapping.js", "mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js" diff --git a/readme.md b/readme.md index 6586bd24f898..0547e4a65470 100644 --- a/readme.md +++ b/readme.md @@ -58,8 +58,8 @@ Configuration: CHROME_PATH: Explicit path of intended Chrome binary. If set must point to an executable of a build of Chromium version 54.0 or later. By default, any detected Chrome Canary or Chrome (stable) will be launched. [default: ""] - --perf Use a performance-test-only configuration [boolean] --port The port to use for the debugging protocol. Use 0 for a random port [default: 0] + --preset Use a built-in configuration. [choices: "full", "perf", "mixed-content"] --hostname The hostname to use for the debugging protocol. [default: "localhost"] --max-wait-for-load The timeout (in milliseconds) to wait before the page is considered done loading and the run should continue. WARNING: Very high values can lead to large traces and instability [default: 45000] diff --git a/typings/config.d.ts b/typings/config.d.ts index d10ccaf5f1f6..7ddb48c8166f 100644 --- a/typings/config.d.ts +++ b/typings/config.d.ts @@ -45,7 +45,7 @@ declare global { blockedUrlPatterns?: string[]; blankPage?: string; blankDuration?: number; - gatherers: GathererJson[]; + gatherers?: GathererJson[]; } export type GathererJson = { diff --git a/typings/externs.d.ts b/typings/externs.d.ts index 7ff952e86bab..02f1099dc2c7 100644 --- a/typings/externs.d.ts +++ b/typings/externs.d.ts @@ -74,6 +74,7 @@ declare global { listAllAudits: boolean; listTraceCategories: boolean; configPath?: string; + preset?: 'full'|'mixed-content'|'perf'; perf: boolean; mixedContent: boolean; verbose: boolean;