feat: Add new errors-in-console audit and tests (#2836)

lighthouse-cli/test/smokehouse/dbw-config.js

@@ -18,6 +18,7 @@ module.exports = {
       'dom-size',
       'link-blocking-first-paint',
       'script-blocking-first-paint',
+      'errors-in-console',
     ],
   },
 };

lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js

@@ -13,6 +13,16 @@ module.exports = [
     initialUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
     url: 'http://localhost:10200/dobetterweb/dbw_tester.html',
     audits: {
+      'errors-in-console': {
+        score: false,
+        rawValue: 3,
+        displayValue: '3',
+        details: {
+          items: {
+            length: 3,
+          },
+        },
+      },
       'is-on-https': {
         score: false,
         extendedInfo: {

lighthouse-core/audits/errors-in-console.js

@@ -0,0 +1,61 @@
+/**
+ * @license Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * @fileoverview Audits a page to determine whether it contains console errors.
+ * This is done by collecting Chrome console log messages and filtering out the non-error ones.
+ */
+
+const Audit = require('./audit');
+
+class ErrorLogs extends Audit {
+  /**
+   * @return {!AuditMeta}
+   */
+  static get meta() {
+    return {
+      category: 'ErrorLogs',
+      name: 'errors-in-console',
+      description: 'No browser errors logged to the console',
+      helpText: 'Errors logged to the console indicate unresolved problems. ' +
+        'They can come from network request failures and other browser concerns.',
+      failureDescription: 'Browser errors were logged to the console',
+      requiredArtifacts: ['ChromeConsoleMessages'],
+    };
+  }
+
+  /**
+   * @param {!Artifacts} artifacts
+   * @return {!AuditResult}
+   */
+  static audit(artifacts) {
+    const entries = artifacts.ChromeConsoleMessages;
+    const tableRows = entries.filter(log => log.entry.level === 'error').map(item => {
+      return {
+        source: item.entry.source,
+        description: item.entry.text,
+        url: item.entry.url,
+      };
+    });
+
+    const headings = [
+      {key: 'url', itemType: 'url', text: 'URL'},
+      {key: 'description', itemType: 'text', text: 'Description'},
+    ];
+
+    const details = Audit.makeTableDetails(headings, tableRows);
+    const numErrors = tableRows.length;
+
+    return {
+      score: numErrors === 0,
+      rawValue: numErrors,
+      details,
+    };
+  }
+}
+
+module.exports = ErrorLogs;

lighthouse-core/config/default.js

@@ -72,6 +72,7 @@ module.exports = {
     'speed-index-metric',
     'screenshot-thumbnails',
     'estimated-input-latency',
+    'errors-in-console',
     'time-to-first-byte',
     'first-interactive',
     'consistently-interactive',
@@ -238,7 +239,6 @@ module.exports = {
         {id: 'dom-size', weight: 0, group: 'perf-info'},
         {id: 'critical-request-chains', weight: 0, group: 'perf-info'},
         {id: 'user-timings', weight: 0, group: 'perf-info'},
-
         {id: 'screenshot-thumbnails', weight: 0},
       ],
     },
@@ -300,6 +300,7 @@ module.exports = {
         {id: 'deprecations', weight: 1},
         {id: 'manifest-short-name-length', weight: 1},
         {id: 'password-inputs-can-be-pasted-into', weight: 1},
+        {id: 'errors-in-console', weight: 1},
         {id: 'image-aspect-ratio', weight: 1},
       ],
     },

lighthouse-core/test/audits/errors-in-console-test.js

@@ -0,0 +1,94 @@
+/**
+ * @license Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-env mocha */
+
+const ErrorLogsAudit = require('../../audits/errors-in-console.js');
+const assert = require('assert');
+
+describe('Console error logs audit', () => {
+  it('passes when no console messages were found', () => {
+    const auditResult = ErrorLogsAudit.audit({
+      ChromeConsoleMessages: [],
+    });
+    assert.equal(auditResult.rawValue, 0);
+    assert.equal(auditResult.score, true);
+    assert.ok(!auditResult.displayValue, 0);
+    assert.equal(auditResult.details.items.length, 0);
+  });
+
+  it('filter out the non error logs', () => {
+    const auditResult = ErrorLogsAudit.audit({
+      ChromeConsoleMessages: [
+        {
+          entry: {
+            level: 'info',
+            source: 'network',
+            text: 'This is a simple info msg',
+          },
+        },
+      ],
+    });
+    assert.equal(auditResult.rawValue, 0);
+    assert.equal(auditResult.score, true);
+    assert.equal(auditResult.details.items.length, 0);
+  });
+
+  it('fails when error logs are found ', () => {
+    const auditResult = ErrorLogsAudit.audit({
+      ChromeConsoleMessages: [
+        {
+          entry: {
+            level: 'error',
+            source: 'network',
+            text: 'The server responded with a status of 404 (Not Found)',
+            url: 'http://www.example.com/favicon.ico',
+          },
+        }, {
+          entry: {
+            level: 'error',
+            source: 'network',
+            text: 'WebSocket connection failed: Unexpected response code: 500',
+            url: 'http://www.example.com/wsconnect.ws',
+          },
+        },
+      ],
+    });
+    assert.equal(auditResult.rawValue, 2);
+    assert.equal(auditResult.score, false);
+    assert.equal(auditResult.details.items.length, 2);
+    assert.equal(auditResult.details.items[0][0].type, 'url');
+    assert.equal(auditResult.details.items[0][0].text, 'http://www.example.com/favicon.ico');
+    assert.equal(auditResult.details.items[0][1].type, 'text');
+    assert.equal(auditResult.details.items[0][1].text,
+      'The server responded with a status of 404 (Not Found)');
+    assert.equal(auditResult.details.items[1][0].type, 'url');
+    assert.equal(auditResult.details.items[1][0].text, 'http://www.example.com/wsconnect.ws');
+    assert.equal(auditResult.details.items[1][1].type, 'text');
+    assert.equal(auditResult.details.items[1][1].text,
+      'WebSocket connection failed: Unexpected response code: 500');
+  });
+
+  it('handle the case when some logs fields are undefined', () => {
+    const auditResult = ErrorLogsAudit.audit({
+      ChromeConsoleMessages: [
+        {
+          entry: {
+            level: 'error',
+          },
+        },
+      ],
+    });
+    assert.equal(auditResult.rawValue, 1);
+    assert.equal(auditResult.score, false);
+    assert.equal(auditResult.details.items.length, 1);
+    // url is undefined
+    assert.strictEqual(auditResult.details.items[0][0].text, undefined);
+    // text is undefined
+    assert.strictEqual(auditResult.details.items[0][1].text, undefined);
+  });
+});