core(is-crawlable): make sure that page is not blocked by robots.txt file

lighthouse-core/audits/seo/is-crawlable.js

@@ -6,6 +6,8 @@
 'use strict';
 
 const Audit = require('../audit');
+const robotsParser = require('robots-parser');
+const URL = require('../../lib/url-shim');
 const BLOCKLIST = new Set([
   'noindex',
   'none',
@@ -65,7 +67,7 @@ class IsCrawlable extends Audit {
       failureDescription: 'Page is blocked from indexing',
       helpText: 'The "Robots" directives tell crawlers how your content should be indexed. ' +
       '[Learn more](https://developers.google.com/search/reference/robots_meta_tag).',
-      requiredArtifacts: ['MetaRobots'],
+      requiredArtifacts: ['MetaRobots', 'RobotsTxt'],
     };
   }
 
@@ -96,8 +98,22 @@ class IsCrawlable extends Audit {
             hasBlockingDirective(h.value))
           .forEach(h => blockingDirectives.push({source: `${h.name}: ${h.value}`}));
 
+        if (artifacts.RobotsTxt.content) {
+          const robotsFileUrl = new URL('/robots.txt', mainResource.url);
+          const robotsTxt = robotsParser(robotsFileUrl.href, artifacts.RobotsTxt.content);
+
+          if (!robotsTxt.isAllowed(mainResource.url)) {
+            blockingDirectives.push({
+              source: {
+                type: 'url',
+                text: robotsFileUrl.href,
+              },
+            });
+          }
+        }
+
         const headings = [
-          {key: 'source', itemType: 'code', text: 'Source'},
+          {key: 'source', itemType: 'code', text: 'Blocking Directive Source'},
         ];
         const details = Audit.makeTableDetails(headings, blockingDirectives);
 

lighthouse-core/config/default.js

@@ -45,6 +45,7 @@ module.exports = {
       'seo/hreflang',
       'seo/embedded-content',
       'seo/canonical',
+      'seo/robots-txt',
       'fonts',
     ],
   },

lighthouse-core/gather/gatherers/seo/robots-txt.js

@@ -0,0 +1,36 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const Gatherer = require('../gatherer');
+
+/* global fetch, URL, location */
+
+function getRobotsTxtContent() {
+  return fetch(new URL('/robots.txt', location.href))
+    .then(response => {
+      if (!response.ok) {
+        return {status: response.status, content: null};
+      }
+
+      return response.text()
+        .then(content => ({status: response.status, content}));
+    })
+    .catch(_ => ({status: null, content: null}));
+}
+
+
+class RobotsTxt extends Gatherer {
+  /**
+   * @param {{driver: !Driver}} options Run options
+   * @return {!Promise<!{code: number, content: string}>}
+   */
+  afterPass(options) {
+    return options.driver.evaluateAsync(`(${getRobotsTxtContent.toString()}())`);
+  }
+}
+
+module.exports = RobotsTxt;

lighthouse-core/test/audits/seo/is-crawlable-test.js

@@ -30,6 +30,7 @@ describe('SEO: Is page crawlable audit', () => {
         devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
         requestMainResource: () => Promise.resolve(mainResource),
         MetaRobots: robotsValue,
+        RobotsTxt: {},
       };
 
       return IsCrawlableAudit.audit(artifacts).then(auditResult => {
@@ -49,6 +50,7 @@ describe('SEO: Is page crawlable audit', () => {
       devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
       requestMainResource: () => Promise.resolve(mainResource),
       MetaRobots: 'all, noarchive',
+      RobotsTxt: {},
     };
 
     return IsCrawlableAudit.audit(artifacts).then(auditResult => {
@@ -64,6 +66,7 @@ describe('SEO: Is page crawlable audit', () => {
       devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
       requestMainResource: () => Promise.resolve(mainResource),
       MetaRobots: null,
+      RobotsTxt: {},
     };
 
     return IsCrawlableAudit.audit(artifacts).then(auditResult => {
@@ -102,6 +105,7 @@ describe('SEO: Is page crawlable audit', () => {
         devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
         requestMainResource: () => Promise.resolve(mainResource),
         MetaRobots: null,
+        RobotsTxt: {},
       };
 
       return IsCrawlableAudit.audit(artifacts).then(auditResult => {
@@ -124,21 +128,23 @@ describe('SEO: Is page crawlable audit', () => {
       devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
       requestMainResource: () => Promise.resolve(mainResource),
       MetaRobots: null,
+      RobotsTxt: {},
     };
 
     return IsCrawlableAudit.audit(artifacts).then(auditResult => {
       assert.equal(auditResult.rawValue, true);
     });
   });
 
-  it('succeeds when there is no robots header', () => {
+  it('succeeds when there is no robots header and robots.txt is unavailable', () => {
     const mainResource = {
       responseHeaders: [],
     };
     const artifacts = {
       devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
       requestMainResource: () => Promise.resolve(mainResource),
       MetaRobots: null,
+      RobotsTxt: {},
     };
 
     return IsCrawlableAudit.audit(artifacts).then(auditResult => {
@@ -157,29 +163,120 @@ describe('SEO: Is page crawlable audit', () => {
       devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
       requestMainResource: () => Promise.resolve(mainResource),
       MetaRobots: null,
+      RobotsTxt: {},
     };
 
     return IsCrawlableAudit.audit(artifacts).then(auditResult => {
       assert.equal(auditResult.rawValue, true);
     });
   });
 
+  it('fails when page is blocked from indexing by robots.txt', () => {
+    const robotsTxts = [
+      {
+        content: `User-agent: *
+        Disallow: /`,
+      },
+      {
+        content: `User-agent: *
+        Disallow: /test/page.html`,
+      },
+      {
+        content: `User-agent: *
+        Disallow:
+
+        User-agent: *
+        Disallow: /`,
+      },
+      {
+        content: `User-agent: *
+        Disallow: /one/
+        Disallow: /two/
+        Disallow: /test/
+        Allow: page.html
+        # Allow: /test/page.html
+        Allow: /test/page.html /someother/url.html`,
+      },
+    ];
+
+    const allRuns = robotsTxts.map(robotsTxt => {
+      const mainResource = {
+        url: 'http://example.com/test/page.html',
+        responseHeaders: [],
+      };
+      const artifacts = {
+        devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
+        requestMainResource: () => Promise.resolve(mainResource),
+        MetaRobots: null,
+        RobotsTxt: robotsTxt,
+      };
+
+      return IsCrawlableAudit.audit(artifacts).then(auditResult => {
+        assert.equal(auditResult.rawValue, false);
+        assert.equal(auditResult.details.items.length, 1);
+      });
+    });
+
+    return Promise.all(allRuns);
+  });
+
+  it('succeeds when page is allowed by robots.txt', () => {
+    const robotsTxts = [
+      {
+        content: `User-agent: SomeBot
+        Disallow: /`,
+      },
+      {
+        content: `User-agent: *
+        Disallow: /_/
+        Disallow: /search?q=*
+        Disallow: /test/
+        Allow: /test/page.html`,
+      },
+    ];
+
+    const allRuns = robotsTxts.map(robotsTxt => {
+      const mainResource = {
+        url: 'http://example.com/test/page.html',
+        responseHeaders: [],
+      };
+      const artifacts = {
+        devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
+        requestMainResource: () => Promise.resolve(mainResource),
+        MetaRobots: null,
+        RobotsTxt: robotsTxt,
+      };
+
+      return IsCrawlableAudit.audit(artifacts).then(auditResult => {
+        assert.equal(auditResult.rawValue, true);
+      });
+    });
+
+    return Promise.all(allRuns);
+  });
+
   it('returns all failing items', () => {
     const mainResource = {
+      url: 'http://example.com/test/page.html',
       responseHeaders: [
         {name: 'x-robots-tag', value: 'none'},
         {name: 'x-robots-tag', value: 'noindex'},
       ],
     };
+    const robotsTxt = {
+      content: `User-agent: *
+      Disallow: /`,
+    };
     const artifacts = {
       devtoolsLogs: {[IsCrawlableAudit.DEFAULT_PASS]: []},
       requestMainResource: () => Promise.resolve(mainResource),
       MetaRobots: 'noindex',
+      RobotsTxt: robotsTxt,
     };
 
     return IsCrawlableAudit.audit(artifacts).then(auditResult => {
       assert.equal(auditResult.rawValue, false);
-      assert.equal(auditResult.details.items.length, 3);
+      assert.equal(auditResult.details.items.length, 4);
     });
   });
 });

package.json

@@ -96,6 +96,7 @@
     "parse-cache-control": "1.0.1",
     "raven": "^2.2.1",
     "rimraf": "^2.6.1",
+    "robots-parser": "^1.0.2",
     "semver": "^5.3.0",
     "speedline": "1.3.0",
     "update-notifier": "^2.1.0",

yarn.lock

@@ -3602,6 +3602,10 @@ rimraf@~2.2.6:
   version "2.2.8"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582"
 
+robots-parser@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.npmjs.org/robots-parser/-/robots-parser-1.0.2.tgz#9ebe25b1a2c52773cbe6f1dbe90ebc9518089009"
+
 run-async@^2.2.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.3.0.tgz#0371ab4ae0bdd720d4166d7dfda64ff7a445a6c0"