Create prepareSitemap Batfish helper to improve sitemap (#427)

* Create prepareSitemap Batfish helper

* Update CHANGELOG.md

* Clean up, add `docsPath`, and `outputDirectory`

* Exclude `splitPage` from sitemap

* Assert sitemap

* Extract urls

* Sort sitemap

* Remove snapshot

* Update sitemap.test.js

* Update batfish-helpers.md

* Update batfish-helpers.md

* Copyedit

* Update package-lock.json

.travis.yml

@@ -6,3 +6,5 @@ notifications:
 scripts:
   - npm test
   - npm run build-docs
+  # run sitemap tests after build to assert sitemap is correct
+  - jest src/helpers/batfish/__tests__/sitemap.test.js

CHANGELOG.md

@@ -5,6 +5,13 @@
 - Enforce use of React PureComponents to improve performance. [#414](https://github.com/mapbox/dr-ui/pull/414)
 - Improve color contrast on code and anchor elements. [#434](https://github.com/mapbox/dr-ui/pull/434)
 - Enable eslint-plugin-jsx-a11y. [#435](https://github.com/mapbox/dr-ui/pull/435)
+- Create `prepareSitemap` Batfish helper to improve sitemap. [#427](https://github.com/mapbox/dr-ui/pull/427). See [`prepareSitemap`](https://mapbox.github.io/dr-ui/guides/batfish-helpers/#prepare-sitemap) for instructions on how to add the `dataSelector` to your Batfish configuration. You will also want to add the following to the top of any redirect files so that they will be excluded from the sitemap:
+
+```js
+/*---
+hideFromSearchEngines: true
+---*/
+```
 
 ## 3.3.1
 

docs/batfish.config.js

@@ -2,7 +2,8 @@ const path = require('path');
 const {
   buildNavigation,
   buildFilters,
-  buildSplitPages
+  buildSplitPages,
+  prepareSitemap
 } = require('../src/helpers/batfish/index.js');
 
 const siteBasePath = '/dr-ui';
@@ -22,6 +23,7 @@ const addPages = [
 
 module.exports = () => {
   return {
+    siteOrigin: 'https://mapbox.github.io',
     siteBasePath: siteBasePath,
     outputDirectory: path.join(__dirname, '_site/'),
     temporaryDirectory: path.join(__dirname, '_site_tmp/'),
@@ -47,10 +49,16 @@ module.exports = () => {
         require('../src/plugins/make-table-scroll')
       ]
     },
+    sitemap: {
+      // generated by the sitemapIgnore dataSelector
+      ignoreFile: 'docs/_site_tmp/data/sitemap-ignore.js'
+    },
     dataSelectors: {
       navigation: (data) => buildNavigation({ siteBasePath, data, addPages }),
       filters: (data) => buildFilters(data),
       splitPages: (data) => buildSplitPages(data),
+      sitemapIgnore: ({ pages }) =>
+        prepareSitemap({ pages, siteBasePath, docsPath: 'docs' }),
       sync: (data) => {
         /* syncs data to fixtures to properly test batfish selectors */
         const sortBy = (key) => {

docs/src/pages/guides/batfish-helpers.md

@@ -10,6 +10,7 @@ prependJs:
   - "import navigation from '@mapbox/batfish/data/navigation'; // eslint-disable-line"
   - "import topics from '@mapbox/batfish/data/filters'; // eslint-disable-line"
   - "import splitPages from '@mapbox/batfish/data/split-pages'; // eslint-disable-line"
+  - "import sitemapIgnore from '@mapbox/batfish/data/sitemap-ignore'; // eslint-disable-line"
 ---
 
 Dr. UI has two functions that are used in `dataSelectors` in `batfish.config.js` to help build page metadata and site hierarchy. Each data selector has tests to assert the shape of the data.
@@ -252,6 +253,69 @@ class PageShell extends React.Component {
 {{JSON.stringify(splitPages, null,2)}}
 ```
 
+## Prepare sitemap
+
+`prepareSitemap` creates a data file that lists all files with `hideFromSearchEngines: true` or `splitPage: true` set in the page's frontMatter. Use this generated file with Batfish's [`sitemap.ignoreFile`](https://github.com/mapbox/batfish/blob/main/docs/configuration.md#sitemap) option to generate a sitemap that excludes any paths in the prepared file.
+
+### Arguments
+
+- `data`, object. Provided by the data selector.
+- `siteBasePath`, string. The site's `siteBasePath` as defined in the Batfish config.
+- `outputDirectory`, string. Batfish's `outputDirectory`. Default `_site`.
+- `docsPath`, string. The directory that `batfish.config.js` lives in, if not in the root. (It's unlikely that you will use this option.)
+
+### Set up in batfish.config.js
+
+```js
+const { prepareSitemap } = require('@mapbox/dr-ui/helpers/batfish/index.js');
+
+module.exports = () => {
+  return {
+    sitemap: {
+      // generated by the sitemapIgnore dataSelector
+      ignoreFile: '_site_tmp/data/sitemap-ignore.js'
+    },
+    dataSelectors: {
+      sitemapIgnore: ({ pages }) => prepareSitemap({ pages, siteBasePath })
+    }
+  };
+};
+```
+
+#### Advanced usage
+
+For sites that generate demos, like mapbox-gl-js and mts-docs, you can exclude the assets folder:
+
+```js
+const { prepareSitemap } = require('@mapbox/dr-ui/helpers/batfish/index.js');
+
+module.exports = () => {
+  return {
+    sitemap: {
+      // generated by the sitemapIgnore dataSelector
+      ignoreFile: '_site_tmp/data/sitemap-ignore.js'
+    },
+    dataSelectors: {
+      sitemapIgnore: ({ pages }) => [
+        ...prepareSitemap({ pages, siteBasePath }),
+        // exclude demos generated by webpack loader
+        `${process.cwd()}/_site/assets/`
+      ]
+    }
+  };
+};
+```
+
+### Output
+
+- The shape is an array of paths that should be excluded from the sitemap.
+
+#### Sample
+
+```json
+{{JSON.stringify(sitemapIgnore, null,2)}}
+```
+
 ## Troubleshooting
 
 If data is not building as you expect:

docs/src/pages/index.js

@@ -1,3 +1,7 @@
+/*---
+hideFromSearchEngines: true
+---*/
+
 import { prefixUrl } from '@mapbox/batfish/modules/prefix-url';
 import { createRedirect } from '../../../src/helpers/create-redirect';
 

src/helpers/batfish/__tests__/fixtures/data.json

@@ -148,7 +148,8 @@
         "prependJs": [
           "import navigation from '@mapbox/batfish/data/navigation'; // eslint-disable-line",
           "import topics from '@mapbox/batfish/data/filters'; // eslint-disable-line",
-          "import splitPages from '@mapbox/batfish/data/split-pages'; // eslint-disable-line"
+          "import splitPages from '@mapbox/batfish/data/split-pages'; // eslint-disable-line",
+          "import sitemapIgnore from '@mapbox/batfish/data/sitemap-ignore'; // eslint-disable-line"
         ]
       }
     },
@@ -275,7 +276,9 @@
     {
       "filePath": "./docs/src/pages/index.js",
       "path": "/dr-ui/",
-      "frontMatter": {}
+      "frontMatter": {
+        "hideFromSearchEngines": true
+      }
     },
     {
       "filePath": "./node_modules/@mapbox/batfish/dist/webpack/default-not-found.js",

src/helpers/batfish/__tests__/sitemap.test.js

@@ -0,0 +1,94 @@
+const prepareSitemap = require('../sitemap');
+const { pages } = require('./fixtures/data.json');
+const { existsSync, readFileSync } = require('fs');
+
+const siteMap = './docs/_site/sitemap.xml';
+const siteBasePath = '/dr-ui';
+
+describe('prepareSitemap', () => {
+  it('works', () => {
+    const ignoreFile = prepareSitemap({ pages, siteBasePath }).map((f) =>
+      f.replace(process.cwd(), '')
+    );
+    expect(ignoreFile).toEqual([
+      '/_site/guides/split-pages/sections/how-to/',
+      '/_site/guides/split-pages/sections/intro/',
+      '/_site/guides/split-pages/sections/limitations/',
+      '/_site/guides/split-pages/sections/tag-debugger/',
+      '/_site/index.html'
+    ]);
+  });
+
+  it('advanced, exclude demos generated by webpack loader', () => {
+    const ignoreFile = [
+      ...prepareSitemap({ pages, siteBasePath }),
+      `${process.cwd()}/_site/assets/`
+    ].map((f) => f.replace(process.cwd(), ''));
+    expect(ignoreFile).toEqual([
+      '/_site/guides/split-pages/sections/how-to/',
+      '/_site/guides/split-pages/sections/intro/',
+      '/_site/guides/split-pages/sections/limitations/',
+      '/_site/guides/split-pages/sections/tag-debugger/',
+      '/_site/index.html',
+      '/_site/assets/'
+    ]);
+  });
+
+  it('with docsPath', () => {
+    const ignoreFile = prepareSitemap({
+      pages,
+      siteBasePath,
+      docsPath: 'docs'
+    }).map((f) => f.replace(process.cwd(), ''));
+    expect(ignoreFile).toEqual([
+      '/docs/_site/guides/split-pages/sections/how-to/',
+      '/docs/_site/guides/split-pages/sections/intro/',
+      '/docs/_site/guides/split-pages/sections/limitations/',
+      '/docs/_site/guides/split-pages/sections/tag-debugger/',
+      '/docs/_site/index.html'
+    ]);
+  });
+
+  it('with outputDirectory', () => {
+    const ignoreFile = prepareSitemap({
+      pages,
+      siteBasePath,
+      outputDirectory: '_batfish_site'
+    }).map((f) => f.replace(process.cwd(), ''));
+    expect(ignoreFile).toEqual([
+      '/_batfish_site/guides/split-pages/sections/how-to/',
+      '/_batfish_site/guides/split-pages/sections/intro/',
+      '/_batfish_site/guides/split-pages/sections/limitations/',
+      '/_batfish_site/guides/split-pages/sections/tag-debugger/',
+      '/_batfish_site/index.html'
+    ]);
+  });
+
+  // travis is configured to run this test file again after build
+  // this test asserts the URLs in the sitemap
+  if (existsSync(siteMap)) {
+    it('sitemap', () => {
+      const contents = readFileSync(siteMap, 'utf-8')
+        .match(
+          /(?<=(<loc>))(.*)+?(?=(<\/loc>))/gm // eslint-disable-line
+        )
+        .sort();
+      expect(contents).toEqual([
+        'https://mapbox.github.io/dr-ui/changelog',
+        'https://mapbox.github.io/dr-ui/components',
+        'https://mapbox.github.io/dr-ui/examples',
+        'https://mapbox.github.io/dr-ui/examples/example-1',
+        'https://mapbox.github.io/dr-ui/examples/example-2',
+        'https://mapbox.github.io/dr-ui/examples/example-3',
+        'https://mapbox.github.io/dr-ui/examples/example-4',
+        'https://mapbox.github.io/dr-ui/examples/example-5',
+        'https://mapbox.github.io/dr-ui/guides',
+        'https://mapbox.github.io/dr-ui/guides/a11y',
+        'https://mapbox.github.io/dr-ui/guides/batfish-helpers',
+        'https://mapbox.github.io/dr-ui/guides/multi-structured',
+        'https://mapbox.github.io/dr-ui/guides/page-layout',
+        'https://mapbox.github.io/dr-ui/guides/split-pages'
+      ]);
+    });
+  }
+});

src/helpers/batfish/index.js

@@ -1,9 +1,11 @@
 const { buildNavigation } = require('./navigation');
 const { buildFilters } = require('./filters');
 const { buildSplitPages } = require('./split-pages');
+const prepareSitemap = require('./sitemap');
 
 module.exports = {
   buildNavigation,
   buildFilters,
-  buildSplitPages
+  buildSplitPages,
+  prepareSitemap
 };

src/helpers/batfish/sitemap.js

@@ -0,0 +1,32 @@
+const { join } = require('path');
+
+function prepareSitemap({
+  pages,
+  siteBasePath,
+  docsPath = '',
+  outputDirectory = '_site'
+}) {
+  // find pages with `hideFromSearchEngines: true` in frontMatter or `splitPage: true`
+  // we do not want these pages to appear in the sitemap
+  return pages
+    .filter(
+      ({ frontMatter }) =>
+        frontMatter.hideFromSearchEngines || frontMatter.splitPage
+    )
+    .map(({ path }) => {
+      // fix root index path to return full path
+      if (path === `${siteBasePath}/`) {
+        return path.replace(
+          join(siteBasePath, '/'),
+          join(process.cwd(), docsPath, outputDirectory, 'index.html')
+        );
+      } else {
+        return path.replace(
+          siteBasePath,
+          join(process.cwd(), docsPath, outputDirectory)
+        );
+      }
+    });
+}
+
+module.exports = prepareSitemap;