[WIP]: feat(gatsby): use lokijs for in-memory database

benchmarks/query/README.md

@@ -0,0 +1,5 @@
+# Query benchmark
+
+Stress tests creating lots of queries.
+
+Defaults to building a site with 5k pages split evenly amongst 10 types. Set the `NUM_PAGES` environment variable to change the number of pages, and `NUM_TYPES` to change the number of types they're split over. E.g to create a site with 5 types, each with 200 pages, do `NUM_TYPES=5 NUM_PAGES=1000 gatsby build`

benchmarks/query/bin/runQueryTiming.sh

@@ -0,0 +1,8 @@
+# Run the build (after purging .cache) and output the amount of time
+# taken by the query execution phase
+#
+# run with `bin/runQueryTiming.sh`
+
+output=$(rm -rf .cache && gatsby build | grep "run graphql queries")
+echo $output | cut -d' ' -f 6
+

benchmarks/query/gatsby-node.js

@@ -0,0 +1,117 @@
+const _ = require(`lodash`)
+const faker = require(`faker`)
+const fs = require(`fs`)
+
+let NUM_PAGES = 5000
+if (process.env.NUM_PAGES) {
+  NUM_PAGES = process.env.NUM_PAGES
+}
+
+let NUM_TYPES = 1
+if (process.env.NUM_TYPES) {
+  NUM_TYPES = process.env.NUM_TYPES
+}
+
+function newTypeName() {
+  return _.capitalize(_.camelCase(faker.lorem.word()))
+}
+
+let types = []
+
+// Create NUM_PAGES nodes, split over NUM_TYPES types. Each node has
+// the bare minimum of content
+exports.sourceNodes = ({ actions: { createNode } }) => {
+  for (var i = 0; i < NUM_TYPES; i++) {
+    types.push(newTypeName())
+  }
+  // Create markdown nodes
+  const pagesPerType = NUM_PAGES / NUM_TYPES
+
+  let step = 0
+
+  _.forEach(types, typeName => {
+    for (var i = 0; i < pagesPerType; i++) {
+      step++
+      const id = `${typeName}${step.toString()}`
+      createNode({
+        id,
+        parent: null,
+        children: [],
+        internal: {
+          type: typeName,
+          nestedId: id,
+          content: faker.lorem.word(),
+          contentDigest: step.toString(),
+        },
+      })
+    }
+  })
+}
+
+// Total hack. It would be nice if we could programatically generate
+// graphQL per component. But in the meantime, we just generate the
+// actual component js file with the graphql
+function createPageTemplateJs(typeName) {
+  const lowerTypeName = _.lowerFirst(typeName)
+  return `
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  const node = data["${lowerTypeName}"]
+  return (
+    <div>
+      <h1>{node.id}. Not much ey</h1>
+    </div>
+  )
+}
+
+export const query = graphql\`
+  query($id: String!) {
+    ${lowerTypeName}(internal: { nestedId: { eq: $id } }) {
+      id
+    }
+  }
+\`
+`
+}
+
+function allTypeQuery(typeName) {
+  return `
+{
+  all${typeName}(sort: { fields: [id] }) {
+    edges {
+      node {
+        id
+      }
+    }
+  }
+}
+`
+}
+
+// Create a page for each node, and write out a new component js for
+// each different type to .cache/${typeName}Template.js
+async function createTypePages({ graphql, actions }, typeName) {
+  const templateSrc = createPageTemplateJs(typeName)
+  const templateFilename = `./.cache/${typeName}Template.js`
+  fs.writeFileSync(templateFilename, templateSrc)
+  let result = await graphql(allTypeQuery(typeName))
+  _.forEach(result.data[`all${typeName}`].edges, edge => {
+    const { node } = edge
+    actions.createPage({
+      path: `/${typeName}/${node.id}/`,
+      component: require.resolve(templateFilename),
+      context: {
+        id: node.id,
+        useQueryIndex: true,
+      },
+    })
+  })
+}
+
+exports.createPages = async args => {
+  _.forEach(types, typeName => {
+    createTypePages(args, typeName)
+  })
+}

benchmarks/query/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "query-benchmark",
+  "description": "Benchmarks for Gatsby query performance",
+  "license": "MIT",
+  "scripts": {
+    "develop": "gatsby develop",
+    "build": "gatsby build",
+    "serve": "gatsby serve"
+  },
+  "dependencies": {
+    "faker": "^4.1.0",
+    "gatsby": "next",
+    "lodash": "^4.17.11",
+    "react": "^16.3.2",
+    "react-dom": "^16.3.2"
+  }
+}

benchmarks/query/recording.md

@@ -0,0 +1,88 @@
+## Summary
+
+Key findings:
+
+- loki without indexes is overall slightly faster than master, except when there are many types
+- loki with indexes is about 2x faster on sites with 10k pages, and 5x faster with 20k pages. But is ever so slightly slower when those pages are split across 100 types.
+
+Overall, loki is a big win for sites with lots of pages of the same type. For smaller sites, the difference is negligible.
+
+## Benchmarks
+
+Performed on 2018 13" MBP. 4-core 2.7 GHz Intel Core i7. 16 GB 2133 MHz LPDDR3
+
+### Gatsby master
+
+- Gatsby: master
+
+```
+query $ NUM_TYPES=1 NUM_PAGES=10000 bin/runQueryTiming.sh
+21.135
+```
+
+```
+query $ NUM_TYPES=100 NUM_PAGES=10000 bin/runQueryTiming.sh
+13.112
+```
+
+```
+query $ NUM_TYPES=1 NUM_PAGES=20000 bin/runQueryTiming.sh
+67.812
+```
+
+```
+query $ NUM_TYPES=100 NUM_PAGES=20000 bin/runQueryTiming.sh
+24.656
+```
+
+### Gatsby loki without index
+
+- Gatsby:loki
+- Index = false
+- loki nested index patch
+
+```
+query $ NUM_TYPES=1 NUM_PAGES=10000 bin/runQueryTiming.sh
+14.834
+```
+
+```
+query $ NUM_TYPES=100 NUM_PAGES=10000 bin/runQueryTiming.sh
+14.676
+```
+
+```
+query $ NUM_TYPES=1 NUM_PAGES=20000 bin/runQueryTiming.sh
+58.377
+```
+
+```
+query $ NUM_TYPES=100 NUM_PAGES=20000 bin/runQueryTiming.sh
+27.486
+```
+
+### Gatsby loki with index
+
+- Gatsby:loki
+- Index = true
+- loki nested index patch
+
+```
+query $ NUM_TYPES=1 NUM_PAGES=10000 bin/runQueryTiming.sh
+8.126
+```
+
+```
+query $ NUM_TYPES=100 NUM_PAGES=10000 bin/runQueryTiming.sh
+15.050
+```
+
+```
+query $ NUM_TYPES=1 NUM_PAGES=20000 bin/runQueryTiming.sh
+12.797
+```
+
+```
+query $ NUM_TYPES=100 NUM_PAGES=20000 bin/runQueryTiming.sh
+27.020
+```

benchmarks/query/src/pages/index.js

@@ -0,0 +1,3 @@
+import React from "react"
+
+export default () => <div>Hello world!</div>

packages/gatsby-source-contentful/src/gatsby-node.js

@@ -216,17 +216,16 @@ exports.sourceNodes = async (
 // Check if there are any ContentfulAsset nodes and if gatsby-image is installed. If so,
 // add fragments for ContentfulAsset and gatsby-image. The fragment will cause an error
 // if there's not ContentfulAsset nodes and without gatsby-image, the fragment is useless.
-exports.onPreExtractQueries = async ({ store, getNodes }) => {
+exports.onPreExtractQueries = async ({ store, getNodesByType }) => {
   const program = store.getState().program
 
   const CACHE_DIR = path.resolve(
     `${program.directory}/.cache/contentful/assets/`
   )
   await fs.ensureDir(CACHE_DIR)
 
-  const nodes = getNodes()
-
-  if (!nodes.some(n => n.internal.type === `ContentfulAsset`)) {
+  const nodes = getNodesByType(`ContentfulAsset`)
+  if (!nodes || !nodes.length || nodes.length === 0) {
     return
   }
 

packages/gatsby-transformer-remark/src/extend-node-type.js

@@ -62,7 +62,7 @@ const withPathPrefix = (url, pathPrefix) =>
 const ASTPromiseMap = new Map()
 
 module.exports = (
-  { type, store, pathPrefix, getNode, getNodes, cache, reporter },
+  { type, store, pathPrefix, getNode, getNodesByType, cache, reporter },
   pluginOptions
 ) => {
   if (type.name !== `MarkdownRemark`) {
@@ -74,7 +74,13 @@ module.exports = (
 
   return new Promise((resolve, reject) => {
     // Setup Remark.
-    const { commonmark = true, footnotes = true, pedantic = true, gfm = true, blocks } = pluginOptions
+    const {
+      commonmark = true,
+      footnotes = true,
+      pedantic = true,
+      gfm = true,
+      blocks,
+    } = pluginOptions
     const remarkOptions = {
       gfm,
       commonmark,
@@ -113,7 +119,7 @@ module.exports = (
       } else {
         const ASTGenerationPromise = new Promise(async resolve => {
           if (process.env.NODE_ENV !== `production` || !fileNodes) {
-            fileNodes = getNodes().filter(n => n.internal.type === `File`)
+            fileNodes = getNodesByType(`File`)
           }
           const ast = await new Promise((resolve, reject) => {
             // Use Bluebird's Promise function "each" to run remark plugins serially.
@@ -180,7 +186,7 @@ module.exports = (
               // typegen plugins just modify the auto-generated types to add derived fields
               // as well as computationally expensive fields.
               if (process.env.NODE_ENV !== `production` || !fileNodes) {
-                fileNodes = getNodes().filter(n => n.internal.type === `File`)
+                fileNodes = getNodesByType(`File`)
               }
               // Use Bluebird's Promise function "each" to run remark plugins serially.
               Promise.each(pluginOptions.plugins, plugin => {
@@ -249,10 +255,16 @@ module.exports = (
           const addSlugToUrl = function(node) {
             if (node.url) {
               if (_.get(markdownNode, pathToSlugField) === undefined) {
-                console.warn(`Skipping TableOfContents. Field '${pathToSlugField}' missing from markdown node`)
+                console.warn(
+                  `Skipping TableOfContents. Field '${pathToSlugField}' missing from markdown node`
+                )
                 return null
               }
-              node.url = [pathPrefix, _.get(markdownNode, pathToSlugField), node.url]
+              node.url = [
+                pathPrefix,
+                _.get(markdownNode, pathToSlugField),
+                node.url,
+              ]
                 .join(`/`)
                 .replace(/\/\//g, `/`)
             }

packages/gatsby-transformer-screenshot/src/gatsby-node.js

@@ -16,13 +16,11 @@ const screenshotQueue = new Queue(
 )
 
 exports.onPreBootstrap = (
-  { store, cache, actions, createNodeId, getNodes, createContentDigest },
+  { store, cache, actions, createNodeId, getNodesByType, createContentDigest },
   pluginOptions
 ) => {
   const { createNode, touchNode } = actions
-  const screenshotNodes = getNodes().filter(
-    n => n.internal.type === `Screenshot`
-  )
+  const screenshotNodes = getNodesByType(`Screenshot`)
 
   if (screenshotNodes.length === 0) {
     return null

packages/gatsby-transformer-sharp/src/gatsby-node.js

@@ -3,14 +3,13 @@ const fs = require(`fs-extra`)
 exports.onCreateNode = require(`./on-node-create`)
 exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`)
 
-exports.onPreExtractQueries = async ({ store, getNodes }) => {
+exports.onPreExtractQueries = async ({ store, getNodesByType }) => {
   const program = store.getState().program
 
   // Check if there are any ImageSharp nodes. If so add fragments for ImageSharp.
   // The fragment will cause an error if there are no ImageSharp nodes.
-  const nodes = getNodes()
-
-  if (!nodes.some(n => n.internal.type === `ImageSharp`)) {
+  const nodes = getNodesByType(`ImageSharp`)
+  if (!nodes || !nodes.length || nodes.length === 0) {
     return
   }
 

packages/gatsby/package.json

@@ -83,6 +83,7 @@
     "json-stringify-safe": "^5.0.1",
     "kebab-hash": "^0.1.2",
     "lodash": "^4.17.10",
+    "lokijs": "^1.5.5",
     "md5": "^2.2.1",
     "md5-file": "^3.1.1",
     "mime": "^2.2.0",