request all (if more than 1000) blog posts from Contentful

.babelrc

@@ -0,0 +1,8 @@
+{
+  "presets": ["babel-preset-gatsby"],
+  "env": {
+    "test": {
+      "plugins": ["require-context-hook"]
+    }
+  }
+}

src/functions/blogMethods/__tests__/filter-posts-to-process.test.js

@@ -1,5 +1,5 @@
 /* eslint-disable no-console */
-const FilterPostsToProcess = require('../../../../src/functions/blogMethods/filter-posts-to-process');
+const FilterPostsToProcess = require('../filter-posts-to-process');
 
 const originalConsoleInfo = console.info;
 beforeEach(() => {

src/functions/blogMethods/__tests__/get-all-contentful-blog-post.test.js

@@ -0,0 +1,33 @@
+const GetAllContentfulBlogPosts = require('../get-all-contentful-blog-posts');
+
+const posts = [...Array(2 ** 16).keys()].map(i => ({ slug: `post-${i}` }));
+
+const mockGetEntries = jest.fn(({ skip, limit }) => ({
+  skip,
+  limit,
+  total: posts.length,
+  items: posts.slice(skip, skip + limit),
+}));
+const environment = { getEntries: mockGetEntries };
+
+beforeEach(() => {
+  mockGetEntries.mockClear();
+});
+
+it('requests blog posts', async () => {
+  await GetAllContentfulBlogPosts(environment);
+  mockGetEntries.mock.calls.map(([{ content_type }]) => {
+    expect(content_type).toBe('blogPost');
+  });
+});
+
+it('requests until it has all entries and returns them', async () => {
+  expect(await GetAllContentfulBlogPosts(environment)).toEqual(posts);
+});
+
+it('does not exceed the contentful entry limit', async () => {
+  await GetAllContentfulBlogPosts(environment);
+  mockGetEntries.mock.calls.map(([{ limit }]) => {
+    expect(limit).toBeLessThanOrEqual(1000);
+  });
+});

src/functions/blogMethods/__tests__/parse-xml-to-json.test.js

@@ -1,4 +1,4 @@
-const ParseXMLToJSON = require('../../../../src/functions/blogMethods/parse-xml-to-json');
+const ParseXMLToJSON = require('../parse-xml-to-json');
 const { readFileSync } = require('fs');
 const { resolve } = require('path');
 

src/functions/blogMethods/__tests__/publish-to-contentful.test.js

@@ -1,5 +1,5 @@
 /* eslint-disable no-console */
-const PublishToContentful = require('../../../../src/functions/blogMethods/publish-to-contentful');
+const PublishToContentful = require('../publish-to-contentful');
 const until = require('async-wait-until');
 
 const originalConsoleInfo = console.info;

src/functions/blogMethods/__tests__/transform-html-to-markdown.test.js

@@ -1,4 +1,4 @@
-const TransformHTMLToMarkdown = require('../../../../src/functions/blogMethods/transform-html-to-markdown');
+const TransformHTMLToMarkdown = require('../transform-html-to-markdown');
 
 const tweetBlockquote = `
   <blockquote class="twitter-tweet">

src/functions/blogMethods/__tests__/transform-meta-data.test.js

@@ -1,4 +1,4 @@
-const TransformMetaData = require('../../../../src/functions/blogMethods/transform-meta-data');
+const TransformMetaData = require('../transform-meta-data');
 
 const headerImage = {
   sys: {

src/functions/blogMethods/__tests__/validate-mdx.test.js

@@ -1,4 +1,4 @@
-const ValidateMdx = require('../../../../src/functions/blogMethods/validate-mdx');
+const ValidateMdx = require('../validate-mdx');
 
 it.each`
   description      | mdx

src/functions/blogMethods/custom-mdx/__tests__/add-frontmatter.test.js

@@ -1,4 +1,4 @@
-const AddFrontMatter = require('../../../../../src/functions/blogMethods/custom-mdx/add-frontmatter');
+const AddFrontMatter = require('../add-frontmatter');
 
 const post = {
   title: 'Blog Title',

src/functions/blogMethods/custom-mdx/__tests__/index.test.js

@@ -1,18 +1,10 @@
-const TransformCustomMdx = require('../../../../../src/functions/blogMethods/custom-mdx');
-const TransformImageData = require('../../../../../src/functions/blogMethods/custom-mdx/transform-image-data');
+const TransformCustomMdx = require('..');
+const TransformImageData = require('../transform-image-data');
 
-jest.mock(
-  '../../../../../src/functions/blogMethods/custom-mdx/add-frontmatter',
-);
-jest.mock(
-  '../../../../../src/functions/blogMethods/custom-mdx/transform-iframes',
-);
-jest.mock(
-  '../../../../../src/functions/blogMethods/custom-mdx/transform-image-data',
-);
-jest.mock(
-  '../../../../../src/functions/blogMethods/custom-mdx/transform-strings',
-);
+jest.mock('../add-frontmatter');
+jest.mock('../transform-iframes');
+jest.mock('../transform-image-data');
+jest.mock('../transform-strings');
 
 const post = {
   slug: 'Post',

src/functions/blogMethods/custom-mdx/__tests__/transform-iframes.test.js

@@ -1,4 +1,4 @@
-const TransformIframes = require('../../../../../src/functions/blogMethods/custom-mdx/transform-iframes');
+const TransformIframes = require('../transform-iframes');
 const Got = require('got');
 
 jest.mock('got');

src/functions/blogMethods/custom-mdx/__tests__/transform-image-data.test.js

@@ -1,4 +1,4 @@
-jest.mock('../../../../../src/functions/utils/is-prod', () => true);
+jest.mock('../../../utils/is-prod', () => true);
 
 const mockUploadToContentful = jest.fn(({ name }) => {
   return {
@@ -7,16 +7,13 @@ const mockUploadToContentful = jest.fn(({ name }) => {
   };
 });
 
-jest.mock(
-  '../../../../../src/functions/blogMethods/custom-mdx/upload-image-to-contentful',
-  () => mockUploadToContentful,
-);
+jest.mock('../upload-image-to-contentful', () => mockUploadToContentful);
 
 beforeEach(() => {
   mockUploadToContentful.mockClear();
 });
 
-const TransformImageData = require('../../../../../src/functions/blogMethods/custom-mdx/transform-image-data');
+const TransformImageData = require('../transform-image-data');
 
 const img1 = {
   caption: 'Image 1 caption',

src/functions/blogMethods/custom-mdx/__tests__/transform-strings.test.js

@@ -1,4 +1,4 @@
-const TransformStrings = require('../../../../../src/functions/blogMethods/custom-mdx/transform-strings');
+const TransformStrings = require('../transform-strings');
 
 it.each([
   'https://medium.com/yld-blog/',

src/functions/blogMethods/__tests__/upload-image-to-contentful.test.js → src/functions/blogMethods/custom-mdx/__tests__/upload-image-to-contentful.test.js

@@ -1,4 +1,4 @@
-const UploadImageToContentful = require('../../../../src/functions/blogMethods/custom-mdx/upload-image-to-contentful');
+const UploadImageToContentful = require('../upload-image-to-contentful');
 
 const mockPublish = jest.fn(function publish() {
   return {

src/functions/blogMethods/custom-mdx/transform-strings.js

@@ -11,9 +11,9 @@ const postPathSegment = new RegExp(`(${slug.source}-${hexUid.source})?`);
 const path = new RegExp(`${blogPath.source}(\\/${postPathSegment.source})?`);
 
 const mdLink = new RegExp(`\\(${origin.source}${path.source}\\)`, 'gi');
-const mdLinkFull = /\(https?:\/\/medium\.com\/yld(-engineering)?-blog(\/((?<slug>[^)]+)-[0-9a-f]{8,})?)?\)/;
+const mdLinkFull = /\(https?:\/\/medium\.com\/yld(-engineering)?-blog(\/((?<slug>[^)]+)-[0-9a-f]{8,})?)?\)/gi;
 // composite mdLink and mdLinkFull are provided because one may find either more legible - this ensures they are kept in sync
-assert.strictEqual(mdLink.source, mdLinkFull.source);
+assert.deepStrictEqual(mdLink, mdLinkFull);
 
 const transformStrings = content =>
   content.replace(mdLink, function() {

src/functions/blogMethods/get-all-contentful-blog-posts.js

@@ -0,0 +1,19 @@
+const getAllBlogPosts = async environment => {
+  let total = Number.POSITIVE_INFINITY;
+  let items = [];
+
+  while (items.length < total) {
+    const { items: cmsPosts, total: cmsTotal } = await environment.getEntries({
+      content_type: 'blogPost',
+      limit: 1000,
+      skip: items.length,
+    });
+
+    items.push(...cmsPosts);
+    total = cmsTotal;
+  }
+
+  return items;
+};
+
+module.exports = getAllBlogPosts;

src/functions/blogMethods/index.js

@@ -6,6 +6,7 @@ const TransformHtmlToMd = require('./transform-html-to-markdown');
 const TransformCustomMDX = require('./custom-mdx');
 const PublishToContentful = require('./publish-to-contentful');
 const FilterPostsToProcess = require('./filter-posts-to-process');
+const GetAllContentfulBlogPosts = require('./get-all-contentful-blog-posts');
 const ValidateMdx = require('./validate-mdx');
 const TransformMetaData = require('./transform-meta-data');
 
@@ -25,11 +26,7 @@ const syncMediumToContentful = async data => {
 
   const { allFields, requiredFields } = GetContentTypeFields(contentType);
 
-  const { items: cmsBlogPosts } = await environment.getEntries({
-    limit: 1000,
-    content_type: 'blogPost',
-  });
-
+  const cmsBlogPosts = await GetAllContentfulBlogPosts(environment);
   const parsedPostsInJson = await ParseXMLToJSON(data);
 
   const postsToProcess = FilterPostsToProcess(parsedPostsInJson, {