From c8924512a34391ce92715a2b61fc4b0b91a9e10f Mon Sep 17 00:00:00 2001 From: Alexander Akait Date: Thu, 22 Oct 2020 14:20:08 +0300 Subject: [PATCH] feat: persistent cache between compilations (webpack@5 only) (#541) --- .gitignore | 1 + src/index.js | 208 +++++++++++++++------ test/CopyPlugin.test.js | 70 ++++++- test/__snapshots__/CopyPlugin.test.js.snap | 84 ++++++--- 4 files changed, 279 insertions(+), 84 deletions(-) diff --git a/.gitignore b/.gitignore index 3162bc45..82f19b2c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ npm-debug.log* /reports /node_modules /test/fixtures/\[special\$directory\] +/test/outputs .DS_Store Thumbs.db diff --git a/src/index.js b/src/index.js index fb9e8aa9..9ece267a 100644 --- a/src/index.js +++ b/src/index.js @@ -37,8 +37,58 @@ class CopyPlugin { this.options = options.options || {}; } + static async createSnapshot(compilation, startTime, dependency) { + if (!compilation.fileSystemInfo) { + return; + } + + // eslint-disable-next-line consistent-return + return new Promise((resolve, reject) => { + compilation.fileSystemInfo.createSnapshot( + startTime, + [dependency], + // eslint-disable-next-line no-undefined + undefined, + // eslint-disable-next-line no-undefined + undefined, + null, + (error, snapshot) => { + if (error) { + reject(error); + + return; + } + + resolve(snapshot); + } + ); + }); + } + + static async checkSnapshotValid(compilation, snapshot) { + if (!compilation.fileSystemInfo) { + return; + } + + // eslint-disable-next-line consistent-return + return new Promise((resolve, reject) => { + compilation.fileSystemInfo.checkSnapshotValid( + snapshot, + (error, isValid) => { + if (error) { + reject(error); + + return; + } + + resolve(isValid); + } + ); + }); + } + // eslint-disable-next-line class-methods-use-this - async runPattern(compiler, compilation, logger, inputPattern) { + static async runPattern(compiler, compilation, logger, cache, inputPattern) { const pattern = typeof inputPattern === 'string' ? { from: inputPattern } @@ -49,7 +99,6 @@ class CopyPlugin { pattern.to = path.normalize( typeof pattern.to !== 'undefined' ? pattern.to : '' ); - pattern.context = path.normalize( typeof pattern.context !== 'undefined' ? !path.isAbsolute(pattern.context) @@ -266,64 +315,109 @@ class CopyPlugin { compilation.fileDependencies.add(file.absoluteFrom); } - logger.debug(`reading "${file.absoluteFrom}" to write to assets`); + let source; - let data; + if (cache) { + const cacheEntry = await cache.getPromise(file.relativeFrom, null); - try { - data = await readFile(inputFileSystem, file.absoluteFrom); - } catch (error) { - compilation.errors.push(error); + if (cacheEntry) { + const isValidSnapshot = await CopyPlugin.checkSnapshotValid( + compilation, + cacheEntry.snapshot + ); - return; + if (isValidSnapshot) { + ({ source } = cacheEntry); + } + } } - if (pattern.transform) { - logger.log(`transforming content for "${file.absoluteFrom}"`); - - if (pattern.cacheTransform) { - const cacheDirectory = pattern.cacheTransform.directory - ? pattern.cacheTransform.directory - : typeof pattern.cacheTransform === 'string' - ? pattern.cacheTransform - : findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir(); - let defaultCacheKeys = { - version, - transform: pattern.transform, - contentHash: crypto.createHash('md4').update(data).digest('hex'), - }; - - if (typeof pattern.cacheTransform.keys === 'function') { - defaultCacheKeys = await pattern.cacheTransform.keys( - defaultCacheKeys, - file.absoluteFrom - ); - } else { - defaultCacheKeys = { - ...defaultCacheKeys, - ...pattern.cacheTransform.keys, + if (!source) { + let startTime; + + if (cache) { + startTime = Date.now(); + } + + logger.debug(`reading "${file.absoluteFrom}" to write to assets`); + + let data; + + try { + data = await readFile(inputFileSystem, file.absoluteFrom); + } catch (error) { + compilation.errors.push(error); + + return; + } + + if (pattern.transform) { + logger.log(`transforming content for "${file.absoluteFrom}"`); + + if (pattern.cacheTransform) { + const cacheDirectory = pattern.cacheTransform.directory + ? pattern.cacheTransform.directory + : typeof pattern.cacheTransform === 'string' + ? pattern.cacheTransform + : findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir(); + let defaultCacheKeys = { + version, + transform: pattern.transform, + contentHash: crypto + .createHash('md4') + .update(data) + .digest('hex'), }; - } - const cacheKeys = serialize(defaultCacheKeys); + if (typeof pattern.cacheTransform.keys === 'function') { + defaultCacheKeys = await pattern.cacheTransform.keys( + defaultCacheKeys, + file.absoluteFrom + ); + } else { + defaultCacheKeys = { + ...defaultCacheKeys, + ...pattern.cacheTransform.keys, + }; + } - try { - const result = await cacache.get(cacheDirectory, cacheKeys); + const cacheKeys = serialize(defaultCacheKeys); - logger.debug( - `getting cached transformation for "${file.absoluteFrom}"` - ); + try { + const result = await cacache.get(cacheDirectory, cacheKeys); - ({ data } = result); - } catch (_ignoreError) { - data = await pattern.transform(data, file.absoluteFrom); + logger.debug( + `getting cached transformation for "${file.absoluteFrom}"` + ); - logger.debug(`caching transformation for "${file.absoluteFrom}"`); + ({ data } = result); + } catch (_ignoreError) { + data = await pattern.transform(data, file.absoluteFrom); + + logger.debug( + `caching transformation for "${file.absoluteFrom}"` + ); - await cacache.put(cacheDirectory, cacheKeys, data); + await cacache.put(cacheDirectory, cacheKeys, data); + } + } else { + data = await pattern.transform(data, file.absoluteFrom); } - } else { - data = await pattern.transform(data, file.absoluteFrom); + } + + source = new RawSource(data); + + if (cache) { + const snapshot = await CopyPlugin.createSnapshot( + compilation, + startTime, + file.relativeFrom + ); + + await cache.storePromise(file.relativeFrom, null, { + source, + snapshot, + }); } } @@ -349,7 +443,7 @@ class CopyPlugin { { resourcePath: file.absoluteFrom }, file.webpackTo, { - content: data, + content: source.source(), context: pattern.context, } ); @@ -374,7 +468,7 @@ class CopyPlugin { } // eslint-disable-next-line no-param-reassign - file.data = data; + file.source = source; // eslint-disable-next-line no-param-reassign file.targetPath = normalizePath(file.webpackTo); // eslint-disable-next-line no-param-reassign @@ -392,6 +486,10 @@ class CopyPlugin { compiler.hooks.thisCompilation.tap(pluginName, (compilation) => { const logger = compilation.getLogger('copy-webpack-plugin'); + const cache = compilation.getCache + ? compilation.getCache('CopyWebpackPlugin') + : // eslint-disable-next-line no-undefined + undefined; compilation.hooks.additionalAssets.tapAsync( 'copy-webpack-plugin', @@ -404,7 +502,13 @@ class CopyPlugin { assets = await Promise.all( this.patterns.map((item) => limit(async () => - this.runPattern(compiler, compilation, logger, item) + CopyPlugin.runPattern( + compiler, + compilation, + logger, + cache, + item + ) ) ) ); @@ -426,12 +530,10 @@ class CopyPlugin { absoluteFrom, targetPath, webpackTo, - data, + source, force, } = asset; - const source = new RawSource(data); - // For old version webpack 4 /* istanbul ignore if */ if (typeof compilation.emitAsset !== 'function') { diff --git a/test/CopyPlugin.test.js b/test/CopyPlugin.test.js index f7273a7c..898f0607 100644 --- a/test/CopyPlugin.test.js +++ b/test/CopyPlugin.test.js @@ -1,6 +1,7 @@ import path from 'path'; import webpack from 'webpack'; +import del from 'del'; import CopyPlugin from '../src'; @@ -633,9 +634,72 @@ describe('CopyPlugin', () => { .then(done) .catch(done); }); + }); + + describe('cache', () => { + it('should work with the "memory" cache', async () => { + const compiler = getCompiler({ + cache: { + type: 'memory', + }, + }); + + new CopyPlugin({ + patterns: [ + { + from: path.resolve(__dirname, './fixtures/directory'), + }, + ], + }).apply(compiler); + + const { stats } = await compile(compiler); + + if (webpack.version[0] === '4') { + expect( + Object.keys(stats.compilation.assets).filter( + (assetName) => stats.compilation.assets[assetName].emitted + ).length + ).toBe(5); + } else { + expect(stats.compilation.emittedAssets.size).toBe(5); + } + + expect(readAssets(compiler, stats)).toMatchSnapshot('assets'); + expect(stats.compilation.errors).toMatchSnapshot('errors'); + expect(stats.compilation.warnings).toMatchSnapshot('warnings'); + + await new Promise(async (resolve) => { + const { stats: newStats } = await compile(compiler); + + if (webpack.version[0] === '4') { + expect( + Object.keys(newStats.compilation.assets).filter( + (assetName) => newStats.compilation.assets[assetName].emitted + ).length + ).toBe(4); + } else { + expect(newStats.compilation.emittedAssets.size).toBe(0); + } - it('should work and do not emit unchanged assets', async () => { - const compiler = getCompiler(); + expect(readAssets(compiler, newStats)).toMatchSnapshot('assets'); + expect(newStats.compilation.errors).toMatchSnapshot('errors'); + expect(newStats.compilation.warnings).toMatchSnapshot('warnings'); + + resolve(); + }); + }); + + it('should work with the "filesystem" cache', async () => { + const cacheDirectory = path.resolve(__dirname, './outputs/.cache'); + + await del(cacheDirectory); + + const compiler = getCompiler({ + cache: { + type: 'filesystem', + cacheDirectory, + }, + }); new CopyPlugin({ patterns: [ @@ -671,7 +735,7 @@ describe('CopyPlugin', () => { ).length ).toBe(4); } else { - expect(newStats.compilation.emittedAssets.size).toBe(4); + expect(newStats.compilation.emittedAssets.size).toBe(0); } expect(readAssets(compiler, newStats)).toMatchSnapshot('assets'); diff --git a/test/__snapshots__/CopyPlugin.test.js.snap b/test/__snapshots__/CopyPlugin.test.js.snap index 4c768dc6..9f77923e 100644 --- a/test/__snapshots__/CopyPlugin.test.js.snap +++ b/test/__snapshots__/CopyPlugin.test.js.snap @@ -1,5 +1,61 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`CopyPlugin cache should work with the "filesystem" cache: assets 1`] = ` +Object { + ".dottedfile": "dottedfile contents +", + "directoryfile.txt": "new", + "nested/deep-nested/deepnested.txt": "", + "nested/nestedfile.txt": "", +} +`; + +exports[`CopyPlugin cache should work with the "filesystem" cache: assets 2`] = ` +Object { + ".dottedfile": "dottedfile contents +", + "directoryfile.txt": "new", + "nested/deep-nested/deepnested.txt": "", + "nested/nestedfile.txt": "", +} +`; + +exports[`CopyPlugin cache should work with the "filesystem" cache: errors 1`] = `Array []`; + +exports[`CopyPlugin cache should work with the "filesystem" cache: errors 2`] = `Array []`; + +exports[`CopyPlugin cache should work with the "filesystem" cache: warnings 1`] = `Array []`; + +exports[`CopyPlugin cache should work with the "filesystem" cache: warnings 2`] = `Array []`; + +exports[`CopyPlugin cache should work with the "memory" cache: assets 1`] = ` +Object { + ".dottedfile": "dottedfile contents +", + "directoryfile.txt": "new", + "nested/deep-nested/deepnested.txt": "", + "nested/nestedfile.txt": "", +} +`; + +exports[`CopyPlugin cache should work with the "memory" cache: assets 2`] = ` +Object { + ".dottedfile": "dottedfile contents +", + "directoryfile.txt": "new", + "nested/deep-nested/deepnested.txt": "", + "nested/nestedfile.txt": "", +} +`; + +exports[`CopyPlugin cache should work with the "memory" cache: errors 1`] = `Array []`; + +exports[`CopyPlugin cache should work with the "memory" cache: errors 2`] = `Array []`; + +exports[`CopyPlugin cache should work with the "memory" cache: warnings 1`] = `Array []`; + +exports[`CopyPlugin cache should work with the "memory" cache: warnings 2`] = `Array []`; + exports[`CopyPlugin logging should logging when "from" is a directory: logs 1`] = ` Object { "logs": Array [ @@ -76,31 +132,3 @@ Object { ], } `; - -exports[`CopyPlugin watch mode should work and do not emit unchanged assets: assets 1`] = ` -Object { - ".dottedfile": "dottedfile contents -", - "directoryfile.txt": "new", - "nested/deep-nested/deepnested.txt": "", - "nested/nestedfile.txt": "", -} -`; - -exports[`CopyPlugin watch mode should work and do not emit unchanged assets: assets 2`] = ` -Object { - ".dottedfile": "dottedfile contents -", - "directoryfile.txt": "new", - "nested/deep-nested/deepnested.txt": "", - "nested/nestedfile.txt": "", -} -`; - -exports[`CopyPlugin watch mode should work and do not emit unchanged assets: errors 1`] = `Array []`; - -exports[`CopyPlugin watch mode should work and do not emit unchanged assets: errors 2`] = `Array []`; - -exports[`CopyPlugin watch mode should work and do not emit unchanged assets: warnings 1`] = `Array []`; - -exports[`CopyPlugin watch mode should work and do not emit unchanged assets: warnings 2`] = `Array []`;