From 300f914ba8bb94d9c399699d126d81aba0b22142 Mon Sep 17 00:00:00 2001 From: Minh Nguyen Cong Date: Tue, 6 Feb 2024 12:35:23 +0100 Subject: [PATCH] feat: Support overwrite/skip folder download (#516) * feat: Support overwrite/skip folder download * Update docs * Update logic * Update logic * Update folders.test.js --- src/commands/folders/download.js | 131 +++++++++++++++------ test/commands/folders.test.js | 189 +++++++++++++++++++++++++++++++ 2 files changed, 287 insertions(+), 33 deletions(-) diff --git a/src/commands/folders/download.js b/src/commands/folders/download.js index b3fc609b..fb3a0fb5 100644 --- a/src/commands/folders/download.js +++ b/src/commands/folders/download.js @@ -22,13 +22,15 @@ const utils = require('../../util'); * @private */ function saveFileToDisk(folderPath, file, stream) { - let output; try { output = fs.createWriteStream(path.join(folderPath, file.path)); stream.pipe(output); } catch (ex) { - throw new BoxCLIError(`Error downloading file ${file.id} to ${file.path}`, ex); + throw new BoxCLIError( + `Error downloading file ${file.id} to ${file.path}`, + ex + ); } /* eslint-disable promise/avoid-new */ @@ -44,11 +46,16 @@ class FoldersDownloadCommand extends BoxCommand { async run() { const { flags, args } = this.parse(FoldersDownloadCommand); - this.maxDepth = flags.hasOwnProperty('depth') && flags.depth >= 0 ? flags.depth : Number.POSITIVE_INFINITY; + this.outputPath = null; + this.maxDepth = + flags.hasOwnProperty('depth') && flags.depth >= 0 + ? flags.depth + : Number.POSITIVE_INFINITY; + this.overwrite = flags.overwrite; - let outputPath; let id = args.id; let outputFinalized = Promise.resolve(); + let rootItemPath = null; let destinationPath; if (flags.destination) { @@ -69,30 +76,36 @@ class FoldersDownloadCommand extends BoxCommand { } /* eslint-enable no-sync */ - let spinner = ora('Starting download').start(); + this.spinner = ora('Starting download').start(); if (flags.zip) { - let fileName = `folders-download-${id}-${dateTime.format(new Date(), 'YYYY-MM-DDTHH_mm_ss_SSS')}.zip`; - outputPath = path.join(destinationPath, fileName); - outputFinalized = this._setupZip(outputPath); + this.overwrite = true; + let fileName = `folders-download-${id}-${dateTime.format( + new Date(), + 'YYYY-MM-DDTHH_mm_ss_SSS' + )}.zip`; + rootItemPath = fileName; + outputFinalized = this._setupZip(path.join(destinationPath, fileName)); } try { + this.outputPath = destinationPath; for await (let item of this._getItems(id, '')) { if (item.type === 'folder' && !this.zip) { - // Set output path to the top-level folder, which is the first item in the generator - outputPath = outputPath || path.join(destinationPath, item.path); + rootItemPath = rootItemPath || item.path; - spinner.text = `Creating folder ${item.id} at ${item.path}`; + this.spinnerLog(`Creating folder ${item.id} at ${item.path}`); try { await mkdirp(path.join(destinationPath, item.path)); } catch (ex) { - throw new BoxCLIError(`Folder ${item.path} could not be created`, ex); + throw new BoxCLIError( + `Folder ${item.path} could not be created`, + ex + ); } } else if (item.type === 'file') { - - spinner.text = `Downloading file ${item.id} to ${item.path}`; + this.spinnerLog(`Downloading file ${item.id} to ${item.path}`); let stream = await this.client.files.getReadStream(item.id); if (this.zip) { @@ -104,7 +117,7 @@ class FoldersDownloadCommand extends BoxCommand { } } } catch (err) { - spinner.stop(); + this.spinner.stop(); throw err; } @@ -112,7 +125,19 @@ class FoldersDownloadCommand extends BoxCommand { this.zip.finalize(); } await outputFinalized; - spinner.succeed(`Downloaded folder ${id} to ${outputPath}`); + this.spinner.succeed( + `${this.bufferLog || ''}\nDownloaded folder ${id} to ${path.join( + this.outputPath, + rootItemPath + )}`.trim() + ); + } + + spinnerLog(message, preserveText = false) { + this.spinner.text = `${this.bufferLog || ''}\n${message}`.trim(); + if (preserveText) { + this.bufferLog = this.spinner.text; + } } /** @@ -124,7 +149,6 @@ class FoldersDownloadCommand extends BoxCommand { * @private */ async* _getItems(folderId, folderPath) { - let folder = await this.client.folders.get(folderId); folderPath = path.join(folderPath, folder.name); @@ -137,19 +161,52 @@ class FoldersDownloadCommand extends BoxCommand { let folderItems = folder.item_collection.entries; if (folder.item_collection.total_count > folderItems.length) { - let iterator = await this.client.folders.getItems(folderId, { usemarker: true, fields: 'type,id,name' }); + let iterator = await this.client.folders.getItems(folderId, { + usemarker: true, + fields: 'type,id,name', + }); folderItems = { [Symbol.asyncIterator]: () => iterator }; } for await (let item of folderItems) { - if (item.type === 'folder' && folderPath.split(path.sep).length <= this.maxDepth) { - yield* this._getItems(item.id, folderPath); + if (item.type === 'folder') { + // We only recurse this folder by one of the following conditions: + // 1. The overwrite flag is true. We will download all files and folders within the provided depth (overwite). + // 2. The folder does not exist. We will download all files and folders within the provided depth. + // 3. The folder exists and overwrite is false, we only download files and folders not existing, within the provided depth. + /* eslint-disable no-sync */ + if ( + folderPath.split(path.sep).length <= this.maxDepth + ) { + /* eslint-enable no-sync */ + yield* this._getItems(item.id, folderPath); + } else { + // If the folder exists and overwrite is false, we skip the folder. + this.spinnerLog( + `Skipping folder ${item.name} (${item.id}) at ${folderPath} because reached max depth of ${this.maxDepth}`, + true + ); + } } else if (item.type === 'file') { - yield { - type: 'file', - id: item.id, - name: item.name, - path: path.join(folderPath, item.name), - }; + // We only download file if overwrite is true or the file does not exist. + // Skip downloading if overwrite is false and the file exists. + /* eslint-disable no-sync */ + if ( + this.overwrite || + !fs.existsSync(path.join(this.outputPath, folderPath, item.name)) + ) { + /* eslint-enable no-sync */ + yield { + type: 'file', + id: item.id, + name: item.name, + path: path.join(folderPath, item.name), + }; + } else { + this.spinnerLog( + `Skipping file ${item.name} (${item.id}) at ${folderPath} because it already exists and overwrite is disabled`, + true + ); + } } } } @@ -163,20 +220,22 @@ class FoldersDownloadCommand extends BoxCommand { * @private */ _setupZip(destinationPath) { - // Set up archive stream this.zip = archiver('zip', { - zlib: { level: 9 } // Use the best available compression + zlib: { level: 9 }, // Use the best available compression }); let output; try { output = fs.createWriteStream(destinationPath); } catch (ex) { - throw new BoxCLIError(`Could not write to destination path ${destinationPath}`, ex); + throw new BoxCLIError( + `Could not write to destination path ${destinationPath}`, + ex + ); } - this.zip.on('error', err => { + this.zip.on('error', (err) => { throw new BoxCLIError('Error writing to zip file', err); }); @@ -199,7 +258,7 @@ FoldersDownloadCommand.flags = { ...BoxCommand.flags, destination: flags.string({ description: 'The destination folder to download the Box folder into', - parse: utils.parsePath + parse: utils.parsePath, }), zip: flags.boolean({ description: 'Download the folder into a single .zip archive', @@ -209,7 +268,13 @@ FoldersDownloadCommand.flags = { 'Number of levels deep to recurse when downloading the folder tree', }), 'create-path': flags.boolean({ - description: 'Recursively creates a path to a directory if it does not exist', + description: + 'Recursively creates a path to a directory if it does not exist', + allowNo: true, + default: true, + }), + overwrite: flags.boolean({ + description: '[default: true] Overwrite the folder if it already exists.', allowNo: true, default: true, }), @@ -221,7 +286,7 @@ FoldersDownloadCommand.args = [ required: true, hidden: false, description: 'ID of the folder to download', - } + }, ]; module.exports = FoldersDownloadCommand; diff --git a/test/commands/folders.test.js b/test/commands/folders.test.js index ab77190d..6cbed4e8 100644 --- a/test/commands/folders.test.js +++ b/test/commands/folders.test.js @@ -1773,6 +1773,195 @@ describe('Folders', () => { assert.deepEqual(actualContents, expectedContents); assert.equal(ctx.stdout, ''); }); + + test + .nock(TEST_API_ROOT, api => api + .get(`/2.0/folders/${folderID}`) + .reply(200, getFolderFixture) + .get('/2.0/folders/22222') + .reply(200, getSubfolderFixture) + .get('/2.0/files/77777/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/77777` }) + .get('/2.0/files/44444/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/44444` }) + .get('/2.0/files/55555/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/55555` }) + ) + .nock(TEST_DOWNLOAD_ROOT, api => api + .get('/44444') + .reply(200, expectedContents['file 1.txt']) + .get('/55555') + .reply(200, expectedContents['file 2.txt']) + .get('/77777') + .reply(200, expectedContents.subfolder['subfolder file 1.txt']) + ) + .do(() => { + /* eslint-disable no-sync */ + const folderPath = path.join(destination, folderName); + if (fs.existsSync(destination)) { + fs.removeSync(destination); + } + fs.mkdirSync(destination); + fs.mkdirSync(folderPath); + fs.writeFileSync(path.join(folderPath, 'file 1.txt'), 'test123'); + /* eslint-enable no-sync */ + }) + .stdout() + .stderr() + .command([ + 'folders:download', + folderID, + `--destination=${destination}`, + '--no-color', + '--token=test', + ]) + .it('should download and overwrite existing folder', async(ctx) => { + let folderPath = path.join(destination, folderName); + let actualContents = getDirectoryContents(folderPath); + await fs.remove(destination); + + assert.deepEqual(actualContents, expectedContents); + assert.equal(ctx.stdout, ''); + }); + + test + .nock(TEST_API_ROOT, api => api + .get(`/2.0/folders/${folderID}`) + .reply(200, getFolderFixture) + .get('/2.0/folders/22222') + .reply(200, getSubfolderFixture) + .get('/2.0/files/77777/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/77777` }) + .get('/2.0/files/55555/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/55555` }) + ) + .nock(TEST_DOWNLOAD_ROOT, api => api + .get('/55555') + .reply(200, expectedContents['file 2.txt']) + .get('/77777') + .reply(200, expectedContents.subfolder['subfolder file 1.txt']) + ) + .do(() => { + /* eslint-disable no-sync */ + const folderPath = path.join(destination, folderName); + if (fs.existsSync(destination)) { + fs.removeSync(destination); + } + fs.mkdirSync(destination); + fs.mkdirSync(folderPath); + fs.writeFileSync(path.join(folderPath, 'file 1.txt'), 'test123'); + /* eslint-enable no-sync */ + }) + .stdout() + .stderr() + .command([ + 'folders:download', + folderID, + `--destination=${destination}`, + '--no-color', + '--no-overwrite', + '--token=test', + ]) + .it('should not overwrite existing folder when --no-overwrite flag is passed', async(ctx) => { + let folderPath = path.join(destination, folderName); + let actualContents = getDirectoryContents(folderPath); + await fs.remove(destination); + assert.equal(actualContents['file 1.txt'], 'test123'); + assert.equal(ctx.stdout, ''); + }); + + test + .nock(TEST_API_ROOT, api => api + .get(`/2.0/folders/${folderID}`) + .reply(200, getFolderFixture) + .get('/2.0/files/55555/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/55555` }) + ) + .nock(TEST_DOWNLOAD_ROOT, api => api + .get('/55555') + .reply(200, expectedContents['file 2.txt']) + ) + .do(() => { + /* eslint-disable no-sync */ + const folderPath = path.join(destination, folderName); + if (fs.existsSync(destination)) { + fs.removeSync(destination); + } + fs.mkdirSync(destination); + fs.mkdirSync(folderPath); + fs.mkdirSync(path.join(folderPath, 'subfolder')); + fs.writeFileSync(path.join(folderPath, 'file 1.txt'), 'test123'); + /* eslint-enable no-sync */ + }) + .stdout() + .stderr() + .command([ + 'folders:download', + folderID, + `--destination=${destination}`, + '--no-color', + '--no-overwrite', + '--depth=0', + '--token=test', + ]) + .it('should not overwrite existing file and folder in root folder when --no-overwrite and --depth=0 flag is passed', async(ctx) => { + let folderPath = path.join(destination, folderName); + let actualContents = getDirectoryContents(folderPath); + await fs.remove(destination); + assert.equal(actualContents['file 1.txt'], 'test123'); + assert.equal(Object.keys(actualContents.subfolder).length, 0); + assert.equal(ctx.stdout, ''); + }); + + test + .nock(TEST_API_ROOT, api => api + .get(`/2.0/folders/${folderID}`) + .reply(200, getFolderFixture) + .get('/2.0/folders/22222') + .reply(200, getSubfolderFixture) + .get('/2.0/files/77777/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/77777` }) + .get('/2.0/files/55555/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/55555` }) + ) + .nock(TEST_DOWNLOAD_ROOT, api => api + .get('/55555') + .reply(200, expectedContents['file 2.txt']) + .get('/77777') + .reply(200, expectedContents.subfolder['subfolder file 1.txt']) + ) + .do(() => { + /* eslint-disable no-sync */ + const folderPath = path.join(destination, folderName); + if (fs.existsSync(destination)) { + fs.removeSync(destination); + } + fs.mkdirSync(destination); + fs.mkdirSync(folderPath); + fs.mkdirSync(path.join(folderPath, 'subfolder')); + fs.writeFileSync(path.join(folderPath, 'file 1.txt'), 'test123'); + /* eslint-enable no-sync */ + }) + .stdout() + .stderr() + .command([ + 'folders:download', + folderID, + `--destination=${destination}`, + '--no-color', + '--no-overwrite', + '--depth=10', + '--token=test', + ]) + .it('should not overwrite existing file and folder in folder recursively when --no-overwrite and --depth=10 flag is passed', async(ctx) => { + let folderPath = path.join(destination, folderName); + let actualContents = getDirectoryContents(folderPath); + await fs.remove(destination); + assert.equal(actualContents['file 1.txt'], 'test123'); + assert.equal(Object.keys(actualContents.subfolder).length, 1); + assert.equal(ctx.stdout, ''); + }); + }); describe('folders:locks', () => {