Skip to content

Commit

Permalink
feat: Support overwrite/skip folder download (#516)
Browse files Browse the repository at this point in the history 
* feat: Support overwrite/skip folder download

* Update docs

* Update logic

* Update logic

* Update folders.test.js
  • Loading branch information
@congminh1254
congminh1254 authored Feb 6, 2024
1 parent b900fdb commit 300f914
Show file tree
Hide file tree
Showing 2 changed files with 287 additions and 33 deletions.
131 changes: 98 additions & 33 deletions src/commands/folders/download.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,15 @@ const utils = require('../../util');
* @private
*/
function saveFileToDisk(folderPath, file, stream) {

let output;
try {
output = fs.createWriteStream(path.join(folderPath, file.path));
stream.pipe(output);
} catch (ex) {
throw new BoxCLIError(`Error downloading file ${file.id} to ${file.path}`, ex);
throw new BoxCLIError(
`Error downloading file ${file.id} to ${file.path}`,
ex
);
}

/* eslint-disable promise/avoid-new */
Expand All @@ -44,11 +46,16 @@ class FoldersDownloadCommand extends BoxCommand {
async run() {
const { flags, args } = this.parse(FoldersDownloadCommand);

this.maxDepth = flags.hasOwnProperty('depth') && flags.depth >= 0 ? flags.depth : Number.POSITIVE_INFINITY;
this.outputPath = null;
this.maxDepth =
flags.hasOwnProperty('depth') && flags.depth >= 0
? flags.depth
: Number.POSITIVE_INFINITY;
this.overwrite = flags.overwrite;

let outputPath;
let id = args.id;
let outputFinalized = Promise.resolve();
let rootItemPath = null;

let destinationPath;
if (flags.destination) {
Expand All @@ -69,30 +76,36 @@ class FoldersDownloadCommand extends BoxCommand {
}
/* eslint-enable no-sync */

let spinner = ora('Starting download').start();
this.spinner = ora('Starting download').start();

if (flags.zip) {
let fileName = `folders-download-${id}-${dateTime.format(new Date(), 'YYYY-MM-DDTHH_mm_ss_SSS')}.zip`;
outputPath = path.join(destinationPath, fileName);
outputFinalized = this._setupZip(outputPath);
this.overwrite = true;
let fileName = `folders-download-${id}-${dateTime.format(
new Date(),
'YYYY-MM-DDTHH_mm_ss_SSS'
)}.zip`;
rootItemPath = fileName;
outputFinalized = this._setupZip(path.join(destinationPath, fileName));
}

try {
this.outputPath = destinationPath;
for await (let item of this._getItems(id, '')) {
if (item.type === 'folder' && !this.zip) {

// Set output path to the top-level folder, which is the first item in the generator
outputPath = outputPath || path.join(destinationPath, item.path);
rootItemPath = rootItemPath || item.path;

spinner.text = `Creating folder ${item.id} at ${item.path}`;
this.spinnerLog(`Creating folder ${item.id} at ${item.path}`);
try {
await mkdirp(path.join(destinationPath, item.path));
} catch (ex) {
throw new BoxCLIError(`Folder ${item.path} could not be created`, ex);
throw new BoxCLIError(
`Folder ${item.path} could not be created`,
ex
);
}
} else if (item.type === 'file') {

spinner.text = `Downloading file ${item.id} to ${item.path}`;
this.spinnerLog(`Downloading file ${item.id} to ${item.path}`);
let stream = await this.client.files.getReadStream(item.id);

if (this.zip) {
Expand All @@ -104,15 +117,27 @@ class FoldersDownloadCommand extends BoxCommand {
}
}
} catch (err) {
spinner.stop();
this.spinner.stop();
throw err;
}

if (this.zip) {
this.zip.finalize();
}
await outputFinalized;
spinner.succeed(`Downloaded folder ${id} to ${outputPath}`);
this.spinner.succeed(
`${this.bufferLog || ''}\nDownloaded folder ${id} to ${path.join(
this.outputPath,
rootItemPath
)}`.trim()
);
}

spinnerLog(message, preserveText = false) {
this.spinner.text = `${this.bufferLog || ''}\n${message}`.trim();
if (preserveText) {
this.bufferLog = this.spinner.text;
}
}

/**
Expand All @@ -124,7 +149,6 @@ class FoldersDownloadCommand extends BoxCommand {
* @private
*/
async* _getItems(folderId, folderPath) {

let folder = await this.client.folders.get(folderId);
folderPath = path.join(folderPath, folder.name);

Expand All @@ -137,19 +161,52 @@ class FoldersDownloadCommand extends BoxCommand {

let folderItems = folder.item_collection.entries;
if (folder.item_collection.total_count > folderItems.length) {
let iterator = await this.client.folders.getItems(folderId, { usemarker: true, fields: 'type,id,name' });
let iterator = await this.client.folders.getItems(folderId, {
usemarker: true,
fields: 'type,id,name',
});
folderItems = { [Symbol.asyncIterator]: () => iterator };
}
for await (let item of folderItems) {
if (item.type === 'folder' && folderPath.split(path.sep).length <= this.maxDepth) {
yield* this._getItems(item.id, folderPath);
if (item.type === 'folder') {
// We only recurse this folder by one of the following conditions:
// 1. The overwrite flag is true. We will download all files and folders within the provided depth (overwite).
// 2. The folder does not exist. We will download all files and folders within the provided depth.
// 3. The folder exists and overwrite is false, we only download files and folders not existing, within the provided depth.
/* eslint-disable no-sync */
if (
folderPath.split(path.sep).length <= this.maxDepth
) {
/* eslint-enable no-sync */
yield* this._getItems(item.id, folderPath);
} else {
// If the folder exists and overwrite is false, we skip the folder.
this.spinnerLog(
`Skipping folder ${item.name} (${item.id}) at ${folderPath} because reached max depth of ${this.maxDepth}`,
true
);
}
} else if (item.type === 'file') {
yield {
type: 'file',
id: item.id,
name: item.name,
path: path.join(folderPath, item.name),
};
// We only download file if overwrite is true or the file does not exist.
// Skip downloading if overwrite is false and the file exists.
/* eslint-disable no-sync */
if (
this.overwrite ||
!fs.existsSync(path.join(this.outputPath, folderPath, item.name))
) {
/* eslint-enable no-sync */
yield {
type: 'file',
id: item.id,
name: item.name,
path: path.join(folderPath, item.name),
};
} else {
this.spinnerLog(
`Skipping file ${item.name} (${item.id}) at ${folderPath} because it already exists and overwrite is disabled`,
true
);
}
}
}
}
Expand All @@ -163,20 +220,22 @@ class FoldersDownloadCommand extends BoxCommand {
* @private
*/
_setupZip(destinationPath) {

// Set up archive stream
this.zip = archiver('zip', {
zlib: { level: 9 } // Use the best available compression
zlib: { level: 9 }, // Use the best available compression
});

let output;
try {
output = fs.createWriteStream(destinationPath);
} catch (ex) {
throw new BoxCLIError(`Could not write to destination path ${destinationPath}`, ex);
throw new BoxCLIError(
`Could not write to destination path ${destinationPath}`,
ex
);
}

this.zip.on('error', err => {
this.zip.on('error', (err) => {
throw new BoxCLIError('Error writing to zip file', err);
});

Expand All @@ -199,7 +258,7 @@ FoldersDownloadCommand.flags = {
...BoxCommand.flags,
destination: flags.string({
description: 'The destination folder to download the Box folder into',
parse: utils.parsePath
parse: utils.parsePath,
}),
zip: flags.boolean({
description: 'Download the folder into a single .zip archive',
Expand All @@ -209,7 +268,13 @@ FoldersDownloadCommand.flags = {
'Number of levels deep to recurse when downloading the folder tree',
}),
'create-path': flags.boolean({
description: 'Recursively creates a path to a directory if it does not exist',
description:
'Recursively creates a path to a directory if it does not exist',
allowNo: true,
default: true,
}),
overwrite: flags.boolean({
description: '[default: true] Overwrite the folder if it already exists.',
allowNo: true,
default: true,
}),
Expand All @@ -221,7 +286,7 @@ FoldersDownloadCommand.args = [
required: true,
hidden: false,
description: 'ID of the folder to download',
}
},
];

module.exports = FoldersDownloadCommand;
Loading

0 comments on commit 300f914

Please sign in to comment.