diff --git a/app/cmd/cli/arguments.js b/app/cmd/cli/arguments.js index 0a1fb2e..cd448e9 100644 --- a/app/cmd/cli/arguments.js +++ b/app/cmd/cli/arguments.js @@ -29,6 +29,12 @@ const cmdArgumentsDefinition = [ multiple: true, type: String, description: 'Define what range should be performed for each visualization from the configuration object. If not defined, the tool will perform visualization for all defined ranges. Each range is identified by its "id" value.' + }, + { + name: 'strictFilter', + defaultOption: false, + type: Boolean, + description: 'If set, the tool will perform visualization only for the ranges defined in the rangeFilter argument and not for whole folder content (incremental approach).' } ]; diff --git a/app/modules/io/csv-data-reader.js b/app/modules/io/csv-data-reader.js index ba0a71a..738dc00 100644 --- a/app/modules/io/csv-data-reader.js +++ b/app/modules/io/csv-data-reader.js @@ -14,6 +14,20 @@ const readAllCsvFilesForVisualization = async (visualization) => { return csvItems.flatMap(item => item); } +const readFilteredCsvFilesForVisualization = async (visualization, range) => { + CONSOLE_LOG.info(`Reading extracted data for visualization (${visualization.id})`); + let files = fs.readdirSync(visualization.output) + let csvItems = []; + for (const file of files) { + const label = file.match(/\d{4}-\d{2}-\d{2}/g)[0]; + if (range.filterKeys.some(key => key === label)) { + let csvContent = await csv().fromFile(path.resolve(visualization.output, file)); + csvItems.push(mapLabel(file, csvContent)); + } + } + return csvItems.flatMap(item => item); +} + const mapLabel = (file, csvContent) => { return csvContent.map(item => { // use YYYY-MM-DD from file name as a label value - data per BD @@ -23,5 +37,6 @@ const mapLabel = (file, csvContent) => { } module.exports = { - readAllCsvFilesForVisualization + readAllCsvFilesForVisualization, + readFilteredCsvFilesForVisualization } \ No newline at end of file diff --git a/app/modules/service/visualize-service.js b/app/modules/service/visualize-service.js index 33f3692..442b880 100644 --- a/app/modules/service/visualize-service.js +++ b/app/modules/service/visualize-service.js @@ -1,6 +1,6 @@ const { updateSection, updatePage } = require("../bookkit/client/bookkit-client"); const createUu5String = require("../bookkit/visualization/visualization"); -const { readAllCsvFilesForVisualization } = require("../io/csv-data-reader"); +const { readAllCsvFilesForVisualization, readFilteredCsvFilesForVisualization } = require("../io/csv-data-reader"); const { CONSOLE_LOG } = require("../logger/logger"); const {download} = require("../archive-data-downloader/archive-data-downloader-helper"); const { emptyDir } = require("../io/fs-helper"); @@ -18,7 +18,13 @@ const _processRanges = async (cmdArgs, configuration, visualization, token) => { for (const [index, range] of visualization.ranges.entries()) { CONSOLE_LOG.info(`Processing range id: ${range.id} (${index+1} of ${visualization.ranges.length})`); let command = await download(visualization, range, cmdArgs); - let data = await readAllCsvFilesForVisualization(visualization); + let data = []; + if (cmdArgs.strictFilter) { + CONSOLE_LOG.info("Running in strict filter mode."); + data = await readFilteredCsvFilesForVisualization(visualization, range); + } else { + data = await readAllCsvFilesForVisualization(visualization); + } let uu5StringContent = createUu5String(visualization, range, data, command); CONSOLE_LOG.info(`Uploading data into bookkit.`); await updateSection(configuration.bookkit.uri, visualization.pageCode, range.sectionCode, uu5StringContent, token); diff --git a/readme.md b/readme.md index 924f1ec..de517fe 100644 --- a/readme.md +++ b/readme.md @@ -104,5 +104,8 @@ Define what visualization should be performed from the configuration object. If ### --rangeFilter string[] Define what range should be performed for each visualization from the configuration object. If not defined, the tool will perform visualization for all defined ranges. Each range is identified by its "id" value. +### --strictFilter +If set, the tool will perform visualization only for the ranges defined in the rangeFilter argument and not for whole folder content (incremental approach). + ## Logs logs are automatically stored to the ```%HOME%/.archive-data-visualizer/logs``` folder \ No newline at end of file