Strict range filtering (year range fix)

app/cmd/cli/arguments.js

@@ -29,6 +29,12 @@ const cmdArgumentsDefinition = [
     multiple: true,
     type: String,
     description: 'Define what range should be performed for each visualization from the configuration object. If not defined, the tool will perform visualization for all defined ranges. Each range is identified by its "id" value.'
+  },
+  {
+    name: 'strictFilter',
+    defaultOption: false,
+    type: Boolean,
+    description: 'If set, the tool will perform visualization only for the ranges defined in the rangeFilter argument and not for whole folder content (incremental approach).'
   }
 ];
 

app/modules/io/csv-data-reader.js

@@ -14,6 +14,20 @@ const readAllCsvFilesForVisualization = async (visualization) => {
     return csvItems.flatMap(item => item);
 }
 
+const readFilteredCsvFilesForVisualization = async (visualization, range) => {
+    CONSOLE_LOG.info(`Reading extracted data for visualization (${visualization.id})`);
+    let files = fs.readdirSync(visualization.output)
+    let csvItems = [];
+    for (const file of files) {
+        const label = file.match(/\d{4}-\d{2}-\d{2}/g)[0];
+        if (range.filterKeys.some(key => key === label)) {
+            let csvContent = await csv().fromFile(path.resolve(visualization.output, file));
+            csvItems.push(mapLabel(file, csvContent));
+        }
+    }
+    return csvItems.flatMap(item => item);
+}
+
 const mapLabel = (file, csvContent) => {
     return csvContent.map(item => {
         // use YYYY-MM-DD from file name as a label value - data per BD
@@ -23,5 +37,6 @@ const mapLabel = (file, csvContent) => {
 }
 
 module.exports = {
-    readAllCsvFilesForVisualization
+    readAllCsvFilesForVisualization,
+    readFilteredCsvFilesForVisualization
 }

app/modules/service/visualize-service.js

@@ -1,6 +1,6 @@
 const { updateSection, updatePage } = require("../bookkit/client/bookkit-client");
 const createUu5String = require("../bookkit/visualization/visualization");
-const { readAllCsvFilesForVisualization } = require("../io/csv-data-reader");
+const { readAllCsvFilesForVisualization, readFilteredCsvFilesForVisualization } = require("../io/csv-data-reader");
 const { CONSOLE_LOG } = require("../logger/logger");
 const {download} = require("../archive-data-downloader/archive-data-downloader-helper");
 const { emptyDir } = require("../io/fs-helper");
@@ -18,7 +18,13 @@ const _processRanges = async (cmdArgs, configuration, visualization, token) => {
     for (const [index, range] of visualization.ranges.entries()) {
         CONSOLE_LOG.info(`Processing range id: ${range.id} (${index+1} of ${visualization.ranges.length})`);
         let command = await download(visualization, range, cmdArgs);
-        let data = await readAllCsvFilesForVisualization(visualization);
+        let data = [];
+        if (cmdArgs.strictFilter) {
+            CONSOLE_LOG.info("Running in strict filter mode.");
+            data = await readFilteredCsvFilesForVisualization(visualization, range);
+        } else {
+            data = await readAllCsvFilesForVisualization(visualization);
+        }
         let uu5StringContent = createUu5String(visualization, range, data, command);
         CONSOLE_LOG.info(`Uploading data into bookkit.`);
         await updateSection(configuration.bookkit.uri, visualization.pageCode, range.sectionCode, uu5StringContent, token);

readme.md

@@ -104,5 +104,8 @@ Define what visualization should be performed from the configuration object. If
 ### --rangeFilter string[]
 Define what range should be performed for each visualization from the configuration object. If not defined, the tool will perform visualization for all defined ranges. Each range is identified by its "id" value.
 
+### --strictFilter
+If set, the tool will perform visualization only for the ranges defined in the rangeFilter argument and not for whole folder content (incremental approach).
+
 ## Logs
 logs are automatically stored to the ```%HOME%/.archive-data-visualizer/logs``` folder