Merge pull request #170 from ezpaarse-project:feature/supported-repor…

…ts-tweaks Feature/supported reports tweaks
ezpaarse-project · May 17, 2024 · 8aeda8a · 8aeda8a
2 parents 3efacaf + caa1f8b
commit 8aeda8a
Show file tree

Hide file tree

Showing 8 changed files with 163 additions and 11 deletions.
diff --git a/api/lib/entities/harvest-session.dto.js b/api/lib/entities/harvest-session.dto.js
@@ -24,7 +24,7 @@ const schema = {
   }),
   beginDate: Joi.string().regex(/^\d{4}-\d{2}$/),
   endDate: Joi.string().regex(/^\d{4}-\d{2}$/),
-  reportTypes: Joi.array().items(Joi.string()),
+  reportTypes: Joi.array().items(Joi.string().lowercase().trim()),
   timeout: Joi.number(),
   allowFaulty: Joi.boolean(),
   downloadUnsupported: Joi.boolean(),

diff --git a/api/lib/entities/harvest-session.service.js b/api/lib/entities/harvest-session.service.js
@@ -41,6 +41,7 @@ const { queryToPrismaFilter } = require('../controllers/utils');
 /* eslint-enable max-len */
 
 const DEFAULT_HARVESTED_REPORTS = new Set(config.get('counter.defaultHarvestedReports'));
+const defaultHarvestedReports = Array.from(DEFAULT_HARVESTED_REPORTS).map((r) => r.toLowerCase());
 
 module.exports = class HarvestSessionService extends BasePrismaService {
   /** @type {BasePrismaService.TransactionFnc<HarvestSessionService>} */
@@ -367,7 +368,7 @@ module.exports = class HarvestSessionService extends BasePrismaService {
       }
 
       // Get report types
-      let reportTypes = Array.from(new Set(session.reportTypes));
+      let reportTypes = Array.from(new Set(session.reportTypes)).map((r) => r?.toLowerCase?.());
 
       /** @type {(SushiCredentials & { endpoint: SushiEndpoint, institution: Institution })[]} */
       let credentialsToHarvest = [];
@@ -430,7 +431,12 @@ module.exports = class HarvestSessionService extends BasePrismaService {
           }
 
           // Get supported reports
-          const { supportedReportsUpdatedAt } = endpoint;
+          const {
+            supportedReportsUpdatedAt,
+            ignoredReports,
+            additionalReports,
+          } = endpoint;
+
           const oneMonthAgo = subMonths(new Date(), 1);
 
           let supportedReports = [];
@@ -465,14 +471,26 @@ module.exports = class HarvestSessionService extends BasePrismaService {
           }
 
           if (reportTypes.includes('all')) {
-            reportTypes = Array.from(DEFAULT_HARVESTED_REPORTS);
+            reportTypes = defaultHarvestedReports;
           }
 
-          const supportedReportsSet = new Set(supportedReports);
+          const supportedReportsSet = new Set([
+            // If there's no support list available, assume the default list is supported
+            ...(supportedReports.length > 0 ? supportedReports : defaultHarvestedReports),
+            ...additionalReports,
+          ]);
+          const ignoredReportsSet = new Set(ignoredReports);
+
+          if (!session.downloadUnsupported) {
+            // Filter supported reports based on session params
+            if (supportedReportsSet.size > 0) {
+              reportTypes = reportTypes.filter((reportId) => supportedReportsSet.has(reportId));
+            }
 
-          // Filter supported report based on session params
-          if (!session.downloadUnsupported && supportedReportsSet.size > 0) {
-            reportTypes = reportTypes.filter((reportId) => supportedReportsSet.has(reportId));
+            // Remove reports that should be ignored
+            if (ignoredReportsSet.size > 0) {
+              reportTypes = reportTypes.filter((reportId) => !ignoredReportsSet.has(reportId));
+            }
           }
 
           // Add jobs

diff --git a/api/lib/entities/sushi-endpoints.dto.js b/api/lib/entities/sushi-endpoints.dto.js
@@ -32,6 +32,9 @@ const schema = {
   testedReport: Joi.string().allow('').empty(null).lowercase(),
   tags: Joi.array().items(Joi.string()),
 
+  ignoredReports: Joi.array().items(Joi.string().lowercase()),
+  additionalReports: Joi.array().items(Joi.string().lowercase()),
+
   credentials: Joi.array().items(Joi.object()),
 
   disabledUntil: Joi.date().allow(null),

diff --git a/api/lib/services/processors/harvest/steps/insert.js b/api/lib/services/processors/harvest/steps/insert.js
@@ -56,6 +56,8 @@ function list2object(list, opts = {}) {
  * @param {import('..').ProcessorStepParam} params
  */
 module.exports = async function process(params) {
+  const now = new Date();
+
   const {
     task: {
       data: task,
@@ -166,6 +168,8 @@ module.exports = async function process(params) {
       X_Endpoint_ID: credentials.endpoint.id,
       X_Package: credentials.packages,
       X_Tags: credentials.tags,
+      X_Harvested_At: now,
+      X_Endpoint_Name: credentials.endpoint.vendor,
       X_Endpoint_Tags: credentials.endpoint.tags,
       X_Harvest_ID: sessionId,
 

diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma
@@ -410,6 +410,10 @@ model SushiEndpoint {
   paramSeparator            String?
   /// List report IDs that are supported by the endpoint
   supportedReports          String[]
+  /// List of report IDs that should be ignored, even if the endpoint indicates that they are supported
+  ignoredReports            String[]
+  /// Additional report IDs to be added to the list of supported reports provided by the endpoint
+  additionalReports         String[]
   /// Date on which the list of supported reports was last updated
   supportedReportsUpdatedAt DateTime?
   /// Report used when testing endpoint

diff --git a/front/components/EndpointForm.vue b/front/components/EndpointForm.vue
@@ -63,7 +63,38 @@
                 :items="supportedReports"
                 placeholder="pr"
                 outlined
-              />
+              >
+                <template #prepend-item>
+                  <v-list-item>
+                    <v-list-item-content>
+                      <v-list-item-subtitle>
+                        {{ $t('reports.supportedReportsOnPlatform') }}
+                      </v-list-item-subtitle>
+                    </v-list-item-content>
+                  </v-list-item>
+                </template>
+
+                <template #no-data>
+                  <v-list-item>
+                    <v-list-item-content>
+                      <v-list-item-title v-if="supportedReportsSearch">
+                        <i18n path="noMatchFor">
+                          <template #search>
+                            <strong>{{ supportedReportsSearch }}</strong>
+                          </template>
+
+                          <template #key>
+                            <kbd>{{ $t('enterKey') }}</kbd>
+                          </template>
+                        </i18n>
+                      </v-list-item-title>
+                      <v-list-item-title v-else>
+                        {{ $t('reports.supportedReportsUnavailable') }}
+                      </v-list-item-title>
+                    </v-list-item-content>
+                  </v-list-item>
+                </template>
+              </v-combobox>
             </v-col>
 
             <v-col cols="4">
@@ -125,6 +156,73 @@
 
       <v-divider />
 
+      <v-card-title>
+        {{ $t('endpoints.harvestedReports') }}
+      </v-card-title>
+
+      <v-card-text>
+        <p>{{ $t('endpoints.ignoredReportsDesc') }}</p>
+
+        <v-combobox
+          v-model="endpointForm.ignoredReports"
+          :search-input.sync="supportedReportsSearch"
+          :items="supportedReports"
+          :label="$t('endpoints.ignoredReports')"
+          multiple
+          small-chips
+          deletable-chips
+          outlined
+        >
+          <template #prepend-item>
+            <v-list-item>
+              <v-list-item-content>
+                <v-list-item-subtitle>
+                  {{ $t('reports.supportedReportsOnPlatform') }}
+                </v-list-item-subtitle>
+              </v-list-item-content>
+            </v-list-item>
+          </template>
+
+          <template #no-data>
+            <v-list-item>
+              <v-list-item-content>
+                <v-list-item-title v-if="supportedReportsSearch">
+                  <i18n path="noMatchFor">
+                    <template #search>
+                      <strong>{{ supportedReportsSearch }}</strong>
+                    </template>
+
+                    <template #key>
+                      <kbd>{{ $t('enterKey') }}</kbd>
+                    </template>
+                  </i18n>
+                </v-list-item-title>
+                <v-list-item-title v-else>
+                  {{ $t('reports.supportedReportsUnavailable') }}
+                </v-list-item-title>
+              </v-list-item-content>
+            </v-list-item>
+          </template>
+        </v-combobox>
+
+        <p>{{ $t('endpoints.additionalReportsDesc') }}</p>
+
+        <v-combobox
+          v-model="endpointForm.additionalReports"
+          :label="$t('endpoints.additionalReports')"
+          multiple
+          small-chips
+          deletable-chips
+          outlined
+        >
+          <template #append>
+            <div />
+          </template>
+        </v-combobox>
+      </v-card-text>
+
+      <v-divider />
+
       <v-card-title>
         {{ $t('endpoints.queryParameters') }}
       </v-card-title>
@@ -227,13 +325,16 @@ export default {
       saving: false,
       valid: false,
       formTitle: '',
+      supportedReportsSearch: '',
 
       supportedReports: [],
 
       endpointForm: {
         id: null,
         params: [],
         tags: [],
+        ignoredReports: [],
+        additionalReports: [],
         vendor: '',
         sushiUrl: '',
         description: '',
@@ -277,6 +378,12 @@ export default {
       this.endpointForm.id = data.id;
       this.endpointForm.params = Array.isArray(data.params) ? data.params : [];
       this.endpointForm.tags = Array.isArray(data.tags) ? data.tags : [];
+      this.endpointForm.ignoredReports = (
+        Array.isArray(data.ignoredReports) ? data.ignoredReports : []
+      );
+      this.endpointForm.additionalReports = (
+        Array.isArray(data.additionalReports) ? data.additionalReports : []
+      );
 
       this.endpointForm.vendor = data.vendor || '';
       this.endpointForm.sushiUrl = data.sushiUrl || '';

diff --git a/front/locales/en.json b/front/locales/en.json
@@ -210,6 +210,11 @@
     "requireRequestorId": "Requestor ID is required",
     "requireApiKey": "An API key is required",
     "isSushiCompliant": "The endpoint is SUSHI compliant",
+    "harvestedReports": "Harvested reports",
+    "ignoredReportsDesc": "The reports listed below will be considered as unsupported, even if they are present in the list provided by the endpoint.",
+    "ignoredReports": "Ignored reports",
+    "additionalReportsDesc": "The reports listed below will be considered as supported, even if they are not present in the list provided by the endpoint.",
+    "additionalReports": "Additional reports",
     "ignoreReportValidation": "Ignore report validation",
     "description": "Description",
     "paramSeparatorDesc": "ezMESURE automatically adds {0}, {1} and {2} parameters while harvesting. If the endpoint does not accept the standard separator {separator} for this multivalued fields, please enter the separator here.",
@@ -583,6 +588,7 @@
     "checkCredentials": "Check credentials",
     "supportedReports": "Supported reports",
     "supportedReportsOnPlatform": "Reports supported by the publisher platform",
+    "supportedReportsUnavailable": "The list of supported reports is not available for this publisher platform.",
     "failedToFetchReports": "Failed to fetch the list of available reports",
     "onlyShowMasterReports": "Only show master reports"
   },
@@ -870,5 +876,7 @@
   "invalidate": "Invalidate",
   "reason": "Reason: {reason}",
   "groupBy": "Group by",
-  "indeterminate": "indeterminate"
+  "indeterminate": "indeterminate",
+  "noMatchFor": "No results matching \"{search}\". Press {key} to add it.",
+  "enterKey": "enter"
 }
diff --git a/front/locales/fr.json b/front/locales/fr.json
@@ -210,6 +210,11 @@
     "requireRequestorId": "Un requestor ID est nécessaire",
     "requireApiKey": "Une clé d'API est nécessaire",
     "isSushiCompliant": "Le point d'accès est conforme SUSHI",
+    "harvestedReports": "Rapports moissonnés",
+    "ignoredReportsDesc": "Les rapports saisis ci-dessous seront considérés comme non-supportés, même s'ils sont présents dans la liste fournie par le point d'accès.",
+    "ignoredReports": "Rapports ignorés",
+    "additionalReportsDesc": "Les rapports saisis ci-dessous seront considérés comme supportés, même s'ils ne se trouvent pas dans la liste fournie par le point d'accès.",
+    "additionalReports": "Rapports additionnels",
     "ignoreReportValidation": "Ignorer la validation du rapport",
     "description": "Description",
     "paramSeparatorDesc": "ezMESURE ajoute automatiquement les paramètres {0}, {1} et {2} lors du moissonnage. Si le point d'accès n'accepte pas le séparateur standard {separator} pour ces champs multivalués, renseignez le séparateur ici.",
@@ -584,6 +589,7 @@
     "checkCredentials": "Vérifier les identifiants",
     "supportedReports": "Rapports supportés",
     "supportedReportsOnPlatform": "Rapports supportés par la plateforme éditeur",
+    "supportedReportsUnavailable": "La liste des rapports supportés n'est pas disponible pour cette plateforme éditeur.",
     "failedToFetchReports": "Impossible de récupérer la liste des rapports disponibles",
     "onlyShowMasterReports": "Afficher uniquement les rapports maîtres"
   },
@@ -870,5 +876,7 @@
   "invalidate": "Invalider",
   "reason": "Raison : {reason}",
   "groupBy": "Regrouper par",
-  "indeterminate": "indéterminée"
+  "indeterminate": "indéterminée",
+  "noMatchFor": "Aucune correspondance pour \"{search}\". Appuyez sur {key} pour l'ajouter.",
+  "enterKey": "entrée"
 }