Merge pull request #64 from demokratie-live/alpha/master

Alpha/master
demokratie-live · May 11, 2020 · 0b37f52 · 0b37f52
2 parents d8af1d0 + eaf1f7e
commit 0b37f52
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+### 0.1.18
+
+- [Fix] fix named polls matching
+
 ### 0.1.17
 
 - [Fix] Upgrade @democracy-deutschland/bundestag.io-definitions to fix toggle named poll

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "bundestag.io",
-  "version": "0.1.17",
+  "version": "0.1.18",
   "main": "src/index.js",
   "author": "Ulf Gebhardt",
   "license": "Apache2",

diff --git a/src/importer/importNamedPolls.js b/src/importer/importNamedPolls.js
@@ -1,5 +1,6 @@
 import { Scraper } from '@democracy-deutschland/scapacra';
 import { NamedPollScraper } from '@democracy-deutschland/scapacra-bt';
+import url from 'url';
 
 import {
   PROCEDURE as PROCEDURE_DEFINITIONS,
@@ -29,9 +30,14 @@ export default async () => {
       // "http://dipbt.bundestag.de:80/dip21/btd/19/010/1901038.pdf
       // The named poll scraper returns them like so:
       // http://dip21.bundestag.de/dip21/btd/19/010/1901038.pdf
-      const findSpotUrls = dataPackage.data.documents.map(document =>
-        document.replace('http://dip21.bundestag.de/', 'http://dipbt.bundestag.de:80/'),
-      );
+      const findSpotUrls = dataPackage.data.documents.map(document => {
+        return {
+          'history.findSpotUrl': {
+            $regex: `.*${url.parse(document).path}.*`,
+          },
+        };
+        return document.replace('http://dip21.bundestag.de/', 'http://dipbt.bundestag.de:80/');
+      });
 
       let procedures;
       // Only match those which are not an Änderungsantrag
@@ -46,7 +52,7 @@ export default async () => {
       ) {
         // Find matching Procedures
         procedures = await Procedure.find({
-          'history.findSpotUrl': { $all: findSpotUrls },
+          $and: findSpotUrls,
           'history.decision': {
             $elemMatch: {
               type: PROCEDURE_DEFINITIONS.HISTORY.DECISION.TYPE.NAMENTLICHE_ABSTIMMUNG,

diff --git a/src/importer/importProcedures.js b/src/importer/importProcedures.js
@@ -36,9 +36,11 @@ const ensureArray = element => {
 
 const saveProcedure = async ({ procedureData }) => {
   // Transform History
-  const process = _.isArray(procedureData.VORGANGSABLAUF.VORGANGSPOSITION)
+  let process = _.isArray(procedureData.VORGANGSABLAUF.VORGANGSPOSITION)
     ? procedureData.VORGANGSABLAUF.VORGANGSPOSITION
     : [procedureData.VORGANGSABLAUF.VORGANGSPOSITION];
+  // TODO check why some e are undefined
+  process = process.filter(e => e);
   const history = process.map(e => {
     const flow = {
       assignment: e.ZUORDNUNG.trim(),