generate_dataset_annif.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate dataset Annif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows: 12243\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>maa1</th>\n",
       "      <th>maa2</th>\n",
       "      <th>ppn</th>\n",
       "      <th>jvu_1100</th>\n",
       "      <th>taal_1500_publ</th>\n",
       "      <th>taal_1500_orig</th>\n",
       "      <th>isbn_2000</th>\n",
       "      <th>unesco_1121</th>\n",
       "      <th>unesco_1122</th>\n",
       "      <th>nur_codes_5061</th>\n",
       "      <th>brinkman_520x</th>\n",
       "      <th>brinkman_520x_ppn</th>\n",
       "      <th>onix_7880</th>\n",
       "      <th>prim_auteur_3000</th>\n",
       "      <th>sec_auteur_3011</th>\n",
       "      <th>titel_4000</th>\n",
       "      <th>ondertitel_4000</th>\n",
       "      <th>samenvatting_4207</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>322079640</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.789054e+12</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>321</td>\n",
       "      <td>levensbeschrijvingen</td>\n",
       "      <td>075613816</td>\n",
       "      <td>9789054292692</td>\n",
       "      <td>Han/van@Bree$aut$!069567727!Han van Bree 1957-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>De @geest van het Oude Loo</td>\n",
       "      <td>Juliana en haar vriendenkring 1947-1957</td>\n",
       "      <td>In het boek wordt de hofcrisis van 1956 voor h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>33015673X</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>fra</td>\n",
       "      <td>9.789490e+12</td>\n",
       "      <td>b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>736</td>\n",
       "      <td>perceptie | esthetiek</td>\n",
       "      <td>075618451 | 075605503</td>\n",
       "      <td>9789490334086</td>\n",
       "      <td>Gilles@Deleuze$aut$!06860873X!Gilles Deleuze 1...</td>\n",
       "      <td>Walter/van der@Star$trl$!125379315!Walter van ...</td>\n",
       "      <td>@Francis Bacon</td>\n",
       "      <td>logica van de gewaarwording</td>\n",
       "      <td>Esthetische analyse van het werk van de Britse...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  maa1 maa2        ppn  jvu_1100 taal_1500_publ taal_1500_orig     isbn_2000  \\\n",
       "0    A   Aa  322079640      2015            ned            NaN  9.789054e+12   \n",
       "1    A   Aa  33015673X      2015            ned            fra  9.789490e+12   \n",
       "\n",
       "  unesco_1121 unesco_1122 nur_codes_5061          brinkman_520x  \\\n",
       "0           7         NaN            321   levensbeschrijvingen   \n",
       "1           b         NaN            736  perceptie | esthetiek   \n",
       "\n",
       "       brinkman_520x_ppn      onix_7880  \\\n",
       "0              075613816  9789054292692   \n",
       "1  075618451 | 075605503  9789490334086   \n",
       "\n",
       "                                    prim_auteur_3000  \\\n",
       "0     Han/van@Bree$aut$!069567727!Han van Bree 1957-   \n",
       "1  Gilles@Deleuze$aut$!06860873X!Gilles Deleuze 1...   \n",
       "\n",
       "                                     sec_auteur_3011  \\\n",
       "0                                                NaN   \n",
       "1  Walter/van der@Star$trl$!125379315!Walter van ...   \n",
       "\n",
       "                   titel_4000                          ondertitel_4000  \\\n",
       "0  De @geest van het Oude Loo  Juliana en haar vriendenkring 1947-1957   \n",
       "1              @Francis Bacon              logica van de gewaarwording   \n",
       "\n",
       "                                   samenvatting_4207  \n",
       "0  In het boek wordt de hofcrisis van 1956 voor h...  \n",
       "1  Esthetische analyse van het werk van de Britse...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import GGC-data\n",
    "df = pd.read_csv('../data/original_source_data/vraag_20190620.txt', sep='\\t')\n",
    "print('Number of rows: ' + str(len(df)))\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess GGC-data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine title, subtitle and summary and tidy up data\n",
    "Combine `titel_4000`, `ondertitel_4000` and `samenvatting_4207` values into a new column called `samenvatting_plus_titel`. \n",
    "\n",
    "Convert `brinkman_520x` and `brinkman_520x_ppn` string values into a list of strings split at `|` e.g. `\"perceptie | esthetiek\"` becomes `[perceptie, esthetiek]`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>maa1</th>\n",
       "      <th>maa2</th>\n",
       "      <th>ppn</th>\n",
       "      <th>jvu_1100</th>\n",
       "      <th>taal_1500_publ</th>\n",
       "      <th>taal_1500_orig</th>\n",
       "      <th>isbn_2000</th>\n",
       "      <th>unesco_1121</th>\n",
       "      <th>unesco_1122</th>\n",
       "      <th>nur_codes_5061</th>\n",
       "      <th>brinkman_520x</th>\n",
       "      <th>brinkman_520x_ppn</th>\n",
       "      <th>onix_7880</th>\n",
       "      <th>prim_auteur_3000</th>\n",
       "      <th>sec_auteur_3011</th>\n",
       "      <th>titel_4000</th>\n",
       "      <th>ondertitel_4000</th>\n",
       "      <th>samenvatting_4207</th>\n",
       "      <th>samenvatting_plus_titel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>322079640</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.789054e+12</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>321</td>\n",
       "      <td>[levensbeschrijvingen]</td>\n",
       "      <td>[075613816]</td>\n",
       "      <td>9789054292692</td>\n",
       "      <td>Han/van@Bree$aut$!069567727!Han van Bree 1957-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>De @geest van het Oude Loo</td>\n",
       "      <td>Juliana en haar vriendenkring 1947-1957</td>\n",
       "      <td>In het boek wordt de hofcrisis van 1956 voor h...</td>\n",
       "      <td>De  geest van het Oude Loo Juliana en haar vri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>33015673X</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>fra</td>\n",
       "      <td>9.789490e+12</td>\n",
       "      <td>b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>736</td>\n",
       "      <td>[perceptie, esthetiek]</td>\n",
       "      <td>[075618451, 075605503]</td>\n",
       "      <td>9789490334086</td>\n",
       "      <td>Gilles@Deleuze$aut$!06860873X!Gilles Deleuze 1...</td>\n",
       "      <td>Walter/van der@Star$trl$!125379315!Walter van ...</td>\n",
       "      <td>@Francis Bacon</td>\n",
       "      <td>logica van de gewaarwording</td>\n",
       "      <td>Esthetische analyse van het werk van de Britse...</td>\n",
       "      <td>Francis Bacon logica van de gewaarwording Est...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  maa1 maa2        ppn  jvu_1100 taal_1500_publ taal_1500_orig     isbn_2000  \\\n",
       "0    A   Aa  322079640      2015            ned            NaN  9.789054e+12   \n",
       "1    A   Aa  33015673X      2015            ned            fra  9.789490e+12   \n",
       "\n",
       "  unesco_1121 unesco_1122 nur_codes_5061           brinkman_520x  \\\n",
       "0           7         NaN            321  [levensbeschrijvingen]   \n",
       "1           b         NaN            736  [perceptie, esthetiek]   \n",
       "\n",
       "        brinkman_520x_ppn      onix_7880  \\\n",
       "0             [075613816]  9789054292692   \n",
       "1  [075618451, 075605503]  9789490334086   \n",
       "\n",
       "                                    prim_auteur_3000  \\\n",
       "0     Han/van@Bree$aut$!069567727!Han van Bree 1957-   \n",
       "1  Gilles@Deleuze$aut$!06860873X!Gilles Deleuze 1...   \n",
       "\n",
       "                                     sec_auteur_3011  \\\n",
       "0                                                NaN   \n",
       "1  Walter/van der@Star$trl$!125379315!Walter van ...   \n",
       "\n",
       "                   titel_4000                          ondertitel_4000  \\\n",
       "0  De @geest van het Oude Loo  Juliana en haar vriendenkring 1947-1957   \n",
       "1              @Francis Bacon              logica van de gewaarwording   \n",
       "\n",
       "                                   samenvatting_4207  \\\n",
       "0  In het boek wordt de hofcrisis van 1956 voor h...   \n",
       "1  Esthetische analyse van het werk van de Britse...   \n",
       "\n",
       "                             samenvatting_plus_titel  \n",
       "0  De  geest van het Oude Loo Juliana en haar vri...  \n",
       "1   Francis Bacon logica van de gewaarwording Est...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace NaN by a space (' ') in titel_4000 and ondertitel_4000 (subtitles) before merging data from the three columns.\n",
    "df['titel_4000'].fillna(' ', axis=0, inplace=True)\n",
    "df['ondertitel_4000'].fillna(' ', axis=0, inplace=True)\n",
    "\n",
    "# Remove @ from title and create new column of combined values.\n",
    "df['samenvatting_plus_titel'] = df['titel_4000'].str.replace('@', ' ') + ' ' + df['ondertitel_4000'] + ' ' + df['samenvatting_4207']\n",
    "\n",
    "# Convert string with '|' (pipes) into list of Brinkman subjects and brinkman subjects' ppn's.\n",
    "df['brinkman_520x_ppn'] = df['brinkman_520x_ppn'].str.replace(' ', '').str.split('|')\n",
    "df['brinkman_520x'] = df['brinkman_520x'].str.replace(' \\| ', '|', regex=True).str.split('|')\n",
    "\n",
    "# Remove errors. (In exploratory research stage found problems with entries (mostly assigned subjects not in thesaurus))\n",
    "df = df[~df.ppn.isin(['406248214', '418852146', '421344679', '394609565', '405599978', '406344175', '420510567'])]\n",
    "\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create 2 individual DF's, one of entries with subjects refering to content and one with subjects refering to form"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Brinkman subjects that refer to the form of an entry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ppn</th>\n",
       "      <th>btr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>075598612</td>\n",
       "      <td>adresboeken</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>075656876</td>\n",
       "      <td>anekdoten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>374639280</td>\n",
       "      <td>antiquariaatscatalogi</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         ppn                    btr\n",
       "0  075598612            adresboeken\n",
       "1  075656876              anekdoten\n",
       "2  374639280  antiquariaatscatalogi"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import Brinkman subjects that refer to the form of an entry (e.g. autobiography)\n",
    "df_vorm = pd.read_csv('../data/original_source_data/btr_vorm.tsv', sep='\\t')\n",
    "df_vorm.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to two lists so we can use them to subtract these subjects from the assigned subjects using sets.\n",
    "ppn_vormtrefwoorden = df_vorm['ppn'].values.tolist()\n",
    "vormtrefwoorden = df_vorm['btr'].values.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create GGC-dataset with only content subjects (`df_content`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove Brinkman subjects and subjects' PPN that refer to form.\n",
    "df_content = df\n",
    "df_content['brinkman_520x_ppn'] = df_content['brinkman_520x_ppn'].apply(lambda row: list(set(row) - set(ppn_vormtrefwoorden)))\n",
    "df_content['brinkman_520x'] = df_content['brinkman_520x'].apply(lambda row: list(set(row) - set(vormtrefwoorden)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove empty lists i.e. remove entries which only have subjects refering to form.\n",
    "df_content = df_content[df_content.astype(str)['brinkman_520x_ppn'] != '[]']\n",
    "df_content = df_content[df_content.astype(str)['brinkman_520x'] != '[]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>maa1</th>\n",
       "      <th>maa2</th>\n",
       "      <th>ppn</th>\n",
       "      <th>jvu_1100</th>\n",
       "      <th>taal_1500_publ</th>\n",
       "      <th>taal_1500_orig</th>\n",
       "      <th>isbn_2000</th>\n",
       "      <th>unesco_1121</th>\n",
       "      <th>unesco_1122</th>\n",
       "      <th>nur_codes_5061</th>\n",
       "      <th>brinkman_520x</th>\n",
       "      <th>brinkman_520x_ppn</th>\n",
       "      <th>onix_7880</th>\n",
       "      <th>prim_auteur_3000</th>\n",
       "      <th>sec_auteur_3011</th>\n",
       "      <th>titel_4000</th>\n",
       "      <th>ondertitel_4000</th>\n",
       "      <th>samenvatting_4207</th>\n",
       "      <th>samenvatting_plus_titel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>33015673X</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>fra</td>\n",
       "      <td>9.789490e+12</td>\n",
       "      <td>b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>736</td>\n",
       "      <td>[esthetiek, perceptie]</td>\n",
       "      <td>[075618451, 075605503]</td>\n",
       "      <td>9789490334086</td>\n",
       "      <td>Gilles@Deleuze$aut$!06860873X!Gilles Deleuze 1...</td>\n",
       "      <td>Walter/van der@Star$trl$!125379315!Walter van ...</td>\n",
       "      <td>@Francis Bacon</td>\n",
       "      <td>logica van de gewaarwording</td>\n",
       "      <td>Esthetische analyse van het werk van de Britse...</td>\n",
       "      <td>Francis Bacon logica van de gewaarwording Est...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>352699566</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.789462e+12</td>\n",
       "      <td>z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>648</td>\n",
       "      <td>[vakantieverblijven]</td>\n",
       "      <td>[075625156]</td>\n",
       "      <td>9789462080744</td>\n",
       "      <td>Mieke@Dings$aut$!270022139!Mieke Dings 1979-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@Tussen tent en villa</td>\n",
       "      <td>het vakantiepark in Nederland 1920-nu</td>\n",
       "      <td>Het vakantiepark: wie heeft er niet weleens ee...</td>\n",
       "      <td>Tussen tent en villa het vakantiepark in Nede...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>363250565</td>\n",
       "      <td>2016</td>\n",
       "      <td>ned</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.789039e+12</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>320</td>\n",
       "      <td>[Marokkanen]</td>\n",
       "      <td>[07566111X]</td>\n",
       "      <td>9789038898254</td>\n",
       "      <td>Salaheddine@Benchikhi$aut$!296322547!Salaheddi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@Salaheddine punt NL</td>\n",
       "      <td>kom maar op met Nederland</td>\n",
       "      <td>In Salaheddine punt NL vertelt Salaheddine hoe...</td>\n",
       "      <td>Salaheddine punt NL kom maar op met Nederland...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  maa1 maa2        ppn  jvu_1100 taal_1500_publ taal_1500_orig     isbn_2000  \\\n",
       "1    A   Aa  33015673X      2015            ned            fra  9.789490e+12   \n",
       "3    A   Aa  352699566      2015            ned            NaN  9.789462e+12   \n",
       "5    A   Aa  363250565      2016            ned            NaN  9.789039e+12   \n",
       "\n",
       "  unesco_1121 unesco_1122 nur_codes_5061           brinkman_520x  \\\n",
       "1           b         NaN            736  [esthetiek, perceptie]   \n",
       "3           z         NaN            648    [vakantieverblijven]   \n",
       "5           7         NaN            320            [Marokkanen]   \n",
       "\n",
       "        brinkman_520x_ppn      onix_7880  \\\n",
       "1  [075618451, 075605503]  9789490334086   \n",
       "3             [075625156]  9789462080744   \n",
       "5             [07566111X]  9789038898254   \n",
       "\n",
       "                                    prim_auteur_3000  \\\n",
       "1  Gilles@Deleuze$aut$!06860873X!Gilles Deleuze 1...   \n",
       "3       Mieke@Dings$aut$!270022139!Mieke Dings 1979-   \n",
       "5  Salaheddine@Benchikhi$aut$!296322547!Salaheddi...   \n",
       "\n",
       "                                     sec_auteur_3011             titel_4000  \\\n",
       "1  Walter/van der@Star$trl$!125379315!Walter van ...         @Francis Bacon   \n",
       "3                                                NaN  @Tussen tent en villa   \n",
       "5                                                NaN   @Salaheddine punt NL   \n",
       "\n",
       "                         ondertitel_4000  \\\n",
       "1            logica van de gewaarwording   \n",
       "3  het vakantiepark in Nederland 1920-nu   \n",
       "5              kom maar op met Nederland   \n",
       "\n",
       "                                   samenvatting_4207  \\\n",
       "1  Esthetische analyse van het werk van de Britse...   \n",
       "3  Het vakantiepark: wie heeft er niet weleens ee...   \n",
       "5  In Salaheddine punt NL vertelt Salaheddine hoe...   \n",
       "\n",
       "                             samenvatting_plus_titel  \n",
       "1   Francis Bacon logica van de gewaarwording Est...  \n",
       "3   Tussen tent en villa het vakantiepark in Nede...  \n",
       "5   Salaheddine punt NL kom maar op met Nederland...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Complete DF of entries having only assigned subjects refering to the content (5710 rows).\n",
    "#df_content.to_pickle('../data/original_source_data/df_zaaktrefwoorden.pkl')\n",
    "df_content.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create GGC-dataset with only form subjects (`df_form`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove Brinkman subjects and subjects' PPN that refer to content.\n",
    "df_form = df\n",
    "df_form['brinkman_520x_ppn'] = df_form['brinkman_520x_ppn'].apply(lambda row: list(set(row) & set(ppn_vormtrefwoorden)))\n",
    "df_form['brinkman_520x'] = df_form['brinkman_520x'].apply(lambda row: list(set(row) & set(vormtrefwoorden)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove empty lists i.e. remove entries which only have subjects refering to content.\n",
    "df_form = df_form[df_form.astype(str)['brinkman_520x_ppn'] != '[]']\n",
    "df_form = df_form[df_form.astype(str)['brinkman_520x'] != '[]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>maa1</th>\n",
       "      <th>maa2</th>\n",
       "      <th>ppn</th>\n",
       "      <th>jvu_1100</th>\n",
       "      <th>taal_1500_publ</th>\n",
       "      <th>taal_1500_orig</th>\n",
       "      <th>isbn_2000</th>\n",
       "      <th>unesco_1121</th>\n",
       "      <th>unesco_1122</th>\n",
       "      <th>nur_codes_5061</th>\n",
       "      <th>brinkman_520x</th>\n",
       "      <th>brinkman_520x_ppn</th>\n",
       "      <th>onix_7880</th>\n",
       "      <th>prim_auteur_3000</th>\n",
       "      <th>sec_auteur_3011</th>\n",
       "      <th>titel_4000</th>\n",
       "      <th>ondertitel_4000</th>\n",
       "      <th>samenvatting_4207</th>\n",
       "      <th>samenvatting_plus_titel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>322079640</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.789054e+12</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>321</td>\n",
       "      <td>[levensbeschrijvingen]</td>\n",
       "      <td>[075613816]</td>\n",
       "      <td>9789054292692</td>\n",
       "      <td>Han/van@Bree$aut$!069567727!Han van Bree 1957-</td>\n",
       "      <td>NaN</td>\n",
       "      <td>De @geest van het Oude Loo</td>\n",
       "      <td>Juliana en haar vriendenkring 1947-1957</td>\n",
       "      <td>In het boek wordt de hofcrisis van 1956 voor h...</td>\n",
       "      <td>De  geest van het Oude Loo Juliana en haar vri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>352655844</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.789460e+12</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>698</td>\n",
       "      <td>[levensbeschrijvingen]</td>\n",
       "      <td>[075613816]</td>\n",
       "      <td>9789460041228</td>\n",
       "      <td>Anton/van de@Sande$aut$!067525180!Antonius Wil...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@Prins Frederik der Nederlanden 1797-1881</td>\n",
       "      <td>gentleman naast de troon</td>\n",
       "      <td>Beschrijving van het leven van prins Frederik,...</td>\n",
       "      <td>Prins Frederik der Nederlanden 1797-1881 gent...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A</td>\n",
       "      <td>Aa</td>\n",
       "      <td>362837317</td>\n",
       "      <td>2015</td>\n",
       "      <td>ned</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.789047e+12</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>301</td>\n",
       "      <td>[romans en novellen ; oorspr. - Nederlands]</td>\n",
       "      <td>[075629402]</td>\n",
       "      <td>9789046815809</td>\n",
       "      <td>Jan/van der@Mast$aut$!07502943X!Jan van der Ma...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@Agneta</td>\n",
       "      <td></td>\n",
       "      <td>Jacques van Marken (1845-1906), oprichter van ...</td>\n",
       "      <td>Agneta   Jacques van Marken (1845-1906), opri...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  maa1 maa2        ppn  jvu_1100 taal_1500_publ taal_1500_orig     isbn_2000  \\\n",
       "0    A   Aa  322079640      2015            ned            NaN  9.789054e+12   \n",
       "2    A   Aa  352655844      2015            ned            NaN  9.789460e+12   \n",
       "4    A   Aa  362837317      2015            ned            NaN  9.789047e+12   \n",
       "\n",
       "  unesco_1121 unesco_1122 nur_codes_5061  \\\n",
       "0           7         NaN            321   \n",
       "2           7         NaN            698   \n",
       "4           4         NaN            301   \n",
       "\n",
       "                                 brinkman_520x brinkman_520x_ppn  \\\n",
       "0                       [levensbeschrijvingen]       [075613816]   \n",
       "2                       [levensbeschrijvingen]       [075613816]   \n",
       "4  [romans en novellen ; oorspr. - Nederlands]       [075629402]   \n",
       "\n",
       "       onix_7880                                   prim_auteur_3000  \\\n",
       "0  9789054292692     Han/van@Bree$aut$!069567727!Han van Bree 1957-   \n",
       "2  9789460041228  Anton/van de@Sande$aut$!067525180!Antonius Wil...   \n",
       "4  9789046815809  Jan/van der@Mast$aut$!07502943X!Jan van der Ma...   \n",
       "\n",
       "  sec_auteur_3011                                 titel_4000  \\\n",
       "0             NaN                 De @geest van het Oude Loo   \n",
       "2             NaN  @Prins Frederik der Nederlanden 1797-1881   \n",
       "4             NaN                                    @Agneta   \n",
       "\n",
       "                           ondertitel_4000  \\\n",
       "0  Juliana en haar vriendenkring 1947-1957   \n",
       "2                 gentleman naast de troon   \n",
       "4                                            \n",
       "\n",
       "                                   samenvatting_4207  \\\n",
       "0  In het boek wordt de hofcrisis van 1956 voor h...   \n",
       "2  Beschrijving van het leven van prins Frederik,...   \n",
       "4  Jacques van Marken (1845-1906), oprichter van ...   \n",
       "\n",
       "                             samenvatting_plus_titel  \n",
       "0  De  geest van het Oude Loo Juliana en haar vri...  \n",
       "2   Prins Frederik der Nederlanden 1797-1881 gent...  \n",
       "4   Agneta   Jacques van Marken (1845-1906), opri...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Complete DF of entries having only assigned subjects refering to the form (7428 rows).\n",
    "#df_form.to_pickle('../data/original_source_data/df_vormtrefwoorden.pkl')\n",
    "df_form.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Full-text document corpus for Annif\n",
    "[Annif document corpus formats](https://github.com/NatLibFi/Annif/wiki/Document-corpus-formats)\n",
    "\n",
    "All files will be saved into ./data/fulltext_corpus directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Brinkman TSV as python dictionary.\n",
    "dict_subjects = {}\n",
    "with open('data/brinkmanthesaurus_vocab.tsv', mode='r') as infile:\n",
    "    reader = csv.reader(infile, delimiter=\"\\t\")\n",
    "    for row in reader:\n",
    "        dict_subjects[row[1]] = row[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Choose a DF to use to generate full-text corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose DF (df_content, df_form or df) to use to create full-text corpus.\n",
    "#df_corp = df_content\n",
    "df_corp = df_form"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate full-text document corpus text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "De 0 onderstaande toegewezen Brinkmantrefwoorden staan niet in brinkmanthesaurus_vocab.tsv, maar hebben nog wel andere termen toegewezen:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create dict with 'ppn' (of work) and 'brinkman_520x' e.g. {33015673X': ['perceptie', 'esthetiek'], ...}\n",
    "dict_ppn_bk = pd.Series(df_corp.brinkman_520x.values,index=df_corp.ppn).to_dict()\n",
    "\n",
    "# Create dictionary {ppn : [brinkman_id, brinkman_term]}\n",
    "dict_assigned_sub = {}\n",
    "\n",
    "br_not_found = []\n",
    "for ppn, asg_subj in dict_ppn_bk.items():\n",
    "    mult_sub = []\n",
    "    for subj in asg_subj:\n",
    "        try:\n",
    "            mult_sub.append([dict_subjects[subj], subj])\n",
    "        except KeyError:\n",
    "            br_not_found.append(subj)\n",
    "    dict_assigned_sub[ppn] = mult_sub\n",
    "\n",
    "\n",
    "print(f'De {len(set(br_not_found))} onderstaande toegewezen Brinkmantrefwoorden staan niet in brinkmanthesaurus_vocab.tsv, maar hebben nog wel andere termen toegewezen:\\n')   \n",
    "for i in set(br_not_found):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create fulltext corpus .txt files.\n",
    "dict_sum = pd.Series(df_corp.samenvatting_plus_titel.values,index=df_corp.ppn).to_dict()\n",
    "if not os.path.exists(os.path.join('data', 'fulltext_corpus')):\n",
    "    os.makedirs(os.path.join('data', 'fulltext_corpus'))\n",
    "for ppn, summ in dict_sum.items():\n",
    "    filename = ppn + '.txt'\n",
    "    with open(os.path.join(os.path.join('data', 'fulltext_corpus'), filename), mode='w') as sumfile:\n",
    "        sumfile.write(str(summ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate full-text document corpus subject files\n",
    "In simple format (.key files)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "# Create .key file for each ppn. Subject vocabulary as TSV.\n",
    "err = []\n",
    "if not os.path.exists(os.path.join('data', 'fulltext_corpus')):\n",
    "    os.makedirs(os.path.join('data', 'fulltext_corpus'))\n",
    "for ppn, subj_id in dict_assigned_sub.items():\n",
    "    filename = ppn + '.tsv'\n",
    "    with open(os.path.join(os.path.join('data', 'fulltext_corpus'), filename), mode='w') as subfile:\n",
    "        if len(subj_id) > 1:\n",
    "            for subj_nr in subj_id:\n",
    "                subfile.write(subj_nr[0] + '\\t' + subj_nr[1] + '\\n')\n",
    "        elif len(subj_id) == 1:\n",
    "            subfile.write(subj_id[0][0] + '\\t' + subj_id[0][1])\n",
    "        else:\n",
    "            err.append(ppn)\n",
    "\n",
    "# 'err' is a list with PPN's of problematic entries (no clear brinkman identifier)\n",
    "# e.g. after split one of the single terms is not found in te vocabulary.\n",
    "# Should be empty - I've removed them to tidy up GGC-data at the start of this notebook.\n",
    "print(err)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split dataset into train, test, eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split dataset 80-15-5\n",
    "train, test, ev = np.split(df_corp, [int(.8*len(df_corp)), int(.95*len(df_corp))])\n",
    "\n",
    "# Create folders\n",
    "if not os.path.exists(os.path.join(os.path.join('data', 'fulltext_corpus'), 'train')):\n",
    "    os.makedirs(os.path.join(os.path.join('data', 'fulltext_corpus'), 'train'))\n",
    "if not os.path.exists(os.path.join(os.path.join('data', 'fulltext_corpus'), 'test')):\n",
    "    os.makedirs(os.path.join(os.path.join('data', 'fulltext_corpus'), 'test'))\n",
    "if not os.path.exists(os.path.join(os.path.join('data', 'fulltext_corpus'), 'eval')):\n",
    "    os.makedirs(os.path.join(os.path.join('data', 'fulltext_corpus'), 'eval'))\n",
    "\n",
    "# Move selected vocab files to train folder.\n",
    "for ppn in train['ppn'].tolist():\n",
    "    os.rename(os.path.join(os.path.join('data', 'fulltext_corpus'), ppn + '.tsv'), os.path.join(os.path.join(os.path.join('data', 'fulltext_corpus'), 'train'), ppn + '.tsv'))\n",
    "    os.rename(os.path.join(os.path.join('data', 'fulltext_corpus'), ppn + '.txt'), os.path.join(os.path.join(os.path.join('data', 'fulltext_corpus'), 'train'), ppn + '.txt'))\n",
    "\n",
    "# Move selected vocab files to test folder.\n",
    "for ppn in test['ppn'].tolist():\n",
    "    os.rename(os.path.join(os.path.join('data', 'fulltext_corpus'), ppn + '.tsv'), os.path.join(os.path.join(os.path.join('data', 'fulltext_corpus'), 'test'), ppn + '.tsv'))\n",
    "    os.rename(os.path.join(os.path.join('data', 'fulltext_corpus'), ppn + '.txt'), os.path.join(os.path.join(os.path.join('data', 'fulltext_corpus'), 'test'), ppn + '.txt'))\n",
    "    \n",
    "# Move selected vocab files to dev folder.\n",
    "for ppn in ev['ppn'].tolist():\n",
    "    os.rename(os.path.join(os.path.join('data', 'fulltext_corpus'), ppn + '.tsv'), os.path.join(os.path.join(os.path.join('data', 'fulltext_corpus'), 'eval'), ppn + '.tsv'))\n",
    "    os.rename(os.path.join(os.path.join('data', 'fulltext_corpus'), ppn + '.txt'), os.path.join(os.path.join(os.path.join('data', 'fulltext_corpus'), 'eval'), ppn + '.txt'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}