exps/AGRR-2019-full.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "import sys\n",
    "import numpy as npl\n",
    "\n",
    "sys.path.append(\"../\")\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0. Parse data\n",
    "Store data in NER format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/home/eartemov/ae/work/data/AGRR-2019/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "train_path = data_dir + \"train.csv\"\n",
    "valid_path = data_dir + \"dev.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(train_path, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>cV</th>\n",
       "      <th>cR1</th>\n",
       "      <th>cR2</th>\n",
       "      <th>V</th>\n",
       "      <th>R1</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Будучи в прошлый четверг в Софии, он назвал се...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Работа с двухбайтовыми наборами символов — про...</td>\n",
       "      <td>1</td>\n",
       "      <td>92:99</td>\n",
       "      <td>83:91</td>\n",
       "      <td>103:109</td>\n",
       "      <td>127:127</td>\n",
       "      <td>119:124</td>\n",
       "      <td>127:134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Заместитель Генерального секретаря подчеркнул,...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Продажа недвижимости из собственных портфелей ...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Новым является то, что повышенное давление кон...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  class     cV    cR1  \\\n",
       "0  Будучи в прошлый четверг в Софии, он назвал се...      0    NaN    NaN   \n",
       "1  Работа с двухбайтовыми наборами символов — про...      1  92:99  83:91   \n",
       "2  Заместитель Генерального секретаря подчеркнул,...      0    NaN    NaN   \n",
       "3  Продажа недвижимости из собственных портфелей ...      0    NaN    NaN   \n",
       "4  Новым является то, что повышенное давление кон...      0    NaN    NaN   \n",
       "\n",
       "       cR2        V       R1       R2  \n",
       "0      NaN      NaN      NaN      NaN  \n",
       "1  103:109  127:127  119:124  127:134  \n",
       "2      NaN      NaN      NaN      NaN  \n",
       "3      NaN      NaN      NaN      NaN  \n",
       "4      NaN      NaN      NaN      NaN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sup_labels = ['cV', 'cR1', 'cR2', 'V', 'R1', 'R2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_iob( words, tag):\n",
    "    tags = []\n",
    "    if not len(words):\n",
    "        raise ValueError(\"Words should have len > 0.\")\n",
    "    tags.append(\"{}_{}\".format(\"B\", tag))\n",
    "    tags.extend([\"{}_{}\".format(\"I\", tag)] * (len(words) - 1))\n",
    "    return tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_row(row, origin_idx, sup_labels=['cV', 'cR1', 'cR2', 'V', 'R1', 'R2'], word_tokenizer=lambda x: x.split()):\n",
    "    try:\n",
    "        text = row.text\n",
    "    except:\n",
    "        text = row[\"0\"]\n",
    "    len_doc = len(text)\n",
    "    pos = 0\n",
    "    splitted = []\n",
    "    tags = []\n",
    "    cur_pos = 0\n",
    "    cls = 0\n",
    "    try:\n",
    "        cls = int(row[\"class\"])\n",
    "    except:\n",
    "        cls = 0\n",
    "    tokens_pos = dict()\n",
    "    if cls:\n",
    "        for name in sup_labels:\n",
    "            if type(row[name]) is not float:\n",
    "                for b in row[name].split():\n",
    "                    b = b.split(\":\")\n",
    "                    start_pos, end_pos = map(int, b)\n",
    "                    tokens_pos[start_pos] = {\"name\": name, \"text\": text[start_pos:end_pos]}\n",
    "    while pos < len_doc:\n",
    "        token = tokens_pos.get(pos)\n",
    "        if token:\n",
    "            splitted.append(text[cur_pos:pos])\n",
    "            tags.append((\"O\", pos))\n",
    "            pos += len(token[\"text\"])\n",
    "            cur_pos = pos\n",
    "            splitted.append(token[\"text\"])\n",
    "            tags.append((token[\"name\"], pos))\n",
    "        pos += 1\n",
    "    if len(text[cur_pos:pos]):\n",
    "        # pos = min(len_doc, self.max_pos + 20)\n",
    "        splitted.append(text[cur_pos:pos])\n",
    "        tags.append((\"O\", cur_pos))\n",
    "    res_tokens = []\n",
    "    assert len(splitted) == len(tags)\n",
    "    cur_pos = 0\n",
    "    len_word = 0\n",
    "    res_labels = []\n",
    "    res_tokens = []\n",
    "    res_start_pos = []\n",
    "    res_len = []\n",
    "    for s_text, (tag, pos) in zip(splitted, tags):\n",
    "        words = word_tokenizer(s_text)\n",
    "        # words = self.__clean_word(text).split()\n",
    "        if not len(words):\n",
    "            continue\n",
    "        if tag == \"O\":\n",
    "            token_tags = [\"O\"] * len(words)\n",
    "        else:\n",
    "            token_tags = to_iob(words, tag)\n",
    "        assert len(words) == len(token_tags)\n",
    "        for word, tag_ in zip(words, token_tags):\n",
    "            cur_pos = text.find(word, cur_pos)\n",
    "            len_word = len(word)\n",
    "            if len(word):\n",
    "                res_labels.append(tag_)\n",
    "                res_tokens.append(word)\n",
    "                res_start_pos.append(str(cur_pos))\n",
    "                res_len.append(str(len_word))\n",
    "            cur_pos += len_word\n",
    "    return pd.DataFrame({\n",
    "        0: [\" \".join(res_labels)],\n",
    "        1: [\" \".join(res_tokens)],\n",
    "        2: [cls],\n",
    "        \"start_pos\": [\" \".join(res_start_pos)],\n",
    "        \"len\": [\" \".join(res_len)],\n",
    "        \"origin_idx\": origin_idx\n",
    "    }, columns=[0, 1, 2, \"start_pos\", \"len\", \"origin_idx\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prc_df(path, sup_labels):\n",
    "    res = pd.DataFrame(columns=[0, 1, 2, \"start_pos\", \"len\", \"origin_idx\"])\n",
    "    df = pd.read_csv(path, sep=\"\\t\")\n",
    "    for origin_idx, row in tqdm(df.iterrows(), total=len(df), leave=False):\n",
    "        res = res.append(parse_row(row, origin_idx, sup_labels))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                      \r"
     ]
    }
   ],
   "source": [
    "train_df = prc_df(train_path, sup_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>start_pos</th>\n",
       "      <th>len</th>\n",
       "      <th>origin_idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O O O O O O O O O O O O O O O O O O O O O O O</td>\n",
       "      <td>Будучи в прошлый четверг в Софии, он назвал се...</td>\n",
       "      <td>0</td>\n",
       "      <td>0 7 9 17 25 27 34 37 44 49 61 82 84 87 97 102 ...</td>\n",
       "      <td>6 1 7 7 1 6 2 6 4 11 20 1 2 9 4 4 5 7 1 4 7 8 ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O O O O O O O O O O B_cR1 I_cR1 B_cV O B_c...</td>\n",
       "      <td>Работа с двухбайтовыми наборами символов — про...</td>\n",
       "      <td>1</td>\n",
       "      <td>0 7 9 23 32 41 43 50 57 61 75 79 83 89 92 100 ...</td>\n",
       "      <td>6 1 13 8 8 1 6 6 3 13 3 3 5 2 7 2 6 6 1 5 1 2 4 1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O O O O O O O O O O O O O O O O O O O O O ...</td>\n",
       "      <td>Заместитель Генерального секретаря подчеркнул,...</td>\n",
       "      <td>0</td>\n",
       "      <td>0 12 25 35 47 51 60 71 74 81 97 108 110 118 12...</td>\n",
       "      <td>11 12 9 11 3 8 10 2 6 15 10 1 7 2 1 12 5 6 4 1...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O O O O O O O O O O O O O O O O O O O O O O</td>\n",
       "      <td>Продажа недвижимости из собственных портфелей ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0 8 21 24 36 46 56 72 76 78 87 94 97 107 109 1...</td>\n",
       "      <td>7 12 2 11 9 9 15 3 1 8 6 2 9 1 6 1 13 9 8 3 9 ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O O O O O O O O O O O O O O O O O O O O O ...</td>\n",
       "      <td>Новым является то, что повышенное давление кон...</td>\n",
       "      <td>0</td>\n",
       "      <td>0 6 15 19 23 34 43 59 67 73 86 98 102 111 113 ...</td>\n",
       "      <td>5 8 3 3 10 8 15 7 5 12 11 3 8 1 7 5 11 9 1 15 ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0  \\\n",
       "0  O O O O O O O O O O O O O O O O O O O O O O O O O   \n",
       "0  O O O O O O O O O O O O B_cR1 I_cR1 B_cV O B_c...   \n",
       "0  O O O O O O O O O O O O O O O O O O O O O O O ...   \n",
       "0    O O O O O O O O O O O O O O O O O O O O O O O O   \n",
       "0  O O O O O O O O O O O O O O O O O O O O O O O ...   \n",
       "\n",
       "                                                   1  2  \\\n",
       "0  Будучи в прошлый четверг в Софии, он назвал се...  0   \n",
       "0  Работа с двухбайтовыми наборами символов — про...  1   \n",
       "0  Заместитель Генерального секретаря подчеркнул,...  0   \n",
       "0  Продажа недвижимости из собственных портфелей ...  0   \n",
       "0  Новым является то, что повышенное давление кон...  0   \n",
       "\n",
       "                                           start_pos  \\\n",
       "0  0 7 9 17 25 27 34 37 44 49 61 82 84 87 97 102 ...   \n",
       "0  0 7 9 23 32 41 43 50 57 61 75 79 83 89 92 100 ...   \n",
       "0  0 12 25 35 47 51 60 71 74 81 97 108 110 118 12...   \n",
       "0  0 8 21 24 36 46 56 72 76 78 87 94 97 107 109 1...   \n",
       "0  0 6 15 19 23 34 43 59 67 73 86 98 102 111 113 ...   \n",
       "\n",
       "                                                 len origin_idx  \n",
       "0  6 1 7 7 1 6 2 6 4 11 20 1 2 9 4 4 5 7 1 4 7 8 ...          0  \n",
       "0  6 1 13 8 8 1 6 6 3 13 3 3 5 2 7 2 6 6 1 5 1 2 4 1          1  \n",
       "0  11 12 9 11 3 8 10 2 6 15 10 1 7 2 1 12 5 6 4 1...          2  \n",
       "0  7 12 2 11 9 9 15 3 1 8 6 2 9 1 6 1 13 9 8 3 9 ...          3  \n",
       "0  5 8 3 3 10 8 15 7 5 12 11 3 8 1 7 5 11 9 1 15 ...          4  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \r"
     ]
    }
   ],
   "source": [
    "dev_df = prc_df(valid_path, sup_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>start_pos</th>\n",
       "      <th>len</th>\n",
       "      <th>origin_idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O O O O O O O O O O O O O O O O O O O O O ...</td>\n",
       "      <td>Центральная часть лунной тени, где наблюдается...</td>\n",
       "      <td>0</td>\n",
       "      <td>0 12 18 25 31 35 47 54 59 68 77 80 86 88 99 10...</td>\n",
       "      <td>11 5 6 5 3 11 6 4 8 8 2 5 1 10 9 5 5 12 10 1 7...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O B_cV B_cR1 I_cR1 B_cR2 I_cR2 O O B_R1 I_R1 I...</td>\n",
       "      <td>Я превращу твое сердце в траву , а все, что ты...</td>\n",
       "      <td>1</td>\n",
       "      <td>0 2 11 16 23 25 30 32 34 39 43 46 52 54 56 60</td>\n",
       "      <td>1 8 4 6 1 5 1 1 4 3 2 6 1 1 4 1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O B_cR1 B_cV B_cR2 I_cR2 I_cR2 O O B_R1 B_...</td>\n",
       "      <td>В данном примере строки сгруппированы по назва...</td>\n",
       "      <td>1</td>\n",
       "      <td>0 2 9 17 24 38 41 50 56 58 60 68 71 81 88</td>\n",
       "      <td>1 6 7 6 13 2 8 6 1 1 7 2 9 7 1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O O O O O O O O O O</td>\n",
       "      <td>Ассоциация намерена занять на мировом рынке до...</td>\n",
       "      <td>0</td>\n",
       "      <td>0 11 20 27 30 38 44 50 57 63 71 79</td>\n",
       "      <td>10 8 6 2 7 5 5 6 5 7 7 14</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O O O O O O O</td>\n",
       "      <td>Ты сама все портишь-твои слова хороший мой</td>\n",
       "      <td>0</td>\n",
       "      <td>0 3 8 12 25 31 39</td>\n",
       "      <td>2 4 3 12 5 7 3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0  \\\n",
       "0  O O O O O O O O O O O O O O O O O O O O O O O ...   \n",
       "0  O B_cV B_cR1 I_cR1 B_cR2 I_cR2 O O B_R1 I_R1 I...   \n",
       "0  O O O B_cR1 B_cV B_cR2 I_cR2 I_cR2 O O B_R1 B_...   \n",
       "0                            O O O O O O O O O O O O   \n",
       "0                                      O O O O O O O   \n",
       "\n",
       "                                                   1  2  \\\n",
       "0  Центральная часть лунной тени, где наблюдается...  0   \n",
       "0  Я превращу твое сердце в траву , а все, что ты...  1   \n",
       "0  В данном примере строки сгруппированы по назва...  1   \n",
       "0  Ассоциация намерена занять на мировом рынке до...  0   \n",
       "0         Ты сама все портишь-твои слова хороший мой  0   \n",
       "\n",
       "                                           start_pos  \\\n",
       "0  0 12 18 25 31 35 47 54 59 68 77 80 86 88 99 10...   \n",
       "0      0 2 11 16 23 25 30 32 34 39 43 46 52 54 56 60   \n",
       "0          0 2 9 17 24 38 41 50 56 58 60 68 71 81 88   \n",
       "0                 0 11 20 27 30 38 44 50 57 63 71 79   \n",
       "0                                  0 3 8 12 25 31 39   \n",
       "\n",
       "                                                 len origin_idx  \n",
       "0  11 5 6 5 3 11 6 4 8 8 2 5 1 10 9 5 5 12 10 1 7...          0  \n",
       "0                    1 8 4 6 1 5 1 1 4 3 2 6 1 1 4 1          1  \n",
       "0                     1 6 7 6 13 2 8 6 1 1 7 2 9 7 1          2  \n",
       "0                          10 8 6 2 7 5 5 6 5 7 7 14          3  \n",
       "0                                     2 4 3 12 5 7 3          4  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv(data_dir+\"train_parsed.csv\", index=False, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_df.to_csv(data_dir+\"dev_parsed.csv\", index=False, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Create dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from modules import BertNerData as NerData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "\n",
    "train_path = data_dir + \"train_parsed.csv\"\n",
    "valid_path = data_dir + \"dev_parsed.csv\"\n",
    "model_dir = \"/home/aaemeljanov/models/multi_cased_L-12_H-768_A-12/\"\n",
    "init_checkpoint_pt = os.path.join(model_dir, \"pytorch_model.bin\")\n",
    "bert_config_file = os.path.join(model_dir, \"bert_config.json\")\n",
    "vocab_file = os.path.join(model_dir, \"vocab.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:pytorch_pretrained_bert.tokenization:The pre-trained model you are loading is a cased model but you have not set `do_lower_case` to False. We are setting `do_lower_case=False` for you but you may want to check this behavior.\n",
      "INFO:pytorch_pretrained_bert.tokenization:loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt from cache at /home/eartemov/.pytorch_pretrained_bert/96435fa287fbf7e469185f1062386e05a075cadbf6838b74da22bf64b080bc32.99bcd55fc66f4f3360bc49ba472b940b8dcf223ea6a345deb969d607ca900729\n",
      "                                                       \r"
     ]
    }
   ],
   "source": [
    "data = NerData.create(train_path, valid_path, vocab_file, is_cls=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, '[CLS]': 1, 'B_O': 2, 'X': 3, 'B_cR1': 4, 'I_cR1': 5, 'B_cV': 6, 'B_cR2': 7, 'B_R1': 8, 'B_R2': 9, 'I_R2': 10, 'I_cR2': 11, 'I_R1': 12, 'I_cV': 13}\n"
     ]
    }
   ],
   "source": [
    "print(data.label2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.label2idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Create model\n",
    "For creating pytorch model we need to create `NerModel` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.models.bert_models import BertBiLSTMAttnNCRFJoint, BertBiLSTMAttnNCRF, BertBiLSTMNCRF, BertBiLSTMAttnNMTJoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_pretrained_bert.modeling:loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased.tar.gz from cache at /home/eartemov/.pytorch_pretrained_bert/731c19ddf94e294e00ec1ba9a930c69cc2a0fd489b25d3d691373fae4c0986bd.4e367b0d0155d801930846bb6ed98f8a7c23e0ded37888b29caa37009a40c7b9\n",
      "INFO:pytorch_pretrained_bert.modeling:extracting archive file /home/eartemov/.pytorch_pretrained_bert/731c19ddf94e294e00ec1ba9a930c69cc2a0fd489b25d3d691373fae4c0986bd.4e367b0d0155d801930846bb6ed98f8a7c23e0ded37888b29caa37009a40c7b9 to temp dir /tmp/tmpej25z635\n",
      "INFO:pytorch_pretrained_bert.modeling:Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 119547\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build CRF...\n"
     ]
    }
   ],
   "source": [
    "model = BertBiLSTMAttnNCRFJoint.create(\n",
    "    len(data.label2idx), len(data.cls2idx), \"\", \"\",\n",
    "    enc_hidden_dim=1024, rnn_layers=1, num_heads=3, input_dropout=0.5, nbest=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AttnNCRFJointDecoder(\n",
       "  (attn): MultiHeadAttention(\n",
       "    (attention): _MultiHeadAttention(\n",
       "      (attention): ScaledDotProductAttention(\n",
       "        (softmax): Softmax()\n",
       "        (dropout): Dropout(p=0.5)\n",
       "      )\n",
       "    )\n",
       "    (proj): Linear(in_features=192, out_features=1024, bias=True)\n",
       "    (dropout): Dropout(p=0.5)\n",
       "    (layer_norm): LayerNormalization()\n",
       "  )\n",
       "  (linear): Linears(\n",
       "    (linears): ModuleList(\n",
       "      (0): Linear(in_features=1024, out_features=512, bias=True)\n",
       "    )\n",
       "    (output_linear): Linear(in_features=512, out_features=16, bias=True)\n",
       "  )\n",
       "  (crf): NCRF()\n",
       "  (intent_out): PoolingLinearClassifier(\n",
       "    (dropout): Dropout(p=0.5)\n",
       "    (linear): Linears(\n",
       "      (linears): ModuleList(\n",
       "        (0): Linear(in_features=3072, out_features=512, bias=True)\n",
       "      )\n",
       "      (output_linear): Linear(in_features=512, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (intent_loss): CrossEntropyLoss()\n",
       ")"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8158226"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_n_trainable_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Create learner\n",
    "\n",
    "For training our pytorch model we need to create `NerLearner` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules import NerLearner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 150\n",
    "learner = NerLearner(model, data,\n",
    "                     best_model_path=\"/home/eartemov/ae/work/models/AGGR-test.cpt\",\n",
    "                     lr=0.001, clip=5.0, sup_labels=data.id2label[4:],\n",
    "                     t_total=num_epochs * len(data.train_dl))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Learn your NER model\n",
    "Call `learner.fit`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn_crfsuite.metrics import flat_classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Resuming train... Current epoch 0.\n",
      "INFO:root:                                                                        \n",
      "epoch 1, average train epoch loss=93.079\n",
      "\n",
      "INFO:root:on epoch 0 by max_f1: 0.0              \n",
      "INFO:root:on epoch {} classification report:\n",
      "  0%|          | 0/1026 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B_cR1      0.000     0.000     0.000      1382\n",
      "       I_cR1      0.000     0.000     0.000      2022\n",
      "        B_cV      0.000     0.000     0.000      1382\n",
      "       B_cR2      0.000     0.000     0.000      1355\n",
      "        B_R1      0.000     0.000     0.000      1500\n",
      "        B_R2      0.000     0.000     0.000      1473\n",
      "        I_R2      0.000     0.000     0.000      3255\n",
      "       I_cR2      0.000     0.000     0.000      2395\n",
      "        I_R1      0.000     0.000     0.000      1769\n",
      "        I_cV      0.000     0.000     0.000         1\n",
      "\n",
      "   micro avg      0.000     0.000     0.000     16534\n",
      "   macro avg      0.000     0.000     0.000     16534\n",
      "weighted avg      0.000     0.000     0.000     16534\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0      0.666     1.000     0.800      2760\n",
      "           1      0.000     0.000     0.000      1382\n",
      "\n",
      "    accuracy                          0.666      4142\n",
      "   macro avg      0.333     0.500     0.400      4142\n",
      "weighted avg      0.444     0.666     0.533      4142\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:                                                                         \n",
      "epoch 2, average train epoch loss=49.978\n",
      "\n",
      "INFO:root:on epoch 0 by max_f1: 0.0              \n",
      "INFO:root:on epoch {} classification report:\n",
      "  0%|          | 0/1026 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B_cR1      0.000     0.000     0.000      1382\n",
      "       I_cR1      0.000     0.000     0.000      2022\n",
      "        B_cV      0.000     0.000     0.000      1382\n",
      "       B_cR2      0.000     0.000     0.000      1355\n",
      "        B_R1      0.000     0.000     0.000      1500\n",
      "        B_R2      0.000     0.000     0.000      1473\n",
      "        I_R2      0.000     0.000     0.000      3255\n",
      "       I_cR2      0.000     0.000     0.000      2395\n",
      "        I_R1      0.000     0.000     0.000      1769\n",
      "        I_cV      0.000     0.000     0.000         1\n",
      "\n",
      "   micro avg      0.000     0.000     0.000     16534\n",
      "   macro avg      0.000     0.000     0.000     16534\n",
      "weighted avg      0.000     0.000     0.000     16534\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0      0.666     1.000     0.800      2760\n",
      "           1      0.000     0.000     0.000      1382\n",
      "\n",
      "    accuracy                          0.666      4142\n",
      "   macro avg      0.333     0.500     0.400      4142\n",
      "weighted avg      0.444     0.666     0.533      4142\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train loss: 32.578501900473796:   9%|▉         | 91/1026 [00:30<06:40,  2.33it/s]"
     ]
    }
   ],
   "source": [
    "learner.fit(num_epochs, target_metric='f1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(learner.history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Evaluate\n",
    "Create new data loader from existing path."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dl = get_bert_data_loader_for_predict(valid_path, learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "preds_res, preds_cls = learner.predict(dl)\n",
    "# preds_res = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "IOB precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from modules.train.train import validate_step\n",
    "p_rep, p_cls = validate_step(\n",
    "    learner.data.valid_dl, learner.model, learner.data.id2label, learner.data.id2label[1:], learner.data.cls2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "      [CLS]      1.000     1.000     1.000      4142\n",
      "        B_O      0.976     0.975     0.975     74268\n",
      "          X      0.993     0.993     0.993    109877\n",
      "      B_cR1      0.860     0.851     0.855      1382\n",
      "      I_cR1      0.855     0.857     0.856      2022\n",
      "       B_cV      0.956     0.937     0.946      1382\n",
      "      B_cR2      0.893     0.895     0.894      1355\n",
      "       B_R1      0.946     0.923     0.935      1500\n",
      "       B_R2      0.933     0.948     0.940      1473\n",
      "       I_R2      0.932     0.911     0.922      3255\n",
      "      I_cR2      0.858     0.921     0.888      2395\n",
      "       I_R1      0.915     0.922     0.919      1769\n",
      "       I_cV      0.000     0.000     0.000         1\n",
      "\n",
      "avg / total      0.980     0.980     0.980    204821\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# old\n",
    "print(p_rep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# old\n",
    "print(p_cls)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokens report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn_crfsuite.metrics import flat_classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res, first_choicer)\n",
    "true_tokens, true_labels = bert_labels2tokens(dl, [x.labels for x in dl.dataset], first_choicer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert pred_tokens == true_tokens\n",
    "tokens_report = flat_classification_report(true_labels, pred_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### By tokens reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "pl = []\n",
    "for line in pred_labels:\n",
    "    for p in line:\n",
    "        pl.append(p)\n",
    "        \n",
    "tl = []\n",
    "for line in true_labels:\n",
    "    for p in line:\n",
    "        tl.append(p)\n",
    "accuracy_score(tl, pl), f1_score(tl, pl, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tokens_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. To needle format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokens2spans_(tokens_, labels_, start_pos):\n",
    "    pos_idx = 0\n",
    "    res = []\n",
    "    idx_ = 0\n",
    "    while idx_ < len(labels_):\n",
    "        label = labels_[idx_]\n",
    "        if label in [\"I_O\", \"B_O\", \"O\"]:\n",
    "            res.append((tokens_[idx_], \"O\", None))\n",
    "            idx_ += 1\n",
    "        elif label == \"[SEP]\" or label == \"<eos>\":\n",
    "            break\n",
    "        elif label == \"[CLS]\" or label == \"<bos>\":\n",
    "            res.append((tokens_[idx_], label, start_pos[idx_]))\n",
    "            idx_ += 1\n",
    "        else:\n",
    "            span = [tokens_[idx_]]\n",
    "            \n",
    "            try:\n",
    "                pos = start_pos[idx_]\n",
    "            except:\n",
    "                print(\" \".join(tokens_))\n",
    "                print(\" \".join(labels_))\n",
    "                print(tokens_[idx_], labels_[idx_])\n",
    "                print(start_pos[idx_])\n",
    "            try:\n",
    "                span_label = labels_[idx_].split(\"_\")[1]\n",
    "            except IndexError:\n",
    "                print(label, labels_[idx_].split(\"_\"))\n",
    "                span_label = None\n",
    "            idx_ += 1\n",
    "            while idx_ < len(labels_):\n",
    "                if labels_[idx_] not in [\"I_O\", \"B_O\", \"O\"] and labels_[idx_].split(\"_\")[0] == \"I\" and  span_label == labels_[idx_].split(\"_\")[1]:\n",
    "                    span.append(tokens_[idx_])\n",
    "                    idx_ += 1\n",
    "                # Skip one O\n",
    "                elif idx_ + 1 < len(labels_) and labels_[idx_ + 1] not in [\"I_O\", \"B_O\", \"O\"] and labels_[idx_ + 1].split(\"_\")[0] == \"I\" and  span_label == labels_[idx_ + 1].split(\"_\")[1]:\n",
    "                    span.append(tokens_[idx_])\n",
    "                    idx_ += 1\n",
    "                    span.append(tokens_[idx_])\n",
    "                    idx_ += 1\n",
    "                else:\n",
    "                    break\n",
    "            res.append((\" \".join(span), span_label, pos))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "def post_prc_row(text, tokens, labels, cls, start_pos):\n",
    "    # SHIT IN DATASET, fix align\n",
    "    text = text.replace('\\x97', \"U\")\n",
    "    text = text.replace('\\uf076', \"U\")\n",
    "    text = text.replace(\"\\ue405\", \"U\")\n",
    "    text = text.replace(\"\\ue105\", \"U\")\n",
    "    text = text.replace(\"\\ue415\", \"U\")\n",
    "    text = text.replace('\\x07', \"U\")\n",
    "    start_pos = []\n",
    "    pos = 0\n",
    "    for tok in tokens:\n",
    "        if tok.find(\"unk\") > -1:\n",
    "            tok = tok.replace(\"unk\", \"U\")\n",
    "        pos = text.find(tok, max(pos-2, 0))\n",
    "        # SHIT IN DATASET, fix align\n",
    "        if pos == -1:\n",
    "            text = text.replace(\"unk\", \"UUU\")\n",
    "            tok = \"UUU\"\n",
    "            print(tok)\n",
    "        start_pos.append(pos)\n",
    "    spans = tokens2spans_(tokens, labels, start_pos)\n",
    "    res = {}\n",
    "    # cls = 0\n",
    "    if cls:\n",
    "        for token, label, pos in spans:\n",
    "            if label != \"O\":\n",
    "                while any([token.endswith(p) for p in string.punctuation]):\n",
    "                    token = token[:-1]\n",
    "                # cls = 1\n",
    "                pos = int(pos)\n",
    "                if label in res:\n",
    "                    if tok.find(\"unk\") > -1:\n",
    "                        tok = tok.replace(\"unk\", \"U\")\n",
    "                    pp = pos\n",
    "                    pos = text.find(token, max(pos-1, 0))\n",
    "                    if pos == -1:\n",
    "                        print(pp, text[pp:], token, text[pp:].find(token))\n",
    "                        pos = pp\n",
    "                    res[label] += \" \" + \":\".join(map(str, [pos, pos + len(token)]))\n",
    "                    if label == \"R2\":\n",
    "                        res[\"V\"] += \" \" + \":\".join(map(str, [pos, pos]))\n",
    "                else:\n",
    "                    res[label] = \":\".join(map(str, [pos, pos + len(token)]))\n",
    "                    if label == \"R2\":\n",
    "                        res[\"V\"] = \":\".join(map(str, [pos, pos]))\n",
    "    if res.get(\"R1\", None) and res.get(\"R2\", None) is None:\n",
    "        try:\n",
    "            res[\"V\"] = \" \".join([\"{}:{}\".format(f.split(\":\")[0], f.split(\":\")[0]) for f in res.get(\"R1\").split()])\n",
    "        except:\n",
    "            print(res.get(\"R1\"), res.get(\"R1\").split(), )\n",
    "            raise\n",
    "    res[\"text\"] = text\n",
    "    res[\"class\"] = cls\n",
    "    for key in res:\n",
    "        res[key] = [res[key]]\n",
    "    return pd.DataFrame(res, columns=['text', 'class', 'cV', 'cR1', 'cR2', 'V', 'R1', 'R2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def post_prc(origin_path, parsed_path, pred_tokens, pred_labels, pred_cls):\n",
    "    origin_df = pd.read_csv(origin_path, sep=\"\\t\")\n",
    "    parsed_df = pd.read_csv(parsed_path)\n",
    "    res = pd.DataFrame(columns=['text', 'class', 'cV', 'cR1', 'cR2', 'V', 'R1', 'R2'])\n",
    "    idx = 0\n",
    "    for (_, o_row), (_, p_row), tokens, labels, cls in zip(origin_df.iterrows(), parsed_df.iterrows(), pred_tokens, pred_labels, pred_cls):\n",
    "        try:\n",
    "            res = res.append(post_prc_row(o_row.text, tokens, labels, cls, p_row.start_pos.split()))\n",
    "        except:\n",
    "            print(o_row.text)\n",
    "            print(\" \".join(tokens))\n",
    "            print(len(p_row.start_pos.split()), len(tokens))\n",
    "            print(idx)\n",
    "            raise\n",
    "        idx += 1\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_pred = post_prc(\n",
    "    \"/datadrive/AGRR-2019/dev.csv\",\n",
    "    \"/datadrive/AGRR-2019/dev_parsed.csv\",\n",
    "    pred_tokens, pred_labels, preds_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>cV</th>\n",
       "      <th>cR1</th>\n",
       "      <th>cR2</th>\n",
       "      <th>V</th>\n",
       "      <th>R1</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Центральная часть лунной тени, где наблюдается...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Я превращу твое сердце в траву, а все, что ты ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2:10</td>\n",
       "      <td>11:22</td>\n",
       "      <td>23:30</td>\n",
       "      <td>54:54</td>\n",
       "      <td>34:52</td>\n",
       "      <td>54:60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>В данном примере строки сгруппированы по назва...</td>\n",
       "      <td>1</td>\n",
       "      <td>24:37</td>\n",
       "      <td>17:23</td>\n",
       "      <td>38:56</td>\n",
       "      <td>68:68</td>\n",
       "      <td>60:67</td>\n",
       "      <td>68:88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ассоциация намерена занять на мировом рынке до...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ты сама все портишь-твои слова хороший мой</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text class     cV    cR1  \\\n",
       "0  Центральная часть лунной тени, где наблюдается...     0    NaN    NaN   \n",
       "0  Я превращу твое сердце в траву, а все, что ты ...     1   2:10  11:22   \n",
       "0  В данном примере строки сгруппированы по назва...     1  24:37  17:23   \n",
       "0  Ассоциация намерена занять на мировом рынке до...     0    NaN    NaN   \n",
       "0         Ты сама все портишь-твои слова хороший мой     0    NaN    NaN   \n",
       "\n",
       "     cR2      V     R1     R2  \n",
       "0    NaN    NaN    NaN    NaN  \n",
       "0  23:30  54:54  34:52  54:60  \n",
       "0  38:56  68:68  60:67  68:88  \n",
       "0    NaN    NaN    NaN    NaN  \n",
       "0    NaN    NaN    NaN    NaN  "
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_pred.to_csv(\"/datadrive/AGRR-2019/dev_pred.csv\", sep=\"\\t\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Binary classification quality (f1-score): 0.9583778014941302\r\n",
      "Gapping resolution quality (symbol-wise f-measure): 0.9576077060524616\r\n"
     ]
    }
   ],
   "source": [
    "!python3 /datadrive/AGRR-2019/agrr_metrics.py -r /datadrive/AGRR-2019/dev.csv /datadrive/AGRR-2019/dev_pred.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Make prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_path = data_dir + \"test_gold_standard.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \r"
     ]
    }
   ],
   "source": [
    "test_df = prc_df(test_path, sup_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.to_csv(data_dir + \"test_gold_standard_parsed.csv\", index=False, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \r"
     ]
    }
   ],
   "source": [
    "dl = get_bert_data_loader_for_predict(data_dir + \"test_gold_standard_parsed.csv\", learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "preds_res, preds_cls = learner.predict(dl)\n",
    "# preds_res = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res, first_choicer)\n",
    "true_tokens, true_labels = bert_labels2tokens(dl, [x.labels for x in dl.dataset], first_choicer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### See wtf encoding or spaces :("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pred = post_prc(\n",
    "    data_dir + \"test_gold_standard.csv\",\n",
    "    data_dir + \"test_gold_standard_parsed.csv\",\n",
    "    pred_tokens, pred_labels, preds_cls)\n",
    "test_pred.to_csv(\"/home/aaemeljanov/data/AGRR-2019/test_pred.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 /home/aaemeljanov/data/AGRR-2019/agrr_metrics.py -r /home/aaemeljanov/data/AGRR-2019/test_gold_standard.csv /home/aaemeljanov/data/AGRR-2019/test_pred.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Merge train and dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = \"/home/aaemeljanov/data/AGRR-2019/train_parsed.csv\"\n",
    "valid_path = \"/home/aaemeljanov/data/AGRR-2019/dev_parsed.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(train_path)\n",
    "dev_df = pd.read_csv(valid_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "ful_df = train_df.append(dev_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20548"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ful_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "ful_df.to_csv(\"/home/aaemeljanov/data/AGRR-2019/train_dev_parsed.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Train full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from modules import BertNerData as NerData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "\n",
    "data_dir = \"/home/aaemeljanov/data/AGRR-2019/\"\n",
    "train_path = data_dir + \"train_dev_parsed.csv\"\n",
    "valid_path = data_dir + \"test_gold_standard_parsed.csv\"\n",
    "model_dir = \"/home/aaemeljanov/models/multi_cased_L-12_H-768_A-12/\"\n",
    "init_checkpoint_pt = os.path.join(model_dir, \"pytorch_model.bin\")\n",
    "bert_config_file = os.path.join(model_dir, \"bert_config.json\")\n",
    "vocab_file = os.path.join(model_dir, \"vocab.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                       \r"
     ]
    }
   ],
   "source": [
    "data = NerData.create(train_path, valid_path, vocab_file, is_cls=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'I_cV': 13, '[CLS]': 1, 'B_O': 2, 'X': 3, '<pad>': 0, 'B_R1': 8, 'B_cV': 6, 'I_R1': 12, 'I_R2': 10, 'I_cR1': 5, 'B_cR1': 4, 'I_cR2': 11, 'B_cR2': 7, 'B_R2': 9}\n"
     ]
    }
   ],
   "source": [
    "print(data.label2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.label2idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.2. Create model\n",
    "For creating pytorch model we need to create `NerModel` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.models.bert_models import BertBiLSTMAttnNCRFJoint, BertBiLSTMAttnNCRF, BertBiLSTMNCRF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build CRF...\n"
     ]
    }
   ],
   "source": [
    "# for 0.9\n",
    "model = BertBiLSTMAttnNCRFJoint.create(\n",
    "         len(data.label2idx), len(data.cls2idx), bert_config_file, init_checkpoint_pt,\n",
    "         enc_hidden_dim=1024, rnn_layers=1, num_heads=3, input_dropout=0.5, nbest=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.models.bert_models import BertBiLSTMAttnNMTJoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build CRF...\n"
     ]
    }
   ],
   "source": [
    "model = BertBiLSTMAttnNMTJoint.create(\n",
    "        len(data.label2idx), len(data.cls2idx), bert_config_file, init_checkpoint_pt,\n",
    "        enc_hidden_dim=128, rnn_layers=1, dec_embedding_dim=32, dec_hidden_dim=128, input_dropout=0.5, nbest=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NMTJointDecoder(\n",
       "  (crf): NCRF()\n",
       "  (embedding): Embedding(16, 32)\n",
       "  (lstm): LSTM(288, 128, batch_first=True)\n",
       "  (attn): Linear(in_features=128, out_features=128, bias=True)\n",
       "  (slot_out): Linear(in_features=256, out_features=16, bias=True)\n",
       "  (intent_loss): CrossEntropyLoss()\n",
       "  (intent_out): Linears(\n",
       "    (linears): ModuleList(\n",
       "      (0): Linear(in_features=256, out_features=64, bias=True)\n",
       "    )\n",
       "    (output_linear): Linear(in_features=64, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "679007"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_n_trainable_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.3. Create learner\n",
    "\n",
    "For training our pytorch model we need to create `NerLearner` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules import NerLearner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = NerLearner(model, data,\n",
    "                         best_model_path=\"/home/aaemeljanov/models/AGRR-2019/big2.cpt\",\n",
    "                         lr=0.0001, clip=1.0, sup_labels=data.id2label[1:],\n",
    "                         t_total=num_epochs * len(data.train_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = NerLearner(model, data,\n",
    "                         best_model_path=\"/home/aaemeljanov/models/AGRR-2019/nmt.cpt\",\n",
    "                         lr=0.001, clip=5.0, sup_labels=data.id2label[1:],\n",
    "                         t_total=num_epochs * len(data.train_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 50\n",
    "learner = NerLearner(model, data,\n",
    "                     best_model_path=\"/datadrive/AGRR-2019/big_full.cpt\",\n",
    "                     lr=0.001, clip=5.0, sup_labels=data.id2label[1:],\n",
    "                     t_total=num_epochs * len(data.train_dl))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.4. Learn your NER model\n",
    "Call `learner.fit`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learner.fit(num_epochs, target_metric='f1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### How to overfit :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \r"
     ]
    }
   ],
   "source": [
    "dl = get_bert_data_loader_for_predict(valid_path, learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "preds_res, preds_cls = learner.predict(dl)\n",
    "# preds_res = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "IOB precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from modules.train.train import validate_step\n",
    "p_rep, p_cls = validate_step(\n",
    "    learner.data.valid_dl, learner.model, learner.data.id2label, learner.data.id2label[1:], learner.data.cls2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "      [CLS]      1.000     1.000     1.000      4142\n",
      "        B_O      1.000     1.000     1.000     74268\n",
      "          X      1.000     1.000     1.000    109877\n",
      "      B_cR1      0.999     0.999     0.999      1382\n",
      "      I_cR1      1.000     1.000     1.000      2022\n",
      "       B_cV      1.000     1.000     1.000      1382\n",
      "      B_cR2      1.000     0.999     1.000      1355\n",
      "       B_R1      1.000     1.000     1.000      1500\n",
      "       B_R2      1.000     1.000     1.000      1473\n",
      "       I_R2      1.000     1.000     1.000      3255\n",
      "      I_cR2      1.000     1.000     1.000      2395\n",
      "       I_R1      1.000     1.000     1.000      1769\n",
      "       I_cV      1.000     1.000     1.000         1\n",
      "\n",
      "avg / total      1.000     1.000     1.000    204821\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(p_rep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0      1.000     1.000     1.000      2760\n",
      "          1      1.000     1.000     1.000      1382\n",
      "\n",
      "avg / total      1.000     1.000     1.000      4142\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(p_cls)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokens report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn_crfsuite.metrics import flat_classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res)\n",
    "true_tokens, true_labels = bert_labels2tokens(dl, [x.labels for x in dl.dataset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert pred_tokens == true_tokens\n",
    "tokens_report = flat_classification_report(true_labels, pred_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### By tokens reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9651315936335022, 0.9090821842348382)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "pl = []\n",
    "for line in pred_labels:\n",
    "    for p in line:\n",
    "        pl.append(p)\n",
    "        \n",
    "tl = []\n",
    "for line in true_labels:\n",
    "    for p in line:\n",
    "        tl.append(p)\n",
    "accuracy_score(tl, pl), f1_score(tl, pl, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         I_O       0.98      0.98      0.98     37024\n",
      "        I_R1       0.90      0.91      0.90      1488\n",
      "        I_R2       0.90      0.94      0.92      2247\n",
      "       I_cR1       0.85      0.82      0.83      1592\n",
      "       I_cR2       0.88      0.90      0.89      1766\n",
      "        I_cV       0.92      0.93      0.93       680\n",
      "\n",
      "   micro avg       0.97      0.97      0.97     44797\n",
      "   macro avg       0.90      0.92      0.91     44797\n",
      "weighted avg       0.97      0.97      0.97     44797\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(tokens_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "103 книга, для получения сведений - кнопку Справка. книга , для получения сведений -1\n",
      "87 О  log  n   — «довольно дешево». О log n -1\n"
     ]
    }
   ],
   "source": [
    "test_pred = post_prc(\n",
    "    \"/home/aaemeljanov/data/AGRR-2019/test_gold_standard.csv\",\n",
    "    \"/home/aaemeljanov/data/AGRR-2019/test_gold_standard_parsed.csv\",\n",
    "    pred_tokens, pred_labels, preds_cls)\n",
    "test_pred.to_csv(\"/home/aaemeljanov/data/AGRR-2019/test_pred.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Binary classification quality (f1-score): 0.9377777777777778\r\n",
      "Gapping resolution quality (symbol-wise f-measure): 0.8506361125630272\r\n"
     ]
    }
   ],
   "source": [
    "!python3 /home/aaemeljanov/data/AGRR-2019/agrr_metrics.py -r /home/aaemeljanov/data/AGRR-2019/test_gold_standard.csv /home/aaemeljanov/data/AGRR-2019/test_pred.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "tt = pd.read_csv(\"/home/aaemeljanov/data/AGRR-2019/test_gold_standard.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>cV</th>\n",
       "      <th>cR1</th>\n",
       "      <th>cR2</th>\n",
       "      <th>V</th>\n",
       "      <th>R1</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Изобретение относится к судостроению и касаетс...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Эти состояния называют фазами воды, а превраще...</td>\n",
       "      <td>1</td>\n",
       "      <td>14:22</td>\n",
       "      <td>0:13</td>\n",
       "      <td>23:34</td>\n",
       "      <td>81:81</td>\n",
       "      <td>38:78</td>\n",
       "      <td>81:100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>И должен ни единой долькой  Не отступаться от ...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Он потребовал обеспечить полное осуществление ...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>По мнению местного пастора Элла Эбанкса, запре...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  class     cV   cR1  \\\n",
       "0  Изобретение относится к судостроению и касаетс...      0    NaN   NaN   \n",
       "1  Эти состояния называют фазами воды, а превраще...      1  14:22  0:13   \n",
       "2  И должен ни единой долькой  Не отступаться от ...      0    NaN   NaN   \n",
       "3  Он потребовал обеспечить полное осуществление ...      0    NaN   NaN   \n",
       "4  По мнению местного пастора Элла Эбанкса, запре...      0    NaN   NaN   \n",
       "\n",
       "     cR2      V     R1      R2  \n",
       "0    NaN    NaN    NaN     NaN  \n",
       "1  23:34  81:81  38:78  81:100  \n",
       "2    NaN    NaN    NaN     NaN  \n",
       "3    NaN    NaN    NaN     NaN  \n",
       "4    NaN    NaN    NaN     NaN  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_pred = post_prc(\n",
    "    \"/datadrive/AGRR-2019/dev.csv\",\n",
    "    \"/datadrive/AGRR-2019/dev_parsed.csv\",\n",
    "    pred_tokens, pred_labels, preds_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>cV</th>\n",
       "      <th>cR1</th>\n",
       "      <th>cR2</th>\n",
       "      <th>V</th>\n",
       "      <th>R1</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Центральная часть лунной тени, где наблюдается...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Я превращу твое сердце в траву, а все, что ты ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2:10</td>\n",
       "      <td>11:22</td>\n",
       "      <td>23:30</td>\n",
       "      <td>54:54</td>\n",
       "      <td>34:52</td>\n",
       "      <td>54:60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>В данном примере строки сгруппированы по назва...</td>\n",
       "      <td>1</td>\n",
       "      <td>24:37</td>\n",
       "      <td>17:23</td>\n",
       "      <td>38:56</td>\n",
       "      <td>68:68</td>\n",
       "      <td>60:67</td>\n",
       "      <td>68:88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ассоциация намерена занять на мировом рынке до...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ты сама все портишь-твои слова хороший мой</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text class     cV    cR1  \\\n",
       "0  Центральная часть лунной тени, где наблюдается...     0    NaN    NaN   \n",
       "0  Я превращу твое сердце в траву, а все, что ты ...     1   2:10  11:22   \n",
       "0  В данном примере строки сгруппированы по назва...     1  24:37  17:23   \n",
       "0  Ассоциация намерена занять на мировом рынке до...     0    NaN    NaN   \n",
       "0         Ты сама все портишь-твои слова хороший мой     0    NaN    NaN   \n",
       "\n",
       "     cR2      V     R1     R2  \n",
       "0    NaN    NaN    NaN    NaN  \n",
       "0  23:30  54:54  34:52  54:60  \n",
       "0  38:56  68:68  60:67  68:88  \n",
       "0    NaN    NaN    NaN    NaN  \n",
       "0    NaN    NaN    NaN    NaN  "
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_pred.to_csv(\"/datadrive/AGRR-2019/dev_pred_overfit.csv\", sep=\"\\t\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### WTF -> errors with spaces or so on?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Binary classification quality (f1-score): 1.0\r\n",
      "Gapping resolution quality (symbol-wise f-measure): 0.9982856495331368\r\n"
     ]
    }
   ],
   "source": [
    "!python3 /datadrive/AGRR-2019/agrr_metrics.py -r /datadrive/AGRR-2019/dev.csv /datadrive/AGRR-2019/dev_pred_overfit.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.5 Make prediction with model trained on full data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \r"
     ]
    }
   ],
   "source": [
    "dl = get_bert_data_loader_for_predict(\"/datadrive/AGRR-2019/test_gold_standard_parsed.csv\", learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "preds_res, preds_cls = learner.predict(dl)\n",
    "# preds_res = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res, first_choicer)\n",
    "true_tokens, true_labels = bert_labels2tokens(dl, [x.labels for x in dl.dataset], first_choicer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### See wtf encoding or spaces :("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87 О  log  n   — «довольно дешево». О log n -1\n"
     ]
    }
   ],
   "source": [
    "test_pred = post_prc(\n",
    "    \"/datadrive/AGRR-2019/test_gold_standard.csv\",\n",
    "    \"/datadrive/AGRR-2019/test_gold_standard_parsed.csv\",\n",
    "    pred_tokens, pred_labels, preds_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pred.to_csv(\"/datadrive/AGRR-2019/test_gold_standard_pred.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2045"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Binary classification quality (f1-score): 0.9455882352941176\r\n",
      "Gapping resolution quality (symbol-wise f-measure): 0.8590594658512715\r\n"
     ]
    }
   ],
   "source": [
    "!python3 /datadrive/AGRR-2019/agrr_metrics.py -r /datadrive/AGRR-2019/test_gold_standard.csv /datadrive/AGRR-2019/test_gold_standard_pred.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analyze errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"/datadrive/AGRR-2019/test_gold_standard.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = []\n",
    "for (idx_t, row_true), (idx_p, row_pred) in zip(test_df.iterrows(), test_pred.iterrows()):\n",
    "    if any([str(el_true) != str(el_pred) for el_true, el_pred in zip(row_true, row_pred)]):\n",
    "        errors.append((idx_t, row_true, row_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Описание структуры CREATESTRUCT приводится ниже, а описание структуры MDICREATESTRUCT — в разделе, посвященном сообщению WM_MDICREATE\n"
     ]
    }
   ],
   "source": [
    "print(errors[1][1].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class         1\n",
       "cV        32:42\n",
       "cR1        0:31\n",
       "cR2       43:47\n",
       "V         88:88\n",
       "R1        51:85\n",
       "R2       88:133\n",
       "Name: 11, dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors[1][1][1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class          1\n",
       "cV         32:42\n",
       "cR1         0:31\n",
       "cR2          NaN\n",
       "V        121:121\n",
       "R1         51:85\n",
       "R2       121:133\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors[1][2][1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ниже'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors[1][1].text[43:47]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>cV</th>\n",
       "      <th>cR1</th>\n",
       "      <th>cR2</th>\n",
       "      <th>V</th>\n",
       "      <th>R1</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Изобретение относится к судостроению и касаетс...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Эти состояния называют фазами воды, а превраще...</td>\n",
       "      <td>1</td>\n",
       "      <td>14:22</td>\n",
       "      <td>0:13</td>\n",
       "      <td>23:35</td>\n",
       "      <td>81:81</td>\n",
       "      <td>38:78</td>\n",
       "      <td>81:101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>И должен ни единой долькой  Не отступаться от ...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Он потребовал обеспечить полное осуществление ...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>По мнению местного пастора Элла Эбанкса, запре...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text class     cV   cR1  \\\n",
       "0  Изобретение относится к судостроению и касаетс...     0    NaN   NaN   \n",
       "0  Эти состояния называют фазами воды, а превраще...     1  14:22  0:13   \n",
       "0  И должен ни единой долькой  Не отступаться от ...     0    NaN   NaN   \n",
       "0  Он потребовал обеспечить полное осуществление ...     0    NaN   NaN   \n",
       "0  По мнению местного пастора Элла Эбанкса, запре...     0    NaN   NaN   \n",
       "\n",
       "     cR2      V     R1      R2  \n",
       "0    NaN    NaN    NaN     NaN  \n",
       "0  23:35  81:81  38:78  81:101  \n",
       "0    NaN    NaN    NaN     NaN  \n",
       "0    NaN    NaN    NaN     NaN  \n",
       "0    NaN    NaN    NaN     NaN  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diff_test.csv  ner-bert  pytorch-pretrained-BERT\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../../"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \r"
     ]
    }
   ],
   "source": [
    "diff_test_df = prc_df(\"../../diff_test.csv\", sup_labels)\n",
    "\n",
    "diff_test_df.to_csv(\"/datadrive/AGRR-2019/diff_test_parsed.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \r"
     ]
    }
   ],
   "source": [
    "dl = get_bert_data_loader_for_predict(\"/datadrive/AGRR-2019/diff_test_parsed.csv\", learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "preds_res, preds_cls = learner.predict(dl)\n",
    "# preds_res = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res, first_choicer)\n",
    "true_tokens, true_labels = bert_labels2tokens(dl, [x.labels for x in dl.dataset], first_choicer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### See wtf encoding or spaces :("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "dff_test_pred = post_prc(\n",
    "    \"../../diff_test.csv\",\n",
    "    \"/datadrive/AGRR-2019/diff_test_parsed.csv\",\n",
    "    pred_tokens, pred_labels, preds_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "dff_test_pred.to_csv(\"/datadrive/AGRR-2019/diff_test_pred.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>cV</th>\n",
       "      <th>cR1</th>\n",
       "      <th>cR2</th>\n",
       "      <th>V</th>\n",
       "      <th>R1</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Самые крутые либералы стоят сейчас и думают, к...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>В самом конце 1812-го казаки захватили Мариенб...</td>\n",
       "      <td>1</td>\n",
       "      <td>57:63</td>\n",
       "      <td>51:56</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>А вскоре заказчик вновь обратился к изготовите...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Как правило, две сессии продолжительностью по ...</td>\n",
       "      <td>1</td>\n",
       "      <td>55:65</td>\n",
       "      <td>13:16 24:54</td>\n",
       "      <td>58:67</td>\n",
       "      <td>111:111</td>\n",
       "      <td>77:108</td>\n",
       "      <td>111:141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>В первом случае способ обработки получил назва...</td>\n",
       "      <td>1</td>\n",
       "      <td>33:40</td>\n",
       "      <td>0:15</td>\n",
       "      <td>50:64</td>\n",
       "      <td>78:78</td>\n",
       "      <td>66:75</td>\n",
       "      <td>78:95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text class     cV  \\\n",
       "0  Самые крутые либералы стоят сейчас и думают, к...     0    NaN   \n",
       "0  В самом конце 1812-го казаки захватили Мариенб...     1  57:63   \n",
       "0  А вскоре заказчик вновь обратился к изготовите...     0    NaN   \n",
       "0  Как правило, две сессии продолжительностью по ...     1  55:65   \n",
       "0  В первом случае способ обработки получил назва...     1  33:40   \n",
       "\n",
       "           cR1    cR2        V      R1       R2  \n",
       "0          NaN    NaN      NaN     NaN      NaN  \n",
       "0        51:56    NaN      NaN     NaN      NaN  \n",
       "0          NaN    NaN      NaN     NaN      NaN  \n",
       "0  13:16 24:54  58:67  111:111  77:108  111:141  \n",
       "0         0:15  50:64    78:78   66:75    78:95  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dff_test_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.read_csv(\"../../diff_test.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test syntagrus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/aaemeljanov/AGRR-2019/exps/syntagrus_new.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls /home/aaemeljanov/AGRR-2019/exps/syntagrus_new.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = pd.read_csv(\"/home/aaemeljanov/AGRR-2019/exps/syntagrus_new.csv\", sep=\"\\t\", header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "sf = pd.DataFrame(columns=tt.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "sf.text = t[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>cV</th>\n",
       "      <th>cR1</th>\n",
       "      <th>cR2</th>\n",
       "      <th>V</th>\n",
       "      <th>R1</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Нет, желающих слепить \"мегапартию\" навалом и з...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Боевики учли это, а наше командование – нет.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Кладбище напоминало ему Манхэттен, но в Манхэт...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Прогресс не касался лишь принципа движения дет...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Люда росла, и город рос, но как-то вяло, с зап...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text class   cV  cR1  cR2  \\\n",
       "0  Нет, желающих слепить \"мегапартию\" навалом и з...   NaN  NaN  NaN  NaN   \n",
       "1       Боевики учли это, а наше командование – нет.   NaN  NaN  NaN  NaN   \n",
       "2  Кладбище напоминало ему Манхэттен, но в Манхэт...   NaN  NaN  NaN  NaN   \n",
       "3  Прогресс не касался лишь принципа движения дет...   NaN  NaN  NaN  NaN   \n",
       "4  Люда росла, и город рос, но как-то вяло, с зап...   NaN  NaN  NaN  NaN   \n",
       "\n",
       "     V   R1   R2  \n",
       "0  NaN  NaN  NaN  \n",
       "1  NaN  NaN  NaN  \n",
       "2  NaN  NaN  NaN  \n",
       "3  NaN  NaN  NaN  \n",
       "4  NaN  NaN  NaN  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "sf.to_csv(\"/home/aaemeljanov/AGRR-2019/exps/syntagrus_newp.csv\", index=False, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                  \r"
     ]
    }
   ],
   "source": [
    "syntagrus_test_df = prc_df(\"/home/aaemeljanov/AGRR-2019/exps/syntagrus_newp.csv\", sup_labels)\n",
    "\n",
    "syntagrus_test_df.to_csv(\"/home/aaemeljanov/AGRR-2019/exps/syntagrus_new_parsed.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "dl = get_bert_data_loader_for_predict(\"/home/aaemeljanov/AGRR-2019/exps/syntagrus_new_parsed.csv\", learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                             \r"
     ]
    }
   ],
   "source": [
    "preds_res, preds_cls = learner.predict(dl)\n",
    "# preds_res = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res)\n",
    "true_tokens, true_labels = bert_labels2tokens(dl, [x.labels for x in dl.dataset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "syntagrus_test_pred = post_prc(\n",
    "    \"/home/aaemeljanov/AGRR-2019/exps/syntagrus_newp.csv\",\n",
    "    \"/home/aaemeljanov/AGRR-2019/exps/syntagrus_new_parsed.csv\",\n",
    "    pred_tokens, pred_labels, preds_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "syntagrus_test_pred.to_csv(\"/home/aaemeljanov/AGRR-2019/exps/syntagrus_new_pred.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>cV</th>\n",
       "      <th>cR1</th>\n",
       "      <th>cR2</th>\n",
       "      <th>V</th>\n",
       "      <th>R1</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Нет, желающих слепить \"мегапартию\" навалом и з...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Боевики учли это, а наше командование – нет.</td>\n",
       "      <td>1</td>\n",
       "      <td>8:12</td>\n",
       "      <td>0:7</td>\n",
       "      <td>13:16</td>\n",
       "      <td>40:40</td>\n",
       "      <td>20:37</td>\n",
       "      <td>40:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Кладбище напоминало ему Манхэттен, но в Манхэт...</td>\n",
       "      <td>1</td>\n",
       "      <td>51:56</td>\n",
       "      <td>38:50</td>\n",
       "      <td>57:62</td>\n",
       "      <td>80:80</td>\n",
       "      <td>66:77</td>\n",
       "      <td>80:84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Прогресс не касался лишь принципа движения дет...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Люда росла, и город рос, но как-то вяло, с зап...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text class     cV    cR1  \\\n",
       "0  Нет, желающих слепить \"мегапартию\" навалом и з...     0    NaN    NaN   \n",
       "0       Боевики учли это, а наше командование – нет.     1   8:12    0:7   \n",
       "0  Кладбище напоминало ему Манхэттен, но в Манхэт...     1  51:56  38:50   \n",
       "0  Прогресс не касался лишь принципа движения дет...     0    NaN    NaN   \n",
       "0  Люда росла, и город рос, но как-то вяло, с зап...     0    NaN    NaN   \n",
       "\n",
       "     cR2      V     R1     R2  \n",
       "0    NaN    NaN    NaN    NaN  \n",
       "0  13:16  40:40  20:37  40:43  \n",
       "0  57:62  80:80  66:77  80:84  \n",
       "0    NaN    NaN    NaN    NaN  \n",
       "0    NaN    NaN    NaN    NaN  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syntagrus_test_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(117, 117)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sf), len(syntagrus_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}