1. Data Analysis and Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "!python3 -m spacy download en_core_web_sm\n",
    "!python3 -m textblob.download_corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "//anaconda3/lib/python3.7/site-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from pylab import rcParams\n",
    "rcParams['figure.figsize'] = 7, 6\n",
    "\n",
    "from copy import deepcopy\n",
    "import seaborn as sns\n",
    "\n",
    "import sys\n",
    "sys.path.append('./utils')\n",
    "from utils import review_feature\n",
    "rf = review_feature()\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features.csv             output_ranked_train.csv  train.csv\r\n",
      "Preprocessed_Reviews.csv test.csv\r\n",
      "Untitled.ipynb           test_ranked_output.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>answer_option</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Fast and accurate delivery</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>As usual it is genuine</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Behavior of delivery boy is very bad. Delivery...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>fwegwrqdsdvwfg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>These strips were as per my requirment</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1671</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Ft GM</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1672</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>I like</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1673</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Nice price with long expiry</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1674</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Price &amp; Service</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1675</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Good discount</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1676 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        product                                      answer_option  label\n",
       "0     Accucheck                         Fast and accurate delivery      0\n",
       "1     Accucheck                            As usual it is genuine       0\n",
       "2     Accucheck  Behavior of delivery boy is very bad. Delivery...      0\n",
       "3     Accucheck                                     fwegwrqdsdvwfg      0\n",
       "4     Accucheck             These strips were as per my requirment      0\n",
       "...         ...                                                ...    ...\n",
       "1671  Accucheck                                              Ft GM      0\n",
       "1672  Accucheck                                             I like      0\n",
       "1673  Accucheck                        Nice price with long expiry      0\n",
       "1674  Accucheck                                    Price & Service      0\n",
       "1675  Accucheck                                      Good discount      0\n",
       "\n",
       "[1676 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis to understand per product who many informative and how many not informative reviews are there. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>label</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>product</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accucheck</th>\n",
       "      <td>317</td>\n",
       "      <td>85</td>\n",
       "      <td>402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Becadexamin</th>\n",
       "      <td>53</td>\n",
       "      <td>27</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Evion</th>\n",
       "      <td>89</td>\n",
       "      <td>33</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Neurobion</th>\n",
       "      <td>286</td>\n",
       "      <td>137</td>\n",
       "      <td>423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SevenseascodLiverOil</th>\n",
       "      <td>60</td>\n",
       "      <td>22</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Shelcal</th>\n",
       "      <td>262</td>\n",
       "      <td>126</td>\n",
       "      <td>388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Supradyn</th>\n",
       "      <td>50</td>\n",
       "      <td>23</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shampoo</th>\n",
       "      <td>57</td>\n",
       "      <td>49</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>1174</td>\n",
       "      <td>502</td>\n",
       "      <td>1676</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "label                    0    1   All\n",
       "product                              \n",
       "Accucheck              317   85   402\n",
       "Becadexamin             53   27    80\n",
       "Evion                   89   33   122\n",
       "Neurobion              286  137   423\n",
       "SevenseascodLiverOil    60   22    82\n",
       "Shelcal                262  126   388\n",
       "Supradyn                50   23    73\n",
       "shampoo                 57   49   106\n",
       "All                   1174  502  1676"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_analysis = pd.crosstab(df['product'],df['label'], margins='All')\n",
    "label_analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a1efa8898>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAFqCAYAAADhiBq7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZxcVZn/8c+XGAnDkrCENUACgmwhDQZEQEDg54CiLAYBUQIuiCLigg46oyCKCm4zLoMiOzKQSGBAZRwwghD2BJqEGJAIUVoYCRECCIkkPL8/7qmk0qne0l11b598369Xv7ruqVtdT29PnXruWRQRmJlZXtYoOwAzMxt4Tu5mZhlycjczy5CTu5lZhpzczcwy5ORuZpah15UdAMBGG20Uo0ePLjsMM7NBZcaMGc9GxMhG91UiuY8ePZrp06eXHYaZ2aAi6U9d3eeyjJlZhpzczcwy5ORuZpahStTczWz19eqrr9LR0cGiRYvKDqWyhg0bxqhRoxg6dGivH+Pkbmal6ujoYN1112X06NFIKjucyokIFixYQEdHB2PGjOn141yWMbNSLVq0iA033NCJvQuS2HDDDfv8zsbJ3cxK58TevVX5+Ti5m5n10WWXXcZTTz3V8L4vf/nL/OY3v+n28YsXL+bggw+mra2NSZMmNSPEwVdzH33mr3p13rxvvrPJkZhZM/T2f7y3mpELLrvsMnbZZRc233zzle4755xzenz8gw8+yKuvvkp7e/uAx1bjnruZrdbmzZvHjjvuyEc+8hF23nln3v72t/PKK68A0N7ezl577cWuu+7KkUceyXPPPce1117L9OnTOf7442lra1t2bs2JJ57ItddeCxSz78866yx23313xo4dyyOPPMIzzzzD+9//ftrb22lra+OPf/wjU6dOZbfddmPs2LF88IMfZPHixf3+vpzczWy199hjj3Hqqacye/ZsRowYwZQpUwA44YQTOO+885g5cyZjx47lK1/5ChMmTGD8+PFcddVVtLe3s9Zaa3X7tTfaaCMeeOABPvaxj/Htb3+bjTfemIsuuoi3vvWttLe3s8UWW3DiiScyadIkZs2axZIlS7jgggv6/T05uZvZam/MmDG0tbUB8KY3vYl58+axcOFCnn/+efbff38AJk6cyO23397nr33UUUet8HU7e/TRRxkzZgzbb799v56nMyd3M1vtrbnmmstuDxkyhCVLlgz41+7q60bEgD1XPSd3M7MGhg8fzvrrr88dd9wBwJVXXrmsF7/uuuvy4osvDsjz7LDDDsybN4+5c+eu9Dz9MehGy5iZtcrll1/OKaecwssvv8w222zDpZdeChQXTU855RTWWmst7r777h7r7t0ZNmwYl156KUcffTRLlixhjz324JRTTul37GrWW4K+GD9+fPR2PXcPhTTLy5w5c9hxxx3LDqPyGv2cJM2IiPGNzu+xLCNpmKT7JD0kabakr6T2MZLulfSYpEmSXp/a10zHc9P9o/v9XZmZWZ/0pua+GDgwIsYBbcAhkvYCzgO+FxHbAc8BH0rnfwh4LiLeAHwvnWdmZi3UY3KPwkvpcGj6COBA4NrUfjlwRLp9eDom3X+QvHCEmVlL9Wq0jKQhktqBZ4BbgD8Cz0dEbVxPB7BFur0F8CRAun8hsGGDr3mypOmSps+fP79/34WZma2gV8k9IpZGRBswCtgTaHT1o3ZltlEvfaWrthFxYUSMj4jxI0c23LzbzMxWUZ/GuUfE88BtwF7ACEm1oZSjgNoSaR3AlgDp/uHA3wYiWDMz653ejJYZKWlEur0WcDAwB7gVmJBOmwjckG7fmI5J9/82qjDe0sysC3vvvXeP59xxxx3svPPODRcLq6LeTGLaDLhc0hCKF4PJEfFLSb8HrpH0NeBB4OJ0/sXAlZLmUvTYj21C3GaWq7OHD/DXW9jjKXfddVeP51x11VWcccYZnHTSSQMRVdP1ZrTMzIjYLSJ2jYhdIuKc1P54ROwZEW+IiKMjYnFqX5SO35Duf7zZ34SZWX+ss846ANx2220ccMABTJgwgR122IHjjz+eiOCiiy5i8uTJnHPOOcvaPve5z7HLLrswduzYpm240R9efsDMrM6DDz7I7Nmz2Xzzzdlnn3248847+fCHP8y0adM47LDDmDBhAlOmTKG9vZ2HHnqIZ599lj322IP99tuPzTbbrOzwl/HCYWZmdfbcc09GjRrFGmusQVtbW8NleqdNm8Zxxx3HkCFD2GSTTdh///25//77Wx9sN5zczczq9Gb538EwRsTJ3cysj/bbbz8mTZrE0qVLmT9/Prfffjt77rln2WGtwDV3q7y+bJjs1UCtFY488kjuvvtuxo0bhyTOP/98Nt1007LDWoGX/LXKc3LPm5f87Z0BX/LXzMwGHyd3M7MMObmbmWXIyd3MSleFa39Vtio/Hyd3MyvVsGHDWLBggRN8FyKCBQsWMGzYsD49zkMhzaxUo0aNoqOjA2/a07Vhw4YxatSoPj3Gyd3MSjV06FDGjBlTdhjZcVnGzCxDTu5mZhlycjczy5CTu5lZhpzczcwy5ORuZpYhD4U0W814ZdXVg3vuZmYZcnI3M8uQk7uZWYZ6TO6StpR0q6Q5kmZLOj21ny3pL5La08c76h7zBUlzJT0q6Z+b+Q2YmdnKenNBdQnw2Yh4QNK6wAxJt6T7vhcR364/WdJOwLHAzsDmwG8kbR8RSwcycDMz61qPPfeIeDoiHki3XwTmAFt085DDgWsiYnFEPAHMBaq1LbiZWeb6VHOXNBrYDbg3NX1C0kxJl0haP7VtATxZ97AOun8xMDOzAdbr5C5pHWAK8KmIeAG4ANgWaAOeBr5TO7XBw1dahV/SyZKmS5rudZzNzAZWr5K7pKEUif2qiLgOICL+GhFLI+I14KcsL710AFvWPXwU8FTnrxkRF0bE+IgYP3LkyP58D2Zm1klvRssIuBiYExHfrWvfrO60I4GH0+0bgWMlrSlpDLAdcN/AhWxmZj3pzWiZfYAPALMktae2LwLHSWqjKLnMAz4KEBGzJU0Gfk8x0uZUj5QxM2utHpN7REyjcR39pm4ecy5wbj/iMjOzfvAMVTOzDDm5m5llyMndzCxDTu5mZhlycjczy5CTu5lZhpzczcwy5ORuZpYhJ3czsww5uZuZZcjJ3cwsQ07uZmYZcnI3M8uQk7uZWYac3M3MMuTkbmaWISd3M7MMObmbmWXIyd3MLENO7mZmGXJyNzPLkJO7mVmGnNzNzDLk5G5mlqEek7ukLSXdKmmOpNmSTk/tG0i6RdJj6fP6qV2Svi9prqSZknZv9jdhZmYr6k3PfQnw2YjYEdgLOFXSTsCZwNSI2A6Ymo4BDgW2Sx8nAxcMeNRmZtatHpN7RDwdEQ+k2y8Cc4AtgMOBy9NplwNHpNuHA1dE4R5ghKTNBjxyMzPrUp9q7pJGA7sB9wKbRMTTULwAABun07YAnqx7WEdqMzOzFul1cpe0DjAF+FREvNDdqQ3aosHXO1nSdEnT58+f39swzMysF3qV3CUNpUjsV0XEdan5r7VyS/r8TGrvALase/go4KnOXzMiLoyI8RExfuTIkasav5mZNdCb0TICLgbmRMR36+66EZiYbk8EbqhrPyGNmtkLWFgr35iZWWu8rhfn7AN8AJglqT21fRH4JjBZ0oeAPwNHp/tuAt4BzAVeBk4a0IjNzKxHPSb3iJhG4zo6wEENzg/g1H7GZWZm/eAZqmZmGXJyNzPLkJO7mVmGnNzNzDLk5G5mliEndzOzDDm5m5llyMndzCxDTu5mZhlycjczy5CTu5lZhpzczcwy5ORuZpYhJ3czsww5uZuZZcjJ3cwsQ07uZmYZcnI3M8uQk7uZWYac3M3MMuTkbmaWISd3M7MMObmbmWXIyd3MLEM9JndJl0h6RtLDdW1nS/qLpPb08Y66+74gaa6kRyX9c7MCNzOzrvWm534ZcEiD9u9FRFv6uAlA0k7AscDO6TH/KWnIQAVrZma902Nyj4jbgb/18usdDlwTEYsj4glgLrBnP+IzM7NV0J+a+yckzUxlm/VT2xbAk3XndKS2lUg6WdJ0SdPnz5/fjzDMzKyzVU3uFwDbAm3A08B3UrsanBuNvkBEXBgR4yNi/MiRI1cxDDMza2SVkntE/DUilkbEa8BPWV566QC2rDt1FPBU/0I0M7O+WqXkLmmzusMjgdpImhuBYyWtKWkMsB1wX/9CNDOzvnpdTydIuho4ANhIUgdwFnCApDaKkss84KMAETFb0mTg98AS4NSIWNqc0M3MrCs9JveIOK5B88XdnH8ucG5/gjIzs/7xDFUzsww5uZuZZcjJ3cwsQz3W3M0GlbOH9/K8hc2Nw6xk7rmbmWXIyd3MLENO7mZmGXJyNzPLkJO7mVmGnNzNzDLkoZBm1lhvh5WCh5ZWkHvuZmYZcnI3M8uQk7uZWYac3M3MMuTkbmaWISd3M7MMObmbmWXIyd3MLENO7mZmGXJyNzPLkJO7mVmGnNzNzDLUY3KXdImkZyQ9XNe2gaRbJD2WPq+f2iXp+5LmSpopafdmBm9mZo31pud+GXBIp7YzgakRsR0wNR0DHApslz5OBi4YmDDNzKwvekzuEXE78LdOzYcDl6fblwNH1LVfEYV7gBGSNhuoYM3MrHdWtea+SUQ8DZA+b5zatwCerDuvI7WZmVkLDfQFVTVoi4YnSidLmi5p+vz58wc4DDOz1duqJve/1sot6fMzqb0D2LLuvFHAU42+QERcGBHjI2L8yJEjVzEMMzNrZFWT+43AxHR7InBDXfsJadTMXsDCWvnGzMxap8c9VCVdDRwAbCSpAzgL+CYwWdKHgD8DR6fTbwLeAcwFXgZOakLMZmbWgx6Te0Qc18VdBzU4N4BT+xuUmZn1j2eompllyMndzCxDPZZlLF+jz/xVr8+d9813NjESMxto7rmbmWXIyd3MLEP5lmXOHt6Hcxc2Lw4zsxK4525mliEndzOzDDm5m5llyMndzCxDTu5mZhlycjczy5CTu5lZhpzczcwy5ORuZpYhJ3czsww5uZuZZcjJ3cwsQ07uZmYZcnI3M8uQk7uZWYbyXc/drMW8baFViXvuZmYZcs/dzKwfevuOrdXv1vqV3CXNA14ElgJLImK8pA2AScBoYB7w3oh4rn9hmplZXwxEWeZtEdEWEePT8ZnA1IjYDpiajs3MrIWaUZY5HDgg3b4cuA34lyY8T6VV9a2ama0e+ttzD+BmSTMknZzaNomIpwHS5437+RxmZtZH/e257xMRT0naGLhF0iO9fWB6MTgZYKuttupnGGaWm+ze/Z49vA/nLuz30/Wr5x4RT6XPzwDXA3sCf5W0GUD6/EwXj70wIsZHxPiRI0f2JwwzM+tklZO7pLUlrVu7DbwdeBi4EZiYTpsI3NDfIM3MrG/6U5bZBLheUu3r/FdE/FrS/cBkSR8C/gwc3f8wzcysL1Y5uUfE48C4Bu0LgIP6E5SZmfWPlx8wM8uQk7uZWYac3M3MMuTkbmaWISd3M7MMObmbmWXI67lb7/R26vQATJs2s/5zz93MLENO7mZmGXJyNzPLkGvuZXMt28yawD13M7MMueduZoNbizfBGCyc3M3K4HKcNZnLMmZmGXJyNzPLkJO7mVmGnNzNzDLk5G5mliEndzOzDDm5m5llyMndzCxDTu5mZhlycjczy1DTkrukQyQ9KmmupDOb9TxmZraypiR3SUOAHwGHAjsBx0naqRnPZWZmK2tWz31PYG5EPB4R/wCuAQ5v0nOZmVknioiB/6LSBOCQiPhwOv4A8OaI+ETdOScDJ6fDNwKPDnAYGwHPDvDXbAbHObAc58AZDDHC6h3n1hExstEdzVryVw3aVngViYgLgQub9PxImh4R45v19QeK4xxYjnPgDIYYwXF2pVllmQ5gy7rjUcBTTXouMzPrpFnJ/X5gO0ljJL0eOBa4sUnPZWZmnTSlLBMRSyR9AvhfYAhwSUTMbsZzdaNpJZ8B5jgHluMcOIMhRnCcDTXlgqqZmZXLM1TNzDLk5G5mliEndzOzDGWT3CVt0KBtTBmxmJnVk7SupHVa+py5XFCVdCdwaES8kI53AiZHxC7lRtZYWn9nE+pGLEXEn8uLaGWS1gTeA4xmxTjPKSumRiRtD3wO2JoV4zywtKASSZ/p7v6I+G6rYumOpN27uz8iHmhVLL0haR/gbJb/zgVERGxTZlydSRoLXAFsQBHjfGBiRDzc7Odu1gzVMnwd+IWkd1IsZ3AFcHy5ITUm6TTgLOCvwGupOYBdSwuqsRuAhcAMYHHJsXTn58CPgZ8CS0uOpbN1yw6gl77TzX0BlP5C2cnFwKcp/jar9juv9xPgMxFxK4CkAyiGRO7d7CfOpucOIOkI4PMU/1BHRcRjJYfUkKS5FGvtLCg7lu5Ieriq73zqSZoREW8qOw5rHUn3RsSby46jJ5IeiohxPbU1w6DvuUv6ASuuW7Me8DhwmiQi4pPlRNatJyl6xFV3l6SxETGr7EB68AtJHweup+4dRkT8rbyQCpI+HxHnN/g7Bajk36ekXSiW6h5Wa4uIK8qLqKFbJX0LuI4Vf+eVKh8Bj0v6EnBlOn4/8EQrnnjQJ3dgeqfjGaVE0TePA7dJ+hUr/mFWov5aZ1/gRElPUMRZq2tWrXw0MX3+XF1bAFWov85Jnzv/nVaSpLOAAyiS+00UezJMoyhzVkmt116/EFcVy0cfBL5C8SIk4HbgpFY8cTZlGUlrA4siYmk6HgKsGREvlxvZytI/0Eoi4iutjqU7krZu1B4Rf2p1LDlIoyUiIv5edixdkTQLGAc8GBHjJG0CXBQR7yo5tBVIGlL7Xx8MJK0HvBYRL7XqOXPouddMBQ4Gaj+8tYCbacGFi76qWhLvSkT8SdI44K2p6Y6IeKjMmBqRNBT4GLBfaroN+ElEvFpaUHVSyehMYO10/BJwXkT8Z6mBNfZKRLwmaUlKSM9QjXdAnc2VdC3FulVzejy7JJ1GyyDpWTxaps+G1b8qRsRLkv6pzIA6k/TvEfEpSb+gcf313SWE1SVJpwMfoXhLCfAzSRdGxA9KDKuRC4ChQC1ZfiC1fbi0iBJJ/0bRwTggIh5PbdsA/yFpg4j4WqkBrmy6pBEUI49mUHSW7is3pIZ2pVht9mJJawCXANfUhkJXiEfL9Fca535a7YKKpDcBP4yIt5Qb2XKS3hQRMyTt3+j+iPhdq2PqjqSZwFtqZYRU+rq7ajX3Mkck9ETSo8C4iFjUqX0t4KGI2L6cyHomaTSwXkTMLDmUbknaD7gaGAFcC3w1IuaWG1XBo2UGxqeAn0uqbQqyGXBMifGsJCJmpM+VSuLdECuOIV5K4122yrZU0rYR8UdY1jOuTD22c2JPba9Ieq3R+WWSdCTw24hYGBHzJI2QdERE/HfZsdVL19TeSXFxcjTFOP2rKEqINwFVedH0aJn+ioj7Je1AMYFJwCNVqbl2Jukw4KusPLtuvVIDW9mlwL2Srk/HR1BMHqmaz1EMjXuc4me5NS0akdALHZIOioip9Y2SDgSeLimm7pwVEbXfNxHxfBoAUKnkDjwG3Ap8KyLuqmu/NvXkq8KjZfor1dc/Q7Fh7EckbQe8MSJ+WXJoK0mTmI4CZkXFfwFpWvq+pD/MiHiw5JAaSksl1L+wV2JGraSdKWb6TqOoYQewB7APcHgJm9h0S9LMzmU3SbMiYmxZMTUiaZ1WjjzpL0nDKUbLvNiy56x4buk1SZMo/nlOiIhdUk3z7ohoKzm0lUi6FTgoIir3thyKYVsR8UKjxdigGpODoOj9RsRvJR3V6P6IuK5Re6tJGga8D9iZ4sVnNnBVo3JN2SRdAjwP/Ijiheg0YP2IOLHMuGq6mgxWU7VJYZL2oLjYW1uGYiHwwVqJtpmyKcsA20bEMZKOg2U1zSrWh6FYIuEmSb+jmpOY/gs4jOU9zRpRnclBAPsDvwUajcEOlo/yKdurwPsi4uCyA+mF04AvAZMoft83A6eWGtGKapPB9qGYaDUpHR9NNScwXgx8PCLuAJC0L0W5s+mDEnJK7v9IvfUAkLQt1V3s6lyKIWbDgNeXHMtKIuKw9LnSSyZHxFnpc1Xq6w1FxFJJL0saHhGVXnYijYw6s+w4uhIRlwNIOhF4W+26mqQfU7wQVc2LtcQOEBHTJLWkNJNTcj8L+DWwpaSrKF7ZTyw1oq5tEBFvLzuInki6AbgGuKGKM31rJG1I8fvfl+LFfRpwTsUWZlsEzJJ0C7BshmpVyghdzb2oqdocDGBzilJHrUS4Tmqrmvsk/YRiqGZQjOC7rbbEcjPXwsmm5g7L/sn3ong7eU9EPFtySA1J+ibFcLMq9jSWSePxj6EYcnYfxVvgX1atVpwS5u3Az1LT8RSThipTBpE0sVF7rSdatq7mXtRUbfiupJMo1nO/NTXtD5xdlZ9nTbq+1pWIJu45kFty34KVN2y4vbyIGktvy9amKBu9SnWHQgLLxhQfSDFb9ZCqxdloyV9J0yNifFePKUMqG24VEY+WHUt3BlGcm7J8AbF7I+L/yoynarIpy0g6j6KXOZsVN8CoXHKPiMGygUPtH/1dFD/b3YFK9YySWyUdC0xOxxOAX5UYz0okvQv4NsU1ljGS2ihKR5UqdwyWOJPFFHMFhgHbS9q+ap25tJTDCay8m1nTy3HZ9NzTNO9dqzK+uSeS1ge2Y8U1s6v2hzmJomf0a4rEeVuVhm+md0BB8c5nbZa/qK8BvFSldxiSZlC8+7ktInZLbVUcP94ozpXGvpdN0oeB04FRQDtFOfbuZpY5VoWku4B7gFks//tsSTkum547xRrpQ6nuCJlluvrDpHprUV9KMYSvMlP56w2md0DAkohY2Gl0bhV7Vo3irKLTKSaD3RMRb0uz06u42uqwiOh2H91mGfTJvW5Sw8tAu6SprDh2vBKjETqp9B+m0u5BEfFrSUdT7FFau+/rEfHFEsNrSNK7qVvyt4Izkx+W9D5gSJo9/Ungrh4eU4bBEueiiFgkCUlrRsQjkt5YdlANXCnpI8AvafEuYWs0+wlaYDrF5IUbKdZruSsd1z6qaFFtxEntD5Ni6nxVHFt3+wud7juklYH0Rhp9dDrw+/RxemqrktMoZqguppgktpBisbuqqY/zauAFqhlnR6pn/zdwSxq2+1QPjynDP4BvUbwzr+WkluzKlVPNfTDtxHQ9xeJBn6IoxTwHDI2Id5QaWCLpwbp667LbjY6rQMXSxG216wHpd/9glerEknar6ro8g10axjkc+HVE/KPseOpJ+iPw5jKGZQ/6skydwbQT05Hp5tlpHOxwiouWVRFd3G50XBUjWD6hZXiZgXThu5I2oyhxXVO1BcNqJG0PnMHKozsqcz1IxeYcMyNiF6jeGPxOZlOUjFsup+Re+Z2YaiQdHBG/geV/mGmSS1WGGY6T9ALFKJS10m3S8bCuH1aabwAPphdKUdTeO5eTSpWurWwKvBe4UMUWdpOiejsx/Rz4MXARFVoTv14U2wA+JGmriPhz2fH0YCnFtcBbafG1wJzKMpXfialG0u0Ur+hnUEybvghYHBETSg1sEEqLw40CllBcpBYVn9CiYl/NzwPHRESl1hZqNCGsiiT9luL3fR8rLudQqfH4Zc5Mzim570GxDsoKOzG1YmnNvkoJ6bPAR1PTlyPi6hJDGtQGQ0KStCPFRLAJwAKKv9UpEfFMqYElWr688ycpNsW+nhaP7uiLrpZLqHiJpqWySe4AkoYyOHZi2oBi49x1KXqdPwPOi5x+GS0k6UfAZRFxf9mxdEXSPRSjT34eEZUb1SHpCZZPCINO11YioirLPC+Tylx7UsR6fxXfraXhpN+gWJ64fsJi03+e2SR3SSc0ao+IK1odS08k/QH4ZkRckqb3nweMj4jKXfwdDCT9nmLPzD9RvEWvrdVTmdEyVSdpT+DJiHg6HU8E3gPMo1iQq2o99w8DX6ZYz18UC4edExGXlBpYJ5KmUaxY+j2KZTxOosi7ZzX9uTNK7j+oOxwGHAQ8UMU6dqMLQZL2q9ryA4OFpK0btUfEn1odS2eSJkfEeyXNosHGJ1V5AZL0AHBwRPxNxR6k11CMeW8Ddqza/1FabmTv2rLOKlaEvSsiqjRfZFnJsH6pCUl3RMRbm/3c2YyWiYjT6o9V7Fl4ZRenl+1JSe8HtomIcyRtRbHet62aKvdQTk+fDys1ip4NqeudHwNcGBFTgCmS2kuMqysdQP2mFy8CT5YUS3cWpaGbj0n6BPAXYONWPHE2PffOUv19ZkTsWHYsnUm6gGIRoQMjYse0iNjNEbFHyaENSnW94tpQzTHAoxGxc6mB9UDSnRGxT9lxAEh6mGIi2BJJjwAn195JSnq4Nqa8KiRdAYyl2Hw8gMMpRs78AaqzZWUa6DGHYh7GVynmYJwfEfc0+7mz6blrxZ1k1qC4gDG560eU6s0RsbukBwEi4jlJlRoSN5h0XllRxS43H+3i9CrZquwA6lwN/E7Ss8ArQG3PzzdQLJVQNX9MHzU3pM+VWkyu7iL/SxT19pbJJrlTrEFdswT4U0R0lBVMD15NU+Rr+72OpG45UOufiHgg9ZiqrjJvmyPi3LTo3mYU7yLrO0qndf3IckREZRba606a8fs5Vt5EqOkzfnNK7n8Gnq5bkGstSaMjYl65YTX0fYpxxBtLOpdi7PO/lRvS4CWpfknVNSg2FZlfUjgrkHRUV3dRLJFRGY1KBRHxhzJi6Uma8bnSi2OVlklIajN+f0qLZ/zmlNx/zorryCxNbZXrwUXEVSo2RTiI4p/8iIiYU3JYg1n9W/ElFLswTSkpls7e1c19VVuWeDA5o+72MIphm0tKiqU7SyLigjKeOJsLqpLaI6KtU9tDETGurJg6q5sF2FDVxhIPNpLWjoi/93ym5UjS7yKi242+W6UKM35z6rnPl/TuiLgRQNLhQMuX2ezBDJaP6tiKYqlfUVxJ/zPFKA/rI0lvAS6mWKdnK0njgI9GxMfLjWylktFKqjKqY7Dp1FFaAxgPbFpSOI3U/69DUXev70k3fYZqTsn9FOAqST9Mxx0UG9NWRkSMAZD0Y+DGiLgpHR9KsVyxrZp/B/6ZYsMWIuKhNBGnCmolozdSlAhvTMfvooKbt21xfyIAAApFSURBVA8iteQJRTlmHvCh0qLppO5//b0U68y/IOlLFNeDvtqKGLIpy9RIWofi+3qxx5NL0mihK0nTI2J8WTENZpLujYg3d9pkpGoluZuB99T+LiWtS7HOTOV2tqqyNArqydo6MoNgmYSZEbGrpH2BrwPfAb4YEW9u9nPnsM0eUOztKWlERLwUES9KWl9S1dbKrnlW0r9JGi1pa0n/SrFSoK2aJyXtDYSk10s6g2LiSJVsRbHlWs0/KDbEsL75CennmN6dfYNiH4SFwIUlxtWV2giZdwI/jogbgJbMackmuQOHRsTztYOIeA6oxLZ1DRwHjKS4yPLfFNORjys1osHtFOBUYAuKclxbOq6SK4H7JJ0t6WzgXqqzOctg0nCZhIj4EvCGEuPqyl8k/YRik5abJK1Ji/JuNmUZFfto7hERi9PxWsD0qk9Bt9VHmjn7Vopa8R3hPVX7bBAuk/BPFJvKz4qIx1RstTg2Im5u9nPndEH1Z8BUSZem45OoaM8ozUj9PMUu8/VrPFdtAkalSfpyN3dHRLTkwlUfLKWYiRx4RvKqGlTLJETEy8B1dcdPA0+34rmzKctExPnA14AdKdaV+TXFlN8qugp4hGLo41coLgZVdqOJCvt7gw8oRk38S1lBNSLpdIrf+0YUZbifSarctP6qi4hzKXYxuwzYt+rLJJQpm7IMgKQ24H0U9a0nKLYx+2H3j2q9ujWeZ9bW867SBIzBKI0+OZ0isU8GvhMV2cIOlpUN31KbZCVpbeDuqqznbvkZ9GWZtDDPsRQXJBcAkyhetN5WamDdq23/97Skd1Ls+zqqxHgGrTSZ5TPA8RRluN3TxfSqESuuLbKU5RNczAbcoE/uFOWNO4B3RcRcAEmfLjekHn0tbSbyWeAHwHpA1WOuHEnfAo6iGAI3NiJeKjmk7lwK3Cvp+nR8BMWsWrOmGPRlGUlHUvTc96aos18DXFSbIWb5kvQaxXodS2i8hd16pQTWhTRaZl+K+G73aBlrpkGf3GtSDfMIivLMgRRv0a9vxZCjvpJ0OXB6bVx+2onpOxHxwXIjs2aRtBcwu9MM1Z0i4t5yI7NcZZPc66U67NHAMVUcXlg/Tb67NstH2nVr99rojrSv5vSI2L3cyCxX2QyFrBcRf4uIn1QxsSdrpN46sOzFKIfrH9Y11Q3bIyJew79zayL/cZXjO8Bdkq6lqBW/Fzi33JCsyR6X9EmgtnHDx4HHS4zHMpdlWWYwkLQTxbUBAVMj4vclh2RNJGljiu0VD6R4QZ8KfKpKY/EtL07uJUlLgG4XEZem5QjWiYgnyo7LzPKQZc296iSdRTE9/gupaSjF2jiWKUnnS1pP0lBJUyU9K+n9Zcdl+XJyL8eRwLtJa6FExFOsuMmz5eftEfECcBjFssTbU2y9ZtYUTu7l+EcaOVEbFrd2yfFY8w1Nn98BXF21HYMsP07u5ZicFvAfIekjwG+An5YckzXXL9L64+MplqYeCSwqOSbLmC+olkTS/wPeTjFa5n8j4paSQ7ImS3MbXoiIpWkTh/Vqe4GaDTQn95JJ2ghYEP5FZE/SLhR7DdRv0HJFeRFZzlyWaSFJe0m6TdJ1knZLW4Y9DPxV0iFlx2fNk0ZI/SB9vA04n+KiullTuOfeQpKmA18EhlMsU3toRNwjaQeKi2xeWyZTkmYB44AHI2KcpE0oVi99V8mhWabcc2+t10XEzRHxc+D/IuIegIh4pOS4rPleSevJLJG0HvAMsE3JMVnGvLZMa9VvivxKp/v8Fipv0yWNoBgVNQN4Cbiv3JAsZy7LtJCkpRQTlwSsBbxcuwsYFhFDu3qs5UPSaIqRMjNLDsUy5uRu1gKSRLHP6zYRcY6krYBNI8K9d2sKJ3ezFpB0AUVZ7sCI2DGNeb85IvYoOTTLlGvuZq3x5ojYPe3IREQ8J+n1ZQdl+fJoGbPWeFXSEJavJzSSFS+wmw0oJ3ez1vg+cD2wsaRzgWnA18sNyXLmmrtZi6TJagexfPetOSWHZBlzcjdrAUnbAh0RsVjSAcCuwBUR8Xy5kVmuXJYxa40pwFJJbwAuAsYA/1VuSJYzJ3ez1ngtIpYARwH/ERGfBjYrOSbLmJO7WWu8Kuk44ATgl6nNM5KtaZzczVrjJOAtwLkR8YSkMXhTdGsiX1A1M8uQZ6iatYCkfYCzga0p/u8ERER42V9rCvfczVogbY79aYrlfpfW2iNiQWlBWdbcczdrjYUR8T9lB2GrD/fczVpA0jeBIcB1wOJae0Q8UFpQljUnd7MWkHRrg+aIiANbHoytFpzczcwy5HHuZi0gaRNJF0v6n3S8k6QPlR2X5cvJ3aw1LgP+F9g8Hf8B+FRp0Vj2nNzNWmOjiJhM2qAjrTOztPuHmK06J3ez1vi7pA1ZvhPTXsDCckOynHmcu1lrfBa4EdhW0p3ASGBCuSFZzjxaxqxFJL0OeCPF0gOPRsSrJYdkGXNZxqwFJD0EfB5YFBEPO7Fbszm5m7XGu4ElwGRJ90s6Q9JWZQdl+XJZxqzFJG0HfAk4PiKGlB2P5ckXVM1aRNJo4L3AMRTDID9fZjyWNyd3sxaQdC/Ftno/B46OiMdLDsky57KMWQtI2iEiHik7Dlt9+IKqWWs857VlrJWc3M1a4zK8toy1kJO7WWt4bRlrKSd3s9bw2jLWUh4tY9Yan8Fry1gLuedu1kSS9pC0adordX/gixR7qN4MdJQanGXNyd2suX4C/CPd3hv4V+BHwHPAhWUFZflzWcasuYZExN/S7WOACyNiCjBFUnuJcVnm3HM3a64haalfgIOA39bd586VNY3/uMya62rgd5KeBV4B7gCQ9AY8WsaayMsPmDVZGva4GXBzRPw9tW0PrJMutJoNOCd3M7MMueZuZpYhJ3czsww5uZv1QNJoSQ+v4mMPkLT3QMdk1hMnd1ttSWrFFncHUExeMmspJ3fLUuptPyLpckkzJV0r6Z8kzZP0ZUnTgKMltUm6J51zvaT10+PfJOkhSXcDp9Z93RMl/bDu+JeSDki3D5H0QHrc1LSt3inApyW1S3prC38EtppzcrecvZFiRuiuwAvAx1P7oojYNyKuAa4A/iWdMws4K51zKfDJiHhLb55I0kjgp8B7ImIcxVZ684AfA9+LiLaIuGOgvjGznji5W86ejIg70+2fAfum25MAJA0HRkTE71L75cB+Ddqv7MVz7QXcHhFPANQtOWBWCid3y1nnSRy147/38Dg1eGzNElb8vxnWi8eYtZyTu+VsK0m1sspxwLT6OyNiIcXeprVa+AeA30XE88BCSbWe/vF1D5sHtElaQ9KWwJ6p/W5gf0ljACRtkNpfBNYdwO/JrFec3C1nc4CJkmYCGwAXNDhnIvCtdE4bcE5qPwn4Ubqg+krd+XcCT1DU578NPAAQEfOBk4HrJD1EKv0AvwCO9AVVazUvP2BZSiNVfhkRu5Qcilkp3HM3M8uQe+5mZhlyz93MLENO7mZmGXJyNzPLkJO7mVmGnNzNzDLk5G5mlqH/DwYGwPf4wDBRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "analysis = label_analysis.reset_index()\n",
    "analysis.columns = ['product','not info', 'info', 'All']\n",
    "analysis.iloc[:-1].plot(x=\"product\", y=[\"not info\", \"info\"], kind=\"bar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lets check the length quality of reviews we have?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['review_len'] = df['answer_option'].apply(lambda x: len(x.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>answer_option</th>\n",
       "      <th>label</th>\n",
       "      <th>review_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Fast and accurate delivery</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>As usual it is genuine</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Behavior of delivery boy is very bad. Delivery...</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>fwegwrqdsdvwfg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>These strips were as per my requirment</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1671</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Ft GM</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1672</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>I like</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1673</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Nice price with long expiry</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1674</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Price &amp; Service</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1675</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Good discount</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1676 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        product                                      answer_option  label  \\\n",
       "0     Accucheck                         Fast and accurate delivery      0   \n",
       "1     Accucheck                            As usual it is genuine       0   \n",
       "2     Accucheck  Behavior of delivery boy is very bad. Delivery...      0   \n",
       "3     Accucheck                                     fwegwrqdsdvwfg      0   \n",
       "4     Accucheck             These strips were as per my requirment      0   \n",
       "...         ...                                                ...    ...   \n",
       "1671  Accucheck                                              Ft GM      0   \n",
       "1672  Accucheck                                             I like      0   \n",
       "1673  Accucheck                        Nice price with long expiry      0   \n",
       "1674  Accucheck                                    Price & Service      0   \n",
       "1675  Accucheck                                      Good discount      0   \n",
       "\n",
       "      review_len  \n",
       "0              4  \n",
       "1              5  \n",
       "2             20  \n",
       "3              1  \n",
       "4              7  \n",
       "...          ...  \n",
       "1671           2  \n",
       "1672           2  \n",
       "1673           5  \n",
       "1674           3  \n",
       "1675           2  \n",
       "\n",
       "[1676 rows x 4 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "checklen = []\n",
    "for i in range(5,50, 5):\n",
    "    checklen.append(len(df[ (df['review_len']>=i-5) & (df['review_len']<i)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[780, 603, 149, 65, 29, 22, 9, 6, 3]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checklen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEoCAYAAABGqrb1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3debgcVbn+/e+dAQhjgCQICRCUCIoIYmQGmZRRE2UQFYkYjHIAQZCXwFFExfNDURmUA0ZRggeEKFNkkCHMKkqYBUQCBhIIJGGMMhl43j/Waik6vSu99+4pO/fnuvrqqlXVVU9P9VStVbVKEYGZmVlX+rU7ADMz62xOFGZmVsqJwszMSjlRmJlZKScKMzMr5URhZmalnCisoSStI+mfkvq3OxarTVJIWr8D4hiZYxnQi2VsJ+nhRsZli3KiWEpJminplbxRf1rSuZJW7O1yI+KJiFgxIt5oRJy1SBoh6WJJ8yW9KOl+SZ9v1voapR0baEk3STq4Acv5fI5/v0bE1SgRcWtEbNDuOPo6J4ql28ciYkVgU+ADwHFtjqdevwJmAesCqwMHAs/0ZEG92ZtdyowDnsvPtpRxojAi4mngGlLCAEDSspJ+IOkJSc9IOlvSoDztIUl7FeYdkPfuN6uuTpC0iqRzJM2R9KSkkyrVUpIel/TBPHxAft178/jBki7rIuQPAedGxL8iYmFE3B0RV+fX7SBpdnHmfPS0Sx4+UdJvJf2fpJeA4/OR1WqF+T+Q38/APP6F/J6fl3SNpHVz+ZmSfli1rt9JOrK730FX68jTQtKXJT2Sp58pSXlaf0k/zPH+Q9Jhlc9f0neB7YCf5CPHnxRWuUut5XUR27rAh4EJwK6S1ihM20HSbElHS5qbv+eDCtP3lHS3pJckzZJ0Yhfr2FfSnVVlR1d+A5L2kPSgpAX5d/S14voLrzk2T18g6WFJO9fx8dviRIQfS+EDmAnskodHAPcDpxemnwZMBVYDVgJ+B/y/PO0E4PzCvHsCf8vDI4EABuTxy4CfAisAw4C/AF/K084Djs7Dk4BHgUMK077aRezXA38A9gfWqZq2AzC75L2eCPwbGEvaURoE3AB8sTD/KcDZeXgsMAN4DzAA+Drwxzxtc+ApoF8eHwK8DKzRRdwBrF+jvMt1FF53BTAYWAeYB+yWp30ZeDB/h6vmz6b4+d8EHFwjjprL6yLubwB/ycP3A0dVfd4LgW8DA4E98mewamH6xvmzfj/pyG9s9W8FWJZ0xPKewrLvBvbOw3OA7fLwqsBm1d83sAHpSHOtwvLf1e7/Wl94tD0AP9r0xaeN5z+BBfnPOg0YnKcJ+FfxTwZsBfwjD6+fX7d8Hj8fOCEPF//8awCvAYMKy/k0cGMeHg9MzcMPAQcDF+bxxysbgxqxrwqcDDwAvAHcA3woT/vPhqPqvRYTxS1V0w8Gbii891nA9nn8amB8Yd5+eUO4biHuj+Thw4CrSj7zrhLF4tYRwLaF6VOAiXn4BnLizeO7UF+iqLm8LuJ+BDgyDx8H3FuYtgPwSmV9uWwusGUXyzoNOLX6t5LHzwK+m4c3Ap4Hls3jTwBfAlauWt5/vm/S73Ju/gwGtvs/1pcernpauo2NiJVIf7YNSXvEAEOB5YE7Jb0g6QXg97mciJhB2kB+TNLywMeBC2osf13SXuacwnJ+SjqyALgZ2E7SO4D+wEXANpJGAquQEsAiIuL5iJgYERuRktE9wGVl1SdVZlWN/xbYStJawPakjdethfdweiH+50jJZHiePhk4IA8fQGo/6a7FrQPg6cLwy0DlxIO1qt5P9XvrSlfLextJ2wDrARfmoguAjSVtWpjt2YhYWGt5kraQdKOkeZJeJB0BDaG2ycBn8vf4OWBKRLyWp+1NOlp5XNLNkraqfnH+XR5J2hmYK+nC/J1aLzlRGBFxM3Au8INcNJ+0l7hRRAzOj1UiNXxX/Jp0dDAGeDD/SavNIh1RDCksZ+W8ga/8sV8GvkLay19A2oBNAG6LiDfriH1+jnstUjXZv0hJDkh1+OQEV3xZ1TJeAK4F9gM+A/w68i5qfg9fKsQ/OCIGRcQf8/T/A8ZI2oRUddRVu0qZxa2jzBxStVPF2lXTe9s99DhS0rpH0tPAn3P5gXW+/gJSFebaEbEKcHZe3iIi4nbgdVK7ymcoJN2IuCMixpB2Mi4jHQXVWsYFEbEtKfkG8L0647QSThRWcRrwEUmb5g30z4BTJQ0DkDRc0q6F+S8EPgocQu2jCSJiDmkD/ENJK0vqJ+ldkj5cmO1mUpXNzXn8pqrxRUj6nqT35QbblXIMMyLiWeDvwHK5EXUgqb5/2Tre/wWkjd/eVe/nbOA4SRvlda8iad/Ce5wN3EHaqF0cEa8sZj3LSFqu8Oi/uHUsxhTgiPz9DAaOrZr+DPDOOpf1NpKWIyXPCaQTHSqPw4HPqr4zxlYCnouIVyVtTkoAZc4DfgIsjIjbchzLSPqspFUi4t/AS6Qqx+p4N5C0k6RlgVdJOztNO017aeJEYQBExDzSn/QbuehYUgPr7UpnB11PaiyszD8H+BOwNanKqCsHAsuQGlyfJ1XzrFmYfjNpY3JLF+O1LA9cCrwAPEbae/x4jutF4L+AnwNPko4wZtdezNtMBUYBz0TEvZXCiLiUtFd6Yf4c/grsXvXayaQG23qqnR4gbcAqj4PqXEdXfkZKxveRGn+vIjUuVzaQpwP75LObzqhzmRVjc4znRcTTlQdwDqmqcLc6lvFfwLclLSCdBFHzSKDgV8D7WPSz/BwwM38+X+at6r6iZUltV/NJR6bDgOPriNEWQ28dYZtZT0janlQFNbKe6rImx7I76YytdRc7cwdSOgV7LulEhkfaHY8lPqIw64VcvXUE8PN2JAlJg/I1BgMkDQe+STraWlIdAtzhJNFZfERh1kOS3gNMB+4lXYfwUhtiWJ5UXbchqZroSuCIdsTSW5Jmkhq6x0bE3W0OxwqcKMzMrJSrnszMrJQThZmZlepzPWcOGTIkRo4c2e4wzMyWKHfeeef8iKi+OBVoQ6KQ9FVS3zpB6mDsINJ59ReSrqy9C/hcRLyeL5w5D/gg8CzwqYiYWbb8kSNHMn369Oa9ATOzPkjS411Na2nVUz597yvA6Ih4H+minf1JFxudGhGjSBdljc8vGQ88HxHrA6fiy/HNzFquHW0UA4BB+fL/5Ul91exEumIX0lWuY/PwmDxOnr5zNzp+MzOzBmhpooiIJ0kduD1BShAvAncCLxR6n5zNW71mDif3hpmnv0i6o5mZmbVIq6ueViUdJaxH6u1zBWr3aVO5uKPW0cMiF35ImiBpuqTp8+bNa1S4ZmZG66uediHd/GZe7gXyElKncoMLPVGOIN01DNLRxdrwn3sbr0Lqq/9tImJSRIyOiNFDh9ZstDczsx5qdaJ4AthS0vK5rWFnUq+iNwL75HnGAZfn4am8dTP3fUh3IfOl5GZmLdTqNoo/kxql7yKdGtuPdK/kY4GjJM0gtUGck19yDrB6Lj8KmNjKeM3MrA/29TR69OjwdRRmZt0j6c6IGF1rWp+7Mrs3Rk68sqXrm3nyni1dn5lZT7ivJzMzK+VEYWZmpZwozMyslBOFmZmVcqIwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK+VEYWZmpZwozMyslBOFmZmVcqIwM7NSThRmZlbKicLMzEo5UZiZWSknCjMzK9XSRCFpA0n3FB4vSTpS0mqSrpP0SH5eNc8vSWdImiHpPkmbtTJeMzNrcaKIiIcjYtOI2BT4IPAycCkwEZgWEaOAaXkcYHdgVH5MAM5qZbxmZtbeqqedgUcj4nFgDDA5l08GxubhMcB5kdwODJa0ZutDNTNberUzUewP/DoPrxERcwDy87BcPhyYVXjN7Fz2NpImSJouafq8efOaGLKZ2dKnLYlC0jLAx4HfLG7WGmWxSEHEpIgYHRGjhw4d2ogQzcwsa9cRxe7AXRHxTB5/plKllJ/n5vLZwNqF140AnmpZlGZm1rZE8WneqnYCmAqMy8PjgMsL5Qfms5+2BF6sVFGZmVlrDGj1CiUtD3wE+FKh+GRgiqTxwBPAvrn8KmAPYAbpDKmDWhiqmZnRhkQRES8Dq1eVPUs6C6p63gAObVFoZmZWg6/MNjOzUk4UZmZWyonCzMxKOVGYmVkpJwozMyvlRGFmZqWcKMzMrJQThZmZlXKiMDOzUk4UZmZWyonCzMxKOVGYmVkpJwozMyvlRGFmZqWcKMzMrJQThZmZlXKiMDOzUi1PFJIGS/qtpL9JekjSVpJWk3SdpEfy86p5Xkk6Q9IMSfdJ2qzV8ZqZLe3qShSS3iNpy8L4IEn/I+kySYd3c52nA7+PiA2BTYCHgInAtIgYBUzL4wC7A6PyYwJwVjfXZWZmvVTvEcX/Ah8rjP8AOAJYDviepGPqWYiklYHtgXMAIuL1iHgBGANMzrNNBsbm4THAeZHcDgyWtGadMZuZWQMMqHO+9wE/BJA0EDgAODIifibpSOBLwCl1LOedwDzgl5I2Ae4kJZw1ImIOQETMkTQszz8cmFV4/excNqfOuJdYIyde2dL1zTx5z5auz8yWHPUeUawAvJSHt8zjl+Txu4B161zOAGAz4KyI+ADwL96qZqpFNcpikZmkCZKmS5o+b968OkMxM7N61JsoHiMlCIBPAHdHxLN5fAiwoM7lzAZmR8Sf8/hvSYnjmUqVUn6eW5h/7cLrRwBPVS80IiZFxOiIGD106NA6QzEzs3rUmyhOBU6SdAfwFeCMwrQdgPvqWUhEPA3MkrRBLtoZeBCYCozLZeOAy/PwVODAfPbTlsCLlSoqMzNrjbraKCLiHEmPAB8CJkbEtMLk54DTurHOw4HzJS1DOlI5iJSwpkgaDzwB7JvnvQrYA5gBvJznNTOzFqq3MZuIuAW4pUb5id1ZYUTcA4yuMWnnGvMGcGh3lm9mZo1VV6KQNJuUJG4Fbo2IvzY1KjMz6xj1HlGcCmwHfBtYXdLzwB/IiQOYHhELmxOimZm1U12N2RHxw4gYGxFDgfcD/w38k9Te8AfgheaFaGZm7dSTvp5eAV7Nj9dI1zo80cigzMysc9TbRnEYqeppO2AocA+pyukY4LaImN+0CM3MrK3qbaM4g3QkcQ7w/YiY3byQzMysk9Rb9XQIcCmpk76Zku7J3X/vU+iXyczM+qB6G7N/GhEHRMS6wLtIvccuB3wHmCPpoSbGaGZmbdSTxuzBVQ+RenQ1M7M+qN7G7ImkhuytgZWBZ4HbgO+TLsS7u1kBmplZe9XbmP1l0llOx5KuzHZVk5nZUqLeTgFHNjkOMzPrUN1qo5C0u6RvSJokaZ1ctr2ktZoTnpmZtVu9bRRrkO4N8UFgJrAecDbpiuyDSFdpH9KcEM3MrJ3qPaL4MbAisGF+FG9Rej01ugg3M7O+od7G7N2AcRExQ1L/qmmz8emxZmZ9VnfaKN7oonwIqXsPMzPrg+pNFLcCh1cdTUR+/gJwQ0OjMjOzjlFv1dOxpAvs/krq8ymAL0p6H/A+YMt6VyhpJrCAdISyMCJGS1oNuAgYSWos3y8inpck4HTSfbNfBj4fEXfVuy4zM+u9evt6+ivpjKfpwOdJG/lPArOALSLi791c744RsWlEVO6dPRGYFhGjgGl5HGB3YFR+TADO6uZ6zMysl+o9oiAiHgU+16Q4xgA75OHJwE2ko5gxwHkREcDtkgZLWjMi5jQpDjMzq9KTTgF7K4BrJd0paUIuW6Oy8c/Pla7Lh5OOWipqnmElaYKk6ZKmz5s3r4mhm5ktfbo8opA0BTguIh7Nw2UiIj5V5zq3iYin8n0srpP0t5J5VaMsFimImARMAhg9evQi083MrOfKqp6GAgPz8DBqbKB7IiKeys9zJV0KbA48U6lSkrQmMDfPPhtYu/DyEcBTjYjDzMzq02WiiIgdC8M7NGJlklYA+kXEgjz8UeDbpO5BxgEn5+fL80umAodJuhDYAnjR7RNmZq1Vb19POwI35Ubl3lgDuDSd9coA4IKI+L2kO4ApksaT+o/aN89/FenU2Bmk02MP6uX6zcysm+o962kaqXroN8CFEfHHnqwsIh4DNqlR/iw1+ovKienQnqzLzMwao96znjYGfkaqKrpN0hOSTpH0weaFZmZmnaDeC+4eiIgTImJDYDPgfOATwB2SZkg6qZlBmplZ+3T7OoqIuCcijouI9YGPA4OA4xoemZmZdYS6r8yuyP0yfRL4FPBhUs+xFzQ4LjMz6xD1nvW0Mqmq6VOkRueFwJXA/sCVEfFa0yI0M7O2qveIYi7pgrtrSJ0CTo2IfzUrKDMz6xz1JoovA5dExEvNDMbMzDpPXYkiIs4FyPeHGEHqVuNeH1WYmfV9dZ/1JOm/gCeBx0l3vNsgl18i6cjmhGdmZu1WV6KQdAzwI9JFdzvx9l5dbyI1cpuZWR9UbxvFocAJEfH9qvtmAzwMvLuxYZmZWaeot+rpHcCdXUx7E1iuMeGYmVmnqTdRzCBdXFfL9sCDjQnHzMw6Tb1VT6cB/yvpdeC3uWxY7hb8KOCLzQjOzMzar97TY38uaVXgBOBbufgq0j0iTowId+FhZtZH1d3XU0ScIulsYCtgCPAc8KeIeLFZwZmZWft1q1PAiFgAXFtdLmlERMxuWFRmZtYxut3NeJGkjSVNBh7t5uv6S7pb0hV5fD1Jf5b0iKSLJC2Ty5fN4zPy9JG9idfMzLqvNFFI+oyk30t6QNIVkrbJ5RtLuhK4B9gT6O6Ni44AHiqMfw84NSJGAc8D43P5eOD5fO+LU/N8ZmbWQl0minxG0/+R+nW6H1gNuF7SYcB0YHPgWGDdiPhOvSuUNIKUXH6ex0W62rtyNtVkYGweHpPHydN3zvObmVmLlLVRHA6cFxGfrxTkPp1OB/4IfCwiXujBOk8D/j9gpTy+OvBCRCzM47OB4Xl4ODALICIWSnoxzz+/B+s1M7MeKKt6ehfwq6qyc0n9PH23J0lC0l7A3IgoXuVd6wgh6phWXO4ESdMlTZ83b153wzIzsxJliWIFYEFVWWV8bg/Xtw3wcUkzgQtJVU6nAYMlVY5uRgBP5eHZpKov8vRVSKflvk1ETIqI0RExeujQoT0MzczMalnc6bFbSxpSGO9H2qPfRtI7ijNGxFWLW1lEHAccByBpB+BrEfFZSb8B9iElj3HA5fklU/P4n/L0GyJikSMKMzNrnsUlih91UX561XgA1b3KdsexwIWSTgLuBs7J5ecAv5I0g3QksX8v1mFmZj1QlijWa+aKI+Im0r0siIjHSGdRVc/zKrBvM+MwM7NyXSaKiHi8lYGYmVln6tWV2WZm1vc5UZiZWSknCjMzK+VEYWZmpcr6enpM0iZ5+ARJa7UuLDMz6xRlRxTDgeXz8DdJV0ybmdlSpuw6ikeBwyStTupzqfoq7bep58psMzNb8pQliuNIXXx/mnTldVdXaUPvr8w2M7MOVXbB3eWSVgPWAp4APknqXsPMzJYipX09RcSbwGxJBwG3RsSzrQnLzMw6xeI6BQQgIiYD5DOftiLd7e454E8R8VTZa83MbMlWV6KQ1A/4CfBF3t4W8YakScDh+ejDzMz6mHovuPs28AXgeGAkMCg/H5/LT2x8aGZm1gnqOqIADgS+HhE/KJQ9AZwiKYCvACc0OjgzM2u/eo8ohgH3dTHtvjzdzMz6oHoTxd/p+u5y+wMPNyYcMzPrNPVWPZ1EulXpOsBvgWdIRxH7AjviW5SamfVZdR1RRMQUYDdgBdL9si8GziD1BbVbRPymnuVIWk7SXyTdK+kBSd/K5etJ+rOkRyRdJGmZXL5sHp+Rp4/s9js0M7Neqbub8Yi4NiK2Ip3x9A5gUERsHRHXdWN9rwE7RcQmwKbAbpK2BL4HnBoRo4DngfF5/vHA8xGxPnBqns/MzFqo2/ejiIg3I2JuT66biOSfeXRgfgSwE6lKC1L/UmPz8Jg8Tp6+syR1d71mZtZzLb9xkaT+ku4B5gLXkXqpfSEiFuZZZpO6OCc/zwLI018EVm9txGZmS7eWJ4qIeCMiNiXd32Jz4D21ZsvPtY4eorpA0gRJ0yVNnzdvXuOCNTOz9t0KNSJeAG4CtgQGS6qcgTUCqPQfNRtYGyBPX4XUx1T1siZFxOiIGD106NBmh25mtlRpaaKQNFTS4Dw8CNgFeAi4EdgnzzYOuDwPT83j5Ok3RMQiRxRmZtY89V5H0ShrApMl9SclqSkRcYWkB0nXaZxEuufFOXn+c4BfSZpBOpLw9RpmZi3W60Qh6WdAv4gYv7h5I+I+4AM1yh8jtVdUl79KuqjPzMzapBFHFDvSxrYOMzNrrl4ninwxnJmZ9VE+EjAzs1J1H1Hk01P3BrblrVuh3gpcUrhYzszM+ph6b4U6DLgWeD8wk9R77FbAocC9kj4aEb7SzcysD6q36ulHpK4ztoiId0bEVhHxTmCLXP6jZgVoZmbtVW+i2AM4NiLuKBbm8eOAPRsdmJmZdYZ6E8WywIIupi0AlmlMOGZm1mnqTRS3A8dKWqFYmMePzdPNzKwPqvesp6NJ/THNknQtb90KdVdSD687NCU6MzNru3pvhXoPMAqYBAwFPkJKFGcDoyLi3qZFaGZmbVX3dRQRMR+Y2MRYzMysA/nKbDMzK9XlEYWkG7qxnIiInRsQj5mZdZiyqqdn63j9msDW1Lg9qZmZ9Q1dJoqI6PI+EJLWIZ0WuxcwHzi18aGZmVkn6FY345LWJ12JfQAwNw//NCJeaUJsZmbWAertFHAj4L9Jd5ubBRwB/CIiXm9ibGZm1gFKz3qS9EFJlwCVW5geTLpu4uyeJAlJa0u6UdJDkh6QdEQuX03SdZIeyc+r5nJJOkPSDEn3Sdqs+2/RzMx6o8tEIelq4C/AesD+EfGeiJgcEW/0Yn0LgaMj4j3AlsChkt5Luj5jWkSMAqbx1vUau5Mu9BsFTADO6sW6zcysB8qOKCrdc6wNnClpbtmjnpVFxJyIuCsPLwAeAoYDY4DJebbJwNg8PAY4L5LbgcGS1uz2uzQzsx4ra6P4VjNXLGkkqTrrz8AaETEHUjLJN0qClERmFV42O5fNqVrWBNIRB+uss04zwzYzW+qUnR7btEQhaUXgYuDIiHhJUpez1gptkYKISaR+qBg9erSv6TAza6CWd+EhaSApSZwfEZfk4mcqVUr5uVKVNZtU9VUxAniqVbGamVmLE4XSocM5wEMRUbx96lRgXB4eB1xeKD8wn/20JfBipYrKzMxao1sX3DXANsDngPsl3ZPLjgdOBqZIGg88QbpeA+Aq0m1YZwAvAwe1NlwzM2tpooiI26jd7gCwSKeCERHAoU0NyszMSrmbcTMzK+VEYWZmpZwozMyslBOFmZmVcqIwM7NSThRmZlbKicLMzEo5UZiZWalWX5ltS5iRE69s6fpmnrxnS9dnZovnIwozMyvlRGFmZqWcKMzMrJQThZmZlXKiMDOzUk4UZmZWyonCzMxKOVGYmVmpVt8z+xeS5kr6a6FsNUnXSXokP6+ayyXpDEkzJN0nabNWxmpmZkmrjyjOBXarKpsITIuIUcC0PA6wOzAqPyYAZ7UoRjMzK2hpooiIW4DnqorHAJPz8GRgbKH8vEhuBwZLWrM1kZqZWUUntFGsERFzAPLzsFw+HJhVmG92LjMzsxbqhETRFdUoi5ozShMkTZc0fd68eU0Oy8xs6dIJieKZSpVSfp6by2cDaxfmGwE8VWsBETEpIkZHxOihQ4c2NVgzs6VNJySKqcC4PDwOuLxQfmA++2lL4MVKFZWZmbVOS+9HIenXwA7AEEmzgW8CJwNTJI0HngD2zbNfBewBzABeBg5qZaxmZpa0NFFExKe7mLRzjXkDOLS5EZmZ2eJ0QtWTmZl1MCcKMzMr5URhZmalnCjMzKxUSxuzzXpj5MQrW7q+mSfv2dL1mXUqH1GYmVkpJwozMyvlRGFmZqWcKMzMrJQThZmZlXKiMDOzUk4UZmZWyonCzMxKOVGYmVkpX5lt1gOtvErcV4hbuzlRmC3B3K2JtYKrnszMrJQThZmZler4RCFpN0kPS5ohaWK74zEzW9p0dBuFpP7AmcBHgNnAHZKmRsSD7Y3MzKq5gb/v6uhEAWwOzIiIxwAkXQiMAZwozKymTmrg76RYekMR0ZQFN4KkfYDdIuLgPP45YIuIOKxqvgnAhDy6AfBwSwOFIcD8Fq+zK50SS6fEAZ0TS6fEAY6llk6JA9oTy7oRMbTWhE4/olCNskUyW0RMAiY1P5zaJE2PiNHtWn9Rp8TSKXFA58TSKXGAY+nkOKCzYoHOb8yeDaxdGB8BPNWmWMzMlkqdnijuAEZJWk/SMsD+wNQ2x2RmtlTp6KqniFgo6TDgGqA/8IuIeKDNYdXStmqvGjollk6JAzonlk6JAxxLLZ0SB3RWLJ3dmG1mZu3X6VVPZmbWZk4UZmZWyonCzKxDSap1iUDLOVH0kKT1JW3c7jiWFJ3yg+8kkvz/q9JJn0mH/GZXhfbH0jFfypJC0oqSLgD+Dry3k37YAJJWaXcMFZK2k3QcQLTxrAlJq0tatl3rL5K0g6TvA0TEm22Mo1N/J+38TD4g6UBJW+VY2vmb3VrSo8BZ7Y4FnCi6Jf+Y5wHvAvYCLmnnD7sobwzPB66TdIWkvSWtlKe1dG9E0hBJlwA3AyPzNTAtJ2k1Sb/Ocdwg6RBJa+Vp/VscS+UzuQFYp12Jy7+TmnEsJ+kc4A/A8cAfJF0kaZs8vWXbyfz9XArcSvpcvtGqdZdxoqiDpGUlnQl8F/gSsAfw+4j4d2Geth0aSnoncC2wFnBZLj4HOEWSIiJaFZ+k/yZdPT8Y2Ak4LCJeb8W6q+JYA7gUGE7qgXgG8HXgvPyZvNHCWCYCTwOrADsCB0bEa61afyEO/05q2wvYGvgYsAupt+r3AqdLGh4Rb7bic5F0IDAXWAfYkvSZ/L3Z661LRPhRxwMYB7wGfKJQNox0dLFMm2P7NDAHGFUo+zrwOPDdPN6vBXEcArxJulhoWfJ1OlXzLFLWpFh2A54HPlAoO4DULczPW/jd7Jc/k8tIF7g2/Xvo6jP372SRdQzIzz8G7in+j0k7g38Frmrh9/QF4FXg2KryFVr5udR6+IiifoujY6MAABATSURBVBcDVwPfldQvH2HcB1xPqtaYUPrqBqvaw9mYtAGcUyj7X+B84DhJG0QT94okVa7w/wXpMxkKDI6IkLShpH1y/W+/yL/2FhhO6n3z6ULZxcBJwBck7daiOC4jdTszBFg/fw8bSTpI0o6Vap9mqFSZ5O9BtPF3UlV980ve+p2s2srfiaT3S/o2pJ4fcvEI4PGIeL3wW74W+BGwm6SP1XgPDY0lx/MLUnXTnpLWySfMXAZcLuni3HYysJEx1K0d2WlJfQDbAQuAV4DbgaOAicCNpD2kMU1e//LAeN7aE6o8H0U6ZF2R1ONuZQ9yQ+Au4Lo83rC9kRqxLJOfx5E2RhOBycDLpHadN4HfA9s2+DNZAfgq8GVgT2CNXP4F4AVSFYIK868KXAlMb8L3Ux3Lmrl8DPAAcDYpmb5GSmBvAtOB/ZoQxzeAU3Is787lx7bhd1Idy4a5fHzJ7+SaRv9O8joHAf/M6yjWDByafyuD8nj//LwW8Dvgby2MZZv8Hd1D2s5cCfwGeIS04/OVRsdSV7ztWOmS+gCWAU4EziNVOfXL5WsDvyXdB6Mph+7AJ/MP6E3gS7ms8oN+f/5RfS6PD6xMB74IvA5sn8t6vREoiyUPX583hjcCHwe2JdUDzyEdlY3qbQx5PROAZ4G7STezWgBck6etmP/8E2vE91FgITC2elozYsnTf5HjuRXYHdgE2CJvEO4lV5H19vsBDsvrmQ7ckmO6K097b/5eWvU76TKWPP2GGr+TjxV+J+s3MBYBawJ/ISXFJwv/3w/l2E7I4wMKr/sUKYHt24pY8vSzgL8BY0kJRfkxDbgT2KoR/59uxdzqFS7pD2BdYL2qsoGkPcg3STdaavQ6dyI1xt6eH7eQ91bz9GGkRPVIoaySRDYinc1xVhNjWStPWzY/jwYuB95f9drPA08AR/Uyhv7AwcD9wNfyn24l0okGC4Aj8nxnkvbCls/jlT3oNYCrgGsb8HmUxfIScHSebxNSlcuHql6/J6ku/IxexjEQOAZ4DDg6v8flSW01/yQdYQ0ALmn272QxsSwgNdJCuoNl2e/kqw3+H62efw8TSA3plXaZIaQquHnAsMp7yM/vJvVi/b0mx3JSYdqQ/H1VjnAqv9sd82uOaGQs9TzcRtFNEfF4RPwD3lYH/G/SF/gKaU+20QaQDtO/QGp4G0LaW6vENBeYAqws6fhcXIntAdJe3erKmhDLoXldr+WzZ6YDh0bEffC2z+lc0me0YS7vaSxvkvZAbwd+GRFzImIB6SynG4Bd8jrPBd4ATi6+OCKeAf4BDJS0cg9jqDeWHSUtExH3AsdHxB3wts/kStIGaj1JA3vxmSwkVd+cTjqTaW5EvEyq8nqSlBAWAhcBg5v8OymL5SnSkQsR8RdS0ujqd/KeYnlv5GWsQGobuR/4FfA1SWtHxHzS9/U8cEZ+yZs5lr/n4cF5Ob1uv+kilmMkjcjrnB8Rv4iIVyovyeU3kj6XdzQqlrq1OjP1hQeF+t08vixwAmkD2pBqlar1DSwMDwL+j1RdMbpQPoT0x3yNXA9cmDYF+EMrYqFw2J7H+1WN3wtc3oA4RpCPFCrfSX6+GJiWh1ck7eW/AezKoof3D9KYaqeyWCr1/sV19696/c2N+H6KMRTK3kuq4tgRWI5UffrjFvxOymL5MLBa1bTqz6Qhv5OqZa5G2qFbl1R1/Hfg0sL0Q0lJ7uBC2TtIG/NvtiCW31bNU/3f2ZGUgL/cyFjqirfVK+xLD9Lh44akw+tngO/nP2JTTmHjrQbjj5Lqws+tmr4hqQ78AVIV0UqkKo8HgENaGUuN+fsD+5AacfdqYBz9qp6nAFcWpq9MqpZ7HDiIlFA3zhus/27wZ1IaSxefyR6kPf4DGhhH5QSD40nVTnOA5/LG6COk+8rfREqUzf6ddBXLw7V+B836neRlvyt/76NIVWGHko4WPgx8gtSO9g3gX8CFwGeBn+fvZ7MWxbIdqV1ku8K8K+Xf7CWkarC1GhlLXfG2eoV95UFqF7ietOfzLC0+GwH4IanB6xN5vLIXO4RUDfJ8fp5Lqnteu4WxFBsD18sbw6+T9qCmULU32eBYHgS+nYcrG+xlSFUuL5IO9+flP+k7m/wdLRJLHl4f2DtvlOaTTp8d1uB1r0hqIziGtAOxc/7snyTVu7fsd9JFLBflZLBBq34npB27BcD78vjQ/P9dSKoS2zWXf4WUSO/JG+Ytm/CZdBXLv3Msu5Eu0DwuJ4hncjybNvM322W87VhpX3mQTgU9gtyI26J1VjZ+G5OOHq4GlstllcbkNUhXmH6NBp962Y1Y+pEOr78FzMobii82+bMZQdob3L1QVmmsXZ50dsuBFE5HbGUsuXxg/s08RdrDP7gJ6658LytWla+TN05H5fG1mv07qSOWw0lHESeRGrCb9jsh7b3fRtrJey/pLKKFeeP8zRrzN3PnarGx5OTxZeAK4LPN/s2WxtvOlS/pD9p0lWRh/V8jHcIfQWo32RMY2gGx9COdAjqMVAe7Z4ti+AipDndUHu8HbAoMacPnUSuWTUh78isBu7QhpmHATOBH7fiN1IjlceAHeXw9YI8mr3NtUmPw1XmDfB3pGpdfko4yK9Wprbg6vSyW+YVYlmv3dxXhs556JfI32WqFs0B+Sdo4jydVX/yOdI1Du2O5nHSh0NhIZ4ld2aIYtiCdYfOIpHVJV9beBezbzPV3I5a7SefjL4iI61sQiwrDA0nJO4BfN3vddcbyJqmKiYj4R0Rc1eQw5pI2zOuQTmneNyIuJ13jMpjUxQnRmo4+y2JZBfhMjuXVFsSyWAMWP4t1mkjdLCxDOusI4H2kDcAeEfH7pS2Wwh97c2C6pEOBb5KuY9irBRugTo0lJK1Oqg/fi3RK9RXAA5VOADsglr+2KpZIp28fRUpQswrrnA68KyKeaHYMnRhLPdTC34o1kKQNSRe7DQKOiYizl+ZYJA0inbUzknRI/82I+EGr4+ikWCS9g9T9w/Kkqo5vRcSZrY6j02Kx7nOiWEIp3c/gCODH8daFOUt1LJJ+RaprPi7a0I13J8aSu65eGfhZB3wmHROLdY8ThfUZudfRTrmRVEfE0uoqpjKdFIt1jxOFmZmV8llPZmZWyonCzMxKOVGYmVkpJwozMyvlRGENJelESfPbHUeRpAmSxtYonympadc3SNpG0l2SXpXU8LNGJEXh8YqkhyQdW7jvc6PXd66k6c1YtnU2X5ltS4MJpLvIXdbi9f6U1FXDrqT7PzTDD0ndqA8iXfF8MqnjwZOasK7v8NYV+LYUcaIwa54NgUkRcXMT1zEzIm7PwzdK2ojUQ27DE0VEPNroZdqSwVVP1nKSVpP0U0nP5GqZP0raomqekHSEpP+RNE/SXEln5qvAi/PtIOm+vJw7JG0uab6kE/P0m4APAuMK1TSfr1rGVyXNlvS8pAslDa7jPewk6c95vc9I+l9JKxZiClL32afndZ5bsqyTJd0v6Z85jvNzlxc9cS+pi4zqdYyRND3H+7Sk7+fO+ZC0Y45xo6rXrCrpdUnj8/giVU+S1smf2XOSXpZ0jaQNCtNvkTSpML5rXtePCmV75/Us38P3bE3mRGEtlTf015O64T4GGEvq6uL6GhvHo0n3TDgAOAX4EqmrkMqyhgNXkap39iFV9ZzP26tH/ot0U6WrgK3yo9ib7X6kG+lMAI4lVd/8z2Lew3uB35O6g96b1OnfZ0hVQJB6rN0qD/8wD3+nZJHD8jr3BI4E3gncIKl/WRxdWId0P/BivPuRbn7zF9L9vb9Fer//L89yM+nOc/tVLesT+fnSWiuStBrpngobkO6bsB/pXtDX5/6uIPUBtl3hZdsDr9YouyvSfbWtE7W7n3M/+tYDOBGYXzJ9PPA6hXuLk6pAHwVOKZQFcEvVay8Dbi+Mn0LaWA8qlO2XX3tioWw6NW7VSro3w6O8/Y58pwFPL+Y9Xgg8QuE+z4X1blX1Hg7r5ufXHxieX7v9YuYN0t3YBpDucfFpUlvI/oV5RLrvwy+rXvsFUoeFq+fx04G/Vc1zDXBFYfxcYHph/DukuzuuVihblXQnwUPz+K45zqF5/BbgJ6Sb9KyYy+4qfvd+dN7DRxTWarsAdwL/kDSgcIbOzcDoqnmvrRp/kHTnuIoPAdfF2zsinNrNeG6MiIVV6xim1HV6VzYHLo2INwplF5M2ftt2c/1I2j1Xv72YlzE7T3p3HS8/nXTjm5eAC4AzI+LCwvR3k44yplQ+7/yZ3wAsR+oWHtKtSTeQtEmOaQjpftoXlax7F9INd14qLHcB6futfJd/BN4Ats1Hk5uT7kM9H9hK0srA+0l3SLQO5cZsa7UhwJakjVu16sbSF6rGXydt3CreQboH9n9ExKuS/tmNeGqtQ6T7bL/exWvWJN3DuLjeNyQ9S7r9a90kfYiU3C4lnbE0l7QHfjtvf69dOYV0859VSNVWX5V0fbx134sh+bmr+2BU2jP+RLoV6adI7Rx7k5JW2Zlile/yUzWmTQOIiAWS7iFVNc0nHcXcR6qy2o60DepHul+3dSgnCmu150hVQYfUmNbdU0ifJt1X+D8kLQes2LPQ6jaH1K5QXG9/0k15nuvmsj5BaqP5VOR6GKU74tXriYiYnl93C3A/cIqkq/PyKvFMIN1hr9o/4D83FppC2ugfn5+vjogFJet+jpTkarW/FF93KykpPAv8IdLNrm4ltU8NBB6MiGfrerfWFk4U1mrTgI+SNnBze7msO4CDJA0qVD99vMZ81UcivfVn4BOSji9UP32S9H+6rZvLGgT8u5Ikss/2JKiI+Lekb5COMD5G2og/DDwJjIyIny1mERcCX5O0F/Bh8q1BS0wjtc08EOX3IbkVOJz0PVyey24hNaavgKudOp4ThTXDMpL2qVF+M3Ae6QyZm5Suin6MtCe+OakR+dRurOc04FDgd5JOJVVFTQReJt1isuJvwK6SdiXt1f6jl3uwJ5H2zi+TdBap3eR7wDUR8aduLus64EhJp5Hueb416SyvnrqY9H6PAabmvfejgV/l9oCrSRvsd5L26PeJfLZRRNwpaQYwiVRFdMVi1vWjHOsNkn5MSkhrkJLMbRFRuTf3raRG+q1JZ7JBqt76N6md6bRevF9rATdmWzOsRLrtZfVjo0g3i9+RtIH8FqnB+nRgFOn0zbpFxJOkU0qHkU7/PJx0Nk9/UuNuxUnAQ6Q97TtIe9s9FhEPALsX1nsS8GvSKbrdXdZVpNNy9yYdAXyYdIpuT2N7k7Snvq2krXLZRcAYYFPS93AJ6bThu1i0HeYiUhvM72Ixp6tGxHxSG8XfgFNJ3+X3Se0l9xXmm5fneZnU0F2J8495lu4ehVmL+cZF1qdI2pa0B7tTRNzY7njM+gInCluiSfoeqRroadKFX98gVS99IDrgVqRmfYHbKGxJtyzpFNE1SGfaXAsc5SRh1jg+ojAzs1JuzDYzs1JOFGZmVsqJwszMSjlRmJlZKScKMzMr5URhZmal/n9LW6JGcDh8uQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "index = np.arange(len(checklen))\n",
    "plt.bar(index, checklen)\n",
    "plt.xlabel('Length of a Review', fontsize=15)\n",
    "plt.ylabel('No. of Reviews', fontsize=15)\n",
    "plt.xticks(index, range(5,50,5), fontsize=15, rotation=30)\n",
    "plt.title('Review Survey Length Analysis')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We have a fair amount of both details reviews (having length >=5) and short reviews (having length <5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing\n",
    "\n",
    "![datapreprocessing](Photos/datapreprocessing.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage1: Language Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Language exception for: 😕🤔😕🤣😕🤣😢😨😔😭😃😁😣😨😁🤦‍♂️😨☺😣😔😨😛🤦‍♂️😢☺👍🙈😨😕😔😊😢😕😔🙈😨😣🤦‍♂️👍😁🤦‍♂️🙈😁🤦‍♂️😂😕😔😊😨👍😂😢👍😂🙈😢👍😂😭👍😂😭😕🙈👍😕😂☺😕😂😂😣😂😕👍😢😕🤦‍♂️😂😢🤔🙈😕🤦‍♂️🙈😢🤔😂🤦‍♂️🙈🤔😔😣😣👆🙈🤔😔😕😕😨🤣😔😕🙈😊😨😣😣🙈😊😕😂🤔😕😢🤔👍🤣😨😕😭😨😕😭😨😕😭😕👍😭😕😕🤣👍😁😭👍👍😭😕😣🙈😕😣😢😣😣😨😣😛😨😭😣😢😭😨😛😨🤣😢😨😂😊🤔😊🙈🙈🙈😭🐂🐎🐷🐎🐂🐈🐱🐮🦓🐱🐄🦌🦔🌹🥀⚘🌸🏵🌲🌷🌸⚘🌱🥔🍑🍓🍆🥝🍐🥑🥕🍆🍏🥝🍓🥔🥝🍆🥔🥑🥝🥝🥝🍒🥔🥑🥝🍏🍏🥑🥔🍐🍏🥑🥔🥔🥔🍆🍏🍏🍓🍇🥕🍏🍓🍏🍓🍏🥔🍐🍐🥔🍐🥔🥔🍐🥔🥑🍓🍏🍓🥝🍓🥑🍓🥑🍓🥑🥕🍐🥕🍏🍓🍇🍑🍈🥑🍇🥑🍏🍍🥑🍓🍏🍓🍇🍆🍎🥔🥝🍓🍐🍒🍇🍓🥑🍌🍏🍆🥝🍆🥝🍆🍏🍓🥑🍓🍎🍄🥝🍒🥥🍒🥝🥔🥦🍓🍆🍅🍏🥔🥝🥕🥑🍓🍏🥔🥝🥔🥝🍓🥘🥙🥘🌮🍕🌮🍕🌯🥘🥚🥩🥓🍖🌯🥘🍟🌭🥓🍳🥓🥪🥚🍖🍗🍕🥙🌭🍔🌭🍿🌭🥙🏌️‍♂️🥋🏓🎳🏓🏏🏓🥅🏌️‍♂️🥊🏌️‍♂️🎳⛳🏏⛳    🤼‍♀️🏍🤼‍♀️🤸‍♂️🤼‍♀️🤾‍♀️🤼‍♂️🏎🤹‍♂️🤺🤼‍♀️🏎🤼‍♂️       🎳🏸🥅🏸🥋🏌️‍♂️🎿🏚🏛🏝🏣🏙🌋🏜🏛🏝         🎙🎤🎶🎧📲🎤🎸📱🎙🎧🎷🎤🎸📻📲🎚🎧🎸🎼🎸📺📷📽📷📽📷📽🔬📽🔬📽🔬    🎸🥁🎙🎤🎚🎧🎸🥁🥁🎸🎤☎️🎸🎻🎸📱🎚🎤🎸🎤☎️🎧🎸🎤🎙🎤🎤☎️📻☎️🎧🎸🎸🎤🎤☎️🎤🎸🎤🎸🎤🎸🎧🎸🎸🎧🎧🎸🎸🎧🎸🎧📻🎸🎸🎧🎧🎸🎧🎸🎧🎸🎤🎸🎤☎️🎧\n"
     ]
    }
   ],
   "source": [
    "bad_reviews = []\n",
    "for indx in df.index:\n",
    "    review = df.at[indx, 'answer_option']\n",
    "    try:\n",
    "        b = rf.language_detection(review)\n",
    "        if b == 'hi' or b == 'mr':\n",
    "            bad_reviews.append(indx)\n",
    "    except:\n",
    "        bad_reviews.append(indx)\n",
    "        print(\"Language exception for:\", review)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Bad Reviews at Stage 1:  3\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of Bad Reviews at Stage 1: \", len(bad_reviews))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>answer_option</th>\n",
       "      <th>label</th>\n",
       "      <th>review_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>Neurobion</td>\n",
       "      <td>अच्छा है</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <td>Neurobion</td>\n",
       "      <td>😕🤔😕🤣😕🤣😢😨😔😭😃😁😣😨😁🤦‍♂️😨☺😣😔😨😛🤦‍♂️😢☺👍🙈😨😕😔😊😢😕😔🙈😨😣🤦‍♂...</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1110</th>\n",
       "      <td>Shelcal</td>\n",
       "      <td>साहब बहुत धाँसू गोली है साला मेरा घुटना ख़राब ...</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        product                                      answer_option  label  \\\n",
       "898   Neurobion                                           अच्छा है      0   \n",
       "1042  Neurobion  😕🤔😕🤣😕🤣😢😨😔😭😃😁😣😨😁🤦‍♂️😨☺😣😔😨😛🤦‍♂️😢☺👍🙈😨😕😔😊😢😕😔🙈😨😣🤦‍♂...      0   \n",
       "1110    Shelcal  साहब बहुत धाँसू गोली है साला मेरा घुटना ख़राब ...      0   \n",
       "\n",
       "      review_len  \n",
       "898            2  \n",
       "1042           5  \n",
       "1110          23  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.index.isin(bad_reviews)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df.index.isin(bad_reviews)].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 2: Gibberish Reviews\n",
    "+ When, we collect data from customers just out of frustation many reviews write gibberish reviews. \n",
    "+ Example: svfsfg, fsdfgdfgfsgsfgdgh, ffgrthyryj, rhrhfas, kuopip\n",
    "+ We need to detection such reviews and filter thoses out.\n",
    "\n",
    "#### Working\n",
    "You could build a model of character to character transitions from a bunch of text in English. So for example, you find out how common it is for there to be a 'h' after a 't' (pretty common). In English, you expect that after a 'q', you'll get a 'u'. If you get a 'q' followed by something other than a 'u', this will happen with very low probability, and hence it should be pretty alarming. Normalize the counts in your tables so that you have a probability. Then for a query, walk through the matrix and compute the product of the transitions you take. Then normalize by the length of the query. When the number is low, you likely have a gibberish query (or something in a different language).\n",
    "\n",
    "If you have a bunch of query logs, you might first make a model of general English text, and then heavily weight your own queries in that model training phase.\n",
    "\n",
    "For background, read about Markov Chains.\n",
    "\n",
    "Source: https://github.com/rrenaud/Gibberish-Detector\n",
    "\n",
    "> I have trained a model for and have stored it's pickle file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_reviews = []\n",
    "for indx in df.index:\n",
    "    review = df.at[indx, 'answer_option']\n",
    "    if rf.gibberish_detection(review, prefix_path = 'utils'):\n",
    "        bad_reviews.append(indx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Bad Reviews at Stage 2:  8\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of Bad Reviews at Stage 2: \", len(bad_reviews))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3, 82, 362, 432, 1449, 1503, 1599, 1668]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>answer_option</th>\n",
       "      <th>label</th>\n",
       "      <th>review_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>fwegwrqdsdvwfg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>qwerwetrjy</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>Neurobion</td>\n",
       "      <td>gehryetw</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>Shelcal</td>\n",
       "      <td>gehryetw</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1449</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>ghyukuyujredfehrbv</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1503</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>VVV. Good</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1599</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>wqwasdbggn</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1668</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Ft GM</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        product       answer_option  label  review_len\n",
       "3     Accucheck      fwegwrqdsdvwfg      0           1\n",
       "82    Accucheck          qwerwetrjy      0           1\n",
       "362   Neurobion            gehryetw      0           1\n",
       "432     Shelcal            gehryetw      0           1\n",
       "1449  Accucheck  ghyukuyujredfehrbv      0           1\n",
       "1503  Accucheck           VVV. Good      0           2\n",
       "1599  Accucheck          wqwasdbggn      0           1\n",
       "1668  Accucheck               Ft GM      0           2"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.index.isin(bad_reviews)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df.index.isin(bad_reviews)].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 3: Profanity Detection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ At times reviewers are not happy with the delivery or service out of frustration they type profanity words in reviews. \n",
    "+ Profanity content also penalises SEO ranking\n",
    "+ We decided to filter out both english and hinglish profanity reviews. \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_reviews = []\n",
    "for indx in df.index:\n",
    "    review = df.at[indx, 'answer_option']\n",
    "    if rf.english_swear_check(review) or rf.hindi_swear_check(review):\n",
    "        bad_reviews.append(indx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Bad Reviews at Stage 3:  3\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of Bad Reviews at Stage 3: \", len(bad_reviews))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>answer_option</th>\n",
       "      <th>label</th>\n",
       "      <th>review_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>STOP SENDING ME SMS you assholes</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1009</th>\n",
       "      <td>Neurobion</td>\n",
       "      <td>MotherFucker I dont want anything</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187</th>\n",
       "      <td>Shelcal</td>\n",
       "      <td>SHELLCALL IS TAKEN BYYOU WIFE, BECAUSE SHE HAS...</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        product                                      answer_option  label  \\\n",
       "140     shampoo                   STOP SENDING ME SMS you assholes      0   \n",
       "1009  Neurobion                  MotherFucker I dont want anything      1   \n",
       "1187    Shelcal  SHELLCALL IS TAKEN BYYOU WIFE, BECAUSE SHE HAS...      1   \n",
       "\n",
       "      review_len  \n",
       "140            6  \n",
       "1009           5  \n",
       "1187          22  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.index.isin(bad_reviews)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df.index.isin(bad_reviews)].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 4: Spelling Correction (Optional Stage not that necessary)\n",
    "+ Trying to improve quality of reviews which have high confidence spell errors. Like withut -> without"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "for indx in df.index:\n",
    "    review = df.at[indx, 'answer_option']\n",
    "    df.at[indx, 'answer_option'] = rf.spell_correct(review, 0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 4.1: Company Tag (Optional Stage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_reviews = []\n",
    "for indx in df.index:\n",
    "    review = df.at[indx, 'answer_option']\n",
    "    if rf.competitive_brand_tag(review):\n",
    "        bad_reviews.append(indx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Bad Reviews at Stage 4.1:  7\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of Bad Reviews at Stage 4.1: \", len(bad_reviews))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>answer_option</th>\n",
       "      <th>label</th>\n",
       "      <th>review_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>amazon products are better</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348</th>\n",
       "      <td>Neurobion</td>\n",
       "      <td>Almost no discount on OTC products Pharmeasy &amp;...</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>Shelcal</td>\n",
       "      <td>papal payment method was not working Product w...</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>SevenseascodLiverOil</td>\n",
       "      <td>Cheaper on amazon</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>941</th>\n",
       "      <td>Neurobion</td>\n",
       "      <td>Amazon has better quality for products</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1057</th>\n",
       "      <td>Neurobion</td>\n",
       "      <td>Almost no discount on OTC products Pharmeasy &amp;...</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1324</th>\n",
       "      <td>Shelcal</td>\n",
       "      <td>grofer is best</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   product                                      answer_option  \\\n",
       "33               Accucheck                         amazon products are better   \n",
       "348              Neurobion  Almost no discount on OTC products Pharmeasy &...   \n",
       "444                Shelcal  papal payment method was not working Product w...   \n",
       "615   SevenseascodLiverOil                                  Cheaper on amazon   \n",
       "941              Neurobion             Amazon has better quality for products   \n",
       "1057             Neurobion  Almost no discount on OTC products Pharmeasy &...   \n",
       "1324               Shelcal                                     grofer is best   \n",
       "\n",
       "      label  review_len  \n",
       "33        0           4  \n",
       "348       0          20  \n",
       "444       0           9  \n",
       "615       0           3  \n",
       "941       0           6  \n",
       "1057      0          20  \n",
       "1324      1           3  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.index.isin(bad_reviews)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df.index.isin(bad_reviews)].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Count of Reviews after preprocessing:  1655\n"
     ]
    }
   ],
   "source": [
    "print(\"Total Count of Reviews after preprocessing: \", len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/Preprocessed_Reviews.csv',index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary\n",
    "+ So far, we have done EDA of Data.\n",
    "+ Preprocessing of reviews in 4 main stages. \n",
    "\n",
    "### Initial Count of Reviews was: 1676\n",
    "### Final Count of Reviews is: 1655\n",
    "\n",
    "+ We have to be strict with are preprocessing so that no bad content is shown on our website. \n",
    "+ By preprocessing we are reducing False positives. \n",
    "\n",
    "### Note: You may feel that only 21 reviews are scrapped but when you will be working on full scale then you will observed that about 7-10% of total reviews would be junk."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}