3.Model Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from joblib import load, dump\n",
    "from copy import deepcopy\n",
    "from statistics import mean\n",
    "\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/Features.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>answer_option</th>\n",
       "      <th>label</th>\n",
       "      <th>review_len</th>\n",
       "      <th>Rn</th>\n",
       "      <th>Rp</th>\n",
       "      <th>Rs</th>\n",
       "      <th>Rc</th>\n",
       "      <th>Rd</th>\n",
       "      <th>Rsc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Fast and accurate delivery</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.232859</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>0.005420</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Expected a longer expire date. Your Product Li...</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>0.596318</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.017615</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>I liked the prompt service</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.319747</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.4215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Good product</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.546925</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>I not needed</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004065</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1650</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>Its not much effective as it has been stated i...</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>-0.300000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.028640</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.3724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1651</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>Liked it very nicely working now my scalp is a...</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>0.166375</td>\n",
       "      <td>0.690000</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.026253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1652</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>its my regular choice</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.009547</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>Good but not very effective</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.234615</td>\n",
       "      <td>0.607692</td>\n",
       "      <td>0.011933</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.4032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1654</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>It is better but not a permanent solution for ...</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1655 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        product                                      answer_option  label  \\\n",
       "0     Accucheck                         Fast and accurate delivery      0   \n",
       "1     Accucheck  Expected a longer expire date. Your Product Li...      0   \n",
       "2     Accucheck                         I liked the prompt service      0   \n",
       "3     Accucheck                                       Good product      0   \n",
       "4     Accucheck                                       I not needed      0   \n",
       "...         ...                                                ...    ...   \n",
       "1650    shampoo  Its not much effective as it has been stated i...      0   \n",
       "1651    shampoo  Liked it very nicely working now my scalp is a...      1   \n",
       "1652    shampoo                              its my regular choice      0   \n",
       "1653    shampoo                        Good but not very effective      0   \n",
       "1654    shampoo  It is better but not a permanent solution for ...      0   \n",
       "\n",
       "      review_len        Rn        Rp        Rs        Rc   Rd     Rsc  \n",
       "0              4  0.232859  0.300000  0.616667  0.005420  1.0  0.0000  \n",
       "1             14  0.596318 -0.100000  0.400000  0.017615  1.0  0.0000  \n",
       "2              5  0.319747  0.600000  0.800000  0.006775  1.0  0.4215  \n",
       "3              2  0.546925  0.700000  0.600000  0.002710  0.0  0.4404  \n",
       "4              3  0.000000  0.000000  0.000000  0.004065  0.0  0.0000  \n",
       "...          ...       ...       ...       ...       ...  ...     ...  \n",
       "1650          12  0.500000 -0.300000  0.800000  0.028640  0.0 -0.3724  \n",
       "1651          11  0.166375  0.690000  0.900000  0.026253  0.0  0.5709  \n",
       "1652           4  0.500000  0.000000  0.076923  0.009547  0.0  0.0000  \n",
       "1653           5  0.000000  0.234615  0.607692  0.011933  0.0 -0.4032  \n",
       "1654          10  0.271683  0.500000  0.500000  0.023866  0.0 -0.1263  \n",
       "\n",
       "[1655 rows x 10 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ranking is a canonical problem for humans. It is easy to classify whether a review is useful (informative) or not. However, ranking reviews on the basis of usefulness, is a complex task. Our ranking methodology is based on this simple education.\n",
    "\n",
    "#### Pairwise ranking approach is applied to rank reviews in the semi-supervised learning method. The pairwise ranking approach looks at a pair of documents at a time in a loss function and predicts a relative ordering. The objective is not to determine the relevance score but to find which document is more relevant than others. This relevance is developed to judge the preference of one review over another.\n",
    "#### In the semi-supervised learning method, mapping is constructed between input and output. This input-output pair in the training model is used to learn the system.\n",
    "#### Review Segregation: We segregated two sets of reviews on which we train our model.\n",
    "+ Set 0 represents reviews with label 0, i.e., ones that are not informative. These include reviews based on delivery, customer support, packaging, etc. These reviews do not describe the product.\n",
    "+ Set 1 represents reviews with label 1, i.e., reviews that are informative and are better than all reviews of Set 0;\n",
    "#### How we segregated and determined labels for reviews:\n",
    "### `Our entire review ranking system is based on the idea that it is easier for humans to binary classify reviews which we call Set 0 and Set 1.`\n",
    "\n",
    "For each product 'Accucheck', 'Becadexamin', 'Evion', 'Neurobion','SevenseascodLiverOil', 'Shelcal', 'Supradyn','shampoo', we asked 10 different people to label reviews as a 1 (informative review) and 0 ( not informative review). Different participants were asked to label so that there is no bias and the model learns to its best."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>label</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>product</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accucheck</th>\n",
       "      <td>310</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Becadexamin</th>\n",
       "      <td>53</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Evion</th>\n",
       "      <td>89</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Neurobion</th>\n",
       "      <td>280</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SevenseascodLiverOil</th>\n",
       "      <td>59</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Shelcal</th>\n",
       "      <td>259</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Supradyn</th>\n",
       "      <td>50</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shampoo</th>\n",
       "      <td>56</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "label                   0    1\n",
       "product                       \n",
       "Accucheck             310   85\n",
       "Becadexamin            53   27\n",
       "Evion                  89   33\n",
       "Neurobion             280  136\n",
       "SevenseascodLiverOil   59   22\n",
       "Shelcal               259  124\n",
       "Supradyn               50   23\n",
       "shampoo                56   49"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_split = pd.crosstab(df['product'],df['label'])\n",
    "data_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the training set:\n",
    "#### We pairwise compared each review of set1 with all reviews of set0 and vice-versa\n",
    "+ (Rx, Ry,1) where x∈Set1 and y∈Set0 → Rx is better than Ry\n",
    "+ (Ry, Rx, 0) where x∈Set1 and y∈Set0 → Ry is worst than Rx\n",
    "<br>\n",
    "\n",
    "#### This now becomes a classification problem.\n",
    "\n",
    "<hr>\n",
    "\n",
    "![PairwiseRanking](Photos/PairwiseRanking.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def building_training_data(df):\n",
    "    A = df[df['label']==1]\n",
    "    A.loc[df['label']==1,'join'] = 'j'\n",
    "    B = df[df['label']==0]\n",
    "    B.loc[df['label']==0,'join'] = 'j'\n",
    "    trainset1 = pd.merge(A,B,how='outer',on='join')\n",
    "    trainset2 = pd.merge(B,A,how='outer',on ='join')\n",
    "\n",
    "    trainset = pd.merge(trainset1,trainset2,how='outer')\n",
    "    return trainset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "//anaconda3/lib/python3.7/site-packages/pandas/core/indexing.py:845: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[key] = _infer_fill_value(value)\n",
      "//anaconda3/lib/python3.7/site-packages/pandas/core/indexing.py:965: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[item] = s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accucheck 395 52700\n",
      "Becadexamin 80 2862\n",
      "Evion 122 5874\n",
      "Neurobion 416 76160\n",
      "SevenseascodLiverOil 81 2596\n",
      "Shelcal 383 64232\n",
      "Supradyn 73 2300\n",
      "shampoo 105 5488\n"
     ]
    }
   ],
   "source": [
    "product_list = df['product'].unique()\n",
    "data_stack = []\n",
    "for product in product_list:\n",
    "    temp = deepcopy(df[df['product']==product].iloc[:,2:])\n",
    "    build_data = building_training_data(temp)\n",
    "    print(product, len(temp), len(build_data))\n",
    "    build_data.drop(columns = ['join','label_y'],inplace=True)\n",
    "    data = build_data.iloc[:,1:]\n",
    "    data['target'] = build_data.iloc[:,0]\n",
    "    data_stack.append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat(data_stack).reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review_len_x</th>\n",
       "      <th>Rn_x</th>\n",
       "      <th>Rp_x</th>\n",
       "      <th>Rs_x</th>\n",
       "      <th>Rc_x</th>\n",
       "      <th>Rd_x</th>\n",
       "      <th>Rsc_x</th>\n",
       "      <th>review_len_y</th>\n",
       "      <th>Rn_y</th>\n",
       "      <th>Rp_y</th>\n",
       "      <th>Rs_y</th>\n",
       "      <th>Rc_y</th>\n",
       "      <th>Rd_y</th>\n",
       "      <th>Rsc_y</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.232859</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>0.005420</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>14</td>\n",
       "      <td>0.596318</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.017615</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>5</td>\n",
       "      <td>0.319747</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.4215</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.546925</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.546925</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212207</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>15</td>\n",
       "      <td>0.689470</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.035800</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1531</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212208</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>7</td>\n",
       "      <td>0.392391</td>\n",
       "      <td>0.316667</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.016706</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2382</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212209</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>25</td>\n",
       "      <td>0.354233</td>\n",
       "      <td>0.233333</td>\n",
       "      <td>0.255556</td>\n",
       "      <td>0.059666</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5927</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212210</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>10</td>\n",
       "      <td>0.467972</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1779</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212211</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>11</td>\n",
       "      <td>0.166375</td>\n",
       "      <td>0.690000</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.026253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5709</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>212212 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        review_len_x      Rn_x  Rp_x      Rs_x      Rc_x  Rd_x   Rsc_x  \\\n",
       "0                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "1                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "2                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "3                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "4                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "...              ...       ...   ...       ...       ...   ...     ...   \n",
       "212207            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "212208            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "212209            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "212210            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "212211            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "\n",
       "        review_len_y      Rn_y      Rp_y      Rs_y      Rc_y  Rd_y   Rsc_y  \\\n",
       "0                  4  0.232859  0.300000  0.616667  0.005420   1.0  0.0000   \n",
       "1                 14  0.596318 -0.100000  0.400000  0.017615   1.0  0.0000   \n",
       "2                  5  0.319747  0.600000  0.800000  0.006775   1.0  0.4215   \n",
       "3                  2  0.546925  0.700000  0.600000  0.002710   0.0  0.4404   \n",
       "4                  2  0.546925  0.700000  0.600000  0.002710   0.0  0.4404   \n",
       "...              ...       ...       ...       ...       ...   ...     ...   \n",
       "212207            15  0.689470  0.700000  0.600000  0.035800   0.0  0.1531   \n",
       "212208             7  0.392391  0.316667  0.600000  0.016706   0.0  0.2382   \n",
       "212209            25  0.354233  0.233333  0.255556  0.059666   0.0  0.5927   \n",
       "212210            10  0.467972  0.500000  0.500000  0.023866   0.0  0.1779   \n",
       "212211            11  0.166375  0.690000  0.900000  0.026253   0.0  0.5709   \n",
       "\n",
       "        target  \n",
       "0            1  \n",
       "1            1  \n",
       "2            1  \n",
       "3            1  \n",
       "4            1  \n",
       "...        ...  \n",
       "212207       0  \n",
       "212208       0  \n",
       "212209       0  \n",
       "212210       0  \n",
       "212211       0  \n",
       "\n",
       "[212212 rows x 15 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Len: 42443   42443\n"
     ]
    }
   ],
   "source": [
    "X = train.iloc[:,:-1].values\n",
    "y = train.iloc[:,-1].values\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test, y_train, y_test = train_test_split(X,y,test_size = 0.2,shuffle = True, stratify = y) \n",
    "print(\"Test Len:\",len(X_test),\" \",len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review_len_x</th>\n",
       "      <th>Rn_x</th>\n",
       "      <th>Rp_x</th>\n",
       "      <th>Rs_x</th>\n",
       "      <th>Rc_x</th>\n",
       "      <th>Rd_x</th>\n",
       "      <th>Rsc_x</th>\n",
       "      <th>review_len_y</th>\n",
       "      <th>Rn_y</th>\n",
       "      <th>Rp_y</th>\n",
       "      <th>Rs_y</th>\n",
       "      <th>Rc_y</th>\n",
       "      <th>Rd_y</th>\n",
       "      <th>Rsc_y</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.232859</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>0.005420</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>14</td>\n",
       "      <td>0.596318</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.017615</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>5</td>\n",
       "      <td>0.319747</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.4215</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.546925</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.546925</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4404</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212207</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>15</td>\n",
       "      <td>0.689470</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.035800</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1531</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212208</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>7</td>\n",
       "      <td>0.392391</td>\n",
       "      <td>0.316667</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.016706</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2382</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212209</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>25</td>\n",
       "      <td>0.354233</td>\n",
       "      <td>0.233333</td>\n",
       "      <td>0.255556</td>\n",
       "      <td>0.059666</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5927</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212210</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>10</td>\n",
       "      <td>0.467972</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1779</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212211</th>\n",
       "      <td>10</td>\n",
       "      <td>0.271683</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.023866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.1263</td>\n",
       "      <td>11</td>\n",
       "      <td>0.166375</td>\n",
       "      <td>0.690000</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.026253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5709</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>212212 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        review_len_x      Rn_x  Rp_x      Rs_x      Rc_x  Rd_x   Rsc_x  \\\n",
       "0                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "1                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "2                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "3                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "4                  5  0.544357  0.52  0.823333  0.006775   0.0  0.0000   \n",
       "...              ...       ...   ...       ...       ...   ...     ...   \n",
       "212207            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "212208            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "212209            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "212210            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "212211            10  0.271683  0.50  0.500000  0.023866   0.0 -0.1263   \n",
       "\n",
       "        review_len_y      Rn_y      Rp_y      Rs_y      Rc_y  Rd_y   Rsc_y  \\\n",
       "0                  4  0.232859  0.300000  0.616667  0.005420   1.0  0.0000   \n",
       "1                 14  0.596318 -0.100000  0.400000  0.017615   1.0  0.0000   \n",
       "2                  5  0.319747  0.600000  0.800000  0.006775   1.0  0.4215   \n",
       "3                  2  0.546925  0.700000  0.600000  0.002710   0.0  0.4404   \n",
       "4                  2  0.546925  0.700000  0.600000  0.002710   0.0  0.4404   \n",
       "...              ...       ...       ...       ...       ...   ...     ...   \n",
       "212207            15  0.689470  0.700000  0.600000  0.035800   0.0  0.1531   \n",
       "212208             7  0.392391  0.316667  0.600000  0.016706   0.0  0.2382   \n",
       "212209            25  0.354233  0.233333  0.255556  0.059666   0.0  0.5927   \n",
       "212210            10  0.467972  0.500000  0.500000  0.023866   0.0  0.1779   \n",
       "212211            11  0.166375  0.690000  0.900000  0.026253   0.0  0.5709   \n",
       "\n",
       "        target  \n",
       "0            1  \n",
       "1            1  \n",
       "2            1  \n",
       "3            1  \n",
       "4            1  \n",
       "...        ...  \n",
       "212207       0  \n",
       "212208       0  \n",
       "212209       0  \n",
       "212210       0  \n",
       "212211       0  \n",
       "\n",
       "[212212 rows x 15 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spot Checking-\n",
    "+ Linear Model\n",
    "+ Non-Linear Model\n",
    "+ Ensemble Model\n",
    "\n",
    "<hr>\n",
    "\n",
    "## Linear Model: Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/shaurya/.local/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy\n",
      " 0.852499572949125\n",
      "Test Accuracy\n",
      " 0.8536389981858021\n",
      "CLASSIFICATION REPORT\n",
      "Training\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.85      0.85     84885\n",
      "           1       0.85      0.85      0.85     84884\n",
      "\n",
      "    accuracy                           0.85    169769\n",
      "   macro avg       0.85      0.85      0.85    169769\n",
      "weighted avg       0.85      0.85      0.85    169769\n",
      "\n",
      "Test \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.85      0.85     21221\n",
      "           1       0.85      0.85      0.85     21222\n",
      "\n",
      "    accuracy                           0.85     42443\n",
      "   macro avg       0.85      0.85      0.85     42443\n",
      "weighted avg       0.85      0.85      0.85     42443\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "classifier = LogisticRegression()\n",
    "classifier.fit(X_train,y_train)\n",
    "print(\"Training Accuracy\\n\", accuracy_score(y_train,classifier.predict(X_train)))\n",
    "print(\"Test Accuracy\\n\", accuracy_score(y_test,classifier.predict(X_test)))\n",
    "\n",
    "print('CLASSIFICATION REPORT')\n",
    "print(\"Training\\n\", classification_report(y_train,classifier.predict(X_train)))\n",
    "print(\"Test \\n\", classification_report(y_test,classifier.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy: 85%\n",
    "### F1-score: 85%"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Non-Linear Model: DecisionTree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy\n",
      " 0.9966366062119704\n",
      "Test Accuracy\n",
      " 0.9836250971891713\n",
      "CLASSIFICATION REPORT\n",
      "Training\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00     84885\n",
      "           1       1.00      0.99      1.00     84884\n",
      "\n",
      "    accuracy                           1.00    169769\n",
      "   macro avg       1.00      1.00      1.00    169769\n",
      "weighted avg       1.00      1.00      1.00    169769\n",
      "\n",
      "Test \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98     21221\n",
      "           1       0.98      0.98      0.98     21222\n",
      "\n",
      "    accuracy                           0.98     42443\n",
      "   macro avg       0.98      0.98      0.98     42443\n",
      "weighted avg       0.98      0.98      0.98     42443\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Decision Tree\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "classifier = DecisionTreeClassifier()\n",
    "classifier.fit(X_train,y_train)\n",
    "\n",
    "print(\"Training Accuracy\\n\", accuracy_score(y_train,classifier.predict(X_train)))\n",
    "print(\"Test Accuracy\\n\", accuracy_score(y_test,classifier.predict(X_test)))\n",
    "\n",
    "print('CLASSIFICATION REPORT')\n",
    "print(\"Training\\n\", classification_report(y_train,classifier.predict(X_train)))\n",
    "print(\"Test \\n\", classification_report(y_test,classifier.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensemble Model: RandomForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy\n",
      " 0.9966366062119704\n",
      "Test Accuracy\n",
      " 0.9878425182008812\n",
      "CLASSIFICATION REPORT\n",
      "Training\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     84885\n",
      "           1       1.00      1.00      1.00     84884\n",
      "\n",
      "    accuracy                           1.00    169769\n",
      "   macro avg       1.00      1.00      1.00    169769\n",
      "weighted avg       1.00      1.00      1.00    169769\n",
      "\n",
      "Test \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99     21221\n",
      "           1       0.99      0.99      0.99     21222\n",
      "\n",
      "    accuracy                           0.99     42443\n",
      "   macro avg       0.99      0.99      0.99     42443\n",
      "weighted avg       0.99      0.99      0.99     42443\n",
      "\n",
      "Test\n",
      "Confusion Matrix: \n",
      " [[20962   259]\n",
      " [  257 20965]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "classifier = RandomForestClassifier(n_estimators=50, n_jobs = -1, oob_score = True,random_state=42)\n",
    "classifier.fit(X_train,y_train)\n",
    "\n",
    "print(\"Training Accuracy\\n\", accuracy_score(y_train,classifier.predict(X_train)))\n",
    "print(\"Test Accuracy\\n\", accuracy_score(y_test,classifier.predict(X_test)))\n",
    "\n",
    "print('CLASSIFICATION REPORT')\n",
    "print(\"Training\\n\", classification_report(y_train,classifier.predict(X_train)))\n",
    "print(\"Test \\n\", classification_report(y_test,classifier.predict(X_test)))\n",
    "\n",
    "print(\"Test\\nConfusion Matrix: \\n\", confusion_matrix(y_test, classifier.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9884254486979366"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Score of the training dataset obtained using an out-of-bag estimate. This attribute exists only when oob_score is True.\n",
    "classifier.oob_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Rd_y</th>\n",
       "      <td>0.192365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rd_x</th>\n",
       "      <td>0.186711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_len_x</th>\n",
       "      <td>0.119960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_len_y</th>\n",
       "      <td>0.109524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rc_y</th>\n",
       "      <td>0.072811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rc_x</th>\n",
       "      <td>0.067608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rsc_x</th>\n",
       "      <td>0.041769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rsc_y</th>\n",
       "      <td>0.039901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rn_y</th>\n",
       "      <td>0.039073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rn_x</th>\n",
       "      <td>0.037902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rp_x</th>\n",
       "      <td>0.026261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rp_y</th>\n",
       "      <td>0.025077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rs_x</th>\n",
       "      <td>0.021099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rs_y</th>\n",
       "      <td>0.019939</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              importance\n",
       "Rd_y            0.192365\n",
       "Rd_x            0.186711\n",
       "review_len_x    0.119960\n",
       "review_len_y    0.109524\n",
       "Rc_y            0.072811\n",
       "Rc_x            0.067608\n",
       "Rsc_x           0.041769\n",
       "Rsc_y           0.039901\n",
       "Rn_y            0.039073\n",
       "Rn_x            0.037902\n",
       "Rp_x            0.026261\n",
       "Rp_y            0.025077\n",
       "Rs_x            0.021099\n",
       "Rs_y            0.019939"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_importances = pd.DataFrame(classifier.feature_importances_,\n",
    "                                   index = train.iloc[:,:-1].columns,\n",
    "                                    columns=['importance']).sort_values('importance',ascending=False)\n",
    "feature_importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['randomforest.joblib']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dump(classifier, 'randomforest.joblib', compress = 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RandomForest Classifier Weights Saved. \n",
    "### Accuracy: 0.98\n",
    "### oob_score: 0.98\n",
    "\n",
    "+ Note, if in your usecase data is too small to split to train-test-split then one can train model on entire data and measure out of bag score. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>\n",
    "\n",
    "## PART 2. Model Ranking Metric\n",
    "\n",
    "### Accuracy of Ranking Methodology\n",
    "+ After sorting the reviews by the review score, we wanted all reviews in Set 1 to be above all reviews of Set 0.\n",
    "+ To test this hypothesis, we developed the following Ranking Metric\n",
    "+ Let the number of 1s in our Dataset be x.\n",
    "### `Ranking Accuracy on Single Product = Number of 1s found in first x positions / x`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = load('randomforest.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Product: Accucheck | Rank Accuracy: 0.9647058823529412\n",
      "Product: Becadexamin | Rank Accuracy: 1.0\n",
      "Product: Evion | Rank Accuracy: 1.0\n",
      "Product: Neurobion | Rank Accuracy: 0.9117647058823529\n",
      "Product: SevenseascodLiverOil | Rank Accuracy: 1.0\n",
      "Product: Shelcal | Rank Accuracy: 0.9435483870967742\n",
      "Product: Supradyn | Rank Accuracy: 1.0\n",
      "Product: shampoo | Rank Accuracy: 1.0\n",
      "Mean Rank Accuracy: 0.9775023719165086\n"
     ]
    }
   ],
   "source": [
    "product_list = df['product'].unique()\n",
    "df['win']=0\n",
    "df['lose']=0\n",
    "df['review_score'] = 0.0\n",
    "df.reset_index(inplace = True, drop = True)\n",
    "\n",
    "\n",
    "def score_giver(C,D):\n",
    "    E = pd.merge(C,D,how='outer',on='j')\n",
    "    E.drop(columns=['j'],inplace = True)\n",
    "    q= classifier.predict(E.values)\n",
    "    return Counter(q)\n",
    "\n",
    "for product in product_list:\n",
    "    data = df[df['product']==product]\n",
    "    for indx in data.index:\n",
    "        review = df.iloc[indx, 3:-3]\n",
    "        review['j'] = 'jn'\n",
    "        C = pd.DataFrame([review])\n",
    "        D = data[data.index!=indx].iloc[:,3:-3]\n",
    "        D['j'] = 'jn'\n",
    "        score = score_giver(C,D)\n",
    "        df.at[indx, 'win'] = 0 if score.get(1) is None else score.get(1)\n",
    "        df.at[indx, 'lose'] = 0 if score.get(0) is None else score.get(0)\n",
    "        df.at[indx, 'review_score'] = float(0 if score.get(1) is None else score.get(1)) / len(data) * 1.0\n",
    "\n",
    "df = df.sort_values(by = ['product','review_score'], ascending = False)\n",
    "\n",
    "r_accuracy =[]\n",
    "for product in product_list:\n",
    "    x = data_split[data_split.index == product][1][0]\n",
    "    number_of_1_in_x = Counter(df[df['product']==product].iloc[:x, ]['label']).get(1)\n",
    "    rank_accuracy = float(number_of_1_in_x*1.0 / x*1.0)\n",
    "    print(\"Product: {} | Rank Accuracy: {}\".format(product, rank_accuracy))\n",
    "    r_accuracy.append(rank_accuracy)\n",
    "print(\"Mean Rank Accuracy: {}\".format(mean(r_accuracy)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>answer_option</th>\n",
       "      <th>label</th>\n",
       "      <th>review_len</th>\n",
       "      <th>Rn</th>\n",
       "      <th>Rp</th>\n",
       "      <th>Rs</th>\n",
       "      <th>Rc</th>\n",
       "      <th>Rd</th>\n",
       "      <th>Rsc</th>\n",
       "      <th>win</th>\n",
       "      <th>lose</th>\n",
       "      <th>review_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1564</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>Wash your head within 3 days for sometimes Or ...</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>1.162791</td>\n",
       "      <td>0.15000</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.073986</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.1823</td>\n",
       "      <td>104</td>\n",
       "      <td>0</td>\n",
       "      <td>0.990476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1615</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>I was diagnosed with Seborrheic Dermatitis a d...</td>\n",
       "      <td>1</td>\n",
       "      <td>79</td>\n",
       "      <td>0.924148</td>\n",
       "      <td>0.00875</td>\n",
       "      <td>0.510000</td>\n",
       "      <td>0.136038</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.7184</td>\n",
       "      <td>103</td>\n",
       "      <td>1</td>\n",
       "      <td>0.980952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1550</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>Like product is as expected Dislike- no immedi...</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0.681312</td>\n",
       "      <td>-0.10000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.026253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.7506</td>\n",
       "      <td>102</td>\n",
       "      <td>2</td>\n",
       "      <td>0.971429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1551</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>It really helps to relieve dandruff and itching</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.20000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.019093</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6865</td>\n",
       "      <td>101</td>\n",
       "      <td>3</td>\n",
       "      <td>0.961905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1568</th>\n",
       "      <td>shampoo</td>\n",
       "      <td>At first the Dandruff disappears - - but it re...</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>0.271280</td>\n",
       "      <td>0.25000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.066826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.6010</td>\n",
       "      <td>96</td>\n",
       "      <td>8</td>\n",
       "      <td>0.914286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>Prompt Delivery</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.364110</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>3</td>\n",
       "      <td>391</td>\n",
       "      <td>0.007595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>courier service</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>3</td>\n",
       "      <td>391</td>\n",
       "      <td>0.007595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>The time/days took to deliver is really bad</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0.369768</td>\n",
       "      <td>-0.70000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.010840</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.5849</td>\n",
       "      <td>0</td>\n",
       "      <td>394</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>The product was delivered after days and too o...</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>0.482174</td>\n",
       "      <td>-1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.025745</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.5423</td>\n",
       "      <td>0</td>\n",
       "      <td>394</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>Accucheck</td>\n",
       "      <td>No SLA, pathetic customer support support was ...</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>0.821540</td>\n",
       "      <td>-0.65000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.013550</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.1280</td>\n",
       "      <td>0</td>\n",
       "      <td>394</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1655 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        product                                      answer_option  label  \\\n",
       "1564    shampoo  Wash your head within 3 days for sometimes Or ...      1   \n",
       "1615    shampoo  I was diagnosed with Seborrheic Dermatitis a d...      1   \n",
       "1550    shampoo  Like product is as expected Dislike- no immedi...      1   \n",
       "1551    shampoo    It really helps to relieve dandruff and itching      1   \n",
       "1568    shampoo  At first the Dandruff disappears - - but it re...      1   \n",
       "...         ...                                                ...    ...   \n",
       "195   Accucheck                                    Prompt Delivery      0   \n",
       "223   Accucheck                                    courier service      0   \n",
       "43    Accucheck        The time/days took to deliver is really bad      0   \n",
       "311   Accucheck  The product was delivered after days and too o...      0   \n",
       "333   Accucheck  No SLA, pathetic customer support support was ...      0   \n",
       "\n",
       "      review_len        Rn       Rp        Rs        Rc   Rd     Rsc  win  \\\n",
       "1564          39  1.162791  0.15000  0.300000  0.073986  1.0 -0.1823  104   \n",
       "1615          79  0.924148  0.00875  0.510000  0.136038  0.0  0.7184  103   \n",
       "1550          12  0.681312 -0.10000  0.400000  0.026253  0.0  0.7506  102   \n",
       "1551           8  0.000000  0.20000  0.200000  0.019093  0.0  0.6865  101   \n",
       "1568          33  0.271280  0.25000  0.333333  0.066826  0.0 -0.6010   96   \n",
       "...          ...       ...      ...       ...       ...  ...     ...  ...   \n",
       "195            2  0.364110  0.00000  0.000000  0.002710  1.0  0.0000    3   \n",
       "223            2  0.000000  0.00000  0.000000  0.002710  1.0  0.0000    3   \n",
       "43             8  0.369768 -0.70000  0.666667  0.010840  1.0 -0.5849    0   \n",
       "311           19  0.482174 -1.00000  1.000000  0.025745  1.0 -0.5423    0   \n",
       "333           13  0.821540 -0.65000  0.800000  0.013550  1.0 -0.1280    0   \n",
       "\n",
       "      lose  review_score  \n",
       "1564     0      0.990476  \n",
       "1615     1      0.980952  \n",
       "1550     2      0.971429  \n",
       "1551     3      0.961905  \n",
       "1568     8      0.914286  \n",
       "...    ...           ...  \n",
       "195    391      0.007595  \n",
       "223    391      0.007595  \n",
       "43     394      0.000000  \n",
       "311    394      0.000000  \n",
       "333    394      0.000000  \n",
       "\n",
       "[1655 rows x 13 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[:, [0,1,-1]].to_csv('data/train_ranked_output.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. Data Analysis and Preprocessing.ipynb\r\n",
      "2. Feature Engineering.ipynb\r\n",
      "3.Model Training.ipynb\r\n",
      "\u001b[34mPhotos\u001b[m\u001b[m\r\n",
      "\u001b[34mdata\u001b[m\u001b[m\r\n",
      "datapipeline.py\r\n",
      "feature_analysis.html\r\n",
      "randomforest.joblib\r\n",
      "requirements.txt\r\n",
      "\u001b[34mutils\u001b[m\u001b[m\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = pd.read_csv('data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}