Solution file Hackerday December 2017_Python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Data Science Project in Python- Build a machine learning algorithm that automatically suggests the right product prices.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = fread('https://s3.amazonaws.com/hackerday.datascience/109/test.tsv')\n",
    "train = fread('https://s3.amazonaws.com/hackerday.datascience/109/train.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gc\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix, hstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.model_selection import train_test_split, cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NUM_BRANDS = 4000\n",
    "NUM_CATEGORIES = 1000\n",
    "NAME_MIN_DF = 10\n",
    "MAX_FEATURES_ITEM_DESCRIPTION = 50000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def handle_missing_inplace(dataset):\n",
    "    dataset['category_name'].fillna(value='missing', inplace=True)\n",
    "    dataset['brand_name'].fillna(value='missing', inplace=True)\n",
    "    dataset['item_description'].fillna(value='missing', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cutting(dataset):\n",
    "    pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]\n",
    "    dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'\n",
    "    pop_category = dataset['category_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def to_categorical(dataset):\n",
    "    dataset['category_name'] = dataset['category_name'].astype('category')\n",
    "    dataset['brand_name'] = dataset['brand_name'].astype('category')\n",
    "    dataset['item_condition_id'] = dataset['item_condition_id'].astype('category')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_table('https://s3.amazonaws.com/hackerday.datascience/109/train.tsv', engine='c')\n",
    "    test = pd.read_table('https://s3.amazonaws.com/hackerday.datascience/109/test.tsv', engine='c')\n",
    "    print('[{}] Finished to load data'.format(time.time() - start_time))\n",
    "    print('Train shape: ', train.shape)\n",
    "    print('Test shape: ', test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nrow_train = train.shape[0]\n",
    "    y = np.log1p(train[\"price\"])\n",
    "    merge: pd.DataFrame = pd.concat([train, test])\n",
    "    submission: pd.DataFrame = test[['test_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del train\n",
    "    del test\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "handle_missing_inplace(merge)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cutting(merge)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_categorical(merge)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cv = CountVectorizer(min_df=NAME_MIN_DF)\n",
    "    X_name = cv.fit_transform(merge['name'])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cv = CountVectorizer()\n",
    "    X_category = cv.fit_transform(merge['category_name'])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tv = TfidfVectorizer(max_features=MAX_FEATURES_ITEM_DESCRIPTION,\n",
    "                         ngram_range=(1, 3),\n",
    "                         stop_words='english')\n",
    "    X_description = tv.fit_transform(merge['item_description'])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lb = LabelBinarizer(sparse_output=True)\n",
    "    X_brand = lb.fit_transform(merge['brand_name'])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_dummies = csr_matrix(pd.get_dummies(merge[['item_condition_id', 'shipping']],\n",
    "                                          sparse=True).values)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sparse_merge = hstack((X_dummies, X_description, X_brand, X_category, X_name)).tocsr()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = sparse_merge[:nrow_train]\n",
    "    X_test = sparse_merge[nrow_train:]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "    d_train = lgb.Dataset(X, label=y, max_bin=8192)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "        'learning_rate': 0.75,\n",
    "        'application': 'regression',\n",
    "        'max_depth': 3,\n",
    "        'num_leaves': 100,\n",
    "        'verbosity': -1,\n",
    "        'metric': 'RMSE',\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = lgb.train(params, train_set=d_train, num_boost_round=3200,  \\\n",
    "    verbose_eval=100) \n",
    "    preds = 0.6*model.predict(X_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Ridge(solver=\"sag\", fit_intercept=True, random_state=205)\n",
    "    model.fit(X, y)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "    preds += 0.4*model.predict(X=X_test)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:Anaconda3]",
   "language": "python",
   "name": "conda-env-Anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}