yelp_rest_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import auc, roc_auc_score, roc_curve, accuracy_score,  recall_score, precision_score\n",
    "from regression_tools.dftransformers import (\n",
    "    ColumnSelector, Identity, FeatureUnion, MapFeature, Intercept)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_metr import plot_roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "from trans_rat import FillPrice, CategoriesTransformScr, RatingNormalizer, FillPriceScr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "import time\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "mc = MongoClient()\n",
    "db  = mc['raw_restaurants']\n",
    "rv_s = db['reviews_scrap']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "colClust_n = ['cl_t0', 'cl_t1', 'cl_t2', 'cl_t3', 'cl_t4', 'cl_t5',  'cl_t7',\n",
    "       'cl_t8', 'cl_t9', 'cl_t10', 'cl_t11', 'cl_t12', 'cl_t13', 'cl_t14',\n",
    "        'cl_t16', 'cl_t17']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#numbers_list = pickle.load(list_unpickle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "clust_model_rf ={}\n",
    "clust_model_gr = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in colClust_n:\n",
    "    filename_gr = 'grboost/mgr_'+col+'.pkl'\n",
    "    with open(filename_gr,'rb') as f:\n",
    "        clust_model_gr[col]= pickle.load( f)\n",
    "    filename_rf = 'rforest/mrf_'+col+'.pkl'\n",
    "    with open(filename_rf,'rb') as f:\n",
    "        clust_model_rf[col] =pickle.load( f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.05, loss='deviance', max_depth=3,\n",
       "              max_features='sqrt', max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=300,\n",
       "              presort='auto', random_state=179, subsample=0.8, verbose=0,\n",
       "              warm_start=False)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clust_model_gr['cl_t3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "rest = db['restaurants']\n",
    "\n",
    "rest_df = pd.DataFrame(list(rest.find({})))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12615, 18)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rest_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### надо почистить дату"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "rest_clean = rest_df.drop(columns= ['_id','coordinates', 'display_phone', \n",
    "                                    'distance','image_url', 'name','location',\n",
    "                                   'is_closed','phone', 'transactions','url'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alias</th>\n",
       "      <th>categories</th>\n",
       "      <th>id</th>\n",
       "      <th>price</th>\n",
       "      <th>rating</th>\n",
       "      <th>rev_API</th>\n",
       "      <th>review_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>maono-seattle</td>\n",
       "      <td>[{'alias': 'hawaiian', 'title': 'Hawaiian'}, {...</td>\n",
       "      <td>gDAO-4m63svPUpii5b8kww</td>\n",
       "      <td>$$</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ohs-sandwiches-seattle-2</td>\n",
       "      <td>[{'alias': 'sandwiches', 'title': 'Sandwiches'...</td>\n",
       "      <td>xzLAynrVb3No3ADUhMxlnQ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>marination-ma-kai-seattle</td>\n",
       "      <td>[{'alias': 'korean', 'title': 'Korean'}, {'ali...</td>\n",
       "      <td>IsvlaFF0EPEhh3ahTuC4xg</td>\n",
       "      <td>$$</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>897</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       alias  \\\n",
       "0              maono-seattle   \n",
       "1   ohs-sandwiches-seattle-2   \n",
       "2  marination-ma-kai-seattle   \n",
       "\n",
       "                                          categories                      id  \\\n",
       "0  [{'alias': 'hawaiian', 'title': 'Hawaiian'}, {...  gDAO-4m63svPUpii5b8kww   \n",
       "1  [{'alias': 'sandwiches', 'title': 'Sandwiches'...  xzLAynrVb3No3ADUhMxlnQ   \n",
       "2  [{'alias': 'korean', 'title': 'Korean'}, {'ali...  IsvlaFF0EPEhh3ahTuC4xg   \n",
       "\n",
       "  price  rating rev_API  review_count  \n",
       "0    $$     4.0       1           788  \n",
       "1   NaN     4.5       1            32  \n",
       "2    $$     4.0       1           897  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rest_clean.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pipeline = Pipeline([FillPriceScr, CategoriesTransformScr, RatingNormalizer ])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr = FillPriceScr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/olga/yelper/trans_rat.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  X[self.priceCol][pd.isnull(X[self.priceCol])] = '$$'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alias</th>\n",
       "      <th>categories</th>\n",
       "      <th>id</th>\n",
       "      <th>RestaurantsPriceRange2</th>\n",
       "      <th>rating</th>\n",
       "      <th>rev_API</th>\n",
       "      <th>review_count</th>\n",
       "      <th>rest_isna</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>maono-seattle</td>\n",
       "      <td>[{'alias': 'hawaiian', 'title': 'Hawaiian'}, {...</td>\n",
       "      <td>gDAO-4m63svPUpii5b8kww</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>788</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ohs-sandwiches-seattle-2</td>\n",
       "      <td>[{'alias': 'sandwiches', 'title': 'Sandwiches'...</td>\n",
       "      <td>xzLAynrVb3No3ADUhMxlnQ</td>\n",
       "      <td>2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      alias  \\\n",
       "0             maono-seattle   \n",
       "1  ohs-sandwiches-seattle-2   \n",
       "\n",
       "                                          categories                      id  \\\n",
       "0  [{'alias': 'hawaiian', 'title': 'Hawaiian'}, {...  gDAO-4m63svPUpii5b8kww   \n",
       "1  [{'alias': 'sandwiches', 'title': 'Sandwiches'...  xzLAynrVb3No3ADUhMxlnQ   \n",
       "\n",
       "   RestaurantsPriceRange2  rating rev_API  review_count  rest_isna  \n",
       "0                       2     4.0       1           788      False  \n",
       "1                       2     4.5       1            32       True  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rest_cl2 = fpr.transform(rest_clean)\n",
    "rest_cl2.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Acai Bowls</td>\n",
       "      <td>0.000840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Active Life</td>\n",
       "      <td>0.004093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Afghan</td>\n",
       "      <td>0.002064</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0         0\n",
       "0   Acai Bowls  0.000840\n",
       "1  Active Life  0.004093\n",
       "2       Afghan  0.002064"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##Acai Bowls\n",
    "mod_cat = pd.read_csv('model_cat.csv')\n",
    "mod_cat.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Acai Bowls',\n",
       " 'Active Life',\n",
       " 'Afghan',\n",
       " 'African',\n",
       " 'American (New)',\n",
       " 'American (Traditional)',\n",
       " 'Arabian',\n",
       " 'Arcades',\n",
       " 'Argentine',\n",
       " 'Art Galleries',\n",
       " 'Arts & Crafts',\n",
       " 'Arts & Entertainment',\n",
       " 'Asian Fusion',\n",
       " 'Automotive',\n",
       " 'Bagels',\n",
       " 'Bakeries',\n",
       " 'Barbeque',\n",
       " 'Bars',\n",
       " 'Beauty & Spas',\n",
       " 'Bed & Breakfast',\n",
       " 'Beer',\n",
       " 'Beer Bar',\n",
       " 'Beer Garden',\n",
       " 'Beer Gardens',\n",
       " 'Belgian',\n",
       " 'Bistros',\n",
       " 'Bowling',\n",
       " 'Brasseries',\n",
       " 'Brazilian',\n",
       " 'Breakfast & Brunch',\n",
       " 'Breweries',\n",
       " 'Brewpubs',\n",
       " 'British',\n",
       " 'Bubble Tea',\n",
       " 'Buffets',\n",
       " 'Burgers',\n",
       " 'Butcher',\n",
       " 'Cafes',\n",
       " 'Cafeteria',\n",
       " 'Cajun/Creole',\n",
       " 'Cambodian',\n",
       " 'Canadian (New)',\n",
       " 'Cantonese',\n",
       " 'Caribbean',\n",
       " 'Casinos',\n",
       " 'Caterers',\n",
       " 'Cheese Shops',\n",
       " 'Cheesesteaks',\n",
       " 'Chicken Shop',\n",
       " 'Chicken Wings',\n",
       " 'Chinese',\n",
       " 'Chocolatiers & Shops',\n",
       " 'Cocktail Bars',\n",
       " 'Coffee & Tea',\n",
       " 'Coffee Roasteries',\n",
       " 'Colombian',\n",
       " 'Comfort Food',\n",
       " 'Convenience Stores',\n",
       " 'Creperies',\n",
       " 'Cuban',\n",
       " 'Cupcakes',\n",
       " 'Custom Cakes',\n",
       " 'Dance Clubs',\n",
       " 'Delicatessen',\n",
       " 'Delis',\n",
       " 'Desserts',\n",
       " 'Dim Sum',\n",
       " 'Diners',\n",
       " 'Dive Bars',\n",
       " 'Do-It-Yourself Food',\n",
       " 'Donairs',\n",
       " 'Donuts',\n",
       " 'Education',\n",
       " 'Ethiopian',\n",
       " 'Ethnic Food',\n",
       " 'Event Planning & Services',\n",
       " 'Falafel',\n",
       " 'Farmers Market',\n",
       " 'Fashion',\n",
       " 'Fast Food',\n",
       " 'Festivals',\n",
       " 'Filipino',\n",
       " 'Fish & Chips',\n",
       " 'Flowers & Gifts',\n",
       " 'Fondue',\n",
       " 'Food',\n",
       " 'Food Court',\n",
       " 'Food Delivery Services',\n",
       " 'Food Stands',\n",
       " 'Food Trucks',\n",
       " 'French',\n",
       " 'Fruits & Veggies',\n",
       " 'Gas Stations',\n",
       " 'Gastropubs',\n",
       " 'Gelato',\n",
       " 'German',\n",
       " 'Gluten-Free',\n",
       " 'Golf',\n",
       " 'Greek',\n",
       " 'Grocery',\n",
       " 'Hakka',\n",
       " 'Halal',\n",
       " 'Hawaiian',\n",
       " 'Health & Medical',\n",
       " 'Health Markets',\n",
       " 'Himalayan/Nepalese',\n",
       " 'Home & Garden',\n",
       " 'Home Services',\n",
       " 'Hookah Bars',\n",
       " 'Hot Dogs',\n",
       " 'Hot Pot',\n",
       " 'Hotels',\n",
       " 'Hotels & Travel',\n",
       " 'Hungarian',\n",
       " 'Ice Cream & Frozen Yogurt',\n",
       " 'Imported Food',\n",
       " 'Indian',\n",
       " 'International',\n",
       " 'International Grocery',\n",
       " 'Internet Cafes',\n",
       " 'Irish',\n",
       " 'Irish Pub',\n",
       " 'Italian',\n",
       " 'Japanese',\n",
       " 'Jazz & Blues',\n",
       " 'Juice Bars & Smoothies',\n",
       " 'Karaoke',\n",
       " 'Kebab',\n",
       " 'Korean',\n",
       " 'Kosher',\n",
       " 'Latin American',\n",
       " 'Lebanese',\n",
       " 'Live/Raw Food',\n",
       " 'Local Flavor',\n",
       " 'Local Services',\n",
       " 'Lounges',\n",
       " 'Malaysian',\n",
       " 'Meat Shops',\n",
       " 'Mediterranean',\n",
       " 'Mexican',\n",
       " 'Middle Eastern',\n",
       " 'Modern European',\n",
       " 'Mongolian',\n",
       " 'Moroccan',\n",
       " 'Music Venues',\n",
       " 'New Mexican Cuisine',\n",
       " 'Nightlife',\n",
       " 'Noodles',\n",
       " 'Organic Stores',\n",
       " 'Pakistani',\n",
       " 'Pan Asian',\n",
       " 'Party & Event Planning',\n",
       " 'Pasta Shops',\n",
       " 'Patisserie/Cake Shop',\n",
       " 'Performing Arts',\n",
       " 'Persian/Iranian',\n",
       " 'Personal Chefs',\n",
       " 'Peruvian',\n",
       " 'Pizza',\n",
       " 'Poke',\n",
       " 'Polish',\n",
       " 'Pool Halls',\n",
       " 'Portuguese',\n",
       " 'Poutineries',\n",
       " 'Pretzels',\n",
       " 'Professional Services',\n",
       " 'Pubs',\n",
       " 'Ramen',\n",
       " 'Restaurants',\n",
       " 'Russian',\n",
       " 'Salad',\n",
       " 'Salvadoran',\n",
       " 'Sandwiches',\n",
       " 'Seafood',\n",
       " 'Seafood Markets',\n",
       " 'Shaved Ice',\n",
       " 'Shopping',\n",
       " 'Shopping Centers',\n",
       " 'Smokehouse',\n",
       " 'Soul Food',\n",
       " 'Soup',\n",
       " 'Southern',\n",
       " 'Spanish',\n",
       " 'Specialty Food',\n",
       " 'Sports Bars',\n",
       " 'Sri Lankan',\n",
       " 'Steakhouses',\n",
       " 'Street Vendors',\n",
       " 'Sushi Bars',\n",
       " 'Szechuan',\n",
       " 'Tacos',\n",
       " 'Taiwanese',\n",
       " 'Tapas Bars',\n",
       " 'Tapas/Small Plates',\n",
       " 'Tea Rooms',\n",
       " 'Tex-Mex',\n",
       " 'Thai',\n",
       " 'Turkish',\n",
       " 'Vegan',\n",
       " 'Vegetarian',\n",
       " 'Venues & Event Spaces',\n",
       " 'Vietnamese',\n",
       " 'Waffles',\n",
       " 'Whiskey Bars',\n",
       " 'Wine & Spirits',\n",
       " 'Wine Bars',\n",
       " 'Wineries',\n",
       " 'Wraps',\n",
       " 'num_of_categ']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mod_fe = mod_cat.iloc[:,0].tolist()\n",
    "mod_fe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rest_cl2' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-034ca24f9ba2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmod_fe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmod_cat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mrest_cl2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'rest_cl2' is not defined"
     ]
    }
   ],
   "source": [
    "rest_cl2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('model_feat.pkl', 'wb') as f:\n",
    "    pickle.dump(mod_fe,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('model_feat.pkl', 'rb') as f:\n",
    "    m2 = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Acai Bowls',\n",
       " 'Active Life',\n",
       " 'Afghan',\n",
       " 'African',\n",
       " 'American (New)',\n",
       " 'American (Traditional)',\n",
       " 'Arabian',\n",
       " 'Arcades',\n",
       " 'Argentine',\n",
       " 'Art Galleries',\n",
       " 'Arts & Crafts',\n",
       " 'Arts & Entertainment',\n",
       " 'Asian Fusion',\n",
       " 'Automotive',\n",
       " 'Bagels',\n",
       " 'Bakeries',\n",
       " 'Barbeque',\n",
       " 'Bars',\n",
       " 'Beauty & Spas',\n",
       " 'Bed & Breakfast',\n",
       " 'Beer',\n",
       " 'Beer Bar',\n",
       " 'Beer Garden',\n",
       " 'Beer Gardens',\n",
       " 'Belgian',\n",
       " 'Bistros',\n",
       " 'Bowling',\n",
       " 'Brasseries',\n",
       " 'Brazilian',\n",
       " 'Breakfast & Brunch',\n",
       " 'Breweries',\n",
       " 'Brewpubs',\n",
       " 'British',\n",
       " 'Bubble Tea',\n",
       " 'Buffets',\n",
       " 'Burgers',\n",
       " 'Butcher',\n",
       " 'Cafes',\n",
       " 'Cafeteria',\n",
       " 'Cajun/Creole',\n",
       " 'Cambodian',\n",
       " 'Canadian (New)',\n",
       " 'Cantonese',\n",
       " 'Caribbean',\n",
       " 'Casinos',\n",
       " 'Caterers',\n",
       " 'Cheese Shops',\n",
       " 'Cheesesteaks',\n",
       " 'Chicken Shop',\n",
       " 'Chicken Wings',\n",
       " 'Chinese',\n",
       " 'Chocolatiers & Shops',\n",
       " 'Cocktail Bars',\n",
       " 'Coffee & Tea',\n",
       " 'Coffee Roasteries',\n",
       " 'Colombian',\n",
       " 'Comfort Food',\n",
       " 'Convenience Stores',\n",
       " 'Creperies',\n",
       " 'Cuban',\n",
       " 'Cupcakes',\n",
       " 'Custom Cakes',\n",
       " 'Dance Clubs',\n",
       " 'Delicatessen',\n",
       " 'Delis',\n",
       " 'Desserts',\n",
       " 'Dim Sum',\n",
       " 'Diners',\n",
       " 'Dive Bars',\n",
       " 'Do-It-Yourself Food',\n",
       " 'Donairs',\n",
       " 'Donuts',\n",
       " 'Education',\n",
       " 'Ethiopian',\n",
       " 'Ethnic Food',\n",
       " 'Event Planning & Services',\n",
       " 'Falafel',\n",
       " 'Farmers Market',\n",
       " 'Fashion',\n",
       " 'Fast Food',\n",
       " 'Festivals',\n",
       " 'Filipino',\n",
       " 'Fish & Chips',\n",
       " 'Flowers & Gifts',\n",
       " 'Fondue',\n",
       " 'Food',\n",
       " 'Food Court',\n",
       " 'Food Delivery Services',\n",
       " 'Food Stands',\n",
       " 'Food Trucks',\n",
       " 'French',\n",
       " 'Fruits & Veggies',\n",
       " 'Gas Stations',\n",
       " 'Gastropubs',\n",
       " 'Gelato',\n",
       " 'German',\n",
       " 'Gluten-Free',\n",
       " 'Golf',\n",
       " 'Greek',\n",
       " 'Grocery',\n",
       " 'Hakka',\n",
       " 'Halal',\n",
       " 'Hawaiian',\n",
       " 'Health & Medical',\n",
       " 'Health Markets',\n",
       " 'Himalayan/Nepalese',\n",
       " 'Home & Garden',\n",
       " 'Home Services',\n",
       " 'Hookah Bars',\n",
       " 'Hot Dogs',\n",
       " 'Hot Pot',\n",
       " 'Hotels',\n",
       " 'Hotels & Travel',\n",
       " 'Hungarian',\n",
       " 'Ice Cream & Frozen Yogurt',\n",
       " 'Imported Food',\n",
       " 'Indian',\n",
       " 'International',\n",
       " 'International Grocery',\n",
       " 'Internet Cafes',\n",
       " 'Irish',\n",
       " 'Irish Pub',\n",
       " 'Italian',\n",
       " 'Japanese',\n",
       " 'Jazz & Blues',\n",
       " 'Juice Bars & Smoothies',\n",
       " 'Karaoke',\n",
       " 'Kebab',\n",
       " 'Korean',\n",
       " 'Kosher',\n",
       " 'Latin American',\n",
       " 'Lebanese',\n",
       " 'Live/Raw Food',\n",
       " 'Local Flavor',\n",
       " 'Local Services',\n",
       " 'Lounges',\n",
       " 'Malaysian',\n",
       " 'Meat Shops',\n",
       " 'Mediterranean',\n",
       " 'Mexican',\n",
       " 'Middle Eastern',\n",
       " 'Modern European',\n",
       " 'Mongolian',\n",
       " 'Moroccan',\n",
       " 'Music Venues',\n",
       " 'New Mexican Cuisine',\n",
       " 'Nightlife',\n",
       " 'Noodles',\n",
       " 'Organic Stores',\n",
       " 'Pakistani',\n",
       " 'Pan Asian',\n",
       " 'Party & Event Planning',\n",
       " 'Pasta Shops',\n",
       " 'Patisserie/Cake Shop',\n",
       " 'Performing Arts',\n",
       " 'Persian/Iranian',\n",
       " 'Personal Chefs',\n",
       " 'Peruvian',\n",
       " 'Pizza',\n",
       " 'Poke',\n",
       " 'Polish',\n",
       " 'Pool Halls',\n",
       " 'Portuguese',\n",
       " 'Poutineries',\n",
       " 'Pretzels',\n",
       " 'Professional Services',\n",
       " 'Pubs',\n",
       " 'Ramen',\n",
       " 'Restaurants',\n",
       " 'Russian',\n",
       " 'Salad',\n",
       " 'Salvadoran',\n",
       " 'Sandwiches',\n",
       " 'Seafood',\n",
       " 'Seafood Markets',\n",
       " 'Shaved Ice',\n",
       " 'Shopping',\n",
       " 'Shopping Centers',\n",
       " 'Smokehouse',\n",
       " 'Soul Food',\n",
       " 'Soup',\n",
       " 'Southern',\n",
       " 'Spanish',\n",
       " 'Specialty Food',\n",
       " 'Sports Bars',\n",
       " 'Sri Lankan',\n",
       " 'Steakhouses',\n",
       " 'Street Vendors',\n",
       " 'Sushi Bars',\n",
       " 'Szechuan',\n",
       " 'Tacos',\n",
       " 'Taiwanese',\n",
       " 'Tapas Bars',\n",
       " 'Tapas/Small Plates',\n",
       " 'Tea Rooms',\n",
       " 'Tex-Mex',\n",
       " 'Thai',\n",
       " 'Turkish',\n",
       " 'Vegan',\n",
       " 'Vegetarian',\n",
       " 'Venues & Event Spaces',\n",
       " 'Vietnamese',\n",
       " 'Waffles',\n",
       " 'Whiskey Bars',\n",
       " 'Wine & Spirits',\n",
       " 'Wine Bars',\n",
       " 'Wineries',\n",
       " 'Wraps',\n",
       " 'num_of_categ']"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alias</th>\n",
       "      <th>id</th>\n",
       "      <th>RestaurantsPriceRange2</th>\n",
       "      <th>stars</th>\n",
       "      <th>rev_API</th>\n",
       "      <th>review_count</th>\n",
       "      <th>rest_isna</th>\n",
       "      <th>Acai Bowls</th>\n",
       "      <th>Active Life</th>\n",
       "      <th>Afghan</th>\n",
       "      <th>...</th>\n",
       "      <th>Venues &amp; Event Spaces</th>\n",
       "      <th>Vietnamese</th>\n",
       "      <th>Waffles</th>\n",
       "      <th>Whiskey Bars</th>\n",
       "      <th>Wine &amp; Spirits</th>\n",
       "      <th>Wine Bars</th>\n",
       "      <th>Wineries</th>\n",
       "      <th>Wraps</th>\n",
       "      <th>num_of_categ</th>\n",
       "      <th>st_over_am</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>maono-seattle</td>\n",
       "      <td>gDAO-4m63svPUpii5b8kww</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>788</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.016000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ohs-sandwiches-seattle-2</td>\n",
       "      <td>xzLAynrVb3No3ADUhMxlnQ</td>\n",
       "      <td>2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.140621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>marination-ma-kai-seattle</td>\n",
       "      <td>IsvlaFF0EPEhh3ahTuC4xg</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>897</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>0.016000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ittos-tapas-seattle-4</td>\n",
       "      <td>Dr9CxgdePozHGwU4beaYGQ</td>\n",
       "      <td>2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>221</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>0.020362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pizzeria-credo-seattle</td>\n",
       "      <td>BeGo3TSuhF7fOv0dzp0s1g</td>\n",
       "      <td>2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>405</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.018000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 217 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       alias                      id  RestaurantsPriceRange2  \\\n",
       "0              maono-seattle  gDAO-4m63svPUpii5b8kww                       2   \n",
       "1   ohs-sandwiches-seattle-2  xzLAynrVb3No3ADUhMxlnQ                       2   \n",
       "2  marination-ma-kai-seattle  IsvlaFF0EPEhh3ahTuC4xg                       2   \n",
       "3      ittos-tapas-seattle-4  Dr9CxgdePozHGwU4beaYGQ                       2   \n",
       "4     pizzeria-credo-seattle  BeGo3TSuhF7fOv0dzp0s1g                       2   \n",
       "\n",
       "   stars rev_API  review_count  rest_isna  Acai Bowls  Active Life  Afghan  \\\n",
       "0    4.0       1           788      False       False        False   False   \n",
       "1    4.5       1            32       True       False        False   False   \n",
       "2    4.0       1           897      False       False        False   False   \n",
       "3    4.5       1           221      False       False        False   False   \n",
       "4    4.5       1           405      False       False        False   False   \n",
       "\n",
       "      ...      Venues & Event Spaces  Vietnamese  Waffles  Whiskey Bars  \\\n",
       "0     ...                      False       False    False         False   \n",
       "1     ...                      False       False    False         False   \n",
       "2     ...                      False       False    False         False   \n",
       "3     ...                      False       False    False         False   \n",
       "4     ...                      False       False    False         False   \n",
       "\n",
       "   Wine & Spirits  Wine Bars  Wineries  Wraps  num_of_categ  st_over_am  \n",
       "0           False      False     False  False             2    0.016000  \n",
       "1           False      False     False  False             2    0.140621  \n",
       "2           False      False     False  False             3    0.016000  \n",
       "3           False      False     False  False             3    0.020362  \n",
       "4           False      False     False  False             2    0.018000  \n",
       "\n",
       "[5 rows x 217 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ct = CategoriesTransformScr(mod_fe)\n",
    "rn = RatingNormalizer(starCol = 'rating')\n",
    "rest_cx = rn.transform(ct.transform(rest_cl2))\n",
    "rest_cx.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "### i forgot to include review_count to model when train\n",
    "colPred1 = rest_cx.columns[2:4].union( rest_cx.columns[6:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('colPred213', 'rb') as f:\n",
    "    colPred = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([], dtype='object')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colPred[colPred != colPred1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([], dtype='object')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colPred1[colPred != colPred1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Acai Bowls', 'Active Life', 'Afghan', 'African', 'American (New)',\n",
       "       'American (Traditional)', 'Arabian', 'Arcades', 'Argentine',\n",
       "       'Art Galleries',\n",
       "       ...\n",
       "       'Waffles', 'Whiskey Bars', 'Wine & Spirits', 'Wine Bars', 'Wineries',\n",
       "       'Wraps', 'num_of_categ', 'rest_isna', 'st_over_am', 'stars'],\n",
       "      dtype='object', length=213)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colPred1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = rest_cx[colPred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Acai Bowls</th>\n",
       "      <th>Active Life</th>\n",
       "      <th>Afghan</th>\n",
       "      <th>African</th>\n",
       "      <th>American (New)</th>\n",
       "      <th>American (Traditional)</th>\n",
       "      <th>Arabian</th>\n",
       "      <th>Arcades</th>\n",
       "      <th>Argentine</th>\n",
       "      <th>Art Galleries</th>\n",
       "      <th>...</th>\n",
       "      <th>Wine &amp; Spirits</th>\n",
       "      <th>Wine Bars</th>\n",
       "      <th>Wineries</th>\n",
       "      <th>Wraps</th>\n",
       "      <th>num_of_categ</th>\n",
       "      <th>price</th>\n",
       "      <th>rest_isna</th>\n",
       "      <th>review_count</th>\n",
       "      <th>st_over_am</th>\n",
       "      <th>stars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 214 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Acai Bowls, Active Life, Afghan, African, American (New), American (Traditional), Arabian, Arcades, Argentine, Art Galleries, Arts & Crafts, Arts & Entertainment, Asian Fusion, Automotive, Bagels, Bakeries, Barbeque, Bars, Beauty & Spas, Bed & Breakfast, Beer, Beer Bar, Beer Garden, Beer Gardens, Belgian, Bistros, Bowling, Brasseries, Brazilian, Breakfast & Brunch, Breweries, Brewpubs, British, Bubble Tea, Buffets, Burgers, Butcher, Cafes, Cafeteria, Cajun/Creole, Cambodian, Canadian (New), Cantonese, Caribbean, Casinos, Caterers, Cheese Shops, Cheesesteaks, Chicken Shop, Chicken Wings, Chinese, Chocolatiers & Shops, Cocktail Bars, Coffee & Tea, Coffee Roasteries, Colombian, Comfort Food, Convenience Stores, Creperies, Cuban, Cupcakes, Custom Cakes, Dance Clubs, Delicatessen, Delis, Desserts, Dim Sum, Diners, Dive Bars, Do-It-Yourself Food, Donairs, Donuts, Education, Ethiopian, Ethnic Food, Event Planning & Services, Falafel, Farmers Market, Fashion, Fast Food, Festivals, Filipino, Fish & Chips, Flowers & Gifts, Fondue, Food, Food Court, Food Delivery Services, Food Stands, Food Trucks, French, Fruits & Veggies, Gas Stations, Gastropubs, Gelato, German, Gluten-Free, Golf, Greek, Grocery, ...]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 214 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "clt0 = clust_model_rf['cl_t0'].predict_proba(X)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.67499515, 0.2146956 , 0.71224975, ..., 0.43123169, 0.73124487,\n",
       "       0.32225622])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clt0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "r0 = clust_model_rf['cl_t0']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Restaurants Prediction\n",
    "make 2 dictionary, then 2 dataframe with cluster predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 19.7 s, sys: 1.42 s, total: 21.2 s\n",
      "Wall time: 12.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "Y_rf = pd.DataFrame()\n",
    "Y_gr = pd.DataFrame()\n",
    "Y_rf['alias'] = rest_cx['alias']\n",
    "Y_gr['alias'] = rest_cx['alias']\n",
    "for col in colClust_n:\n",
    "    Y_rf[col] = clust_model_rf[col].predict_proba(X)[:,1]\n",
    "    Y_gr[col] = clust_model_gr[col].predict_proba(X)[:,1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alias</th>\n",
       "      <th>cl_t0</th>\n",
       "      <th>cl_t1</th>\n",
       "      <th>cl_t2</th>\n",
       "      <th>cl_t3</th>\n",
       "      <th>cl_t4</th>\n",
       "      <th>cl_t5</th>\n",
       "      <th>cl_t7</th>\n",
       "      <th>cl_t8</th>\n",
       "      <th>cl_t9</th>\n",
       "      <th>cl_t10</th>\n",
       "      <th>cl_t11</th>\n",
       "      <th>cl_t12</th>\n",
       "      <th>cl_t13</th>\n",
       "      <th>cl_t14</th>\n",
       "      <th>cl_t16</th>\n",
       "      <th>cl_t17</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>maono-seattle</td>\n",
       "      <td>0.929968</td>\n",
       "      <td>0.992710</td>\n",
       "      <td>0.412188</td>\n",
       "      <td>0.016085</td>\n",
       "      <td>0.837059</td>\n",
       "      <td>0.919372</td>\n",
       "      <td>0.010737</td>\n",
       "      <td>0.035540</td>\n",
       "      <td>0.050482</td>\n",
       "      <td>0.009742</td>\n",
       "      <td>0.096255</td>\n",
       "      <td>2.309888e-03</td>\n",
       "      <td>0.074948</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.272277</td>\n",
       "      <td>0.037797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ohs-sandwiches-seattle-2</td>\n",
       "      <td>0.172442</td>\n",
       "      <td>0.302586</td>\n",
       "      <td>0.010211</td>\n",
       "      <td>0.004146</td>\n",
       "      <td>0.025425</td>\n",
       "      <td>0.049360</td>\n",
       "      <td>0.001897</td>\n",
       "      <td>0.002025</td>\n",
       "      <td>0.005802</td>\n",
       "      <td>0.000076</td>\n",
       "      <td>0.005502</td>\n",
       "      <td>1.293178e-09</td>\n",
       "      <td>0.001371</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005165</td>\n",
       "      <td>0.000869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>marination-ma-kai-seattle</td>\n",
       "      <td>0.953005</td>\n",
       "      <td>0.995034</td>\n",
       "      <td>0.428013</td>\n",
       "      <td>0.003022</td>\n",
       "      <td>0.780916</td>\n",
       "      <td>0.921224</td>\n",
       "      <td>0.004737</td>\n",
       "      <td>0.012592</td>\n",
       "      <td>0.024337</td>\n",
       "      <td>0.342402</td>\n",
       "      <td>0.059423</td>\n",
       "      <td>7.697930e-04</td>\n",
       "      <td>0.050621</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.290787</td>\n",
       "      <td>0.010165</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       alias     cl_t0     cl_t1     cl_t2     cl_t3  \\\n",
       "0              maono-seattle  0.929968  0.992710  0.412188  0.016085   \n",
       "1   ohs-sandwiches-seattle-2  0.172442  0.302586  0.010211  0.004146   \n",
       "2  marination-ma-kai-seattle  0.953005  0.995034  0.428013  0.003022   \n",
       "\n",
       "      cl_t4     cl_t5     cl_t7     cl_t8     cl_t9    cl_t10    cl_t11  \\\n",
       "0  0.837059  0.919372  0.010737  0.035540  0.050482  0.009742  0.096255   \n",
       "1  0.025425  0.049360  0.001897  0.002025  0.005802  0.000076  0.005502   \n",
       "2  0.780916  0.921224  0.004737  0.012592  0.024337  0.342402  0.059423   \n",
       "\n",
       "         cl_t12    cl_t13  cl_t14    cl_t16    cl_t17  \n",
       "0  2.309888e-03  0.074948     0.0  0.272277  0.037797  \n",
       "1  1.293178e-09  0.001371     0.0  0.005165  0.000869  \n",
       "2  7.697930e-04  0.050621     0.0  0.290787  0.010165  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_gr.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_gr['rating'] = rest_cx.stars\n",
    "Y_rf['rating'] = rest_cx.stars\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alias</th>\n",
       "      <th>cl_t0</th>\n",
       "      <th>cl_t1</th>\n",
       "      <th>cl_t2</th>\n",
       "      <th>cl_t3</th>\n",
       "      <th>cl_t4</th>\n",
       "      <th>cl_t5</th>\n",
       "      <th>cl_t7</th>\n",
       "      <th>cl_t8</th>\n",
       "      <th>cl_t9</th>\n",
       "      <th>cl_t10</th>\n",
       "      <th>cl_t11</th>\n",
       "      <th>cl_t12</th>\n",
       "      <th>cl_t13</th>\n",
       "      <th>cl_t14</th>\n",
       "      <th>cl_t16</th>\n",
       "      <th>cl_t17</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>maono-seattle</td>\n",
       "      <td>0.674995</td>\n",
       "      <td>0.899633</td>\n",
       "      <td>0.120200</td>\n",
       "      <td>0.000990</td>\n",
       "      <td>0.382802</td>\n",
       "      <td>0.447516</td>\n",
       "      <td>0.004385</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.017935</td>\n",
       "      <td>0.000892</td>\n",
       "      <td>0.008470</td>\n",
       "      <td>0.001098</td>\n",
       "      <td>0.006509</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.044169</td>\n",
       "      <td>0.005382</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ohs-sandwiches-seattle-2</td>\n",
       "      <td>0.214696</td>\n",
       "      <td>0.499646</td>\n",
       "      <td>0.024044</td>\n",
       "      <td>0.000478</td>\n",
       "      <td>0.016095</td>\n",
       "      <td>0.032686</td>\n",
       "      <td>0.003058</td>\n",
       "      <td>0.000668</td>\n",
       "      <td>0.008900</td>\n",
       "      <td>0.000158</td>\n",
       "      <td>0.008909</td>\n",
       "      <td>0.005692</td>\n",
       "      <td>0.003408</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>0.001514</td>\n",
       "      <td>0.000721</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>marination-ma-kai-seattle</td>\n",
       "      <td>0.712250</td>\n",
       "      <td>0.925625</td>\n",
       "      <td>0.098066</td>\n",
       "      <td>0.000551</td>\n",
       "      <td>0.396129</td>\n",
       "      <td>0.579794</td>\n",
       "      <td>0.003652</td>\n",
       "      <td>0.004784</td>\n",
       "      <td>0.006309</td>\n",
       "      <td>0.004804</td>\n",
       "      <td>0.010670</td>\n",
       "      <td>0.001601</td>\n",
       "      <td>0.006939</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>0.033887</td>\n",
       "      <td>0.005368</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ittos-tapas-seattle-4</td>\n",
       "      <td>0.576311</td>\n",
       "      <td>0.812275</td>\n",
       "      <td>0.047496</td>\n",
       "      <td>0.000632</td>\n",
       "      <td>0.211996</td>\n",
       "      <td>0.292265</td>\n",
       "      <td>0.003506</td>\n",
       "      <td>0.021161</td>\n",
       "      <td>0.010951</td>\n",
       "      <td>0.001633</td>\n",
       "      <td>0.004954</td>\n",
       "      <td>0.000692</td>\n",
       "      <td>0.026224</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>0.049079</td>\n",
       "      <td>0.057404</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pizzeria-credo-seattle</td>\n",
       "      <td>0.614117</td>\n",
       "      <td>0.815253</td>\n",
       "      <td>0.066160</td>\n",
       "      <td>0.000453</td>\n",
       "      <td>0.215199</td>\n",
       "      <td>0.225385</td>\n",
       "      <td>0.139105</td>\n",
       "      <td>0.002124</td>\n",
       "      <td>0.002613</td>\n",
       "      <td>0.000515</td>\n",
       "      <td>0.004208</td>\n",
       "      <td>0.000699</td>\n",
       "      <td>0.003090</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>0.032402</td>\n",
       "      <td>0.020409</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       alias     cl_t0     cl_t1     cl_t2     cl_t3  \\\n",
       "0              maono-seattle  0.674995  0.899633  0.120200  0.000990   \n",
       "1   ohs-sandwiches-seattle-2  0.214696  0.499646  0.024044  0.000478   \n",
       "2  marination-ma-kai-seattle  0.712250  0.925625  0.098066  0.000551   \n",
       "3      ittos-tapas-seattle-4  0.576311  0.812275  0.047496  0.000632   \n",
       "4     pizzeria-credo-seattle  0.614117  0.815253  0.066160  0.000453   \n",
       "\n",
       "      cl_t4     cl_t5     cl_t7     cl_t8     cl_t9    cl_t10    cl_t11  \\\n",
       "0  0.382802  0.447516  0.004385  0.008207  0.017935  0.000892  0.008470   \n",
       "1  0.016095  0.032686  0.003058  0.000668  0.008900  0.000158  0.008909   \n",
       "2  0.396129  0.579794  0.003652  0.004784  0.006309  0.004804  0.010670   \n",
       "3  0.211996  0.292265  0.003506  0.021161  0.010951  0.001633  0.004954   \n",
       "4  0.215199  0.225385  0.139105  0.002124  0.002613  0.000515  0.004208   \n",
       "\n",
       "     cl_t12    cl_t13    cl_t14    cl_t16    cl_t17  rating  \n",
       "0  0.001098  0.006509  0.000033  0.044169  0.005382     4.0  \n",
       "1  0.005692  0.003408  0.000032  0.001514  0.000721     4.5  \n",
       "2  0.001601  0.006939  0.000032  0.033887  0.005368     4.0  \n",
       "3  0.000692  0.026224  0.000030  0.049079  0.057404     4.5  \n",
       "4  0.000699  0.003090  0.000089  0.032402  0.020409     4.5  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_rf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('thres_35_gr', 'rb') as f:\n",
    "    th_gr35 = pickle.load(f)\n",
    "with open('thres_35_rf','rb') as f:\n",
    "    th_rf35 = pickle.load(f )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cl_t0': 0.7145996009074023,\n",
       " 'cl_t1': 0.9251774972382272,\n",
       " 'cl_t2': 0.09672319260149016,\n",
       " 'cl_t3': 0.01,\n",
       " 'cl_t4': 0.3525910562359607,\n",
       " 'cl_t5': 0.3448134487670461,\n",
       " 'cl_t7': 0.10512960889284231,\n",
       " 'cl_t8': 0.09037281891030602,\n",
       " 'cl_t9': 0.051410468261200035,\n",
       " 'cl_t10': 0.16208938583548552,\n",
       " 'cl_t11': 0.09687667183805175,\n",
       " 'cl_t12': 0.06795537384334152,\n",
       " 'cl_t13': 0.10770567432929509,\n",
       " 'cl_t14': 0.01,\n",
       " 'cl_t16': 0.039693625572722824,\n",
       " 'cl_t17': 0.024528574099973412}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "th_rf35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_rating(row, colClust_n, thres):\n",
    "    r = False\n",
    "    for col in colClust_n:\n",
    "        u_col = 'u'+col\n",
    "        if row[u_col] and row[col] >= thres[col]:\n",
    "            r = True\n",
    "    return r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "cl_0 = Y_rf['cl_t0'] > th_rf35['cl_t0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Y_rf' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-50-e45fb45013c8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mY_rf_bool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mY_rf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcolClust_n\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mY_rf_bool\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mY_rf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mth_rf35\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mY_rf_bool\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'base3'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mY_rf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrating\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'Y_rf' is not defined"
     ]
    }
   ],
   "source": [
    "Y_rf_bool = Y_rf.copy()\n",
    "for col in colClust_n:\n",
    "    Y_rf_bool[col] = Y_rf[col] > th_rf35[col]\n",
    "\n",
    "Y_rf_bool['base3'] = Y_rf.rating < 3\n",
    "Y_rf_bool['base35'] = Y_rf.rating < 3.5\n",
    "Y_rf_bool['base4'] = Y_rf.rating < 4\n",
    "Y_rf_bool['base4.5'] = Y_rf.rating < 4.5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alias</th>\n",
       "      <th>cl_t0</th>\n",
       "      <th>cl_t1</th>\n",
       "      <th>cl_t2</th>\n",
       "      <th>cl_t3</th>\n",
       "      <th>cl_t4</th>\n",
       "      <th>cl_t5</th>\n",
       "      <th>cl_t7</th>\n",
       "      <th>cl_t8</th>\n",
       "      <th>cl_t9</th>\n",
       "      <th>...</th>\n",
       "      <th>cl_t12</th>\n",
       "      <th>cl_t13</th>\n",
       "      <th>cl_t14</th>\n",
       "      <th>cl_t16</th>\n",
       "      <th>cl_t17</th>\n",
       "      <th>rating</th>\n",
       "      <th>base3</th>\n",
       "      <th>base35</th>\n",
       "      <th>base4</th>\n",
       "      <th>base4.5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>willmotts-ghost-seattle</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>pho-than-brothers-seattle-6</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>boss-drive-in-seattle</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>drunkys-two-shoe-bbq-seattle-3</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>southside-pizza-seattle</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              alias  cl_t0  cl_t1  cl_t2  cl_t3  cl_t4  cl_t5  \\\n",
       "101         willmotts-ghost-seattle  False  False  False  False  False  False   \n",
       "144     pho-than-brothers-seattle-6   True   True  False  False  False   True   \n",
       "152           boss-drive-in-seattle   True   True  False  False  False  False   \n",
       "171  drunkys-two-shoe-bbq-seattle-3   True   True  False  False  False   True   \n",
       "202         southside-pizza-seattle  False  False  False  False  False  False   \n",
       "\n",
       "     cl_t7  cl_t8  cl_t9   ...     cl_t12  cl_t13  cl_t14  cl_t16  cl_t17  \\\n",
       "101  False  False  False   ...      False   False   False   False   False   \n",
       "144  False  False  False   ...       True   False   False   False   False   \n",
       "152  False   True  False   ...      False   False   False   False   False   \n",
       "171  False  False  False   ...      False   False   False   False   False   \n",
       "202  False  False  False   ...      False   False   False   False   False   \n",
       "\n",
       "     rating  base3  base35  base4  base4.5  \n",
       "101     3.0  False    True   True     True  \n",
       "144     3.0  False    True   True     True  \n",
       "152     3.0  False    True   True     True  \n",
       "171     3.0  False    True   True     True  \n",
       "202     3.0  False    True   True     True  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_rf_bool[Y_rf_bool['base35']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Y_rf_bool' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-49-7faae747ac39>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_rf_bool\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'base35'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mY_rf_bool\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'cl_t0'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'Y_rf_bool' is not defined"
     ]
    }
   ],
   "source": [
    "accuracy_score(Y_rf_bool['base35'],Y_rf_bool['cl_t0'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make prediction???\n",
    "1. Create user profile = set of clusters that this user care\n",
    "2. Ask location \n",
    "3. Get rests from yelp for this location:\n",
    "    - ask yelp api search\n",
    "    - save this in database\n",
    "4. Get rest predictions \n",
    "    - clean database\n",
    "    - predict rests for clusters\n",
    "5. Print results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from scraping.yelp_api import *\n",
    "\n",
    "### looking just on first 100\n",
    "offset_lim = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = search(API_KEY, DEFAULT_TERM, '416 Occidental Ave S, Seattle, WA', 50)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "d0 = pd.DataFrame( response.get('businesses', None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1 = pd.DataFrame( response.get('businesses', None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((50, 16), (50, 16))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d0.shape, d1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 16)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc = pd.concat([d0,d1], axis= 0, ignore_index=True)\n",
    "dc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(response.get('businesses', None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "d += search(API_KEY, DEFAULT_TERM, '416 Occidental Ave S, Seattle, WA', 100).get('businesses', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['alias', 'categories', 'coordinates', 'display_phone', 'distance', 'id',\n",
       "       'image_url', 'is_closed', 'location', 'name', 'phone', 'price',\n",
       "       'rating', 'review_count', 'transactions', 'url'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from concept_proof import cleaning_data, ComboModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['alias', 'categories', 'coordinates', 'display_phone', 'distance', 'id',\n",
       "       'image_url', 'is_closed', 'location', 'name', 'phone', 'price',\n",
       "       'rating', 'review_count', 'transactions', 'url'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/olga/yelper/trans_rat.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  X[self.priceCol][pd.isnull(X[self.priceCol])] = '$$'\n"
     ]
    }
   ],
   "source": [
    "x = cleaning_data(dc, mod_fe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['cl_t0', 'cl_t1', 'cl_t2', 'cl_t3', 'cl_t4', 'cl_t5', 'cl_t7', 'cl_t8', 'cl_t9', 'cl_t10', 'cl_t11', 'cl_t12', 'cl_t13', 'cl_t14', 'cl_t16', 'cl_t17'])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clust_model_rf.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('colPred213', 'rb') as f:\n",
    "    colPred_pr = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RangeIndex(start=0, stop=100, step=1)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alias</th>\n",
       "      <th>id</th>\n",
       "      <th>RestaurantsPriceRange2</th>\n",
       "      <th>stars</th>\n",
       "      <th>review_count</th>\n",
       "      <th>rest_isna</th>\n",
       "      <th>Acai Bowls</th>\n",
       "      <th>Active Life</th>\n",
       "      <th>Afghan</th>\n",
       "      <th>African</th>\n",
       "      <th>...</th>\n",
       "      <th>Venues &amp; Event Spaces</th>\n",
       "      <th>Vietnamese</th>\n",
       "      <th>Waffles</th>\n",
       "      <th>Whiskey Bars</th>\n",
       "      <th>Wine &amp; Spirits</th>\n",
       "      <th>Wine Bars</th>\n",
       "      <th>Wineries</th>\n",
       "      <th>Wraps</th>\n",
       "      <th>num_of_categ</th>\n",
       "      <th>st_over_am</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the-pink-door-seattle-4</td>\n",
       "      <td>VOPdG8llLPaga9iJxXcMuQ</td>\n",
       "      <td>2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4044</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>0.018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>taylor-shellfish-oyster-bar-seattle-2</td>\n",
       "      <td>p1TOF1XkXzLx3a-LF3lDlw</td>\n",
       "      <td>2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>528</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>japonessa-sushi-cocina-seattle</td>\n",
       "      <td>L8RRAd-JZ0Bd4MER0yyX-g</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4143</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>radiator-whiskey-seattle</td>\n",
       "      <td>ps9ZlLoLYTfK87IjW9REfg</td>\n",
       "      <td>2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1087</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>0.018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>girin-korean-ssam-bar-seattle</td>\n",
       "      <td>3F5WWcqqboa-NHbYFs4LAw</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>340</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>0.016</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 216 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   alias                      id  \\\n",
       "0                the-pink-door-seattle-4  VOPdG8llLPaga9iJxXcMuQ   \n",
       "1  taylor-shellfish-oyster-bar-seattle-2  p1TOF1XkXzLx3a-LF3lDlw   \n",
       "2         japonessa-sushi-cocina-seattle  L8RRAd-JZ0Bd4MER0yyX-g   \n",
       "3               radiator-whiskey-seattle  ps9ZlLoLYTfK87IjW9REfg   \n",
       "4          girin-korean-ssam-bar-seattle  3F5WWcqqboa-NHbYFs4LAw   \n",
       "\n",
       "   RestaurantsPriceRange2  stars  review_count  rest_isna  Acai Bowls  \\\n",
       "0                       2    4.5          4044      False       False   \n",
       "1                       2    4.5           528      False       False   \n",
       "2                       2    4.0          4143      False       False   \n",
       "3                       2    4.5          1087      False       False   \n",
       "4                       2    4.0           340      False       False   \n",
       "\n",
       "   Active Life  Afghan  African     ...      Venues & Event Spaces  \\\n",
       "0        False   False    False     ...                      False   \n",
       "1        False   False    False     ...                      False   \n",
       "2        False   False    False     ...                      False   \n",
       "3        False   False    False     ...                      False   \n",
       "4        False   False    False     ...                      False   \n",
       "\n",
       "   Vietnamese  Waffles  Whiskey Bars  Wine & Spirits  Wine Bars  Wineries  \\\n",
       "0       False    False         False           False       True     False   \n",
       "1       False    False         False           False      False     False   \n",
       "2       False    False         False           False      False     False   \n",
       "3       False    False         False           False      False     False   \n",
       "4       False    False         False           False      False     False   \n",
       "\n",
       "   Wraps  num_of_categ  st_over_am  \n",
       "0  False             3       0.018  \n",
       "1  False             2       0.018  \n",
       "2  False             2       0.016  \n",
       "3  False             2       0.018  \n",
       "4  False             3       0.016  \n",
       "\n",
       "[5 rows x 216 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "pr0 = clust_model_rf['cl_t0'].predict_proba(x[colPred_pr])[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = ComboModel('rforest/mrf_', th_rf35, colPred_pr, colClust_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cl_t0': 0.7145996009074023,\n",
       " 'cl_t1': 0.9251774972382272,\n",
       " 'cl_t2': 0.09672319260149016,\n",
       " 'cl_t3': 0.01,\n",
       " 'cl_t4': 0.3525910562359607,\n",
       " 'cl_t5': 0.3448134487670461,\n",
       " 'cl_t7': 0.10512960889284231,\n",
       " 'cl_t8': 0.09037281891030602,\n",
       " 'cl_t9': 0.051410468261200035,\n",
       " 'cl_t10': 0.16208938583548552,\n",
       " 'cl_t11': 0.09687667183805175,\n",
       " 'cl_t12': 0.06795537384334152,\n",
       " 'cl_t13': 0.10770567432929509,\n",
       " 'cl_t14': 0.01,\n",
       " 'cl_t16': 0.039693625572722824,\n",
       " 'cl_t17': 0.024528574099973412}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.model_thres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "olka = ['cl_t0', 'cl_t2', 'cl_t9' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#rf.predict(x,olka)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = pd.DataFrame()\n",
    "        #Y['alias'] = X['alias']\n",
    "answer = np.array(len(x)*[False])\n",
    "for col in olka:\n",
    "    Y[col] = clust_model_rf[col].predict_proba(x[colPred_pr])[:,1]\n",
    "    answer = answer|(Y[col]>th_rf35[col])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 249 ms, sys: 49.1 ms, total: 298 ms\n",
      "Wall time: 336 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "answer = rf.predict(x, olka)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dc[answer]['rating'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['cl_t0', 'cl_t1', 'cl_t2', 'cl_t3', 'cl_t4', 'cl_t5', 'cl_t7', 'cl_t8', 'cl_t9', 'cl_t10', 'cl_t11', 'cl_t12', 'cl_t13', 'cl_t14', 'cl_t16', 'cl_t17'])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clust_model_rf.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "picky = ['cl_t0', 'cl_t1', 'cl_t2', 'cl_t3', 'cl_t4', 'cl_t5', 'cl_t7', 'cl_t8', 'cl_t9', 'cl_t10', 'cl_t11', 'cl_t12', 'cl_t13', 'cl_t14', 'cl_t16', 'cl_t17']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.28 s, sys: 230 ms, total: 1.51 s\n",
      "Wall time: 1.79 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "a_p = rf.predict(x, picky)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alias</th>\n",
       "      <th>categories</th>\n",
       "      <th>coordinates</th>\n",
       "      <th>display_phone</th>\n",
       "      <th>distance</th>\n",
       "      <th>id</th>\n",
       "      <th>image_url</th>\n",
       "      <th>is_closed</th>\n",
       "      <th>location</th>\n",
       "      <th>name</th>\n",
       "      <th>phone</th>\n",
       "      <th>price</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_count</th>\n",
       "      <th>transactions</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the-pink-door-seattle-4</td>\n",
       "      <td>[{'alias': 'italian', 'title': 'Italian'}, {'a...</td>\n",
       "      <td>{'latitude': 47.61028, 'longitude': -122.3425}</td>\n",
       "      <td>(206) 443-3241</td>\n",
       "      <td>1521.522544</td>\n",
       "      <td>VOPdG8llLPaga9iJxXcMuQ</td>\n",
       "      <td>https://s3-media1.fl.yelpcdn.com/bphoto/c_vzRF...</td>\n",
       "      <td>False</td>\n",
       "      <td>{'address1': '1919 Post Alley', 'address2': ''...</td>\n",
       "      <td>The Pink Door</td>\n",
       "      <td>+12064433241</td>\n",
       "      <td>$$</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4044</td>\n",
       "      <td>[]</td>\n",
       "      <td>https://www.yelp.com/biz/the-pink-door-seattle...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>taylor-shellfish-oyster-bar-seattle-2</td>\n",
       "      <td>[{'alias': 'seafood', 'title': 'Seafood'}, {'a...</td>\n",
       "      <td>{'latitude': 47.59884, 'longitude': -122.33268}</td>\n",
       "      <td>(206) 501-4060</td>\n",
       "      <td>39.496086</td>\n",
       "      <td>p1TOF1XkXzLx3a-LF3lDlw</td>\n",
       "      <td>https://s3-media2.fl.yelpcdn.com/bphoto/he49IR...</td>\n",
       "      <td>False</td>\n",
       "      <td>{'address1': '410 Occidental Ave S', 'address2...</td>\n",
       "      <td>Taylor Shellfish Oyster Bar</td>\n",
       "      <td>+12065014060</td>\n",
       "      <td>$$</td>\n",
       "      <td>4.5</td>\n",
       "      <td>528</td>\n",
       "      <td>[]</td>\n",
       "      <td>https://www.yelp.com/biz/taylor-shellfish-oyst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>japonessa-sushi-cocina-seattle</td>\n",
       "      <td>[{'alias': 'japanese', 'title': 'Japanese'}, {...</td>\n",
       "      <td>{'latitude': 47.60809, 'longitude': -122.33899}</td>\n",
       "      <td>(206) 971-7979</td>\n",
       "      <td>1168.535744</td>\n",
       "      <td>L8RRAd-JZ0Bd4MER0yyX-g</td>\n",
       "      <td>https://s3-media3.fl.yelpcdn.com/bphoto/vucCrk...</td>\n",
       "      <td>False</td>\n",
       "      <td>{'address1': '1400 1st Ave', 'address2': '', '...</td>\n",
       "      <td>Japonessa Sushi Cocina</td>\n",
       "      <td>+12069717979</td>\n",
       "      <td>$$</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4143</td>\n",
       "      <td>[restaurant_reservation]</td>\n",
       "      <td>https://www.yelp.com/biz/japonessa-sushi-cocin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>radiator-whiskey-seattle</td>\n",
       "      <td>[{'alias': 'newamerican', 'title': 'American (...</td>\n",
       "      <td>{'latitude': 47.60888, 'longitude': -122.34036}</td>\n",
       "      <td>(206) 467-4268</td>\n",
       "      <td>1299.145186</td>\n",
       "      <td>ps9ZlLoLYTfK87IjW9REfg</td>\n",
       "      <td>https://s3-media4.fl.yelpcdn.com/bphoto/DgO6qK...</td>\n",
       "      <td>False</td>\n",
       "      <td>{'address1': '94 Pike St', 'address2': 'Ste 30...</td>\n",
       "      <td>Radiator Whiskey</td>\n",
       "      <td>+12064674268</td>\n",
       "      <td>$$</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1087</td>\n",
       "      <td>[]</td>\n",
       "      <td>https://www.yelp.com/biz/radiator-whiskey-seat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>girin-korean-ssam-bar-seattle</td>\n",
       "      <td>[{'alias': 'korean', 'title': 'Korean'}, {'ali...</td>\n",
       "      <td>{'latitude': 47.59812, 'longitude': -122.33188}</td>\n",
       "      <td>(206) 257-4259</td>\n",
       "      <td>55.901362</td>\n",
       "      <td>3F5WWcqqboa-NHbYFs4LAw</td>\n",
       "      <td>https://s3-media1.fl.yelpcdn.com/bphoto/F_ZEPs...</td>\n",
       "      <td>False</td>\n",
       "      <td>{'address1': '501 Stadium Pl S', 'address2': '...</td>\n",
       "      <td>Girin Korean Ssam Bar</td>\n",
       "      <td>+12062574259</td>\n",
       "      <td>$$</td>\n",
       "      <td>4.0</td>\n",
       "      <td>340</td>\n",
       "      <td>[pickup]</td>\n",
       "      <td>https://www.yelp.com/biz/girin-korean-ssam-bar...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   alias  \\\n",
       "0                the-pink-door-seattle-4   \n",
       "1  taylor-shellfish-oyster-bar-seattle-2   \n",
       "2         japonessa-sushi-cocina-seattle   \n",
       "3               radiator-whiskey-seattle   \n",
       "4          girin-korean-ssam-bar-seattle   \n",
       "\n",
       "                                          categories  \\\n",
       "0  [{'alias': 'italian', 'title': 'Italian'}, {'a...   \n",
       "1  [{'alias': 'seafood', 'title': 'Seafood'}, {'a...   \n",
       "2  [{'alias': 'japanese', 'title': 'Japanese'}, {...   \n",
       "3  [{'alias': 'newamerican', 'title': 'American (...   \n",
       "4  [{'alias': 'korean', 'title': 'Korean'}, {'ali...   \n",
       "\n",
       "                                       coordinates   display_phone  \\\n",
       "0   {'latitude': 47.61028, 'longitude': -122.3425}  (206) 443-3241   \n",
       "1  {'latitude': 47.59884, 'longitude': -122.33268}  (206) 501-4060   \n",
       "2  {'latitude': 47.60809, 'longitude': -122.33899}  (206) 971-7979   \n",
       "3  {'latitude': 47.60888, 'longitude': -122.34036}  (206) 467-4268   \n",
       "4  {'latitude': 47.59812, 'longitude': -122.33188}  (206) 257-4259   \n",
       "\n",
       "      distance                      id  \\\n",
       "0  1521.522544  VOPdG8llLPaga9iJxXcMuQ   \n",
       "1    39.496086  p1TOF1XkXzLx3a-LF3lDlw   \n",
       "2  1168.535744  L8RRAd-JZ0Bd4MER0yyX-g   \n",
       "3  1299.145186  ps9ZlLoLYTfK87IjW9REfg   \n",
       "4    55.901362  3F5WWcqqboa-NHbYFs4LAw   \n",
       "\n",
       "                                           image_url  is_closed  \\\n",
       "0  https://s3-media1.fl.yelpcdn.com/bphoto/c_vzRF...      False   \n",
       "1  https://s3-media2.fl.yelpcdn.com/bphoto/he49IR...      False   \n",
       "2  https://s3-media3.fl.yelpcdn.com/bphoto/vucCrk...      False   \n",
       "3  https://s3-media4.fl.yelpcdn.com/bphoto/DgO6qK...      False   \n",
       "4  https://s3-media1.fl.yelpcdn.com/bphoto/F_ZEPs...      False   \n",
       "\n",
       "                                            location  \\\n",
       "0  {'address1': '1919 Post Alley', 'address2': ''...   \n",
       "1  {'address1': '410 Occidental Ave S', 'address2...   \n",
       "2  {'address1': '1400 1st Ave', 'address2': '', '...   \n",
       "3  {'address1': '94 Pike St', 'address2': 'Ste 30...   \n",
       "4  {'address1': '501 Stadium Pl S', 'address2': '...   \n",
       "\n",
       "                          name         phone price  rating  review_count  \\\n",
       "0                The Pink Door  +12064433241    $$     4.5          4044   \n",
       "1  Taylor Shellfish Oyster Bar  +12065014060    $$     4.5           528   \n",
       "2       Japonessa Sushi Cocina  +12069717979    $$     4.0          4143   \n",
       "3             Radiator Whiskey  +12064674268    $$     4.5          1087   \n",
       "4        Girin Korean Ssam Bar  +12062574259    $$     4.0           340   \n",
       "\n",
       "               transactions                                                url  \n",
       "0                        []  https://www.yelp.com/biz/the-pink-door-seattle...  \n",
       "1                        []  https://www.yelp.com/biz/taylor-shellfish-oyst...  \n",
       "2  [restaurant_reservation]  https://www.yelp.com/biz/japonessa-sushi-cocin...  \n",
       "3                        []  https://www.yelp.com/biz/radiator-whiskey-seat...  \n",
       "4                  [pickup]  https://www.yelp.com/biz/girin-korean-ssam-bar...  "
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc[a_p].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_p.to_html?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"alias\":\"taylor-shellfish-oyster-bar-seattle-2\",\"categories\":[{\"alias\":\"seafood\",\"title\":\"Seafood\"},{\"alias\":\"raw_food\",\"title\":\"Live\\\\/Raw Food\"}],\"coordinates\":{\"latitude\":47.59884,\"longitude\":-122.33268},\"display_phone\":\"(206) 501-4060\",\"distance\":39.496086001,\"id\":\"p1TOF1XkXzLx3a-LF3lDlw\",\"image_url\":\"https:\\\\/\\\\/s3-media2.fl.yelpcdn.com\\\\/bphoto\\\\/he49IR0G1wrjO1WI-fxkAg\\\\/o.jpg\",\"is_closed\":false,\"location\":{\"address1\":\"410 Occidental Ave S\",\"address2\":\"\",\"address3\":\"\",\"city\":\"Seattle\",\"zip_code\":\"98104\",\"country\":\"US\",\"state\":\"WA\",\"display_address\":[\"410 Occidental Ave S\",\"Seattle, WA 98104\"]},\"name\":\"Taylor Shellfish Oyster Bar\",\"phone\":\"+12065014060\",\"price\":\"$$\",\"rating\":4.5,\"review_count\":528,\"transactions\":[],\"url\":\"https:\\\\/\\\\/www.yelp.com\\\\/biz\\\\/taylor-shellfish-oyster-bar-seattle-2?adjust_creative=KIqjD5y9rnzq9ev6ruaTHQ&utm_campaign=yelp_api_v3&utm_medium=api_v3_business_search&utm_source=KIqjD5y9rnzq9ev6ruaTHQ\"}'"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_p.loc[1,:].to_json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['alias', 'categories', 'coordinates', 'display_phone', 'distance', 'id',\n",
       "       'image_url', 'is_closed', 'location', 'name', 'phone', 'price',\n",
       "       'rating', 'review_count', 'transactions', 'url'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_p.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "colShow = ['name', 'rating', 'location' , 'url']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<table border=\"1\" class=\"dataframe\">\\n  <thead>\\n    <tr style=\"text-align: right;\">\\n      <th></th>\\n      <th>name</th>\\n      <th>rating</th>\\n      <th>location</th>\\n      <th>url</th>\\n    </tr>\\n  </thead>\\n  <tbody>\\n    <tr>\\n      <th>0</th>\\n      <td>The Pink Door</td>\\n      <td>4.5</td>\\n      <td>{\\'address1\\': \\'1919 Post Alley\\', \\'address2\\': \\'\\'...</td>\\n      <td>https://www.yelp.com/biz/the-pink-door-seattle...</td>\\n    </tr>\\n    <tr>\\n      <th>1</th>\\n      <td>Taylor Shellfish Oyster Bar</td>\\n      <td>4.5</td>\\n      <td>{\\'address1\\': \\'410 Occidental Ave S\\', \\'address2...</td>\\n      <td>https://www.yelp.com/biz/taylor-shellfish-oyst...</td>\\n    </tr>\\n    <tr>\\n      <th>2</th>\\n      <td>Japonessa Sushi Cocina</td>\\n      <td>4.0</td>\\n      <td>{\\'address1\\': \\'1400 1st Ave\\', \\'address2\\': \\'\\', \\'...</td>\\n      <td>https://www.yelp.com/biz/japonessa-sushi-cocin...</td>\\n    </tr>\\n    <tr>\\n      <th>3</th>\\n      <td>Radiator Whiskey</td>\\n      <td>4.5</td>\\n      <td>{\\'address1\\': \\'94 Pike St\\', \\'address2\\': \\'Ste 30...</td>\\n      <td>https://www.yelp.com/biz/radiator-whiskey-seat...</td>\\n    </tr>\\n    <tr>\\n      <th>4</th>\\n      <td>Girin Korean Ssam Bar</td>\\n      <td>4.0</td>\\n      <td>{\\'address1\\': \\'501 Stadium Pl S\\', \\'address2\\': \\'...</td>\\n      <td>https://www.yelp.com/biz/girin-korean-ssam-bar...</td>\\n    </tr>\\n  </tbody>\\n</table>'"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_p[colShow].to_html()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.yelp.com/biz/the-pink-door-seattle-4?adjust_creative=KIqjD5y9rnzq9ev6ruaTHQ&utm_campaign=yelp_api_v3&utm_medium=api_v3_business_search&utm_source=KIqjD5y9rnzq9ev6ruaTHQ'"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc.loc[0]['url']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "dc['ur2'] = dc['url'].str.split('?').map(lambda x : x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.yelp.com/biz/the-pink-door-seattle-4'"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dc['ur2'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### making small dataset for offline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: collection_names is deprecated. Use list_collection_names instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['reviews', 'reviews_scrap', 'users', 'restaurants']"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db.collection_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "yl_rests = db['restaurants']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "yl_rest_df = pd.DataFrame(list(yl_rests.find({})))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "tttt = yl_rest_df.iloc[:100,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "tav = yl_rest_df[yl_rest_df.alias =='tablas-woodstone-taverna-mill-creek']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtav = cleaning_data(tav, mod_fe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cl_t0\n",
      "cl_t2\n",
      "cl_t4\n",
      "cl_t5\n",
      "cl_t16\n"
     ]
    }
   ],
   "source": [
    "for x in colClust_n:\n",
    "    if rf.predict(xtav, [x])[0] :\n",
    "        print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}