proof_of_concept.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "import string\n",
    "import re\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "from pyspark.ml.clustering import KMeans, KMeansModel, KMeansSummary\n",
    "from pyspark.ml.feature import CountVectorizer, CountVectorizerModel, IDF, IDFModel\n",
    "from pyspark.sql.functions import udf\n",
    "from pyspark.sql.types import ArrayType, StringType, BooleanType, StructType, StructField\n",
    "from pyspark.ml import Pipeline, PipelineModel\n",
    "\n",
    "from nlp_cl_start import data_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark as ps\n",
    "spark = (ps.sql.SparkSession.builder\n",
    "        .master(\"local[4]\")\n",
    "        .appName(\"yelp_academic\")\n",
    "        .getOrCreate()\n",
    "        )\n",
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#x = PipelineModel.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - hive</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://10.5.81.94:4042\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v2.3.2</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[4]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>PySparkShell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x10ddddc18>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#p = x.load('tfidfkmeans/')\n",
    "km1 = KMeansModel.load('kmeans/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "No training summary available for this KMeansModel",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-123-ea81e14b9387>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mkm1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.2/libexec/python/pyspark/ml/clustering.py\u001b[0m in \u001b[0;36msummary\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    343\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m             raise RuntimeError(\"No training summary available for this %s\" %\n\u001b[0;32m--> 345\u001b[0;31m                                self.__class__.__name__)\n\u001b[0m\u001b[1;32m    346\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: No training summary available for this KMeansModel"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#pipe = PipelineModel.load('tfidf/')\n",
    "pipe = PipelineModel.load('pipe-idf-5-7-18/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#us_ex = pd.read_csv('user_from_scr_clidf18.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0,  1,  2,  4,  5,  7,  8,  9, 10, 11, 12, 13, 16, 17]),\n",
       " array([71, 81, 30, 47, 38, 20, 17, 22,  8, 20,  4, 10,  9,  8]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#np.unique(us_ex.user_cl, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "import time\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "mc = MongoClient()\n",
    "db  = mc['raw_restaurants']\n",
    "rv_s = db['reviews_scrap']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_test = list(rv_s.find({'user_id': '_3KTfz0hcPl_yq4jOyOPiA'}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_df =pd.DataFrame(user_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    a   b\n",
       "0   1   2\n",
       "1   9  20\n",
       "2  17  26\n",
       "3  16   2"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame([{'a':1, 'b':2 },{'a':9, 'b':20 },{'a':17, 'b':26 },{'a':16, 'b':2 }])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddd = spark.createDataFrame(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['_id', 'alias', 'category', 'date', 'id', 'rating', 'text', 'user_id'], dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_df2 = u_df.drop(columns=['_id', 'alias', 'category', 'date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>rating</th>\n",
       "      <th>text</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ErsYo_9v8--feFCy92tymQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Fantastic Thai food with warm service. Authent...</td>\n",
       "      <td>_3KTfz0hcPl_yq4jOyOPiA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>W1NnSItt-SwWHrzkRtZ44g</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Fantastic Vietnamese sandwich: great flavors a...</td>\n",
       "      <td>_3KTfz0hcPl_yq4jOyOPiA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3Q5uzROHYl1i2B_t2sIl7w</td>\n",
       "      <td>3.0</td>\n",
       "      <td>A fine dining restaurant that is in the midst ...</td>\n",
       "      <td>_3KTfz0hcPl_yq4jOyOPiA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-GYe-Aq7NcNJ3u-U6fmwMg</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Great restaurant with an interesting mix of in...</td>\n",
       "      <td>_3KTfz0hcPl_yq4jOyOPiA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pPlsW81ntC2qbF0QOe6pLQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>We always have a good experience here with Ani...</td>\n",
       "      <td>_3KTfz0hcPl_yq4jOyOPiA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       id rating  \\\n",
       "0  ErsYo_9v8--feFCy92tymQ    5.0   \n",
       "1  W1NnSItt-SwWHrzkRtZ44g    5.0   \n",
       "2  3Q5uzROHYl1i2B_t2sIl7w    3.0   \n",
       "3  -GYe-Aq7NcNJ3u-U6fmwMg    5.0   \n",
       "4  pPlsW81ntC2qbF0QOe6pLQ    5.0   \n",
       "\n",
       "                                                text                 user_id  \n",
       "0  Fantastic Thai food with warm service. Authent...  _3KTfz0hcPl_yq4jOyOPiA  \n",
       "1  Fantastic Vietnamese sandwich: great flavors a...  _3KTfz0hcPl_yq4jOyOPiA  \n",
       "2  A fine dining restaurant that is in the midst ...  _3KTfz0hcPl_yq4jOyOPiA  \n",
       "3  Great restaurant with an interesting mix of in...  _3KTfz0hcPl_yq4jOyOPiA  \n",
       "4  We always have a good experience here with Ani...  _3KTfz0hcPl_yq4jOyOPiA  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_sp = spark.createDataFrame(u_df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nlp_cl_start import data_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_token = data_tokenizer(u_sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_pred = pipe.transform(u_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['id',\n",
       " 'rating',\n",
       " 'text',\n",
       " 'user_id',\n",
       " 'token',\n",
       " 'vectors',\n",
       " 'features',\n",
       " 'prediction']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u_pred.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "uu = u_pred.filter((u_pred.rating == '2.0')|(u_pred.rating == '1.0')).select( 'prediction').toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1, 10])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(uu.prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pipe_spar import  cluster_user_by_review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[id: string, rating: string, text: string, user_id: string, token: array<string>]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "up = cluster_user_by_review(u_token, pipe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 19.6 ms, sys: 8.93 ms, total: 28.6 ms\n",
      "Wall time: 2.12 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "up.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(user_id='_3KTfz0hcPl_yq4jOyOPiA', user_cl=4),\n",
       " Row(user_id='_3KTfz0hcPl_yq4jOyOPiA', user_cl=10),\n",
       " Row(user_id='_3KTfz0hcPl_yq4jOyOPiA', user_cl=7),\n",
       " Row(user_id='_3KTfz0hcPl_yq4jOyOPiA', user_cl=5),\n",
       " Row(user_id='_3KTfz0hcPl_yq4jOyOPiA', user_cl=0),\n",
       " Row(user_id='_3KTfz0hcPl_yq4jOyOPiA', user_cl=1)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "up.take(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scraping.add_by_scrap import scrap_by_users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "kk= scrap_by_users('https://www.yelp.com/user_details?userid=KbU6Q0oy5X2mTShnRXiggg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/Cellar/apache-spark/2.3.2/libexec/python/pyspark/sql/session.py:340: UserWarning: inferring schema from dict is deprecated,please use pyspark.sql.Row instead\n",
      "  warnings.warn(\"inferring schema from dict is deprecated,\"\n"
     ]
    }
   ],
   "source": [
    "revs = spark.createDataFrame(kk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_user_clust_from_yelp(reviews):\n",
    "    df = spark.createDataFrame(reviews)\n",
    "    df = df.filter((df.rating == '2.0')|(df.rating == '1.0'))\n",
    "    df = data_tokenizer(df)\n",
    "    clusters = cluster_user_by_review(df, pipe)\n",
    "    return clusters\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/Cellar/apache-spark/2.3.2/libexec/python/pyspark/sql/session.py:340: UserWarning: inferring schema from dict is deprecated,please use pyspark.sql.Row instead\n",
      "  warnings.warn(\"inferring schema from dict is deprecated,\"\n"
     ]
    }
   ],
   "source": [
    "cl = new_user_clust_from_yelp(kk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(user_id='KbU6Q0oy5X2mTShnRXiggg', user_cl=4)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cl.first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "r1 = \"\"\"This review is based only on the drinks and service. \n",
    "\n",
    "Our group got seated at a booth. The decor and ambiance was pretty cool and trendy. When the server came to take our drink order, I had asked her what the Whiteclaw drink was listed under the Beer, Wine, etc. section of the menu. She had told me that it was like a cider that was a little sweet with a hint of grapefruit. That piqued my interest so I ordered that. \n",
    "\n",
    "When she brought our drinks, she placed a really tall can in front of me. I was pretty surprised and wasn't expecting a can, but it was fine. When I took my first sips, I barely tasted anything. I turned the can around to see the front side and noticed at the bottom it said, \"Sparkling water with a hint of grapefruit\". \n",
    "\n",
    "I was pretty confused and bummed out. I wasn't really sure why she would tell me that the Whiteclaw drink was cider. Cider and sparkling water are two very different drinks. \n",
    "\n",
    "I kept sipping and just tried to enjoy it since it was a huge $10 can. But after a few minutes, I decided to order some wine so I can actually enjoy myself. When a different server came to check up on us, my boyfriend asked if I could get the Anderson Valley Rosé. The server told us that that isn't actually a rosé but a completely different kind of drink. It was confusing, but they do actually have a rosé on the menu, which we ordered, but it's not the Anderson Valley drink listed on the menu. \n",
    "\n",
    "When our original server stopped by, she noticed I had the rosé and wasn't drinking my Whiteclaw sparkling water. She asked, \"Did you not like it?\". I said, \"no, not really.\" And all she said was, \"Well I like it.\" and walked away. Had she told me that it was sparkling water, I definitely wouldn't have ordered that and waste $10. \n",
    "\n",
    "Maybe this was just one of those off-chance, bad experiences. Maybe the food is good? The space is definitely pretty cool to hang with friends after dinner or for happy hour. Not sure if I'd return. There's a lot of other bars that have less confusing drink menus and better service that I would prefer going to.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nlp_cl_start import tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "token = tokenize(r1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/Cellar/apache-spark/2.3.2/libexec/python/pyspark/sql/session.py:340: UserWarning: inferring schema from dict is deprecated,please use pyspark.sql.Row instead\n",
      "  warnings.warn(\"inferring schema from dict is deprecated,\"\n"
     ]
    }
   ],
   "source": [
    "rw_df = spark.createDataFrame([{'token': token, 'user_id' :0},{'token': token, 'user_id' :0},{'token': token, 'user_id' :0} ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = cluster_user_by_review(rw_df, pipe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "ff = g.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(user_id=0, user_cl=17)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ff[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_users(cls):\n",
    "    a = []\n",
    "    for cl in cls:\n",
    "        a.append('cl_t'+str(cl[1]))\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cl_t4', 'cl_t2', 'cl_t17']"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "transform_users(cl.collect())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}