Capstone_project_CNN_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.preprocessing import sequence\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.layers.convolutional import Conv1D, MaxPooling1D, Conv2D, MaxPooling2D, AveragePooling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers import Dropout, Flatten, Bidirectional, Dense, Activation, TimeDistributed\n",
    "from keras.models import Model, Sequential\n",
    "from keras.utils import np_utils\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from string import ascii_lowercase\n",
    "from collections import Counter\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import Doc2Vec\n",
    "from gensim.models import doc2vec\n",
    "from gensim.models import KeyedVectors\n",
    "import itertools, nltk, snowballstemmer, re\n",
    "\n",
    "LabeledSentence = doc2vec.LabeledSentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LabeledLineSentence(object):\n",
    "    def __init__(self, sources):\n",
    "        self.sources = sources\n",
    "        \n",
    "        flipped = {}\n",
    "        \n",
    "        # make sure that keys are unique\n",
    "        for key, value in sources.items():\n",
    "            if value not in flipped:\n",
    "                flipped[value] = [key]\n",
    "            else:\n",
    "                raise Exception('Non-unique prefix encountered')\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for source, prefix in self.sources.items():\n",
    "            with utils.smart_open(source) as fin:\n",
    "                for item_no, line in enumerate(fin):\n",
    "                    yield LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no])\n",
    "    \n",
    "    def to_array(self):\n",
    "        self.sentences = []\n",
    "        for source, prefix in self.sources.items():\n",
    "            with utils.smart_open(source) as fin:\n",
    "                for item_no, line in enumerate(fin):\n",
    "                    self.sentences.append(LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no]))\n",
    "        return self.sentences\n",
    "    \n",
    "    def sentences_perm(self):\n",
    "        shuffled = list(self.sentences)\n",
    "        random.shuffle(shuffled)\n",
    "        return shuffled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('deceptive-opinion-spam-corpus.zip', compression='zip', header=0, sep=',', quotechar='\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['polarity'] = np.where(data['polarity']=='positive', 1, 0)\n",
    "data['deceptive'] = np.where(data['deceptive']=='truthful', 1, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_class(c):\n",
    "    if c['polarity'] == 1 and c['deceptive'] == 1:\n",
    "        return [1,1]\n",
    "    elif c['polarity'] == 1 and c['deceptive'] == 0:\n",
    "        return [1,0]\n",
    "    elif c['polarity'] == 0 and c['deceptive'] == 1:\n",
    "        return [0,1]\n",
    "    else:\n",
    "        return [0,0]\n",
    "    \n",
    "def specific_class(c):\n",
    "    if c['polarity'] == 1 and c['deceptive'] == 1:\n",
    "        return \"TRUE_POSITIVE\"\n",
    "    elif c['polarity'] == 1 and c['deceptive'] == 0:\n",
    "        return \"FALSE_POSITIVE\"\n",
    "    elif c['polarity'] == 0 and c['deceptive'] == 1:\n",
    "        return \"TRUE_NEGATIVE\"\n",
    "    else:\n",
    "        return \"FALSE_NEGATIVE\"\n",
    "\n",
    "data['final_class'] = data.apply(create_class, axis=1)\n",
    "data['given_class'] = data.apply(specific_class, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = data['final_class']\n",
    "# encode class values as integers\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(Y)\n",
    "encoded_Y = encoder.transform(Y)\n",
    "# convert integers to dummy variables (i.e. one hot encoded)\n",
    "dummy_y = np_utils.to_categorical(encoded_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "textData = pd.DataFrame(list(data['text'])) # each row is one document; the raw text of the document should be in the 'text_data' column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize stemmer\n",
    "stemmer = snowballstemmer.EnglishStemmer()\n",
    "\n",
    "# grab stopword list, extend it a bit, and then turn it into a set for later\n",
    "stop = stopwords.words('english')\n",
    "stop.extend(['may','also','zero','one','two','three','four','five','six','seven','eight','nine','ten','across','among','beside','however','yet','within']+list(ascii_lowercase))\n",
    "stoplist = stemmer.stemWords(stop)\n",
    "stoplist = set(stoplist)\n",
    "stop = set(sorted(stop + list(stoplist))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove characters and stoplist words, then generate dictionary of unique words\n",
    "textData[0].replace('[!\"#%\\'()*+,-./:;<=>?@\\[\\]^_`{|}~1234567890’”“′‘\\\\\\]',' ',inplace=True,regex=True)\n",
    "wordlist = filter(None, \" \".join(list(set(list(itertools.chain(*textData[0].str.split(' ')))))).split(\" \"))\n",
    "data['stemmed_text_data'] = [' '.join(filter(None,filter(lambda word: word not in stop, line))) for line in textData[0].str.lower().str.split(' ')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove all words that don't occur at least 5 times and then stem the resulting docs\n",
    "minimum_count = 1\n",
    "str_frequencies = pd.DataFrame(list(Counter(filter(None,list(itertools.chain(*data['stemmed_text_data'].str.split(' '))))).items()),columns=['word','count'])\n",
    "low_frequency_words = set(str_frequencies[str_frequencies['count'] < minimum_count]['word'])\n",
    "data['stemmed_text_data'] = [' '.join(filter(None,filter(lambda word: word not in low_frequency_words, line))) for line in data['stemmed_text_data'].str.split(' ')]\n",
    "data['stemmed_text_data'] = [\" \".join(stemmer.stemWords(re.sub('[!\"#%\\'()*+,-./:;<=>?@\\[\\]^_`{|}~1234567890’”“′‘\\\\\\]',' ', next_text).split(' '))) for next_text in data['stemmed_text_data']]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "lmtzr = WordNetLemmatizer()\n",
    "w = re.compile(\"\\w+\",re.I)\n",
    "\n",
    "def label_sentences(df, input_point):\n",
    "    labeled_sentences = []\n",
    "    list_sen = []\n",
    "    for index, datapoint in df.iterrows():\n",
    "        tokenized_words = re.findall(w,datapoint[input_point].lower())\n",
    "        labeled_sentences.append(LabeledSentence(words=tokenized_words, tags=['SENT_%s' %index]))\n",
    "        list_sen.append(tokenized_words)\n",
    "    return labeled_sentences, list_sen\n",
    "\n",
    "def train_doc2vec_model(labeled_sentences):\n",
    "    model = Doc2Vec(min_count=1, window=9, size=512, sample=1e-4, negative=5, workers=7)\n",
    "    model.build_vocab(labeled_sentences)\n",
    "    pretrained_weights = model.wv.syn0\n",
    "    vocab_size, embedding_size = pretrained_weights.shape\n",
    "    model.train(labeled_sentences, total_examples=vocab_size, epochs=400)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Call to deprecated `LabeledSentence` (Class will be removed in 4.0.0, use TaggedDocument instead).\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "textData = data['stemmed_text_data'].to_frame().reset_index()\n",
    "sen, corpus = label_sentences(textData, 'stemmed_text_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc2vec_model = train_doc2vec_model(sen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc2vec_model.save(\"doc2vec_model_opinion_corpus.d2v\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc2vec_model = Doc2Vec.load(\"doc2vec_model_opinion_corpus.d2v\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "tfidf1 = TfidfVectorizer(tokenizer=lambda i:i, lowercase=False, ngram_range=(1,1))\n",
    "result_train1 = tfidf1.fit_transform(corpus)\n",
    "\n",
    "tfidf2 = TfidfVectorizer(tokenizer=lambda i:i, lowercase=False, ngram_range=(1,2))\n",
    "result_train2 = tfidf2.fit_transform(corpus)\n",
    "\n",
    "tfidf3 = TfidfVectorizer(tokenizer=lambda i:i, lowercase=False, ngram_range=(1,3))\n",
    "result_train3 = tfidf3.fit_transform(corpus)\n",
    "\n",
    "svd = TruncatedSVD(n_components=512, n_iter=40, random_state=34)\n",
    "tfidf_data1 = svd.fit_transform(result_train1)\n",
    "tfidf_data2 = svd.fit_transform(result_train2)\n",
    "tfidf_data3 = svd.fit_transform(result_train3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import spacy\n",
    "\n",
    "nlp = spacy.load('en')\n",
    "temp_textData = pd.DataFrame(list(data['text']))\n",
    "\n",
    "overall_pos_tags_tokens = []\n",
    "overall_pos = []\n",
    "overall_tokens = []\n",
    "overall_dep = []\n",
    "\n",
    "for i in range(1600):\n",
    "    doc = nlp(temp_textData[0][i])\n",
    "    given_pos_tags_tokens = []\n",
    "    given_pos = []\n",
    "    given_tokens = []\n",
    "    given_dep = []\n",
    "    for token in doc:\n",
    "        output = \"%s_%s\" % (token.pos_, token.tag_)\n",
    "        given_pos_tags_tokens.append(output)\n",
    "        given_pos.append(token.pos_)\n",
    "        given_tokens.append(token.tag_)\n",
    "        given_dep.append(token.dep_)\n",
    "        \n",
    "    overall_pos_tags_tokens.append(given_pos_tags_tokens)\n",
    "    overall_pos.append(given_pos)\n",
    "    overall_tokens.append(given_tokens)\n",
    "    overall_dep.append(given_dep)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "count = CountVectorizer(tokenizer=lambda i:i, lowercase=False)\n",
    "pos_tags_data = count.fit_transform(overall_pos_tags_tokens).todense()\n",
    "pos_data = count.fit_transform(overall_pos).todense()\n",
    "tokens_data = count.fit_transform(overall_tokens).todense()\n",
    "dep_data = count.fit_transform(overall_dep).todense()\n",
    "min_max_scaler = MinMaxScaler()\n",
    "normalized_pos_tags_data = min_max_scaler.fit_transform(pos_tags_data)\n",
    "normalized_pos_data = min_max_scaler.fit_transform(pos_data)\n",
    "normalized_tokens_data = min_max_scaler.fit_transform(tokens_data)\n",
    "normalized_dep_data = min_max_scaler.fit_transform(dep_data)\n",
    "\n",
    "final_pos_tags_data = np.zeros(shape=(1600, 512)).astype(np.float32)\n",
    "final_pos_data = np.zeros(shape=(1600, 512)).astype(np.float32)\n",
    "final_tokens_data = np.zeros(shape=(1600, 512)).astype(np.float32)\n",
    "final_dep_data = np.zeros(shape=(1600, 512)).astype(np.float32)\n",
    "final_pos_tags_data[:normalized_pos_tags_data.shape[0],:normalized_pos_tags_data.shape[1]] = normalized_pos_tags_data\n",
    "final_pos_data[:normalized_pos_data.shape[0],:normalized_pos_data.shape[1]] = normalized_pos_data\n",
    "final_tokens_data[:normalized_tokens_data.shape[0],:normalized_tokens_data.shape[1]] = normalized_tokens_data\n",
    "final_dep_data[:normalized_dep_data.shape[0],:normalized_dep_data.shape[1]] = normalized_dep_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "370\n"
     ]
    }
   ],
   "source": [
    "maxlength = []\n",
    "for i in range(0,len(sen)):\n",
    "    maxlength.append(len(sen[i][0]))\n",
    "    \n",
    "print(max(maxlength))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   index                                  stemmed_text_data  \\\n",
      "0      0  stay night getaway famili thursday tripl aaa r...   \n",
      "1      1  tripl rate upgrad view room less $ includ brea...   \n",
      "\n",
      "                                 vectorized_comments  \n",
      "0  [-0.78479135, -0.21383545, -0.46632105, -1.006...  \n",
      "1  [-0.9833992, -0.2526004, 0.3219868, -0.0411983...  \n"
     ]
    }
   ],
   "source": [
    "def vectorize_comments(df,d2v_model):\n",
    "    y = []\n",
    "    comments = []\n",
    "    for i in range(0,df.shape[0]):\n",
    "        label = 'SENT_%s' %i\n",
    "        comments.append(d2v_model.docvecs[label])\n",
    "    df['vectorized_comments'] = comments\n",
    "    \n",
    "    return df\n",
    "\n",
    "textData = vectorize_comments(textData,doc2vec_model)\n",
    "print (textData.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/usr/local/lib/python3.6/site-packages/sklearn/grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn import cross_validation\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "X_train, X_test, y_train, y_test = cross_validation.train_test_split(textData[\"vectorized_comments\"].T.tolist(), \n",
    "                                                                     dummy_y, \n",
    "                                                                     test_size=0.1, \n",
    "                                                                     random_state=56)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(textData[\"vectorized_comments\"].T.tolist()).reshape((1,1600,512))\n",
    "y = np.array(dummy_y).reshape((1600,4))\n",
    "X_train2 = np.array(X_train).reshape((1,1440,512))\n",
    "y_train2 = np.array(y_train).reshape((1,1440,4))\n",
    "X_test2 = np.array(X_test).reshape((1,160,512))\n",
    "y_test2 = np.array(y_test).reshape((1,160,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "Xtemp = textData[\"vectorized_comments\"].T.tolist()\n",
    "ytemp = data['given_class']\n",
    "training_indices = []\n",
    "testing_indices = []\n",
    "\n",
    "skf = StratifiedKFold(n_splits=10)\n",
    "skf.get_n_splits(Xtemp, ytemp)\n",
    "\n",
    "for train_index, test_index in skf.split(Xtemp, ytemp):\n",
    "    training_indices.append(train_index)\n",
    "    testing_indices.append(test_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train3 = np.zeros(shape=(1440, max(maxlength)+9, 512)).astype(np.float32)\n",
    "Y_train3 = np.zeros(shape=(1440, 4)).astype(np.float32)\n",
    "X_test3 = np.zeros(shape=(160, max(maxlength)+9, 512)).astype(np.float32)\n",
    "Y_test3 = np.zeros(shape=(160, 4)).astype(np.float32)\n",
    "\n",
    "empty_word = np.zeros(512).astype(np.float32)\n",
    "\n",
    "count_i = 0\n",
    "for i in training_indices[0]:\n",
    "    len1 = len(sen[i][0])\n",
    "    average_vector1 = np.zeros(512).astype(np.float32)\n",
    "    average_vector2 = np.zeros(512).astype(np.float32)\n",
    "    average_vector3 = np.zeros(512).astype(np.float32)\n",
    "    for j in range(max(maxlength)+9):\n",
    "        if j < len1:\n",
    "            X_train3[count_i,j,:] = doc2vec_model[sen[i][0][j]]\n",
    "            average_vector1 += result_train1[i, tfidf1.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
    "            average_vector2 += result_train2[i, tfidf2.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
    "            average_vector3 += result_train3[i, tfidf3.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
    "        elif j == len1:\n",
    "            X_train3[count_i,j,:] = tfidf_data2[i]\n",
    "        elif j == len1+1:\n",
    "            X_train3[count_i,j,:] = tfidf_data3[i]\n",
    "        elif j == len1+2:\n",
    "            X_train3[count_i,j,:] = average_vector2\n",
    "        elif j == len1+3:\n",
    "            X_train3[count_i,j,:] = average_vector3\n",
    "        elif j == len1+4:\n",
    "            X_train3[count_i,j,:] = final_pos_tags_data[i] \n",
    "        elif j == len1+5:\n",
    "            X_train3[count_i,j,:] = final_pos_data[i]\n",
    "        elif j == len1+6:\n",
    "            X_train3[count_i,j,:] = final_tokens_data[i]\n",
    "        elif j == len1+7:\n",
    "            X_train3[count_i,j,:] = final_dep_data[i]\n",
    "        elif j == len1+8:\n",
    "            X_train3[count_i,j,:] = (1/float(3)) * (tfidf_data1[i] + tfidf_data2[i] + tfidf_data3[i])\n",
    "        else:\n",
    "            X_train3[count_i,j,:] = empty_word\n",
    "    \n",
    "    Y_train3[count_i,:] = dummy_y[i]\n",
    "    count_i += 1\n",
    "    \n",
    "\n",
    "count_i = 0\n",
    "for i in testing_indices[0]:\n",
    "    len1 = len(sen[i][0])\n",
    "    average_vector1 = np.zeros(512).astype(np.float32)\n",
    "    average_vector2 = np.zeros(512).astype(np.float32)\n",
    "    average_vector3 = np.zeros(512).astype(np.float32)\n",
    "    for j in range(max(maxlength)+9):\n",
    "        if j < len1:\n",
    "            X_test3[count_i,j,:] = doc2vec_model[sen[i][0][j]]\n",
    "            average_vector1 += result_train1[i, tfidf1.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
    "            average_vector2 += result_train2[i, tfidf2.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]  \n",
    "            average_vector3 += result_train3[i, tfidf3.vocabulary_[sen[i][0][j]]] * doc2vec_model[sen[i][0][j]]\n",
    "        elif j == len1:\n",
    "            X_test3[count_i,j,:] = tfidf_data2[i]\n",
    "        elif j == len1+1:\n",
    "            X_test3[count_i,j,:] = tfidf_data3[i]\n",
    "        elif j == len1+2:\n",
    "            X_test3[count_i,j,:] = average_vector2\n",
    "        elif j == len1+3:\n",
    "            X_test3[count_i,j,:] = average_vector3\n",
    "        elif j == len1+4:\n",
    "            X_test3[count_i,j,:] = final_pos_tags_data[i]\n",
    "        elif j == len1+5:\n",
    "            X_test3[count_i,j,:] = final_pos_data[i]\n",
    "        elif j == len1+6:\n",
    "            X_test3[count_i,j,:] = final_tokens_data[i]\n",
    "        elif j == len1+7:\n",
    "            X_test3[count_i,j,:] = final_dep_data[i]\n",
    "        elif j == len1+8:\n",
    "            X_test3[count_i,j,:] = (1/float(3)) * (tfidf_data1[i] + tfidf_data2[i] + tfidf_data3[i])\n",
    "        else:\n",
    "            X_test3[count_i,j,:] = empty_word\n",
    "     \n",
    "    Y_test3[count_i,:] = dummy_y[i]\n",
    "    count_i += 1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_37 (Conv1D)           (None, 379, 128)          131200    \n",
      "_________________________________________________________________\n",
      "dropout_73 (Dropout)         (None, 379, 128)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_30 (MaxPooling (None, 189, 128)          0         \n",
      "_________________________________________________________________\n",
      "dropout_74 (Dropout)         (None, 189, 128)          0         \n",
      "_________________________________________________________________\n",
      "bidirectional_29 (Bidirectio (None, 200)               183200    \n",
      "_________________________________________________________________\n",
      "dense_28 (Dense)             (None, 4)                 804       \n",
      "=================================================================\n",
      "Total params: 315,204\n",
      "Trainable params: 315,204\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv1D(filters=128, kernel_size=2, padding='same', activation='relu', input_shape=(max(maxlength)+9,512)))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(Dropout(0.3))\n",
    "\n",
    "#model.add(Bidirectional(LSTM(50, dropout=0.3, recurrent_dropout=0.2, return_sequences=True)))\n",
    "model.add(Bidirectional(LSTM(100, dropout=0.3, recurrent_dropout=0.2)))\n",
    "model.add(Dense(4, activation='softmax'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1440 samples, validate on 160 samples\n",
      "Epoch 1/100\n",
      "1440/1440 [==============================] - 18s 13ms/step - loss: 0.5619 - acc: 0.7500 - val_loss: 0.5313 - val_acc: 0.7500\n",
      "Epoch 2/100\n",
      "1440/1440 [==============================] - 10s 7ms/step - loss: 0.5223 - acc: 0.7502 - val_loss: 0.4681 - val_acc: 0.7547\n",
      "Epoch 3/100\n",
      "1440/1440 [==============================] - 10s 7ms/step - loss: 0.4592 - acc: 0.7694 - val_loss: 0.3433 - val_acc: 0.8516\n",
      "Epoch 4/100\n",
      "1440/1440 [==============================] - 1797s 1s/step - loss: 0.3562 - acc: 0.8411 - val_loss: 0.2898 - val_acc: 0.8734\n",
      "Epoch 5/100\n",
      "1440/1440 [==============================] - 23s 16ms/step - loss: 0.2915 - acc: 0.8799 - val_loss: 0.2634 - val_acc: 0.8859\n",
      "Epoch 6/100\n",
      "1440/1440 [==============================] - 506s 351ms/step - loss: 0.2714 - acc: 0.8844 - val_loss: 0.2403 - val_acc: 0.8938\n",
      "Epoch 7/100\n",
      "1440/1440 [==============================] - 25s 17ms/step - loss: 0.2307 - acc: 0.9106 - val_loss: 0.2104 - val_acc: 0.9172\n",
      "Epoch 8/100\n",
      "1440/1440 [==============================] - 2226s 2s/step - loss: 0.2138 - acc: 0.9137 - val_loss: 0.2445 - val_acc: 0.9047\n",
      "Epoch 9/100\n",
      "1440/1440 [==============================] - 25s 18ms/step - loss: 0.2108 - acc: 0.9170 - val_loss: 0.2387 - val_acc: 0.9062\n",
      "Epoch 10/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.1922 - acc: 0.9266 - val_loss: 0.2083 - val_acc: 0.9219\n",
      "Epoch 11/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.1819 - acc: 0.9264 - val_loss: 0.2292 - val_acc: 0.9000\n",
      "Epoch 12/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.1680 - acc: 0.9326 - val_loss: 0.2124 - val_acc: 0.9219\n",
      "Epoch 13/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.1609 - acc: 0.9387 - val_loss: 0.2127 - val_acc: 0.9219\n",
      "Epoch 14/100\n",
      "1440/1440 [==============================] - 35s 24ms/step - loss: 0.1557 - acc: 0.9382 - val_loss: 0.1966 - val_acc: 0.9266\n",
      "Epoch 15/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.1478 - acc: 0.9405 - val_loss: 0.1977 - val_acc: 0.9203\n",
      "Epoch 16/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.1326 - acc: 0.9516 - val_loss: 0.1983 - val_acc: 0.9375\n",
      "Epoch 17/100\n",
      "1440/1440 [==============================] - 37s 26ms/step - loss: 0.1286 - acc: 0.9521 - val_loss: 0.2112 - val_acc: 0.9234\n",
      "Epoch 18/100\n",
      "1440/1440 [==============================] - 37s 26ms/step - loss: 0.1128 - acc: 0.9599 - val_loss: 0.1953 - val_acc: 0.9375\n",
      "Epoch 19/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.1139 - acc: 0.9562 - val_loss: 0.1995 - val_acc: 0.9422\n",
      "Epoch 20/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.1184 - acc: 0.9552 - val_loss: 0.1989 - val_acc: 0.9375\n",
      "Epoch 21/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.1053 - acc: 0.9625 - val_loss: 0.2198 - val_acc: 0.9281\n",
      "Epoch 22/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0937 - acc: 0.9663 - val_loss: 0.2205 - val_acc: 0.9406\n",
      "Epoch 23/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.0929 - acc: 0.9672 - val_loss: 0.2339 - val_acc: 0.9250\n",
      "Epoch 24/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.0853 - acc: 0.9687 - val_loss: 0.2164 - val_acc: 0.9391\n",
      "Epoch 25/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.0831 - acc: 0.9715 - val_loss: 0.2460 - val_acc: 0.9156\n",
      "Epoch 26/100\n",
      "1440/1440 [==============================] - 39s 27ms/step - loss: 0.0879 - acc: 0.9689 - val_loss: 0.2224 - val_acc: 0.9406\n",
      "Epoch 27/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0729 - acc: 0.9752 - val_loss: 0.2373 - val_acc: 0.9219\n",
      "Epoch 28/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.0759 - acc: 0.9717 - val_loss: 0.2370 - val_acc: 0.9219\n",
      "Epoch 29/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.0714 - acc: 0.9748 - val_loss: 0.2420 - val_acc: 0.9312\n",
      "Epoch 30/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.0648 - acc: 0.9757 - val_loss: 0.2326 - val_acc: 0.9312\n",
      "Epoch 31/100\n",
      "1440/1440 [==============================] - 1820s 1s/step - loss: 0.0531 - acc: 0.9804 - val_loss: 0.2313 - val_acc: 0.9328\n",
      "Epoch 32/100\n",
      "1440/1440 [==============================] - 15s 10ms/step - loss: 0.0660 - acc: 0.9771 - val_loss: 0.2562 - val_acc: 0.9328\n",
      "Epoch 33/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0511 - acc: 0.9833 - val_loss: 0.2481 - val_acc: 0.9359\n",
      "Epoch 34/100\n",
      "1440/1440 [==============================] - 752s 522ms/step - loss: 0.0576 - acc: 0.9806 - val_loss: 0.2510 - val_acc: 0.9312\n",
      "Epoch 35/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0670 - acc: 0.9769 - val_loss: 0.2351 - val_acc: 0.9344\n",
      "Epoch 36/100\n",
      "1440/1440 [==============================] - 1964s 1s/step - loss: 0.0620 - acc: 0.9753 - val_loss: 0.2451 - val_acc: 0.9312\n",
      "Epoch 37/100\n",
      "1440/1440 [==============================] - 35s 24ms/step - loss: 0.0508 - acc: 0.9819 - val_loss: 0.2390 - val_acc: 0.9406\n",
      "Epoch 38/100\n",
      "1440/1440 [==============================] - 2002s 1s/step - loss: 0.0463 - acc: 0.9854 - val_loss: 0.2632 - val_acc: 0.9219\n",
      "Epoch 39/100\n",
      "1440/1440 [==============================] - 31s 22ms/step - loss: 0.0500 - acc: 0.9833 - val_loss: 0.2753 - val_acc: 0.9203\n",
      "Epoch 40/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0376 - acc: 0.9878 - val_loss: 0.2753 - val_acc: 0.9250\n",
      "Epoch 41/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0483 - acc: 0.9830 - val_loss: 0.2725 - val_acc: 0.9187\n",
      "Epoch 42/100\n",
      "1440/1440 [==============================] - 35s 25ms/step - loss: 0.0362 - acc: 0.9899 - val_loss: 0.2657 - val_acc: 0.9344\n",
      "Epoch 43/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0320 - acc: 0.9892 - val_loss: 0.2632 - val_acc: 0.9234\n",
      "Epoch 44/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0336 - acc: 0.9889 - val_loss: 0.2689 - val_acc: 0.9312\n",
      "Epoch 45/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0317 - acc: 0.9887 - val_loss: 0.2789 - val_acc: 0.9312\n",
      "Epoch 46/100\n",
      "1440/1440 [==============================] - 39s 27ms/step - loss: 0.0271 - acc: 0.9917 - val_loss: 0.2734 - val_acc: 0.9344\n",
      "Epoch 47/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0264 - acc: 0.9918 - val_loss: 0.3114 - val_acc: 0.9187\n",
      "Epoch 48/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0275 - acc: 0.9911 - val_loss: 0.3009 - val_acc: 0.9281\n",
      "Epoch 49/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0270 - acc: 0.9910 - val_loss: 0.3157 - val_acc: 0.9250\n",
      "Epoch 50/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0253 - acc: 0.9927 - val_loss: 0.3020 - val_acc: 0.9312\n",
      "Epoch 51/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0280 - acc: 0.9915 - val_loss: 0.2999 - val_acc: 0.9219\n",
      "Epoch 52/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0186 - acc: 0.9944 - val_loss: 0.3064 - val_acc: 0.9281\n",
      "Epoch 53/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0233 - acc: 0.9917 - val_loss: 0.3088 - val_acc: 0.9344\n",
      "Epoch 54/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0209 - acc: 0.9918 - val_loss: 0.3198 - val_acc: 0.9312\n",
      "Epoch 55/100\n",
      "1440/1440 [==============================] - 40s 28ms/step - loss: 0.0183 - acc: 0.9948 - val_loss: 0.2984 - val_acc: 0.9297\n",
      "Epoch 56/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0238 - acc: 0.9913 - val_loss: 0.2967 - val_acc: 0.9344\n",
      "Epoch 57/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0156 - acc: 0.9953 - val_loss: 0.2883 - val_acc: 0.9328\n",
      "Epoch 58/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0222 - acc: 0.9934 - val_loss: 0.2772 - val_acc: 0.9375\n",
      "Epoch 59/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0155 - acc: 0.9951 - val_loss: 0.2778 - val_acc: 0.9406\n",
      "Epoch 60/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0194 - acc: 0.9937 - val_loss: 0.2738 - val_acc: 0.9391\n",
      "Epoch 61/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0176 - acc: 0.9944 - val_loss: 0.2858 - val_acc: 0.9406\n",
      "Epoch 62/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0141 - acc: 0.9957 - val_loss: 0.3122 - val_acc: 0.9406\n",
      "Epoch 63/100\n",
      "1440/1440 [==============================] - 38s 26ms/step - loss: 0.0154 - acc: 0.9962 - val_loss: 0.2951 - val_acc: 0.9391\n",
      "Epoch 64/100\n",
      "1440/1440 [==============================] - 37s 26ms/step - loss: 0.0133 - acc: 0.9962 - val_loss: 0.3103 - val_acc: 0.9406\n",
      "Epoch 65/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0112 - acc: 0.9977 - val_loss: 0.3165 - val_acc: 0.9359\n",
      "Epoch 66/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0167 - acc: 0.9939 - val_loss: 0.3319 - val_acc: 0.9312\n",
      "Epoch 67/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0137 - acc: 0.9965 - val_loss: 0.3239 - val_acc: 0.9344\n",
      "Epoch 68/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0159 - acc: 0.9951 - val_loss: 0.3476 - val_acc: 0.9344\n",
      "Epoch 69/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0168 - acc: 0.9951 - val_loss: 0.3449 - val_acc: 0.9297\n",
      "Epoch 70/100\n",
      "1440/1440 [==============================] - 36s 25ms/step - loss: 0.0199 - acc: 0.9944 - val_loss: 0.3233 - val_acc: 0.9297\n",
      "Epoch 71/100\n",
      "1280/1440 [=========================>....] - ETA: 3s - loss: 0.0145 - acc: 0.9955"
     ]
    }
   ],
   "source": [
    "model.fit(X_train3, Y_train3, epochs=100, batch_size=256, validation_data=(X_test3, Y_test3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}