one_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "conda: pytorch_playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch, torchvision\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "# from torch import optim\n",
    "# from torch.autograd import Variable\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# import pandas as pd\n",
    "\n",
    "# torch.set_printoptions(linewidth=120)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import models\n",
    "model = models.CNN()\n",
    "model.load_state_dict(torch.load(\"model_folder/model_2_best_1.pt\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "T = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])\n",
    "mnist_data_train = torchvision.datasets.MNIST('mnist_data',\n",
    "                                              transform=T,\n",
    "                                              download=True, \n",
    "                                              train=True)\n",
    "mnist_data_valid = torchvision.datasets.MNIST('mnist_data',\n",
    "                                             transform=T,\n",
    "                                             download=True, \n",
    "                                             train=False)\n",
    "emnist_data_train = torchvision.datasets.EMNIST('emnist_data',\n",
    "                                               split = \"balanced\",\n",
    "                                               transform=T,\n",
    "                                               download=True,\n",
    "                                               train=True)\n",
    "emnist_data_valid = torchvision.datasets.EMNIST('emnist_data',\n",
    "                                               split = \"balanced\",\n",
    "                                               transform=T,\n",
    "                                               download=True,\n",
    "                                               train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target = list(range(10))\n",
    "# target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bs = 2000\n",
    "# mnist_dataloader_train = torch.utils.data.DataLoader(mnist_data_train, batch_size=bs)\n",
    "# mnist_dataloader_valid = torch.utils.data.DataLoader(mnist_data_valid, batch_size=bs)\n",
    "# emnist_dataloader_train = torch.utils.data.DataLoader(emnist_data_train, batch_size=bs)\n",
    "# emnist_dataloader_valid = torch.utils.data.DataLoader(emnist_data_valid, batch_size=bs)\n",
    "\n",
    "# m_batch = next(iter(mnist_dataloader_train))\n",
    "# m_images, m_labels = m_batch\n",
    "\n",
    "# e_batch = next(iter(emnist_dataloader_train))\n",
    "# e_images, e_labels = e_batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MC_dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch = m_batch\n",
    "# images, labels = batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.eval()\n",
    "# with torch.no_grad():\n",
    "#     preds_eval = model(images)\n",
    "\n",
    "# model.train()\n",
    "# preds_eval_s = F.softmax(preds_eval, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# n = 1000\n",
    "# mc_list = []\n",
    "# with torch.no_grad():\n",
    "#     for i in range(n):\n",
    "#         preds = model(images)\n",
    "#         mc_list.append(preds.unsqueeze(dim=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mc = torch.cat((mc_list), dim=1)\n",
    "# mc_s = F.softmax(mc, dim=2)\n",
    "# mc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# img = 0\n",
    "# mc_s[img].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mc.mean(dim=1).argmax(dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.imshow(images[img].squeeze(), cmap = \"gray\")\n",
    "# print(\"label:\", labels[img])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.stripplot(target, preds_eval[img], jitter=0)\\\n",
    "#     .set_title(\"Standard - Pre-softmax\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# sns.stripplot(target, preds_eval_s[img], jitter=0)\\\n",
    "#     .set_title(\"Standard - Pre-softmax\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.stripplot(target*n, mc[img].reshape(mc[img].numel()), jitter=0.3)\\\n",
    "#     .set_title(\"MC_dropout - Pre-softmax\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig_mc_s = sns.stripplot(target*n, mc_s[img].reshape(mc_s[img].numel()), jitter=0.3)\\\n",
    "#     .set_title(\"MC_dropout - Softmax\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig_mc_b = sns.boxplot(target*n, mc_s[img].reshape(mc_s[img].numel()))\n",
    "# fig_mc_b.set_title(\"MC_dropout - Softmax\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig_mc_b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FINAL TRY?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SET: n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "I = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = mnist_data_train[0]\n",
    "image, label = sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    pred_eval = model(image.unsqueeze(dim=0))\n",
    "\n",
    "model.train()\n",
    "pred_eval_s = F.softmax(pred_eval, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "mc_list = []\n",
    "with torch.no_grad():\n",
    "    for i in range(n):\n",
    "        preds = model(image.unsqueeze(dim=0))\n",
    "        mc_list.append(preds.unsqueeze(dim=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 10, 100])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc = torch.cat((mc_list), dim=2)\n",
    "mc_s = F.softmax(mc, dim=1)\n",
    "# mc_s[img].sum() # =n\n",
    "mc.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GET MC VALUES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.percentile(mc_s[0][4], [0, 25, 50, 75, 100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 10, 100])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc_s.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 20])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_values = []\n",
    "for _image in mc_s:\n",
    "    target_values = []\n",
    "    for _target in _image:\n",
    "        target_values.append(_target.mean())\n",
    "        target_values.append(_target.std())\n",
    "    image_values.append(target_values)\n",
    "mc_tensor = torch.tensor(image_values)\n",
    "mc_tensor.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.eval()\n",
    "# with torch.no_grad():\n",
    "#     preds_eval = model(images)\n",
    "\n",
    "# model.train()\n",
    "# preds_eval_s = F.softmax(preds_eval, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mc_labels = pred_eval_s.argmax(dim=1).eq(label).type_as(pred_eval_s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 30])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc_features = torch.cat((mc_tensor, pred_eval_s), dim=1)\n",
    "mc_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = mc_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now add letters to the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch_e = e_batch\n",
    "# images_e, labels_e = batch_e\n",
    "\n",
    "# model.eval()\n",
    "# with torch.no_grad():\n",
    "#     preds_eval_e = model(images_e)\n",
    "\n",
    "# model.train()\n",
    "# preds_eval_s_e = F.softmax(preds_eval_e, dim=1)\n",
    "\n",
    "# mc_list_e = []\n",
    "# with torch.no_grad():\n",
    "#     for i in range(n):\n",
    "#         preds_e = model(images_e)\n",
    "#         mc_list_e.append(preds_e.unsqueeze(dim=2))\n",
    "        \n",
    "# mc_e = torch.cat((mc_list_e), dim=2)\n",
    "# mc_s_e = F.softmax(mc_e, dim=1)\n",
    "# # mc_s[img].sum() # =n\n",
    "# mc_e.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# image_values_e = []\n",
    "# for _image_e in mc_s_e:\n",
    "#     target_values_e = []\n",
    "#     for _target_e in _image_e:\n",
    "#         target_values_e.append(_target_e.mean())\n",
    "#         target_values_e.append(_target_e.std())\n",
    "#     image_values_e.append(target_values_e)\n",
    "# mc_tensor_e = torch.tensor(image_values_e)\n",
    "# mc_tensor_e.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mc_labels_e = preds_eval_s.argmax(dim=1).eq(labels).type_as(preds_eval_s)\n",
    "# mc_labels_e.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # mc_labels_e = [0] * preds_eval_s.shape[0]\n",
    "# mc_labels_e = torch.zeros(2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mc_features_e = torch.cat((mc_tensor_e, preds_eval_s), dim=1)\n",
    "# mc_features_e.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# mc_features_e.eq(mc_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mc_features_e[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mc_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PLOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# targets = []\n",
    "# for i in range(10):\n",
    "#     for _ in range(n):\n",
    "#         targets.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# img = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig_mc_b = sns.boxplot(targets, mc_s[img].reshape(mc_s[img].numel()))\n",
    "# fig_mc_b.set_title(\"MC_dropout - Softmax\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mc_s_e.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig_mc_b_e = sns.boxplot(targets, mc_s_e[img].reshape(mc_s_e[img].numel()))\n",
    "# fig_mc_b_e.set_title(\"MC_dropout - Softmax - e\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine MNIST and EMNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X = torch.cat((mc_features, mc_features_e))\n",
    "# y = torch.cat((mc_labels, mc_labels_e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # y = np.array(y, dtype=int)\n",
    "# one = 0\n",
    "# zero = 0\n",
    "# for i in y:\n",
    "#     if i == 1:\n",
    "#         one += 1\n",
    "#     if i == 0:\n",
    "#         zero += 1\n",
    "# one, zero"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Only extract 2 highest probabilities with its stdev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# index = reversed(X[0][20:30].argsort())[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "eval probs  -  mc prob mean  -  mc prob std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.cat((X[0][20:30][index], X[0][index*2], X[0][index*2+1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_X_list = []\n",
    "# for i in range(len(X)):\n",
    "#     index = reversed(X[i][20:30].argsort())[:2]\n",
    "#     keep_values = torch.cat((X[i][20:30][index], X[i][index*2], X[i][index*2+1]))\n",
    "#     new_X_list.append(keep_values.unsqueeze(dim=0))\n",
    "# new_X = torch.cat(new_X_list, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SAME BUT NOW WITH 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_X_2_list = []\n",
    "for i in range(len(X)):\n",
    "    index_2 = reversed(X[i][20:30].argsort())[:5]\n",
    "    keep_values_2 = torch.cat((X[i][20:30][index_2], X[i][index_2*2], X[i][index_2*2+1]))\n",
    "    new_X_2_list.append(keep_values_2.unsqueeze(dim=0))\n",
    "new_X_2 = torch.cat(new_X_2_list, dim=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NOW ALL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_X_3_list = []\n",
    "# for i in range(len(X)):\n",
    "#     index_3 = reversed(X[i][20:30].argsort())\n",
    "#     keep_values_3 = torch.cat((X[i][20:30][index_3], X[i][index_3*2], X[i][index_3*2+1]))\n",
    "#     new_X_3_list.append(keep_values_3.unsqueeze(dim=0))\n",
    "# new_X_3 = torch.cat(new_X_3_list, dim=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Only stdev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# newer_X_list = []\n",
    "# for i in range(len(X)):\n",
    "#     stdevs = X[i][[1, 3, 5, 7, 9, 11, 13, 15, 17, 19]]\n",
    "#     newer_X_list.append(stdevs.unsqueeze(dim=0))\n",
    "# newer_X = torch.cat(newer_X_list, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# newer_X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Only stdev + odered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# newerer_X_list = []\n",
    "# for i in range(len(X)):\n",
    "#     stdevs = sorted(X[i][[1, 3, 5, 7, 9, 11, 13, 15, 17, 19]])\n",
    "#     newerer_X_list.append(torch.tensor(stdevs).unsqueeze(dim=0))\n",
    "# newerer_X = torch.cat(newerer_X_list, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sorted(X[i][[1, 3, 5, 7, 9, 11, 13, 15, 17, 19]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# newerer_X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NEW MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import sklearn as sk \n",
    "# sk.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.metrics import balanced_accuracy_score\n",
    "# from sklearn import metrics\n",
    "\n",
    "# from sklearn import model_selection\n",
    "# from sklearn.model_selection import cross_val_score\n",
    "\n",
    "# from sklearn.linear_model import LogisticRegression\n",
    "# from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "# from sklearn.ensemble import RandomForestClassifier\n",
    "# from sklearn.tree import DecisionTreeClassifier\n",
    "# from sklearn.neighbors import KNeighborsClassifier\n",
    "# from sklearn.naive_bayes import GaussianNB\n",
    "# from sklearn.svm import SVC\n",
    "\n",
    "# from datetime import datetime\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_X, test_X, train_y, test_y = train_test_split(newer_X, y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def compare_models(X, Y, kfold_cv=10): \n",
    "#     '''\n",
    "#     Comparing 6 models: LR (linear regression), LDA (logistic discriminant \n",
    "#     analysis), RF (random forest), DT (decision tree), KNN (k-nearest neighbours),\n",
    "#     NB (Naive Bayes classifier) using k-fold crossvalidation. \n",
    "#     A plot will be shown to demonstrate the distribution of the accuracy values.\n",
    "    \n",
    "#     Input: X_train and y_train dataframes, preset: 10 fold CV.\n",
    "    \n",
    "#     Output: Dataframe with the 6 models and their properties.\n",
    "#     '''\n",
    "#     # prepare models and append the various models\n",
    "#     models = []\n",
    "#     models.append(('LR',  LogisticRegression()))\n",
    "#     models.append(('LDA', LinearDiscriminantAnalysis()))\n",
    "#     models.append(('RF',  RandomForestClassifier()))\n",
    "#     models.append(('DT',  DecisionTreeClassifier()))\n",
    "#     models.append(('KNN', KNeighborsClassifier()))\n",
    "#     models.append(('NB',  GaussianNB()))\n",
    "#     models.append(('SVM', SVC()))\n",
    "#     # evaluate each model in turn\n",
    "#     results = []\n",
    "#     names = []\n",
    "#     scoring = 'accuracy' #we use the accuracy as the estimator of the quality of the models\n",
    "#     # train the models using cross validation\n",
    "#     for name, model in models:\n",
    "#         # prepare the cross validation\n",
    "#         kfold = model_selection.KFold(n_splits=kfold_cv)\n",
    "#         # get the cross validation results (where the magic happens)\n",
    "#         cv_results = model_selection.cross_val_score(model, X, Y, cv=kfold, scoring=scoring)\n",
    "#         # append these results to the results list\n",
    "#         results.append(cv_results)\n",
    "#         # append the name to the name list\n",
    "#         names.append(name)\n",
    "#         # put it in a nice format and extract the meand and std\n",
    "#         model_acc = \"{}: \\t{:.4f} ({:.4f})\".format(name, cv_results.mean(), cv_results.std()) \n",
    "#         # print the accuracy values of the models\n",
    "#         print(model_acc)\n",
    "#     # boxplot algorithm comparison, make a boxplot of the results\n",
    "#     fig = plt.figure()\n",
    "#     fig.suptitle('Algorithm Comparison (accuracy)')\n",
    "#     ax = fig.add_subplot(111)\n",
    "#     plt.boxplot(results)\n",
    "#     ax.set_xticklabels(names)\n",
    "#     plt.show()\n",
    "#     # make dataframe with the names of he model and the actual model (to be saved)\n",
    "#     models_df = pd.DataFrame(models, columns = [\"model\", \"model_info\"])\n",
    "#     return models_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare_models(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# compare_models(new_X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# uncertainty_models = compare_models(new_X_2, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare_models(new_X_3, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare_models(newer_X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare_models(newerer_X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rf = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rf.fit(new_X_2, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.bar(range(len(rf.feature_importances_)), rf.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(rf, open(\"model_folder/rf_uncertainty_model.sav\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_model = pickle.load(open(\"model_folder/rf_uncertainty_model.sav\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.], dtype=float32)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_model.predict(new_X_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_eval_s.argmax(dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_X_2[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}