Two_Layer_Neural_Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Design a simple 2 layer NN to train CIFAR10 dataset\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Search and locate the dataset and functions if it is in a remote directory\n",
    "# For simplicity, download the dataset to the current folder and skip this cell\n",
    "import os\n",
    "with os.scandir(path = 'Path to your dataset') as entries:\n",
    "    for entry in entries:\n",
    "        print(entry.name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with os.scandir(path = 'Path to dataloader function') as entries:\n",
    "    for entry in entries:\n",
    "        print(entry.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the 'load_CIFAR' module to this jupyter notebook if it is in a remote directory \n",
    "# For simplicity, download the loader module to the current folder togetherwith the dataset and skip this cell\n",
    "import sys\n",
    "my_path_dir = 'Path to dataloader function'\n",
    "sys.path.insert(0, my_path_dir)\n",
    "mod =  __import__('data_utils') # this is available in cs231 stanford assignment\n",
    "sys.path.pop(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if it is working\n",
    "help(mod.load_CIFAR10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the CIFAR-10 dataset\n",
    "# if this is not working, run the next cell instead\n",
    "cifar10_dir = ' path to the data '\n",
    "X_training, y_training, X_test, y_test = mod.load_CIFAR10(cifar10_dir)\n",
    "print(X_training.shape, y_training.shape, X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shuffle the indices and spare a validation set from training set\n",
    "rand_indices = np.random.permutation(50000)\n",
    "\n",
    "# Split the validation and training data\n",
    "training_indices = rand_indices[:48000]\n",
    "validation_indices = rand_indices[48000:]\n",
    "\n",
    "# re-assign the training and validation sets\n",
    "X_train = X_training[training_indices]\n",
    "X_val = X_training[validation_indices]\n",
    "y_train = y_training[training_indices]\n",
    "y_val = y_training[validation_indices]\n",
    "print(X_train.shape, X_val.shape, y_train.shape, y_val.shape)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape each instance to one long row. Each instance originally has 32 x 3 x 3 attributes\n",
    "X_train = X_train.reshape(48000, -1)\n",
    "X_val = X_val.reshape(2000, -1)\n",
    "X_test = X_test.reshape(10000, -1)\n",
    "print(X_train.shape, X_val.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Zero center the data by substracting the mean of all instances from each instance\n",
    "mean_image = np.mean(X_train, axis=0)\n",
    "X_train -= mean_image\n",
    "X_val -= mean_image\n",
    "X_test -= mean_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# View the data\n",
    "print('Train data shape: ', X_train.shape)\n",
    "print('Train labels shape: ', y_train.shape)\n",
    "print('Validation data shape: ', X_val.shape)\n",
    "print('Validation labels shape: ', y_val.shape)\n",
    "print('Test data shape: ', X_test.shape)\n",
    "print('Test labels shape: ', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a simple 2 layer neural network to train the cifar 10 data\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "class Two_Layer_Neural_Network():\n",
    "    \n",
    "    ''' This is a simple 2 layer Neural Network made for image recognition. The model is run on cifar-10 dataset.\n",
    "    Batch size is randomly picked and training is done with 1000 iterations to overfit the data intentionally. \n",
    "    Forward propagation function uses two inverted drop outs, leaky relu, softmax methods. \n",
    "    The loss function uses negative log likelihood method. Regulation is not used since drop out is used 2 times.\n",
    "    Updating the network parameters is done by Adam's update and are saved to .txt file '''\n",
    "    \n",
    "    def __init__(self, training_data, training_labels, val_data, val_labels, test_data, test_labels, \n",
    "                 batch_size = 400 , std = 1e-3): \n",
    "        # input size is training_data.shape[1]\n",
    "        self.training_data = training_data\n",
    "        self.training_labels = training_labels\n",
    "        self.val_data = val_data\n",
    "        self.val_labels = val_labels\n",
    "        self.test_data = test_data\n",
    "        self.test_labels = test_labels\n",
    "        self.batch_size = batch_size\n",
    "        # define the shape of input data\n",
    "        N, D = training_data.shape\n",
    "        # size of layers is chosen\n",
    "        self.input_size = D\n",
    "        # there are 10 classes in labels\n",
    "        self.output_size = 10\n",
    "        # randomly choose a length for hidden size\n",
    "        # the optimum hidden size should be choosen with a grid or random search between 50 - 2000\n",
    "        self.hidden_size = 200\n",
    "        # initialize the parameters and keep track of them\n",
    "        self.params = {}\n",
    "        # std is chosen 1e-3 since emprically xavier initialization method works between: 1e-2 and 1e-4\n",
    "        # the optimum initialization of std constant should be chosen with a grid search between 1e-1 to 1e-5\n",
    "        self.params['w1'] = std * np.random.randn(self.input_size, self.hidden_size) \n",
    "        self.params['w2'] = std * np.random.randn(self.hidden_size, self.output_size) \n",
    "        # the dot product of input and w1 yields a shape of 2d with [input size axis = 0 , hidden size axis = 1]\n",
    "        self.params['b1'] = np.zeros([self.batch_size, self.hidden_size])\n",
    "        # the dot product of hidden and w2 yields a shape of 2d with [hidden size axis = 0 , output size axis = 1]\n",
    "        self.params['b2'] = np.zeros([self.batch_size, self.output_size])\n",
    "        # keep track of the changing grads \n",
    "        self.grads = {} \n",
    "        self.grads['w1'] = np.zeros(self.params['w1'].shape)\n",
    "        self.grads['w2'] = np.zeros(self.params['w2'].shape)\n",
    "        self.grads['b1'] = np.zeros(self.params['b1'].shape)\n",
    "        self.grads['b2'] = np.zeros(self.params['b2'].shape) \n",
    "        # keep track of loss, parameters while training the network\n",
    "        self.loss_history = []\n",
    "        self.train_accuracy_history = []\n",
    "        self.val_accuracy_history = []\n",
    "    \n",
    "        \n",
    "    def forward_prop(self, input_data):\n",
    "        self.input_data = input_data\n",
    "        # forward pass, first layer \n",
    "        fw1 = np.dot(input_data, self.params['w1']) + self.params['b1']\n",
    "        # use leaky relu as it is the most popular activation function (relu would be just np.max(fw1, 0))\n",
    "        self.alpha_for_leaky_relu = 1e-6\n",
    "        fw1_relu = self.alpha_for_leaky_relu * fw1\n",
    "        # hidden layer is ready\n",
    "        self.hidden_layer = np.maximum(fw1_relu , fw1)\n",
    "        # use inverted drop out for regularization \n",
    "        # traditional regularization loss: reg_constant * np.sum(w**2) for each w added to the loss function\n",
    "        drop_out_mask = (np.random.randn(*self.hidden_layer.shape) < 0.5) / 0.5 # dropout mask\n",
    "        self.hidden_layer *= drop_out_mask  # drop out anything lower than 0.5 and double anything above 0.5\n",
    "        # forward pass second layer \n",
    "        fw2 = np.dot(self.hidden_layer, self.params['w2']) + self.params['b2']\n",
    "        # use inverted drop out for regularization\n",
    "        drop_out_mask2 = (np.random.randn(*fw2.shape) < 0.5) / 0.5 # dropout mask\n",
    "        fw2 *= drop_out_mask2\n",
    "        scores = fw2\n",
    "        # convert the scores to softmax scores \n",
    "        find_max = np.max(scores, axis = 1)\n",
    "        find_max = find_max[:, np.newaxis]\n",
    "        trimmed_scores = scores - find_max\n",
    "        exp_scores = np.exp(trimmed_scores)\n",
    "        self.probs = exp_scores / np.sum(exp_scores, axis = 1)[:, np.newaxis]\n",
    "        # output softmax scores\n",
    "        return self.probs\n",
    "    \n",
    "    \n",
    "    def loss(self, input_labels): # (select columns and press: \" CTRL and / \" to undo the #)\n",
    "        # based on softmax scores, the targets are compared with labels with negative log loss function\n",
    "        corr_probs = self.probs[range(len(self.probs)), input_labels] # for all instances extract the label indices\n",
    "        corr_avg_log_probs = (-np.sum(np.log(corr_probs))) / len(input_labels)\n",
    "        # to avoid inf or zero division errors caused by dtype, change the data structure to float64\n",
    "        data_loss =  corr_avg_log_probs\n",
    "        # since drop out is used for regularization, the regularization loss is not added to total data loss\n",
    "        # reg_loss would be = reg * np.sum(W1*W1) + reg * np.sum(W2*W2) \n",
    "        return data_loss\n",
    "        # if this code outputs inf values, use scikit learn's log loss function  \n",
    "        return data_loss\n",
    "        \n",
    "    \n",
    "    def backward_prop(self, input_labels):\n",
    "        # backward pass \n",
    "        # to calculate the dscores, we create a dloss we use dscores \n",
    "        dscores = np.copy(self.probs)\n",
    "        # the derivative of the combination of softmax and negative log-likelihood is: probs - 1\n",
    "        dscores[range(len(input_labels)), input_labels] -= 1\n",
    "        # calculate the gradient of the average loss\n",
    "        dscores = dscores / len(input_labels)\n",
    "        # db2 has dimensions of N x scores and dscores has dimension of N x scores\n",
    "        self.grads['b2'] += dscores \n",
    "        # backprop w2 and b2\n",
    "        self.grads['w2'] = np.dot(self.hidden_layer.T, dscores)\n",
    "        # backprop hidden layer\n",
    "        dhidden = np.dot(dscores, self.params['w2'].T)\n",
    "        # backprop the ReLU non-linearity\n",
    "        dhidden[self.hidden_layer <= 0] = self.alpha_for_leaky_relu\n",
    "        # continue with db1 and dw1\n",
    "        self.grads['b1'] += dhidden\n",
    "        self.grads['w1'] = np.dot(self.input_data.T, dhidden) \n",
    "        # since we used drop out we are not adding the regularization gradient contribution\n",
    "        # dW2 += 2*reg * W2\n",
    "        # dW1 += 2*reg * W1\n",
    "        return self.grads\n",
    "        \n",
    "        \n",
    "    def train(self, learning_rate = 1e-3, learning_rate_decay = 0.99, training_iter = 1000):\n",
    "        ''' The learning rate, learning rate decay, batch size and number of iterations should be optimized.\n",
    "        Any change in these hyperparameters yield a different result.'''\n",
    "        # train the network with training data and check the training accuracy with validation data\n",
    "        for i in range(training_iter): \n",
    "            # Use SGD to train the model and optimize the parameters \n",
    "            rand_indices = np.random.permutation(len(self.training_data))\n",
    "            batch_indices = rand_indices[:self.batch_size]\n",
    "            training_batch_data = self.training_data[batch_indices]\n",
    "            # assign the batch labels and keep the last training batch data\n",
    "            training_batch_labels =  self.training_labels[batch_indices]\n",
    "            self.last_training_batch_data = training_batch_data\n",
    "            # forward pass with training batch data\n",
    "            self.forward_prop(training_batch_data)\n",
    "            # execute the below loss function when the optimum hyper parameters are found and stop training\n",
    "            total_data_loss = self.loss(training_batch_labels)           \n",
    "            # keep an eye on the training loss and calculate the accuracy of training and validation sets \n",
    "            if i % 50 == 0:\n",
    "                # check the training loss \n",
    "                self.loss_history.append(total_data_loss) # the last training loss is stored\n",
    "                # predict the accuracy from the last batch training batch\n",
    "                predicted_training_batch = self.predict(training_batch_data)\n",
    "                # check training accuracy \n",
    "                predict_train_accuracy = self.accuracy(predicted_training_batch, training_batch_labels)\n",
    "                # store the training accuracy\n",
    "                self.train_accuracy_history.append(predict_train_accuracy) # training accuracy is stored\n",
    "                # predict the accuracy of validation set\n",
    "                # prepare validation set with random indices \n",
    "                rand_ind_for_val = np.random.permutation(len(self.val_data))\n",
    "                one_batch_of_ind = rand_ind_for_val[:self.batch_size]\n",
    "                val_batch_data = self.val_data[one_batch_of_ind]\n",
    "                val_batch_labels = self.val_labels[one_batch_of_ind]\n",
    "                # check validation accuracy \n",
    "                predicted_val_results = self.predict(val_batch_data)\n",
    "                accuracy_val_results = self.accuracy(predicted_val_results, val_batch_labels)\n",
    "                # store validation accuracy\n",
    "                self.val_accuracy_history.append(accuracy_val_results) # validation accuracy is stored\n",
    "                #training loss, training accuracy and validation accuracy are all stored for each check point\n",
    "           \n",
    "            backward = self.backward_prop(training_batch_labels) # dw1, db1, dw1, dw2 stored in grads dictionary\n",
    "            # update the parameters\n",
    "            # REMINDER Adam update's general formula for one parameter is:\n",
    "            # m = beta1*m + (1-beta1)*dx\n",
    "            # v = beta2*v + (1-beta2)*(dx**2)\n",
    "            # x += - learning_rate * m / (np.sqrt(v) + eps)\n",
    "            eps = 1e-8\n",
    "            beta1 = 0.9 \n",
    "            beta2 = 0.999 \n",
    "            ms = {}\n",
    "            vs = {}\n",
    "            for k in self.params.keys():\n",
    "                ms[k] = np.zeros(self.params[k].shape)\n",
    "                vs[k] = np.zeros(self.params[k].shape)\n",
    "            \n",
    "            for k in self.params.keys():\n",
    "                ms[k] = (beta1 * ms[k]) + (1 - beta1) * self.grads[k]\n",
    "                vs[k] = (beta2 * vs[k]) + (1 - beta2) * np.square(self.grads[k])\n",
    "                self.params[k] += -learning_rate * ms[k] / (np.sqrt(vs[k]) + eps)\n",
    "                \n",
    "                   \n",
    "            # reminder for vanilla update:\n",
    "            # for k in ['w1', 'b1', 'w2', 'b2']:\n",
    "            #    self.params[k] += -learning_rate * grads[k]\n",
    "            \n",
    "            # Decay learning rate\n",
    "            learning_rate *= learning_rate_decay\n",
    "            \n",
    "#         after finding the optimum number of iterations and tuning all hyper-parameters, \n",
    "#         retrain this network with these properties and then save the model weights to the current directory\n",
    "#         for k in self.params.keys():\n",
    "#             name = str(k) + '.txt'\n",
    "#             np.savetxt(name, self.params[k])\n",
    "            \n",
    "            \n",
    "    def predict(self, data_for_prediction): # prediction is made and targets are returned\n",
    "        predictions_list = np.zeros(len(data_for_prediction))\n",
    "        for i in range(len(data_for_prediction)):\n",
    "            # Use the trained weights for prediction. No dropouts, no activation functions, or softmax       \n",
    "            hidden = np.maximum(0, np.dot(data_for_prediction[i], self.params['w1']) + self.params['b1'][0])     \n",
    "            scores = np.dot(hidden, self.params['w2']) + self.params['b2'][0]\n",
    "            prediction =  np.argmax(scores) #  np.argmax(scores, axis=1) if multiple instances are loaded\n",
    "            predictions_list[i] = prediction\n",
    "        return predictions_list\n",
    "    \n",
    "    \n",
    "    def accuracy(self, predictions, labels): # load the data to be predicted and their labels\n",
    "        # results = self.predict(data_for_prediction)\n",
    "        accuracy = np.mean(predictions == labels)\n",
    "        return accuracy\n",
    "        \n",
    "        \n",
    "    def show_loss(self):\n",
    "        print(self.loss_history)\n",
    "        print(self.val_accuracy_history)\n",
    "        print(self.train_accuracy_history)\n",
    "#         plt.subplot(3,1,1)\n",
    "#         plt.plot(self.loss_history)\n",
    "#         plt.xlabel('each 50 iterations')\n",
    "#         plt.ylabel('loss history')\n",
    "#         plt.subplot(3,1,2)\n",
    "#         plt.plot(self.train_accuracy_history)\n",
    "#         plt.xlabel('each 50 iterations')\n",
    "#         plt.ylabel('training accuracy')\n",
    "#         plt.subplot(3,1,3)\n",
    "#         plt.plot(self.val_accuracy_history)\n",
    "#         plt.xlabel('each 50 iterations')\n",
    "#         plt.ylabel('test accuracy')\n",
    "\n",
    "    \n",
    "    def test_accuracy(self):\n",
    "        self.test_accuracy_history = []\n",
    "        test_predictions = []\n",
    "        for each_instance in self.test_data:\n",
    "            predict = self.predict(each_instance)\n",
    "            test_predictions.append(predict)\n",
    "            test_predictions = np.array(predictions)\n",
    "        # compare the predictions and actual labels\n",
    "        return self.accuracy(test_predictions, self.test_labels)\n",
    "       \n",
    "    \n",
    "\n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "test_NN = Two_Layer_Neural_Network(X_train, y_train, X_val, y_val, X_test, y_test)\n",
    "test_NN.train()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_NN.show_loss()\n",
    "#test_NN.test_accuracy()\n",
    "np.set_printoptions(precision = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}