ml_fashion_mnist_image_classification.py

# -*- coding: utf-8 -*-
"""ML_Fashion_MNIST_Image_Classification.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1xUedGkXtOAGo3k6_qTDPJeJsjVJLCyLd

# Fashion MNIST Image Augmentation Effects on Model Accuracy

This notebook contains data processing, models, experiements and evaluation of the Fashion MNIST data. The analysis in the notebook is focused on the effects image augmentation has on various models and loss types.

**High level summary of results:**
* MLP ANN had the strongest accuracy followed by SVM (nearly identical accuracy) and then kNN.
* All models had similiar reactions to the image augmentations with inverted pixels, resizing, and horizontal flip causing the biggest decline in accuracy.
* The default levels of noise had minimal impact on model accuracy, but as noise levels increased, kNN performed the best. SVM and MLP ANN had the largest drop in accuracy as noise was added.
* MLP ANN was by far the fastest to train and predict.
* Different loss functions from Cross Entropy (Hinge, MSE, MAE) had a lower overall accuracy with small amount of noise and as noise increased, all loss functions performed similar.
"""

# from google.colab import drive
# drive.mount('/content/drive')

##################################################
# Imports
##################################################

import numpy as np
import cv2
import os
import pandas as pd
import matplotlib.pyplot as plt

import torch
from datetime import datetime
from sklearn.model_selection import train_test_split
from skimage import util, filters, transform
from sklearn import preprocessing
from sklearn.svm import SVC
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.neighbors import KNeighborsClassifier
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras import optimizers
from keras.callbacks import ModelCheckpoint, EarlyStopping
import seaborn as sns
from collections import defaultdict
import json

# Setting the device
if torch.cuda.is_available():
    print('GPU enabled!')
    device = torch.device("cuda:0")
else:
    print('You are not using the GPU, activate it following:')
    print('"Runtime" --> "Change runtime type" --> "Hardware accelerator" --> "GPU" --> "Save"')
    device = torch.device("cpu")

##################################################
# Params
##################################################

DATA_BASE_FOLDER = '/data'

# Initialize models_dict that will hold all necessary model information
reports_dict = {
    'ann_ce' : {'loss':'binary_crossentropy', 'noise_report' : {}},
    'ann_hinge' : {'loss':'hinge', 'noise_report' : {}},
    'ann_mse' : {'loss':'mean_squared_error', 'noise_report' : {}},
    'ann_mae' : {'loss':'mean_absolute_error', 'noise_report' : {}}
}
models_dict = {}

history_dict = {}

##################################################
# Utils
##################################################

def custom_classification_report(model, x_test_dict,y_test, model_type):
    """
    model: fit model to use for prediction
    x_test_dict: dictinary of test datasets
    y_test: classification vector
    model_type: string of the name of the model
    """
    class_dict = {}
    pred_scores_dict = {}
    for k, x_test in x_test_dict.items():

      print('Predicting {} {}...'.format(model_type,k))
      
      # Make prediction
      pred_score = model.predict(x_test)
      
      #Apply argmax to pred_score if it is an ANN model
      if model_type == 'ANN':
        pred_score = [np.argmax(x) for x in pred_score]
        pred_score = np.array(pred_score)

      # Find Accuracy, Precision and Recall on the test set and store in dict
      report = classification_report(y_test, pred_score,output_dict=True)
      class_dict[k] = report
      pred_scores_dict[k] = pred_score

      print('Finished predicting {} {}!'.format(model_type,k))
      
    return pred_scores_dict, class_dict

def summarize_report(report_dict,metric, model_type):
  df = pd.DataFrame()
  for k,v in report_dict.items():
    df[k] = pd.DataFrame(v).transpose()[metric]
  df.index = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
              'Shirt', 'Sneaker', 'Bag', 'Ankle boot','accuracy','micro avg','weighted avg']
  df['model'] = model_type
  return df

def plot_confusion_matrix(pred_dict,fig_title):
  """
  pred_dict: a dictionary of predictions for each image augmentation
  fig_title: the title to show over top all subplots
  """
  # Initialize subplots
  fig, axs = plt.subplots(1,6, figsize=(18,3))
  fig.tight_layout(pad=3.0)

  # Loop through each image augmentation prediction and plot the confusion matrix
  col_idx = 0
  for k,y_pred in pred_dict.items():

    # Create confusion matrix
    cm = confusion_matrix(new_y_test, y_pred)
    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    # Plot confusion matrix
    h = axs[col_idx].imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)
    axs[col_idx].set_title(k)
    axs[col_idx].set_xticks(np.arange(len(y_labels)))
    axs[col_idx].set_xticklabels(y_labels, rotation=90)
    axs[col_idx].set_yticks(np.arange(len(y_labels)))
    axs[col_idx].set_yticklabels(y_labels)

    col_idx += 1
  fig.suptitle(fig_title, fontsize=20)

"""# Dataset

The dataset contains 50k train + 10k validation images of 10 different categories ('T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot').

Each image is a 28x28 grayscale, and for simplicity here is flattened into a 784 dimensional vector.
"""

##################################################
# Load dataset
##################################################

x_train = np.load(os.path.join(DATA_BASE_FOLDER, 'train.npy'))
x_valid = np.load(os.path.join(DATA_BASE_FOLDER, 'validation.npy'))
x_test = np.load(os.path.join(DATA_BASE_FOLDER, 'test.npy'))
y_train = pd.read_csv(os.path.join(DATA_BASE_FOLDER, 'train.csv'))['class'].values
y_valid = pd.read_csv(os.path.join(DATA_BASE_FOLDER, 'validation.csv'))['class'].values

y_labels = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

# Plot random images of different classes
plt.figure(figsize=(25, 5))
for idx in range(20):
    plt.subplot(1, 20, idx + 1)
    img = x_train[idx].reshape(28, 28)
    plt.title(f'{y_labels[y_train[idx]]}')
    plt.imshow(img, cmap='gray')
    plt.axis('off')
plt.show()

##################################################
# Additional data splits for upcoming experiments
##################################################

# For the purposes of our study, we created a new test dataset out of the training datset
new_x_train, new_x_test, new_y_train, new_y_test = train_test_split(x_train, 
                                                                    y_train, 
                                                                    test_size=.2, 
                                                                    random_state=101)

# Because SVM and KNN take a long time to run, we randomly sampled a smaller training set from the same training set used with ANN
smaller_x_train, ignore_x_test, smaller_y_train, ignore_y_test = train_test_split(new_x_train, 
                                                                                  new_y_train, 
                                                                                  test_size=30000, 
                                                                                  random_state=101)

y_train_one = np.eye(len(y_labels))[np.array(new_y_train.astype(int)).reshape(-1)]
y_valid_one = np.eye(len(y_labels))[np.array(y_valid.astype(int)).reshape(-1)]
y_test_one = np.eye(len(y_labels))[np.array(new_y_test.astype(int)).reshape(-1)]

##################################################
# Create new test datasets with image augmentation
##################################################

x_test_original = list()
x_test_invert = list() 
x_test_noise = list()
x_test_blur = list()
x_test_eq = list()
x_test_resize = list()
x_test_hflip = list()

#variables used within the loop
zeros = np.zeros((5,28))

# Loop through every row in the test dataset and apply image augmentations
for x in new_x_test:

  # Original image
  x_test_original.append(x)

  # Create inverted image (flip the grayscale so black is white and white is back)
  x_test_invert.append(util.invert(x))

  # Add random noise to image
  x_test_noise.append(util.random_noise(x))

  # Reshape image for next augmentations to work properly
  img = x.reshape(28,28)

  # Create blurred image
  x_test_blur.append(filters.gaussian(img).reshape(-1))
  
  # Create resized image
  img_resize = transform.resize(img,(18, 28))
  img_resize = np.vstack((img_resize, zeros))
  img_resize = np.vstack((zeros, img_resize))
  x_test_resize.append(img_resize.reshape(-1))
  
  # Create horizontal flipped test dataset
  x_test_hflip.append(np.flip(img,axis=1).copy().reshape(-1))

# Pass image lists through a MinMax Scaler to ensure pixel range is 0 to 255
mm_scaler = preprocessing.MinMaxScaler(feature_range=(0, 255))
x_test_original = mm_scaler.fit_transform(x_test_original)
x_test_invert = mm_scaler.fit_transform(x_test_invert)
x_test_noise = mm_scaler.fit_transform(x_test_noise)
x_test_blur = mm_scaler.fit_transform(x_test_blur)
x_test_resize = mm_scaler.fit_transform(x_test_resize)
x_test_hflip = mm_scaler.fit_transform(x_test_hflip)

# Create dict of all image augmentation data
x_test_dict = {
    'original': x_test_original,
    'invert': x_test_invert,
    'noise': x_test_noise,
    'blur': x_test_blur,
    'resize': x_test_resize,
    'hflip': x_test_hflip
}

##################################################
# Create new test datasets with noise augmentations
##################################################

# Initialize MinMaxScaler to ensure all pixels are from 0 to 255
mm_scaler = preprocessing.MinMaxScaler(feature_range=(0, 255))

# Loop through all test images and store 21 different levels of noise to dict
noise_dict = {}
for noise in np.linspace(0, 1, num = 21):
  image_list = list()
  for idx,image in enumerate(new_x_test):
    image_list.append(util.random_noise(image,var=noise))
  image_list = mm_scaler.fit_transform(image_list)
  noise_dict[noise] = image_list

"""# Model

This section contains our model builds for the upcoming eperiments. The following models will be used:

*   Support Vector Machine (SVM)
*   K-Nearest Neighbors (kNN)
*   Multi-Layer Perceptron Artifical Neural Network (ANN)


"""

##################################################
# Create new folder to save results
##################################################

model_iter = datetime.now().strftime("%Y_%m_%d_%H_%M")
newpath = r'/data/{}'.format(model_iter) 
if not os.path.exists(newpath):
    os.makedirs(newpath)

"""## SVM"""

##################################################
# SVM MODEL FIT
##################################################

clf_svm = SVC(C=10, verbose = True, random_state=101)
clf_svm.fit(smaller_x_train,smaller_y_train)
models_dict['svm'] = clf_svm

"""## KNN"""

##################################################
# KNN MODEL BUILD AND FIT
##################################################
clf_knn = KNeighborsClassifier(n_neighbors=10)
clf_knn.fit(smaller_x_train,smaller_y_train)
models_dict['knn'] = clf_knn

"""## ANN"""

##################################################
# ANN MODEL FUNCTION
##################################################

# Function to quickly build new ANN MLP models
def ANNModel(layers_size = [10], activation = 'relu', lr = .00005, 
             dropout_rate = .1, loss = 'binary_crossentropy'):
  model = Sequential()
  for idx, l in enumerate(layers_size):
    if idx+1 == len(layers_size):
      model.add(Dense(l, activation='sigmoid'))
    else:
      model.add(Dense(l, activation=activation))
      model.add(Dropout(dropout_rate))
  adm_opt = optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=None,
                            decay=0.0, amsgrad=False)
  model.compile(loss=loss, optimizer=adm_opt, metrics=['accuracy'])
  return model

##################################################
# ANN MODELS BUILD AND FIT
##################################################

# Hyper Parameters
activation = 'relu'
lr = .00005
epochs = 100
dropout_rate = 0.1
batch_size = 256

# Set callbacks for model fitting and early stopping with patience of 5
checkpoint = ModelCheckpoint("model.h5", monitor='val_loss', verbose=1,
                             save_best_only=True, mode='min')
early = EarlyStopping(monitor='val_loss', mode='min', patience=5)
callback = [checkpoint, early]

# Loop thorugh all ANN models for build and fit
for model_key, report_dict in reports_dict.items():
  if 'ann' in model_key:
    # Build model keeping all hyper parameters fixed except model_loss
    print('Building {}...'.format(model_key))
    model = ANNModel(layers_size = [1400, 128, 10],
                    activation = activation,
                    lr = lr,
                    dropout_rate = dropout_rate,
                    loss = report_dict['loss'])
    
    # Store model in models_dict
    models_dict[model_key] = model
    print('Finished building {}'.format(model_key))
    print("")

    # Fit Model
    print('Fitting {}...'.format(model_key))
    history = model.fit(new_x_train, y_train_one, batch_size=batch_size,
                        epochs=epochs, validation_data=(x_valid, y_valid_one),
                        callbacks=callback)
    
    # Store fit model history into reports_dict
    history_dict[model_key] = history
    print('Finished fitting {}'.format(model_key))
    print("")

##################################################
# Plot Learning Curves of Training and Validation
##################################################

# Initialize subplots
fig, axs = plt.subplots(2,4, figsize = [20,8])
fig.tight_layout(pad=3.0)

# Loop through each model history to plot loss and accuracy learning curves
model_idx = 0
for model_key, history in history_dict.items():

  if 'ann' in model_key:
    
    # Plot learning curves for loss
    axs[0,model_idx].plot(history.history['loss'])
    axs[0,model_idx].plot(history.history['val_loss'])
    axs[0,model_idx].set_title('{} - LOSS'.format(model_key.upper()))
    axs[0,model_idx].set_ylabel('loss')
    axs[0,model_idx].set_xlabel('epoch')
    axs[0,model_idx].legend(['train', 'test'], loc='upper left')

    # Plot learning curves for accuracy
    axs[1,model_idx].plot(history.history['accuracy'])
    axs[1,model_idx].plot(history.history['val_accuracy'])
    axs[1,model_idx].set_title('{} - ACCURACY'.format(model_key.upper()))
    axs[1,model_idx].set_ylabel('accuracy')
    axs[1,model_idx].set_xlabel('epoch')
    _ = axs[1,model_idx].legend(['train', 'test'], loc='upper left')
  
    model_idx += 1
    # plt.savefig('/data/{}/ann_learning_curves_{}.png'.format(model_iter,model_iter))

"""# Experiments and Evaluation

The following reports were conducted below:


*   What model best predicts the Fashion MNIST images: MLP ANN, SVM, or kNN?
*   What model can best predict augmented images they were never trained on?
*   What model is the most robust as levels of noise are added to the images?
* Given the speed of ANN compared to SVM and kNN, is there a way to make the ANN more robust by changing the loss function?
"""

##################################################
# RUN ALL PREDICTIONS FOR EXPERIMENTS BELOW
##################################################

# THIS TAKES ABOUT 20 MINUTES TO RUN 
# SKIP DOWN TWO CELLS FOR SHORTCUT

#SVM
svm_pred_dict, svm_dict = custom_classification_report(models_dict['svm'], x_test_dict,new_y_test,'SVM')
reports_dict['svm'] = {'report' : svm_dict}
reports_dict['svm'].update({'predictions' : svm_pred_dict})

#KNN
knn_pred_dict, knn_dict = custom_classification_report(models_dict['knn'], x_test_dict,new_y_test,'KNN')
reports_dict['knn'] = {'report' : knn_dict}
reports_dict['knn'].update({'predictions' : knn_pred_dict})

#ANN
ann_pred_dict, ann_dict = custom_classification_report(models_dict['ann_ce'], x_test_dict,new_y_test,'ANN')
reports_dict['ann_ce'].update({'report' : ann_dict})
reports_dict['ann_ce'].update({'predictions' : ann_pred_dict})

# To save us from having to run this again...
def default(obj):
    if isinstance(obj, np.ndarray):
        return obj.tolist()
    raise TypeError('Not serializable')

path = '/data/{}/model_reports_{}.json'.format(model_iter,model_iter)
with open(path, 'w') as fp:
    json.dump(reports_dict, fp, default=default)

##################################################
# RUN PREDICTIONS FOR ALL NOISE LEVELS
##################################################

# THIS TAKES ABOUT 90+ MINUTES TO RUN 
# SKIP TO NEXT CELL FOR SHORTCUT

reports_dict['svm'].update({'noise_report' : {}})
reports_dict['knn'].update({'noise_report' : {}})

report_dict = defaultdict(dict)
for k, x_test in noise_dict.items():
  for model_type, model in models_dict.items():
    print(k, model_type)
    print('predicting {}...'.format(model_type))
    pred_score = model.predict(x_test)
    if 'ann' in model_type:
      pred_score = np.array([np.argmax(x) for x in pred_score])
    temp_dict = {k: classification_report(new_y_test, pred_score,output_dict=True)}
    reports_dict[model_type]['noise_report'].update(temp_dict)
    print('completed {}!'.format(model_type))

path = '/data/{}/model_reports_{}.json'.format(model_iter,model_iter)
with open(path, 'w') as fp:
    json.dump(reports_dict, fp, default=default)

##################################################
# PREDICTIONS SHORTCUT - DIRECT JSON LOAD
##################################################

# If you have the csv and json file for the above code, use this to load results
with open('/data/model_reports_2021_01_10_11_59.json') as handle:
    reports_dict = json.loads(handle.read())

"""## Model Comparison
* **Experiment:** What model best predicts the Fashion MNIST images: MLP ANN, SVM, or KNN?

* **Hypothesis:** Given popularity of using deep learning to capture the complexity of image recognition, we believe the ANN will have the highest accuracy.

* **Summary of findings:** All models did well, but ANN had the higest accuracy. The accuracy was as follows:
  * ANN: 88.4%
  * SVM: 87.5%
  * KNN: 81.8%
"""

##################################################
# MODEL ACCURACY COMPARISON
##################################################

# Create summary report data frames for data visualization
svm_df = summarize_report(reports_dict['svm']['report'],'f1-score','svm')
knn_df = summarize_report(reports_dict['knn']['report'],'f1-score','knn')
ann_df = summarize_report(reports_dict['ann_ce']['report'],'f1-score','ann')

# Combine data frames together 
combined_df = pd.concat([svm_df, knn_df, ann_df])

# Data manipulation needed to visualize data
viz_df = combined_df.loc['accuracy',:]
viz_df = pd.melt(viz_df,id_vars=['model'], value_vars=['original', 'invert', 'noise', 'blur', 'resize', 'hflip'])

# Plot model comparisons across data augmentation types
plt.figure(figsize=(20, 6))
bar = viz_df[viz_df['variable']=='original'].plot.bar(x = 'model', y = 'value', 
                                                      legend = False, xlabel = 'Model', 
                                                      ylabel = 'Accuracy', title = 'Accuracy Comaprison By Model', rot=0)
for i, v in enumerate(viz_df[viz_df['variable']=='original']['value']):
    _ = plt.text(i-.17, v-.08, str(round(v,3)), color='black')
plt.rcParams.update({'font.size': 14})
plt.tight_layout()
# plt.savefig('/data/{}/accuracy_comparison_by_model_{}.png'.format(model_iter,model_iter))

##################################################
# MODEL CLASS COMPARISON
##################################################

# Create df for visualization
class_viz_df = combined_df.loc[~combined_df.index.isin(['accuracy', 'micro avg', 'weighted avg']),['original','model']]
class_viz_df.index.name = 'class'
class_viz_df = class_viz_df.reset_index()

# Plot model comparisons across data augmentation types
g = sns.catplot(x='class', y='original', hue='model', data = class_viz_df, kind='bar',height=5, aspect=2, legend=False)
g.fig.suptitle('F1-Score Comaprison of Models By Class')
g.set_ylabels('F1-Score')
_ = g.set_xlabels('Classification')

plt.rcParams.update({'font.size': 14})
plt.legend(loc='upper right', fontsize = 'small')
plt.tight_layout()
# plt.savefig('/data/{}/f1-score_comparison_of_models_by_class_{}.png'.format(model_iter,model_iter))

"""## Image Augmentation Testing
* **Experiment:** What model can best predict augmented images they were never trained on?

* **Hypothesis:** All models will equally struggle to predict image augmentation with inverted and resizing being the most difficult to predict.

* **Summary of findings:** All models struggled to predict image augmentations at relatively equal proportions. Noise and Blur had the best accuracy across models, while inverted, resized and flipped had a large drop in prediction accuracy. Horizontally flipped images did not predict any shoe type well.
"""

##################################################
# Visualize Image Augmentation
##################################################

for k, v in x_test_dict.items():
  print(k)
  plt.figure(figsize=(25, 20))
  for idx in range(20):
      plt.subplot(1, 20, idx + 1)
      img = np.array(v[idx].reshape(28,28))
      plt.title(f'{y_labels[new_y_test[idx]]}')
      plt.imshow(img, cmap='gray')
      plt.axis('off')
  plt.show()

##################################################
# Compare Model Accuracy Across Image Augmentations
##################################################

# Plot model comparisons across data augmentation types
plt.figure(figsize=(20, 6))
g = sns.catplot(x='variable',y='value', hue='model', data = viz_df, kind='bar',height=7, aspect=1.9, legend=False)
g.fig.suptitle('Accuracy of Models Across Image Augmentations')
g.set_ylabels('Accuracy')
_ = g.set_xlabels('Image Augmentations')

plt.rcParams.update({'font.size': 24})
plt.legend(loc='upper right', fontsize = 'small')
plt.tight_layout()
# plt.savefig('/data/{}/accuracy_comparison_of_models_across_img_aug_{}.png'.format(model_iter,model_iter))

##################################################
# F1-Score Classification Report
##################################################

#SVM Classification Report
print("SVM Classification Report - F1-Score")
print(svm_df) 
print("")

#KNN CLassification Report
print("KNN Classification Report - F1-Score")
print(knn_df) 
print("")

# ANN Classification Report
print("ANN Classification Report - F1-Score")
print(ann_df) 
print("")

##################################################
# Plot Confusion Matrices
##################################################
plot_confusion_matrix(reports_dict['svm']['predictions'],'SVM')
# plt.savefig('/data/{}/svm_confusion_matrix_{}.png'.format(model_iter,model_iter))
plot_confusion_matrix(reports_dict['knn']['predictions'], 'KNN')
# plt.savefig('/data/{}/knn_confusion_matrix_{}.png'.format(model_iter,model_iter))
plot_confusion_matrix(reports_dict['ann_ce']['predictions'], 'ANN')
# plt.savefig('/data/{}/ann_confusion_matrix_{}.png'.format(model_iter,model_iter))

"""## Increasing Noise

* **Experiment:** What model is the most robust as levels of noise are added to the images?

* **Hypothesis:** Deep learning methods such as ANN should be the most robust to added noise.

* **Summary of findings:** kNN was the most robust to added levels of noise, but took an extremely long time to run. ANN and SVM were both highly volatile to added levels of noise.
"""

##################################################
# CREATE AND VISUALIZE LEVELS OF IMAGE NOISE
##################################################

fig, axs = plt.subplots(3,7, figsize=(7, 3.8))
idx = 0
for k,image in noise_dict.items():
  i = int(np.floor(idx/7))
  j = idx%7
  axs[i,j].imshow(image[0].reshape(28,28), cmap=plt.cm.gray)
  axs[i,j].set_axis_off()
  axs[i,j].set_title(round(k,2))
  idx += 1 
# plt.savefig('/data/{}/image_noise_{}.png'.format(model_iter,model_iter))

##################################################
# ACCURACY CURVE BY MODEL WITH INCREASING LEVELS OF NOISE
##################################################

plt.figure(figsize=(7, 5))

noise_df = pd.DataFrame()
for k,model in reports_dict.items():
  v = model['noise_report']
  temp_df = pd.DataFrame()
  for k2,v2 in v.items():
    temp_df[k2] = pd.DataFrame(v2).transpose()['f1-score']
  temp_df['model'] = k
  temp_df.index = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 
                   'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot','accuracy',
                   'micro avg','weighted avg']
  noise_df = pd.concat([noise_df,temp_df])

plot_df = noise_df.loc['accuracy',:].melt(id_vars = ['model'])
g = sns.lineplot(data = plot_df[plot_df['model'].isin(['ann_ce','svm','knn'])], x='variable', y='value', hue='model')
xlabels = plot_df['variable'].astype(float).unique().round(2).astype(str)
g.set_title('Accuracy Curve By Model With Increasing Noise')
g.set_ylabel('Accuracy')
g.set_xlabel('Noise Level')
_ = g.set_xticklabels(xlabels, rotation = 50)

plt.rcParams.update({'font.size': 14})
plt.legend(loc='upper right', fontsize = 'small')
plt.tight_layout()
# plt.savefig('/data/{}/accuracy_curve_by_model_incr_noise_{}.png'.format(model_iter,model_iter))

"""## ANN Loss Functions

* **Experiment:** Given the speed of ANN compared to SVM and kNN, is there a way to make the ANN more robust by changing the loss function?

* **Hypothesis:** Cross Entropy will have a higher accuracy with minimal noise, but other loss functions could have higher accuracy when a lot of noise is added.

* **Summary of findings:** Cross Entropy had the highest accuracy with no noise and as noise was added all loss functions had similar accuracy.
"""

##################################################
# ACCURACY CURVE BY LOSS WITH INCREASING LEVELS OF NOISE
##################################################

plt.figure(figsize=(7, 5))

plot_df = noise_df.loc['accuracy',:].melt(id_vars = ['model'])
g = sns.lineplot(data = plot_df[plot_df['model'].str.contains("ann")], x='variable', y='value', hue='model')
xlabels = plot_df['variable'].astype(float).unique().round(2).astype(str)
g.set_title('Accuracy Curve By ANN Loss With Increasing Noise')
g.set_ylabel('Accuracy')
g.set_xlabel('Noise Level')
_ = g.set_xticklabels(xlabels, rotation = 50)

plt.rcParams.update({'font.size': 14})
plt.legend(loc='upper right', fontsize = 'small')
plt.tight_layout()
# plt.savefig('/data/{}/accuracy_curve_by_annloss_incr_noise_{}.png'.format(model_iter,model_iter))

"""# Best Model Evaluation"""

##################################################
# Evaluate the model here
##################################################

# Use this function to evaluate your model
def accuracy(y_pred, y_true):
    '''
    input y_pred: ndarray of shape (N,)
    input y_true: ndarray of shape (N,)
    '''
    return (1.0 * (y_pred == y_true)).mean()

# Report the accuracy in the train and validation sets.
y_train_pred = models_dict['ann_ce'].predict(new_x_train)
y_train_pred = np.array([np.argmax(x) for x in y_train_pred])

y_valid_pred = models_dict['ann_ce'].predict(x_valid)
y_valid_pred = np.array([np.argmax(x) for x in y_valid_pred])

print('Train Accuracy: {}'.format(accuracy(y_train_pred,new_y_train)))
print('Validation Accuracy: {}'.format(accuracy(y_valid_pred,y_valid)))

"""# Send the submission for the challenge"""

##################################################
# Save your test prediction in y_test_pred
##################################################

# Make prediction
y_test_pred = models_dict['ann_ce'].predict(x_test)
y_test_pred = np.array([np.argmax(x) for x in y_test_pred])

# Create submission
submission = pd.read_csv(os.path.join(DATA_BASE_FOLDER, '/sample_submission.csv'))
if y_test_pred is not None:
    submission['class'] = y_test_pred
submission.to_csv('/data/my_submission_{}.csv'.format(model_iter,model_iter), index=False)