data_Loading_Training_Evaluating.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "data Loading-Training-Evaluating.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPbe4vW4hpq36DBFkW3ZrNb",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mr7495/covid19/blob/master/data_Loading_Training_Evaluating.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V_md08dqhQ9n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Code on (github.com/mr7495/covid19)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6JmhSAzXPUaX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!nvidia-smi #show the allocated GPU"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5yqhPi-FS60r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Connect your Google Drive \n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7UAp6XlqPeou",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#upgrade and install essential libraries\n",
        "!pip install keras --upgrade\n",
        "!pip install zipfile36\n",
        "!pip install pydicom"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zw0BhQ7nPihU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7646ecb8-bfc0-4960-f4e6-d67966192ad1"
      },
      "source": [
        "import keras\n",
        "import numpy as np\n",
        "import cv2\n",
        "import os\n",
        "import random\n",
        "import shutil\n",
        "import pandas as pd\n",
        "import csv\n",
        "import pydicom as dicom\n",
        "import zipfile\n",
        "from keras import optimizers\n",
        "from keras.models import Sequential,Model\n",
        "from keras.layers import Dropout, Flatten, Dense,Input\n",
        "from keras.applications.resnet_v2 import ResNet50V2\n",
        "from keras.applications.xception import Xception\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "from keras.applications.imagenet_utils import preprocess_input\n",
        "from keras import backend as K\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras.initializers import RandomNormal"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ROTH36gTQDg4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# This is a link to kaggle dataset. If you have downloaded it already, save it as kaggle.zip in the current directory.\n",
        "# If the link expired, get the new link from https://www.kaggle.com/c/rsna-pneumonia-detection-challenge/data\n",
        "!wget -cO - 'https://storage.googleapis.com/kaggle-competitions-data/kaggle-v2/10338/862042/bundle/archive.zip?GoogleAccessId=web-data@kaggle-161607.iam.gserviceaccount.com&Expires=1587124102&Signature=UbIsEpcNjy3ymL%2BCt5cNunBYytcPNlMjVW4RmBKzzuwTL%2BqGHXDzKGbFM3rsewy6nWa9GJgU5ScP%2FVPFUVJdAU3gsqw7aR6En0AqbLMjZ3JE%2BMducSHY94zyZH%2Fn6LqBOwq%2F3FQmK6OC8Ze0OW5oJyNFD7nATMQU7GxbrarIMH6F6zg%2BmL%2BZF%2B6uqlZhAwYpKKLQtzVm7Tyu04Hse0ODtfKV78U3nREvAifK9CzPTRHzAh8AxIdNunMInOn10U4bzxWN%2F5x3Cex7kP6UHsTyJX2XF98eBrQinlgBuyWLbInpQDJVVl1QGFebCa7CN6lnOO2wEeV8Xy5MN6B%2FwlZvEw%3D%3D&response-content-disposition=attachment%3B+filename%3Drsna-pneumonia-detection-challenge.zip' > kaggle.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7x0sa1zuRPI9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Get the zip file I have shared, that contains the covid-chestxray-dataset images until 12 April\n",
        "# Through the link below get the shared zip file and add it to your drive:\n",
        "# https://drive.google.com/file/d/1Bwn4vTQUUB0tHK5aHh--Rk6eOxs2jg3q/view?usp=sharing\n",
        "archive = zipfile.ZipFile('kaggle.zip') #Extract Kaggle Dataset\n",
        "for file in archive.namelist():\n",
        "     archive.extract(file, './All')\n",
        "archive = zipfile.ZipFile('drive/My Drive/covid-chestxray-dataset.zip') #Extract covid-chestxray-Dataset\n",
        "for file in archive.namelist():\n",
        "     archive.extract(file, './covid-chestxray-dataset')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JzN_Dxg5V14k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fold_num=1 #Select Fold Number"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jbx1ajkDWFt5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "try:\n",
        "  os.mkdir('All/All')\n",
        "except:\n",
        "  pass"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jmmkBuXJ7dui",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/mr7495/covid19 #connect to our repository on GitHub"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2OhNyJFNU_bL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Warning: Our prepared All.csv & train1.csv to train8.csv in each fold, are based on the covid-chestxray-dataset until 12 April.\n",
        "#If you have used https://drive.google.com/file/d/1Bwn4vTQUUB0tHK5aHh--Rk6eOxs2jg3q/view?usp=sharing link to get the covid-chestxray-dataset.zip file(like the cells above), you can use our prepared csv files\n",
        "#But if you want to load the updated covid-chestxray-dataset, you must make some changes to the csvfiles.\n",
        "shutil.copy('covid19/prepared_csv_files/All.csv','All')\n",
        "for i in range(1,9): #Load the 8 training phases csv files of the indicated fold\n",
        "  shutil.copy('covid19/prepared_csv_files/fold{}/train{}.csv'.format(fold_num,i),'.')\n",
        "  globals()['train{}'.format(i)]=[]\n",
        "\n",
        "# The code for creating All.csv and training.csv files is available on  covid19/dataset preparing.ipynb."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sBwVu4XcWJY5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "images=[]\n",
        "for r,d,f in os.walk('All/stage_2_train_images'): #Read the name of the images in both datasets\n",
        "  for file in f:\n",
        "    images.append(os.path.join(r,file))\n",
        "for r,d,f in os.walk('covid-chestxray-dataset/images'):\n",
        "  for file in f:\n",
        "   images.append(os.path.join(r,file))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_bi89iwNWWcR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "csv_all=pd.read_csv('All/All.csv', nrows=None) #Read the CSV file that contains the names of the images with their labels.\n",
        "for index, row in csv_all.iterrows(): #This loop reads the images, converts them to suitable format and saves them in the All directory\n",
        "  if '.png' in row['filename']: #For creating the All.csv we have converted the kaggle dataset images to png format,\n",
        "                                #but some of the images in the other dataset also are in the format of png, so we use try/except here to distinguish which dataset, the annotation in the CSV file belongs to.\n",
        "    try:\n",
        "      png_index=row['filename'].find('.png')\n",
        "      last_name=row['filename'][:png_index]+'.dcm'\n",
        "      ds = dicom.dcmread(os.path.join('All/stage_2_train_images',last_name))\n",
        "      pixel_array_numpy = ds.pixel_array\n",
        "      imgname = last_name[:-4]+'.png'\n",
        "      cv2.imwrite(os.path.join('All/All', imgname), pixel_array_numpy)\n",
        "    except:\n",
        "      png_index=row['filename'].find('.png')\n",
        "      img=cv2.imread(os.path.join('covid-chestxray-dataset/images',row['filename'][:png_index+4])) \n",
        "      gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)\n",
        "      cv2.imwrite(os.path.join('All/All', row['filename'][:png_index+4]), gray)  \n",
        "  else:\n",
        "    img=cv2.imread(os.path.join('covid-chestxray-dataset/images',row['filename']))\n",
        "    gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)\n",
        "    cv2.imwrite(os.path.join('All/All', row['filename']), gray)  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eFNnawesW_b4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "All=[] #Thie list that is readed from All.csv\n",
        "all_train=[] #This list contains the training annotations\n",
        "all_test=[]\n",
        "with open('All/All.csv',newline='', mode='r') as csvfile: #Adding All.csv rows to All list\n",
        "      csvreader = csv.reader(csvfile, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
        "      for row in csvreader:\n",
        "          All.append(row)\n",
        "for i in range(1,9): #Adding training1.csv to training8.csv rows to All_train list. This 1 to 8 indicate the 8 training phases\n",
        "  with open('train{}.csv'.format(i),newline='', mode='r') as csvfile:\n",
        "      csvreader = csv.reader(csvfile, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
        "      for row in csvreader:\n",
        "        all_train.append(row)\n",
        "with open('all_test.csv'.format(i),newline='', mode='w') as csvfile: #Add all the other images that do not belong to the training phases, to the test set\n",
        "    csvwriter = csv.writer(csvfile, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
        "    csvwriter.writerow(['filename','class'])\n",
        "    for row in All:\n",
        "      if row not in all_train:\n",
        "        csvwriter.writerow(row)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "teujwlOSZ1rL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Because we have written our code somehow to only save the epochs with the best validation accuracy during the training,\n",
        "# we created the s_test.csv with 631 images. That is why validating each epoch for 11302 images during training would be terribly time-consuming\n",
        "#so we select a random s_test.csv for evaluating the network during the training process."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mczHOfijXJDA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for i in range(10): #Shuffle the All list\n",
        "  random.shuffle(All)\n",
        "with open('s_test.csv'.format(i),newline='', mode='w') as csvfile: #Create s_test.csv file for evaluating the network during training\n",
        "    csvwriter = csv.writer(csvfile, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
        "    csvwriter.writerow(['filename','class'])\n",
        "    ln=0\n",
        "    lp=0\n",
        "    for row in All:\n",
        "      if row not in all_train:\n",
        "        if row[1]=='COVID-19':\n",
        "          csvwriter.writerow(row)\n",
        "        elif row[1]=='normal':\n",
        "          if ln<300:\n",
        "            csvwriter.writerow(row)\n",
        "            ln+=1\n",
        "        else:\n",
        "          if lp<300:\n",
        "            csvwriter.writerow(row)\n",
        "            lp+=1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EpoYfpASW-m8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#remove the unnecessary file to increase the free space\n",
        "try:\n",
        "  os.remove('kaggle.zip')\n",
        "  shutil.rmtree('All/stage_2_train_images')\n",
        "  shutil.rmtree('All/stage_2_test_images')\n",
        "  shutil.rmtree('covid-chestxray-dataset')\n",
        "except:\n",
        "  pass"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tpjAiyH3bKIu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Here we set the data generators for applying data augmentation methods\n",
        "train_datagen = ImageDataGenerator(horizontal_flip=True,vertical_flip=True,rescale=1./255,zoom_range=0.05,rotation_range=360,width_shift_range=0.05,height_shift_range=0.05,shear_range=0.05)\n",
        "test_datagen = ImageDataGenerator(rescale=1./255)\n",
        "test_df = pd.read_csv(\"s_test.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M8P6i377a2VB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Now our data is ready\n",
        "# We create the neural network in the next level\n",
        "#If you want to use our trained network you can visit this shared folde:\n",
        "#  https://drive.google.com/drive/folders/19R4T-D-bWUkQOh3xy5CkIDAmkLBt8ID7?usp=sharing"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_c3DNNdHbGiB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import keras.backend as k\n",
        "k.clear_session() #Clear keras backend \n",
        "try:\n",
        "  os.mkdir('models')\n",
        "except:\n",
        "  pass\n",
        "full_name='concatenate'\n",
        "classes_number=3 #Number of classes\n",
        "input_tensor=Input(shape=(300,300,3))\n",
        "######################################################################################################\n",
        "base_model1 = Xception(weights='imagenet', include_top=False, input_tensor=input_tensor)\n",
        "features1 = base_model1.output\n",
        "######################################################################################################\n",
        "base_model2 = ResNet50V2(weights='imagenet', include_top=False, input_tensor=input_tensor)\n",
        "features2 = base_model2.output\n",
        "concatenated=keras.layers.concatenate([features1,features2]) #Concatenate the extracted features\n",
        "####################################################################################################\n",
        "conv=keras.layers.Conv2D(1024, (1, 1),padding='same')(concatenated) #add the concatenated features to a convolutional layer\n",
        "feature = Flatten(name='flatten')(conv)\n",
        "dp = Dropout(0.5)(feature) #add dropout\n",
        "preds = Dense(classes_number, activation='softmax', kernel_initializer=RandomNormal(mean=0.0, stddev=0.001))(dp) \n",
        "Concatenated_model = Model(inputs=input_tensor, outputs=preds)\n",
        "#######################################################\n",
        "for layer in Concatenated_model.layers:\n",
        "  layer.trainable = True\n",
        "Concatenated_model.compile(optimizer=optimizers.nadam(lr=0.0001), loss='categorical_crossentropy',metrics=['accuracy'])\n",
        "filepath=\"models/%s-{epoch:02d}-{val_accuracy:.4f}.hdf5\"%full_name \n",
        "checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', save_best_only=True, mode='max') #creating checkpoint to save the best validation accuracy\n",
        "callbacks_list = [checkpoint]\n",
        "\n",
        "for epoch_num in range(1,9): # This loop applies the 8 phases for training\n",
        "  train_df =pd.read_csv(\"train{}.csv\".format(epoch_num)) #Add data generators for parsing images to the network while training\n",
        "  train_generator = train_datagen.flow_from_dataframe(\n",
        "        dataframe=train_df,\n",
        "        directory='All/All',\n",
        "        x_col=\"filename\",\n",
        "        y_col=\"class\",\n",
        "        target_size=(300, 300),\n",
        "        batch_size=20,\n",
        "        class_mode='categorical',shuffle=True)\n",
        "  validation_generator = test_datagen.flow_from_dataframe(\n",
        "          dataframe=test_df,\n",
        "          directory='All/All',\n",
        "          x_col=\"filename\",\n",
        "          y_col=\"class\",\n",
        "          target_size=(300, 300),\n",
        "          batch_size=20,\n",
        "          class_mode='categorical',shuffle=True)\n",
        "  Concatenated_model.fit_generator(train_generator, epochs=100,validation_data=validation_generator,shuffle=True,callbacks=callbacks_list) #start training\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Gx5rM25d4W4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3YWgRtS1d4l8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#After the training is done, we attempt to evaluate our saved networks."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6VBGSO_KeGr2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trained_models=[]\n",
        "for r,d,f in os.walk('models'):\n",
        "  for file in f:\n",
        "    trained_models.append(os.path.join(r,file)) #Add the saved models in the models directory\n",
        "reports={}\n",
        "test_df = pd.read_csv(\"all_test.csv\") #Load the full test CSV file that includes 11302 images\n",
        "validation_generator = test_datagen.flow_from_dataframe( #Set new data generator\n",
        "        dataframe=test_df,\n",
        "        directory='All/All',\n",
        "        x_col=\"filename\",\n",
        "        y_col=\"class\",\n",
        "        target_size=(300, 300),\n",
        "        batch_size=20,\n",
        "        class_mode='categorical',shuffle=True)\n",
        "for trained_model in trained_models:\n",
        "  k.clear_session()\n",
        "  net=keras.models.load_model(trained_model) #load model\n",
        "  covid_label= validation_generator.class_indices['COVID-19'] #get the index of COVUD-19 class \n",
        "  pneu_label= validation_generator.class_indices['pneumonia'] #get the index of pneumonia class \n",
        "  normal_label= validation_generator.class_indices['normal']  #get the index of normal class \n",
        "  tp=0 #True Positives\n",
        "  fp=0 #False Positives\n",
        "  anum=0 #All the images numbers\n",
        "  ###########\n",
        "  wrong_covid=0\n",
        "  correct_covid=0\n",
        "  not_detected_covid=0\n",
        "  covid_num=0\n",
        "  ###########\n",
        "  wrong_pneu=0\n",
        "  correct_pneu=0\n",
        "  not_detected_pneu=0\n",
        "  pneu_num=0\n",
        "  ############\n",
        "  wrong_normal=0\n",
        "  correct_normal=0\n",
        "  not_detected_normal=0\n",
        "  normal_num=0\n",
        "  ##############\n",
        "  wrong_covid_normal=0 #COVID-19 class which has been detected as normal, incorrectly\n",
        "  wrong_covid_pneu=0   #COVID-19 class which has been detected as pneumonia, incorrectly\n",
        "  wrong_pneu_covid=0   #pneumonia class which has been detected as COVID-19 , incorrectly\n",
        "  wrong_pneu_normal=0  #pneumonia class which has been detected as normal , incorrectly\n",
        "  wrong_normal_pneu=0  #normal class which has been detected as pneumonia , incorrectly\n",
        "  wrong_normal_covid=0  #normal class which has been detected as COVID-19 , incorrectly\n",
        "  ################\n",
        "  for num,img_name in enumerate(validation_generator.filenames): #load image\n",
        "    gt_ind=validation_generator.classes[num] #get the loaded image class index\n",
        "    img=cv2.resize(cv2.imread(os.path.join('All','All',img_name)),(300,300)) #resize image\n",
        "    img=img.astype('float32') / 255.0 #scale the image\n",
        "    pred_ind=np.argmax(net.predict(np.expand_dims(img,axis=0))[0]) #get the predicted class index\n",
        "\n",
        "    anum+=1 #count the number of images\n",
        "\n",
        "\n",
        "    if gt_ind==covid_label:\n",
        "      covid_num+=1\n",
        "    if gt_ind==pneu_label:\n",
        "      pneu_num+=1\n",
        "    if gt_ind==normal_label:\n",
        "      normal_num+=1\n",
        "    ##################\n",
        "    if gt_ind==covid_label and pred_ind==covid_label: \n",
        "      correct_covid+=1\n",
        "    if gt_ind==covid_label and pred_ind!=covid_label:\n",
        "      not_detected_covid+=1\n",
        "      if pred_ind==pneu_label:\n",
        "        wrong_covid_pneu+=1\n",
        "      elif pred_ind==normal_label:\n",
        "        wrong_covid_normal+=1\n",
        "    if gt_ind!=covid_label and pred_ind==covid_label:\n",
        "      wrong_covid+=1\n",
        "    ###########################################\n",
        "    if gt_ind==normal_label and pred_ind==normal_label: \n",
        "      correct_normal+=1\n",
        "    if gt_ind==normal_label and pred_ind!=normal_label:\n",
        "      not_detected_normal+=1\n",
        "      if pred_ind==pneu_label:\n",
        "        wrong_normal_pneu+=1\n",
        "      elif pred_ind==covid_label:\n",
        "        wrong_normal_covid+=1\n",
        "    if gt_ind!=normal_label and pred_ind==normal_label:\n",
        "      wrong_normal+=1\n",
        "    ###########################################\n",
        "    if gt_ind==pneu_label and pred_ind==pneu_label: \n",
        "      correct_pneu+=1\n",
        "    if gt_ind==pneu_label and pred_ind!=pneu_label:\n",
        "      not_detected_pneu+=1\n",
        "      if pred_ind==normal_label:\n",
        "        wrong_pneu_normal+=1\n",
        "      elif pred_ind==covid_label:\n",
        "        wrong_pneu_covid+=1\n",
        "    if gt_ind!=pneu_label and pred_ind==pneu_label:\n",
        "      wrong_pneu+=1\n",
        "    ###########################################\n",
        "\n",
        "    if pred_ind==gt_ind:\n",
        "      tp+=1\n",
        "    else:\n",
        "      fp+=1\n",
        "\n",
        "  reports[trained_model[7:]]=[correct_covid,wrong_covid,not_detected_covid,\n",
        "                   correct_pneu,wrong_pneu,not_detected_pneu,\n",
        "                   correct_normal,wrong_normal,not_detected_normal,\n",
        "                   covid_num,pneu_num,normal_num,tp,fp,wrong_covid_normal,wrong_covid_pneu,\n",
        "                   wrong_pneu_covid,wrong_pneu_normal,wrong_normal_pneu,wrong_normal_covid]\n",
        "\n",
        "\n",
        "  print(trained_model[7:])\n",
        "  print('correct_covid: ',correct_covid/covid_num,'%')\n",
        "  print('wrong_covid: ',wrong_covid)\n",
        "  print('not_detected_covid: ',not_detected_covid/covid_num,'%')\n",
        "  print('###########################')\n",
        "\n",
        "with open('{}-fold{}.csv'.format(trained_model[7:],fold_num),newline='', mode='w') as csvfile: #save the evaluation data into csvfile\n",
        "     csv_writer = csv.writer(csvfile, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
        "     csv_writer.writerow(['name','correct_covid','wrong_covid','not_detected_covid',\n",
        "                          'correct_pneu','wrong_pneu','not_detected_pneu',\n",
        "                          'correct_normal','wrong_normal','not_detected_normal',\n",
        "                          'covid_num','pneu_num','normal_num','tp','fp','wrong_covid_normal','wrong_covid_pneu',\n",
        "                  'wrong_pneu_covid','wrong_pneu_normal','wrong_normal_pneu','wrong_normal_covid'])\n",
        "     for key in reports.keys():\n",
        "         row=[key]\n",
        "         row.extend(reports[key])"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}