Train-SGNN-Transformer-Sentence-Model-SimilarityBXENT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train an SGNN-Transformer Sentence Model with SimilarityBXENT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pytest\r\n",
      "pytest-cov\r\n",
      "joblib\r\n",
      "torch==1.0.0\r\n",
      "torchvision==0.2.1\r\n",
      "scikit-learn==0.20.1\r\n",
      "numpy==1.15.4\r\n"
     ]
    }
   ],
   "source": [
    "# !pip install joblib\n",
    "# !echo \"joblib\" >> requirements.txt\n",
    "# !pip freeze | grep -i torch >> requirements.txt\n",
    "# !pip freeze | grep -i numpy >> requirements.txt\n",
    "!cat requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data.read_txt import *\n",
    "from src.data.config import *\n",
    "from src.data.training_data import *\n",
    "from src.data.sgnn_projection_layer import *\n",
    "from src.model.loss import *\n",
    "from src.model.transformer import *\n",
    "from src.model.save_load_model import *\n",
    "from src.training import *\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.metrics import jaccard_similarity_score, f1_score, accuracy_score\n",
    "from joblib import dump, load\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import math\n",
    "import copy\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Step: 239 Loss: 0.079534 Tokens per Sec: 2505.812324\n",
      "Epoch 0 Step: 479 Loss: 0.056101 Tokens per Sec: 2547.705751\n",
      "Epoch 0 Step: 719 Loss: 0.067586 Tokens per Sec: 2507.719794\n",
      "Epoch 0 Step: 959 Loss: 0.053296 Tokens per Sec: 2546.866083\n",
      "Epoch 0 Step: 1199 Loss: 0.060004 Tokens per Sec: 2531.397485\n",
      "Epoch 0 Step: 1439 Loss: 0.074048 Tokens per Sec: 2560.327252\n",
      "Epoch 0 Step: 1679 Loss: 0.063783 Tokens per Sec: 2538.520846\n",
      "Epoch 0 Step: 1919 Loss: 0.079338 Tokens per Sec: 2503.650134\n",
      "Epoch 0 Step: 2159 Loss: 0.067302 Tokens per Sec: 2451.060108\n",
      "Epoch 0 Step: 2399 Loss: 0.055000 Tokens per Sec: 2438.445192\n",
      "Epoch 0 Step: 2639 Loss: 0.073050 Tokens per Sec: 2495.321814\n",
      "Epoch 0 Step: 2879 Loss: 0.074002 Tokens per Sec: 2538.956635\n",
      "Epoch 0 Step: 3119 Loss: 0.052886 Tokens per Sec: 2535.198840\n",
      "Epoch 0 Step: 3359 Loss: 0.094706 Tokens per Sec: 2511.230404\n",
      "Epoch 0 Step: 3599 Loss: 0.086621 Tokens per Sec: 2504.209927\n",
      "Epoch 0 Step: 3839 Loss: 0.062599 Tokens per Sec: 2549.615593\n",
      "Epoch 0 Step: 4079 Loss: 0.057201 Tokens per Sec: 2498.438028\n",
      "Epoch 0 Step: 4319 Loss: 0.050393 Tokens per Sec: 2471.556412\n",
      "Epoch 0 Step: 4559 Loss: 0.092634 Tokens per Sec: 2317.781912\n",
      "Epoch 0 Step: 4799 Loss: 0.050570 Tokens per Sec: 2430.365634\n",
      "Epoch 0 Step: 5039 Loss: 0.070621 Tokens per Sec: 2223.213584\n",
      "Epoch 0 Step: 5279 Loss: 0.055387 Tokens per Sec: 2267.845056\n",
      "Epoch 0 Step: 5519 Loss: 0.054895 Tokens per Sec: 2378.409308\n",
      "Epoch 0 Step: 5759 Loss: 0.056352 Tokens per Sec: 2460.902119\n",
      "Epoch 0 Step: 5999 Loss: 0.048734 Tokens per Sec: 2505.247648\n",
      "Epoch 0 Step: 6239 Loss: 0.049761 Tokens per Sec: 2517.587739\n",
      "Epoch 0 Step: 6479 Loss: 0.085300 Tokens per Sec: 2502.584470\n",
      "Epoch 0 Step: 6719 Loss: 0.071185 Tokens per Sec: 2431.909109\n",
      "Epoch 0 Step: 6959 Loss: 0.055281 Tokens per Sec: 2612.987896\n",
      "Epoch 0 Step: 7199 Loss: 0.070359 Tokens per Sec: 2591.764270\n",
      "Epoch 0 Step: 7439 Loss: 0.104473 Tokens per Sec: 2483.711086\n",
      "Epoch 0 Step: 7679 Loss: 0.061981 Tokens per Sec: 2470.631823\n",
      "Epoch 0 Step: 7919 Loss: 0.099785 Tokens per Sec: 2229.011646\n",
      "Epoch 0 Step: 8159 Loss: 0.065803 Tokens per Sec: 2239.593568\n",
      "Epoch 0 Step: 8399 Loss: 0.061059 Tokens per Sec: 2264.262610\n",
      "Epoch 0 Step: 8639 Loss: 0.055098 Tokens per Sec: 2161.498181\n",
      "Epoch 0 Step: 8879 Loss: 0.055705 Tokens per Sec: 2228.689178\n",
      "Epoch 0 Step: 9119 Loss: 0.082062 Tokens per Sec: 2227.613598\n",
      "Epoch 0 Step: 9359 Loss: 0.049592 Tokens per Sec: 2285.374729\n",
      "Epoch 0 Step: 9599 Loss: 0.051624 Tokens per Sec: 2249.763762\n",
      "Epoch 0 Step: 9839 Loss: 0.074835 Tokens per Sec: 2386.531668\n",
      "Epoch 0 Step: 10079 Loss: 0.042173 Tokens per Sec: 2252.571769\n",
      "Epoch 0 Step: 10319 Loss: 0.054066 Tokens per Sec: 2467.330407\n",
      "Epoch 0 Step: 10559 Loss: 0.052626 Tokens per Sec: 2253.503130\n",
      "Epoch 0 Step: 10799 Loss: 0.053746 Tokens per Sec: 2198.063046\n",
      "Epoch 0 Step: 11039 Loss: 0.058729 Tokens per Sec: 2385.293927\n",
      "Epoch 0 Step: 11279 Loss: 0.058120 Tokens per Sec: 2383.507509\n",
      "Epoch 0 Step: 11519 Loss: 0.095185 Tokens per Sec: 2445.510629\n",
      "Epoch 0 Step: 11759 Loss: 0.054537 Tokens per Sec: 2589.142023\n",
      "Epoch 0 Step: 11999 Loss: 0.050013 Tokens per Sec: 2618.351834\n",
      "Epoch 0 Step: 12239 Loss: 0.054317 Tokens per Sec: 2570.767002\n",
      "Epoch 0 Step: 12479 Loss: 0.053935 Tokens per Sec: 2619.673368\n",
      "Epoch 0 Step: 12719 Loss: 0.048811 Tokens per Sec: 2524.406338\n",
      "Epoch 0 Step: 12959 Loss: 0.076213 Tokens per Sec: 2555.651217\n",
      "Epoch 0 Step: 13199 Loss: 0.056558 Tokens per Sec: 2546.069112\n",
      "Epoch 0 Step: 13439 Loss: 0.060945 Tokens per Sec: 2534.671511\n",
      "Epoch 0 Step: 13679 Loss: 0.046313 Tokens per Sec: 2538.236746\n",
      "Epoch 0 Step: 13919 Loss: 0.063339 Tokens per Sec: 2524.558100\n",
      "Epoch 0 Step: 14159 Loss: 0.058486 Tokens per Sec: 2587.028581\n",
      "Epoch 0 Step: 14399 Loss: 0.062366 Tokens per Sec: 2556.519736\n",
      "Epoch 0 Step: 14639 Loss: 0.061684 Tokens per Sec: 2544.591846\n",
      "Epoch 0 Step: 14879 Loss: 0.054284 Tokens per Sec: 2578.941865\n",
      "Epoch 0 Step: 15119 Loss: 0.044014 Tokens per Sec: 2576.370791\n",
      "Epoch 0 Step: 15359 Loss: 0.051926 Tokens per Sec: 2573.081321\n",
      "Epoch 0 Step: 15599 Loss: 0.050300 Tokens per Sec: 2583.194372\n",
      "Epoch 0 Step: 15839 Loss: 0.107517 Tokens per Sec: 2457.503588\n",
      "Epoch 0 Step: 16079 Loss: 0.055495 Tokens per Sec: 2464.051710\n",
      "Epoch 0 Step: 16319 Loss: 0.059147 Tokens per Sec: 2539.229260\n",
      "Epoch 0 Step: 16559 Loss: 0.057288 Tokens per Sec: 2542.852318\n",
      "Epoch 0 Step: 16799 Loss: 0.048330 Tokens per Sec: 2495.830751\n",
      "Epoch 0 Step: 17039 Loss: 0.055272 Tokens per Sec: 2543.284478\n",
      "Epoch 0 Step: 17279 Loss: 0.052810 Tokens per Sec: 2545.078462\n",
      "Epoch 0 Step: 17519 Loss: 0.068638 Tokens per Sec: 2562.333719\n",
      "Epoch 0 Step: 17759 Loss: 0.069155 Tokens per Sec: 2505.942140\n",
      "Epoch 0 Step: 17999 Loss: 0.059448 Tokens per Sec: 2488.280922\n",
      "Epoch 0 Step: 18239 Loss: 0.063820 Tokens per Sec: 2559.102607\n",
      "Epoch 0 Step: 18479 Loss: 0.048849 Tokens per Sec: 2574.011467\n",
      "Epoch 0 Step: 18719 Loss: 0.040472 Tokens per Sec: 2454.670712\n",
      "Epoch 0 Step: 18959 Loss: 0.078403 Tokens per Sec: 2311.451801\n",
      "Epoch 0 Step: 19199 Loss: 0.046243 Tokens per Sec: 2578.645866\n",
      "Epoch 0 Step: 19439 Loss: 0.053910 Tokens per Sec: 2488.903022\n",
      "Epoch 0 Step: 19679 Loss: 0.053907 Tokens per Sec: 2591.243415\n",
      "Epoch 0 Step: 19919 Loss: 0.050429 Tokens per Sec: 2537.429653\n",
      "Epoch 0 Step: 20159 Loss: 0.069737 Tokens per Sec: 2588.422699\n",
      "Epoch 0 Step: 20399 Loss: 0.046620 Tokens per Sec: 2481.349192\n",
      "Epoch 0 Step: 20639 Loss: 0.057020 Tokens per Sec: 2402.181140\n",
      "Epoch 0 Step: 20879 Loss: 0.055819 Tokens per Sec: 2286.475779\n",
      "Epoch 0 Step: 21119 Loss: 0.052953 Tokens per Sec: 2447.425468\n",
      "Epoch 0 Step: 21359 Loss: 0.070911 Tokens per Sec: 2427.977243\n",
      "Epoch 0 Step: 21599 Loss: 0.047939 Tokens per Sec: 2383.048593\n",
      "Epoch 0 Step: 21839 Loss: 0.056968 Tokens per Sec: 2512.271914\n",
      "Epoch 0 Step: 22079 Loss: 0.057010 Tokens per Sec: 2522.733469\n",
      "Epoch 0 Step: 22319 Loss: 0.058249 Tokens per Sec: 2413.868612\n",
      "Epoch 0 Step: 22559 Loss: 0.058131 Tokens per Sec: 2422.425778\n",
      "Epoch 0 Step: 22799 Loss: 0.053203 Tokens per Sec: 2515.136103\n",
      "Epoch 0 Step: 23039 Loss: 0.062443 Tokens per Sec: 2476.631797\n",
      "Epoch 0 Step: 23279 Loss: 0.048594 Tokens per Sec: 2361.591185\n",
      "Epoch 0 Step: 23519 Loss: 0.059649 Tokens per Sec: 2290.716544\n",
      "Epoch 0 Step: 23759 Loss: 0.061884 Tokens per Sec: 2471.002102\n",
      "Epoch 0 Step: 23999 Loss: 0.059911 Tokens per Sec: 2523.709854\n",
      "2019-01-07 10:05:32  - Saved model to files: ./models_weights/my-model.sklearn.epoch_00000.notebook_run.gpu0 ./models_weights/my-model.pytorch.epoch_00000.notebook_run.gpu0\n",
      "Epoch 1 Step: 239 Loss: 0.051407 Tokens per Sec: 2390.558596\n",
      "Epoch 1 Step: 479 Loss: 0.045738 Tokens per Sec: 2362.694753\n",
      "Epoch 1 Step: 719 Loss: 0.051654 Tokens per Sec: 2486.625190\n",
      "Epoch 1 Step: 959 Loss: 0.040367 Tokens per Sec: 2524.012741\n",
      "Epoch 1 Step: 1199 Loss: 0.040316 Tokens per Sec: 2430.297052\n",
      "Epoch 1 Step: 1439 Loss: 0.048170 Tokens per Sec: 2533.374415\n",
      "Epoch 1 Step: 1679 Loss: 0.060015 Tokens per Sec: 2478.656436\n",
      "Epoch 1 Step: 1919 Loss: 0.062205 Tokens per Sec: 2539.822601\n",
      "Epoch 1 Step: 2159 Loss: 0.049340 Tokens per Sec: 2440.444101\n",
      "Epoch 1 Step: 2399 Loss: 0.053292 Tokens per Sec: 2336.529812\n",
      "Epoch 1 Step: 2639 Loss: 0.053643 Tokens per Sec: 2457.081784\n",
      "Epoch 1 Step: 2879 Loss: 0.049491 Tokens per Sec: 2467.149794\n",
      "Epoch 1 Step: 3119 Loss: 0.049554 Tokens per Sec: 2513.474265\n",
      "Epoch 1 Step: 3359 Loss: 0.055617 Tokens per Sec: 2498.678445\n",
      "Epoch 1 Step: 3599 Loss: 0.068749 Tokens per Sec: 2530.501461\n",
      "Epoch 1 Step: 3839 Loss: 0.062038 Tokens per Sec: 2488.057801\n",
      "Epoch 1 Step: 4079 Loss: 0.051340 Tokens per Sec: 2496.455215\n",
      "Epoch 1 Step: 4319 Loss: 0.060002 Tokens per Sec: 2465.832294\n",
      "Epoch 1 Step: 4559 Loss: 0.059636 Tokens per Sec: 2536.194363\n",
      "Epoch 1 Step: 4799 Loss: 0.055613 Tokens per Sec: 2523.169417\n",
      "Epoch 1 Step: 5039 Loss: 0.070489 Tokens per Sec: 2479.858640\n",
      "Epoch 1 Step: 5279 Loss: 0.065318 Tokens per Sec: 2521.110593\n",
      "Epoch 1 Step: 5519 Loss: 0.044503 Tokens per Sec: 2532.898824\n",
      "Epoch 1 Step: 5759 Loss: 0.061802 Tokens per Sec: 2503.015569\n",
      "Epoch 1 Step: 5999 Loss: 0.046447 Tokens per Sec: 2559.286358\n",
      "Epoch 1 Step: 6239 Loss: 0.050959 Tokens per Sec: 2540.295324\n",
      "Epoch 1 Step: 6479 Loss: 0.044718 Tokens per Sec: 2560.756584\n",
      "Epoch 1 Step: 6719 Loss: 0.083031 Tokens per Sec: 2475.607919\n",
      "Epoch 1 Step: 6959 Loss: 0.050817 Tokens per Sec: 2458.621345\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Step: 7199 Loss: 0.058283 Tokens per Sec: 2483.761672\n",
      "Epoch 1 Step: 7439 Loss: 0.041650 Tokens per Sec: 2565.915078\n",
      "Epoch 1 Step: 7679 Loss: 0.054724 Tokens per Sec: 2570.761111\n",
      "Epoch 1 Step: 7919 Loss: 0.063471 Tokens per Sec: 2581.614505\n",
      "Epoch 1 Step: 8159 Loss: 0.041398 Tokens per Sec: 2505.112011\n",
      "Epoch 1 Step: 8399 Loss: 0.067593 Tokens per Sec: 2515.197903\n",
      "Epoch 1 Step: 8639 Loss: 0.069686 Tokens per Sec: 2382.403511\n",
      "Epoch 1 Step: 8879 Loss: 0.052854 Tokens per Sec: 2432.915548\n",
      "Epoch 1 Step: 9119 Loss: 0.063079 Tokens per Sec: 2556.271612\n",
      "Epoch 1 Step: 9359 Loss: 0.053092 Tokens per Sec: 2636.683076\n",
      "Epoch 1 Step: 9599 Loss: 0.051026 Tokens per Sec: 2478.364087\n",
      "Epoch 1 Step: 9839 Loss: 0.046112 Tokens per Sec: 2519.274698\n",
      "Epoch 1 Step: 10079 Loss: 0.067315 Tokens per Sec: 2521.368723\n",
      "Epoch 1 Step: 10319 Loss: 0.065241 Tokens per Sec: 2527.254976\n",
      "Epoch 1 Step: 10559 Loss: 0.050516 Tokens per Sec: 2547.052082\n",
      "Epoch 1 Step: 10799 Loss: 0.078200 Tokens per Sec: 2499.352493\n",
      "Epoch 1 Step: 11039 Loss: 0.047257 Tokens per Sec: 2551.077381\n",
      "Epoch 1 Step: 11279 Loss: 0.049722 Tokens per Sec: 2551.308482\n",
      "Epoch 1 Step: 11519 Loss: 0.068343 Tokens per Sec: 2571.711724\n",
      "Epoch 1 Step: 11759 Loss: 0.047196 Tokens per Sec: 2559.700910\n",
      "Epoch 1 Step: 11999 Loss: 0.066788 Tokens per Sec: 2577.741516\n",
      "Epoch 1 Step: 12239 Loss: 0.054743 Tokens per Sec: 2549.516416\n",
      "Epoch 1 Step: 12479 Loss: 0.068151 Tokens per Sec: 2479.855004\n",
      "Epoch 1 Step: 12719 Loss: 0.047784 Tokens per Sec: 2545.989345\n",
      "Epoch 1 Step: 12959 Loss: 0.095695 Tokens per Sec: 2534.498852\n",
      "Epoch 1 Step: 13199 Loss: 0.069505 Tokens per Sec: 2633.619189\n",
      "Epoch 1 Step: 13439 Loss: 0.083179 Tokens per Sec: 2560.543702\n",
      "Epoch 1 Step: 13679 Loss: 0.066503 Tokens per Sec: 2588.918321\n",
      "Epoch 1 Step: 13919 Loss: 0.080156 Tokens per Sec: 2529.527799\n",
      "Epoch 1 Step: 14159 Loss: 0.063771 Tokens per Sec: 2546.836344\n",
      "Epoch 1 Step: 14399 Loss: 0.060351 Tokens per Sec: 2593.496200\n",
      "Epoch 1 Step: 14639 Loss: 0.067953 Tokens per Sec: 2569.206589\n",
      "Epoch 1 Step: 14879 Loss: 0.113040 Tokens per Sec: 2572.560768\n",
      "Epoch 1 Step: 15119 Loss: 0.046268 Tokens per Sec: 2595.724925\n",
      "Epoch 1 Step: 15359 Loss: 0.062603 Tokens per Sec: 2531.021319\n",
      "Epoch 1 Step: 15599 Loss: 0.065947 Tokens per Sec: 2553.534655\n",
      "Epoch 1 Step: 15839 Loss: 0.079001 Tokens per Sec: 2597.479194\n",
      "Epoch 1 Step: 16079 Loss: 0.063620 Tokens per Sec: 2546.714638\n",
      "Epoch 1 Step: 16319 Loss: 0.058196 Tokens per Sec: 2535.190095\n",
      "Epoch 1 Step: 16559 Loss: 0.059678 Tokens per Sec: 2536.678588\n",
      "Epoch 1 Step: 16799 Loss: 0.058145 Tokens per Sec: 2550.476096\n",
      "Epoch 1 Step: 17039 Loss: 0.060396 Tokens per Sec: 2575.319038\n",
      "Epoch 1 Step: 17279 Loss: 0.080552 Tokens per Sec: 2572.471047\n",
      "Epoch 1 Step: 17519 Loss: 0.071285 Tokens per Sec: 2529.707258\n",
      "Epoch 1 Step: 17759 Loss: 0.056895 Tokens per Sec: 2547.426151\n",
      "Epoch 1 Step: 17999 Loss: 0.048218 Tokens per Sec: 2593.063103\n",
      "Epoch 1 Step: 18239 Loss: 0.056778 Tokens per Sec: 2537.065968\n",
      "Epoch 1 Step: 18479 Loss: 0.058151 Tokens per Sec: 2529.731622\n",
      "Epoch 1 Step: 18719 Loss: 0.054339 Tokens per Sec: 2618.866030\n",
      "Epoch 1 Step: 18959 Loss: 0.063607 Tokens per Sec: 2648.893333\n",
      "Epoch 1 Step: 19199 Loss: 0.066775 Tokens per Sec: 2547.849351\n",
      "Epoch 1 Step: 19439 Loss: 0.082401 Tokens per Sec: 2603.857850\n",
      "Epoch 1 Step: 19679 Loss: 0.074732 Tokens per Sec: 2502.425841\n",
      "Epoch 1 Step: 19919 Loss: 0.059640 Tokens per Sec: 2534.606155\n",
      "Epoch 1 Step: 20159 Loss: 0.060950 Tokens per Sec: 2458.764701\n",
      "Epoch 1 Step: 20399 Loss: 0.052764 Tokens per Sec: 2526.204512\n",
      "Epoch 1 Step: 20639 Loss: 0.044887 Tokens per Sec: 2521.885355\n",
      "Epoch 1 Step: 20879 Loss: 0.061172 Tokens per Sec: 2444.303380\n",
      "Epoch 1 Step: 21119 Loss: 0.067961 Tokens per Sec: 2468.536441\n",
      "Epoch 1 Step: 21359 Loss: 0.060509 Tokens per Sec: 2453.680797\n",
      "Epoch 1 Step: 21599 Loss: 0.055478 Tokens per Sec: 2435.997056\n",
      "Epoch 1 Step: 21839 Loss: 0.058443 Tokens per Sec: 2327.632899\n",
      "Epoch 1 Step: 22079 Loss: 0.056243 Tokens per Sec: 2377.210601\n",
      "Epoch 1 Step: 22319 Loss: 0.066709 Tokens per Sec: 2331.465677\n",
      "Epoch 1 Step: 22559 Loss: 0.061322 Tokens per Sec: 2415.237599\n",
      "Epoch 1 Step: 22799 Loss: 0.056875 Tokens per Sec: 2346.795214\n",
      "Epoch 1 Step: 23039 Loss: 0.065339 Tokens per Sec: 2439.464302\n",
      "Epoch 1 Step: 23279 Loss: 0.053481 Tokens per Sec: 2396.067003\n",
      "Epoch 1 Step: 23519 Loss: 0.050586 Tokens per Sec: 2418.514226\n",
      "Epoch 1 Step: 23759 Loss: 0.045736 Tokens per Sec: 2407.379213\n",
      "Epoch 1 Step: 23999 Loss: 0.042582 Tokens per Sec: 2421.742509\n",
      "2019-01-07 15:02:49  - Saved model to files: ./models_weights/my-model.sklearn.epoch_00001.notebook_run.gpu0 ./models_weights/my-model.pytorch.epoch_00001.notebook_run.gpu0\n",
      "Epoch 2 Step: 239 Loss: 0.046688 Tokens per Sec: 2334.749321\n",
      "Epoch 2 Step: 479 Loss: 0.060771 Tokens per Sec: 2338.566829\n",
      "Epoch 2 Step: 719 Loss: 0.084039 Tokens per Sec: 2437.313869\n",
      "Epoch 2 Step: 959 Loss: 0.062754 Tokens per Sec: 2369.955917\n",
      "Epoch 2 Step: 1199 Loss: 0.067327 Tokens per Sec: 2405.558306\n",
      "Epoch 2 Step: 1439 Loss: 0.050225 Tokens per Sec: 2500.664551\n",
      "Epoch 2 Step: 1679 Loss: 0.073147 Tokens per Sec: 2432.848753\n",
      "Epoch 2 Step: 1919 Loss: 0.050865 Tokens per Sec: 2420.266216\n",
      "Epoch 2 Step: 2159 Loss: 0.059185 Tokens per Sec: 2370.081525\n",
      "Epoch 2 Step: 2399 Loss: 0.049397 Tokens per Sec: 2404.324433\n",
      "Epoch 2 Step: 2639 Loss: 0.060262 Tokens per Sec: 2359.627281\n",
      "Epoch 2 Step: 2879 Loss: 0.046172 Tokens per Sec: 2425.853783\n",
      "Epoch 2 Step: 3119 Loss: 0.055477 Tokens per Sec: 2423.556707\n",
      "Epoch 2 Step: 3359 Loss: 0.051570 Tokens per Sec: 2487.320221\n",
      "Epoch 2 Step: 3599 Loss: 0.051201 Tokens per Sec: 2375.685673\n",
      "Epoch 2 Step: 3839 Loss: 0.061646 Tokens per Sec: 2473.969128\n",
      "Epoch 2 Step: 4079 Loss: 0.050882 Tokens per Sec: 2465.188843\n",
      "Epoch 2 Step: 4319 Loss: 0.064120 Tokens per Sec: 2427.187288\n",
      "Epoch 2 Step: 4559 Loss: 0.074446 Tokens per Sec: 2417.360452\n",
      "Epoch 2 Step: 4799 Loss: 0.044102 Tokens per Sec: 2461.714460\n",
      "Epoch 2 Step: 5039 Loss: 0.052772 Tokens per Sec: 2425.832669\n",
      "Epoch 2 Step: 5279 Loss: 0.091840 Tokens per Sec: 2446.275124\n",
      "Epoch 2 Step: 5519 Loss: 0.053395 Tokens per Sec: 2528.682993\n",
      "Epoch 2 Step: 5759 Loss: 0.093565 Tokens per Sec: 2482.002705\n",
      "Epoch 2 Step: 5999 Loss: 0.055923 Tokens per Sec: 2366.503512\n",
      "Epoch 2 Step: 6239 Loss: 0.072142 Tokens per Sec: 2491.821111\n",
      "Epoch 2 Step: 6479 Loss: 0.057194 Tokens per Sec: 2470.325670\n",
      "Epoch 2 Step: 6719 Loss: 0.042567 Tokens per Sec: 2401.895461\n",
      "Epoch 2 Step: 6959 Loss: 0.057160 Tokens per Sec: 2377.720727\n",
      "Epoch 2 Step: 7199 Loss: 0.055164 Tokens per Sec: 2486.031797\n",
      "Epoch 2 Step: 7439 Loss: 0.042246 Tokens per Sec: 2526.159291\n",
      "Epoch 2 Step: 7679 Loss: 0.065562 Tokens per Sec: 2500.771012\n",
      "Epoch 2 Step: 7919 Loss: 0.067350 Tokens per Sec: 2447.333133\n",
      "Epoch 2 Step: 8159 Loss: 0.091257 Tokens per Sec: 2455.853030\n",
      "Epoch 2 Step: 8399 Loss: 0.055721 Tokens per Sec: 2435.480819\n",
      "Epoch 2 Step: 8639 Loss: 0.056641 Tokens per Sec: 2432.272113\n",
      "Epoch 2 Step: 8879 Loss: 0.075930 Tokens per Sec: 2458.170303\n",
      "Epoch 2 Step: 9119 Loss: 0.050899 Tokens per Sec: 2462.053520\n",
      "Epoch 2 Step: 9359 Loss: 0.047876 Tokens per Sec: 2458.868336\n",
      "Epoch 2 Step: 9599 Loss: 0.054494 Tokens per Sec: 2418.630576\n",
      "Epoch 2 Step: 9839 Loss: 0.067040 Tokens per Sec: 2352.908012\n",
      "Epoch 2 Step: 10079 Loss: 0.060733 Tokens per Sec: 2506.273102\n",
      "Epoch 2 Step: 10319 Loss: 0.050877 Tokens per Sec: 2510.717972\n",
      "Epoch 2 Step: 10559 Loss: 0.063271 Tokens per Sec: 2463.343620\n",
      "Epoch 2 Step: 10799 Loss: 0.044971 Tokens per Sec: 2471.873992\n",
      "Epoch 2 Step: 11039 Loss: 0.049862 Tokens per Sec: 2452.585538\n",
      "Epoch 2 Step: 11279 Loss: 0.062935 Tokens per Sec: 2399.507766\n",
      "Epoch 2 Step: 11519 Loss: 0.056260 Tokens per Sec: 2468.628162\n",
      "Epoch 2 Step: 11759 Loss: 0.052879 Tokens per Sec: 2398.604098\n",
      "Epoch 2 Step: 11999 Loss: 0.044189 Tokens per Sec: 2428.113216\n",
      "Epoch 2 Step: 12239 Loss: 0.082285 Tokens per Sec: 2383.342534\n",
      "Epoch 2 Step: 12479 Loss: 0.056648 Tokens per Sec: 2466.335210\n",
      "Epoch 2 Step: 12719 Loss: 0.050378 Tokens per Sec: 2427.795649\n",
      "Epoch 2 Step: 12959 Loss: 0.050373 Tokens per Sec: 2446.028509\n",
      "Epoch 2 Step: 13199 Loss: 0.084315 Tokens per Sec: 2421.038602\n",
      "Epoch 2 Step: 13439 Loss: 0.048842 Tokens per Sec: 2491.418401\n",
      "Epoch 2 Step: 13679 Loss: 0.052029 Tokens per Sec: 2464.202697\n",
      "Epoch 2 Step: 13919 Loss: 0.065698 Tokens per Sec: 2591.270501\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2 Step: 14159 Loss: 0.050559 Tokens per Sec: 2484.746421\n",
      "Epoch 2 Step: 14399 Loss: 0.052026 Tokens per Sec: 2415.585749\n",
      "Epoch 2 Step: 14639 Loss: 0.047790 Tokens per Sec: 2351.825495\n",
      "Epoch 2 Step: 14879 Loss: 0.052695 Tokens per Sec: 2520.705337\n",
      "Epoch 2 Step: 15119 Loss: 0.048949 Tokens per Sec: 2410.753010\n",
      "Epoch 2 Step: 15359 Loss: 0.049319 Tokens per Sec: 2444.003767\n",
      "Epoch 2 Step: 15599 Loss: 0.051183 Tokens per Sec: 2481.253671\n",
      "Epoch 2 Step: 15839 Loss: 0.049587 Tokens per Sec: 2459.634469\n",
      "Epoch 2 Step: 16079 Loss: 0.068763 Tokens per Sec: 2447.336916\n",
      "Epoch 2 Step: 16319 Loss: 0.050009 Tokens per Sec: 2486.289250\n",
      "Epoch 2 Step: 16559 Loss: 0.054445 Tokens per Sec: 2356.981375\n",
      "Epoch 2 Step: 16799 Loss: 0.059054 Tokens per Sec: 2341.790913\n",
      "Epoch 2 Step: 17039 Loss: 0.066519 Tokens per Sec: 2438.512803\n",
      "Epoch 2 Step: 17279 Loss: 0.051239 Tokens per Sec: 2461.575375\n",
      "Epoch 2 Step: 17519 Loss: 0.055436 Tokens per Sec: 2459.725543\n",
      "Epoch 2 Step: 17759 Loss: 0.056573 Tokens per Sec: 2379.570515\n",
      "Epoch 2 Step: 17999 Loss: 0.056936 Tokens per Sec: 2403.272830\n",
      "Epoch 2 Step: 18239 Loss: 0.071790 Tokens per Sec: 2428.972721\n",
      "Epoch 2 Step: 18479 Loss: 0.058107 Tokens per Sec: 2316.866480\n",
      "Epoch 2 Step: 18719 Loss: 0.055148 Tokens per Sec: 2393.143962\n",
      "Epoch 2 Step: 18959 Loss: 0.062460 Tokens per Sec: 2424.998935\n",
      "Epoch 2 Step: 19199 Loss: 0.056545 Tokens per Sec: 2405.050859\n",
      "Epoch 2 Step: 19439 Loss: 0.075046 Tokens per Sec: 2408.463331\n",
      "Epoch 2 Step: 19679 Loss: 0.044282 Tokens per Sec: 2418.294812\n",
      "Epoch 2 Step: 19919 Loss: 0.058876 Tokens per Sec: 2325.659865\n",
      "Epoch 2 Step: 20159 Loss: 0.063109 Tokens per Sec: 2336.629202\n",
      "Epoch 2 Step: 20399 Loss: 0.051702 Tokens per Sec: 2339.954408\n",
      "Epoch 2 Step: 20639 Loss: 0.054136 Tokens per Sec: 2385.340782\n",
      "Epoch 2 Step: 20879 Loss: 0.051736 Tokens per Sec: 2351.338167\n",
      "Epoch 2 Step: 21119 Loss: 0.067850 Tokens per Sec: 2361.307043\n",
      "Epoch 2 Step: 21359 Loss: 0.054586 Tokens per Sec: 2307.885518\n",
      "Epoch 2 Step: 21599 Loss: 0.057267 Tokens per Sec: 2409.109919\n",
      "Epoch 2 Step: 21839 Loss: 0.071078 Tokens per Sec: 2351.896344\n",
      "Epoch 2 Step: 22079 Loss: 0.058159 Tokens per Sec: 2326.434952\n",
      "Epoch 2 Step: 22319 Loss: 0.058208 Tokens per Sec: 2339.470203\n",
      "Epoch 2 Step: 22559 Loss: 0.053033 Tokens per Sec: 2409.223971\n",
      "Epoch 2 Step: 22799 Loss: 0.054263 Tokens per Sec: 2269.134667\n",
      "Epoch 2 Step: 23039 Loss: 0.062451 Tokens per Sec: 2453.574068\n",
      "Epoch 2 Step: 23279 Loss: 0.055998 Tokens per Sec: 2397.037174\n",
      "Epoch 2 Step: 23519 Loss: 0.051290 Tokens per Sec: 2351.647618\n",
      "Epoch 2 Step: 23759 Loss: 0.053839 Tokens per Sec: 2448.545267\n",
      "Epoch 2 Step: 23999 Loss: 0.065624 Tokens per Sec: 2426.751436\n",
      "2019-01-07 20:12:47  - Saved model to files: ./models_weights/my-model.sklearn.epoch_00002.notebook_run.gpu0 ./models_weights/my-model.pytorch.epoch_00002.notebook_run.gpu0\n",
      "Epoch 3 Step: 239 Loss: 0.050503 Tokens per Sec: 2206.911807\n",
      "Epoch 3 Step: 479 Loss: 0.056120 Tokens per Sec: 2231.597249\n",
      "Epoch 3 Step: 719 Loss: 0.066362 Tokens per Sec: 2265.060222\n",
      "Epoch 3 Step: 959 Loss: 0.075225 Tokens per Sec: 2252.288086\n",
      "Epoch 3 Step: 1199 Loss: 0.046970 Tokens per Sec: 2322.741021\n",
      "Epoch 3 Step: 1439 Loss: 0.059510 Tokens per Sec: 2284.620380\n",
      "Epoch 3 Step: 1679 Loss: 0.059562 Tokens per Sec: 2338.936927\n",
      "Epoch 3 Step: 1919 Loss: 0.054074 Tokens per Sec: 2278.632688\n",
      "Epoch 3 Step: 2159 Loss: 0.070618 Tokens per Sec: 2299.928325\n",
      "Epoch 3 Step: 2399 Loss: 0.053511 Tokens per Sec: 2322.374124\n",
      "Epoch 3 Step: 2639 Loss: 0.042013 Tokens per Sec: 2424.040278\n",
      "Epoch 3 Step: 2879 Loss: 0.053936 Tokens per Sec: 2311.101310\n",
      "Epoch 3 Step: 3119 Loss: 0.069536 Tokens per Sec: 2346.512658\n",
      "Epoch 3 Step: 3359 Loss: 0.046956 Tokens per Sec: 2388.928893\n",
      "Epoch 3 Step: 3599 Loss: 0.051383 Tokens per Sec: 2372.830809\n",
      "Epoch 3 Step: 3839 Loss: 0.043823 Tokens per Sec: 2377.678152\n",
      "Epoch 3 Step: 4079 Loss: 0.064916 Tokens per Sec: 2357.338115\n",
      "Epoch 3 Step: 4319 Loss: 0.044692 Tokens per Sec: 2417.295109\n",
      "Epoch 3 Step: 4559 Loss: 0.055474 Tokens per Sec: 2274.987232\n",
      "Epoch 3 Step: 4799 Loss: 0.061781 Tokens per Sec: 2323.988887\n",
      "Epoch 3 Step: 5039 Loss: 0.053243 Tokens per Sec: 2326.915328\n",
      "Epoch 3 Step: 5279 Loss: 0.056134 Tokens per Sec: 2314.329776\n",
      "Epoch 3 Step: 5519 Loss: 0.053381 Tokens per Sec: 2324.536284\n",
      "Epoch 3 Step: 5759 Loss: 0.059614 Tokens per Sec: 2388.278434\n",
      "Epoch 3 Step: 5999 Loss: 0.056721 Tokens per Sec: 2303.347859\n",
      "Epoch 3 Step: 6239 Loss: 0.074365 Tokens per Sec: 2378.476925\n",
      "Epoch 3 Step: 6479 Loss: 0.035591 Tokens per Sec: 2375.891351\n",
      "Epoch 3 Step: 6719 Loss: 0.071318 Tokens per Sec: 2318.768479\n",
      "[...] up to epoch 11"
     ]
    }
   ],
   "source": [
    "batch_size = 192\n",
    "train_iters_per_epoch = 24000\n",
    "max_epoch = 11\n",
    "cuda_device_id = 0  # None for CPU, 0 for first GPU, etc.\n",
    "model_suffix = \".notebook_run.gpu0\"\n",
    "epoch_model_name = MY_MODEL_NAME + \".epoch_{}\" + model_suffix\n",
    "preproc_sgnn_sklearn_pipeline, model_trainer = train_model_on_data(\n",
    "    max_epoch, train_iters_per_epoch, batch_size,\n",
    "    preproc_sgnn_sklearn_pipeline=None,\n",
    "    model_trainer=None,\n",
    "    cuda_device_id=cuda_device_id,\n",
    "    plot=False,\n",
    "    epoch_model_name=epoch_model_name\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## License\n",
    "\n",
    "BSD 3-Clause License.\n",
    "\n",
    "\n",
    "Copyright (c) 2018, Guillaume Chevalier\n",
    "\n",
    "All rights reserved.\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}