train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "744d2b39-1db6-4abf-9aba-fff0351e5f12",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7.1+cu110\n",
      "4.10.2\n",
      "2.4\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import transformers\n",
    "from transformers import AutoModelWithLMHead, PreTrainedTokenizerFast\n",
    "from fastai.text.all import *\n",
    "import fastai\n",
    "import re\n",
    "\n",
    "print(torch.__version__)\n",
    "print(transformers.__version__)\n",
    "print( fastai.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f7cdf335",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
      "The tokenizer class you load from this checkpoint is 'GPT2Tokenizer'. \n",
      "The class this function is called from is 'PreTrainedTokenizerFast'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['▁안녕', '하', '세', '요.', '▁한국어', '▁G', 'P', 'T', '-2', '▁입', '니다.', '😤', ':)', 'l^o']\n",
      "위치추적 전자장치(전자발찌) 훼손 전후로 여성 2명을 잇달아 살해한 ᄋ씨(33)에 대해 구속영장을 신청했다.\n",
      "경찰에 따르면 ᄀ씨는 지난달 29일 오후 9시30분쯤 서울 강남구 논현동 자신의 집에서 ᄂ씨를 흉기로 찔러 숨지게 한 혐의를 받고 있다.\n",
      "ᄀ씨의 범행은 B씨가 “여자친구가 있다”며 경찰에 신고하면서 드러났다.\n",
      "B씨도 같은 날 오전 10시경 경찰서에 자진출석해 조사를 받았다.\n",
      "ᄂ씨와 함께 있던 ᄃ(31·여)은 이날 새벽 1시까지 조사에서 “내가 여자 친구와 사귀고 있어 괴롭다”고 진술했다.</d> 부산시는 오는 10월까지 '부산형\n"
     ]
    }
   ],
   "source": [
    "#download model and tokenizer\n",
    "tokenizer = PreTrainedTokenizerFast.from_pretrained(\"skt/kogpt2-base-v2\",\n",
    "  bos_token='</s>', eos_token='</s>', unk_token='<unk>',\n",
    "  pad_token='<pad>', mask_token='<mask>') \n",
    "model = AutoModelWithLMHead.from_pretrained(\"skt/kogpt2-base-v2\")\n",
    "\n",
    "#test tokenizer\n",
    "print(tokenizer.tokenize(\"안녕하세요. 한국어 GPT-2 입니다.😤:)l^o\"))\n",
    "\n",
    "#test model ouput\n",
    "text = \"\"\"위치추적 전자장치(전자발찌) 훼손 전후로 여성 2명을 잇달아 살해한 \"\"\"\n",
    "input_ids = tokenizer.encode(text)\n",
    "gen_ids = model.generate(torch.tensor([input_ids]),\n",
    "                           max_length=128,\n",
    "                           repetition_penalty=2.0,\n",
    "                           pad_token_id=tokenizer.pad_token_id,\n",
    "                           eos_token_id=tokenizer.eos_token_id,\n",
    "                           bos_token_id=tokenizer.bos_token_id,\n",
    "                           use_cache=True\n",
    "                        )\n",
    "generated = tokenizer.decode(gen_ids[0,:].tolist())\n",
    "print(generated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f7cc350a-93fc-4dc0-82fc-08f314a700aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2744438"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('text_data.txt') as f:\n",
    "    lines = f.read()\n",
    "lines=\" \".join(lines.split())\n",
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "35292d3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2422896"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines=re.sub('\\(계속\\).*?[●○]', '', lines)\n",
    "lines=re.sub('[●○]', '', lines)\n",
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ceb9ef2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#model input output tokenizer\n",
    "class TransformersTokenizer(Transform):\n",
    "    def __init__(self, tokenizer): self.tokenizer = tokenizer\n",
    "    def encodes(self, x): \n",
    "        toks = self.tokenizer.tokenize(x)\n",
    "        return tensor(self.tokenizer.convert_tokens_to_ids(toks))\n",
    "    def decodes(self, x): return TitledStr(self.tokenizer.decode(x.cpu().numpy()))\n",
    "\n",
    "#split data\n",
    "train=lines[:int(len(lines)*0.9)]\n",
    "test=lines[int(len(lines)*0.9):]\n",
    "splits = [[0],[1]]\n",
    "\n",
    "#init dataloader\n",
    "tls = TfmdLists([train,test], TransformersTokenizer(tokenizer), splits=splits, dl_type=LMDataLoader)\n",
    "batch,seq_len = 8,256\n",
    "dls = tls.dataloaders(bs=batch, seq_len=seq_len)\n",
    "# dls.show_batch(max_n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d34f1523",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SuggestedLRs(valley=tensor(1.7378e-05))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.534882</td>\n",
       "      <td>3.688004</td>\n",
       "      <td>39.964989</td>\n",
       "      <td>01:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.260040</td>\n",
       "      <td>3.587380</td>\n",
       "      <td>36.139263</td>\n",
       "      <td>01:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.086436</td>\n",
       "      <td>3.552240</td>\n",
       "      <td>34.891392</td>\n",
       "      <td>01:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.974506</td>\n",
       "      <td>3.527494</td>\n",
       "      <td>34.038567</td>\n",
       "      <td>01:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.925448</td>\n",
       "      <td>3.523227</td>\n",
       "      <td>33.893635</td>\n",
       "      <td>01:55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEKCAYAAAAVaT4rAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjOUlEQVR4nO3de3xcdZ3/8ddncm3Se5teaHovtKWltCUoghe0SJHlpiDWRRYURFYX1FVuiwL6WHbZdVWEXS91ufhbkQK10PIDERAQEHVpI73QWyq0TXpL2qZp7pnMfPaPmZRQ0jaXmTkzmffz8ZhHMmfOzPl8CX2fc77nO99j7o6IiGSPUNAFiIhIain4RUSyjIJfRCTLKPhFRLKMgl9EJMso+EVEskxu0AV0x8iRI33SpElBlyEiklFWrVq1191LDl+eEcE/adIkVq5cGXQZIiIZxcy2dbVcXT0iIllGwS8ikmUU/CIiWSYj+vi7Eg6HqaqqoqWlJehSAlNYWEhpaSl5eXlBlyIiGSRpwW9m9wPnAdXuPvuw174JfA8ocfe9vfn8qqoqBg0axKRJkzCzvhecYdydffv2UVVVxeTJk4MuR0QySDK7eh4Ezjl8oZmNBz4ObO/Lh7e0tDBixIisDH0AM2PEiBFZfcYjIr2TtOB395eB/V289EPgRqDP80Fna+h3yPb2i/RnB1vCPLNuF/saWhP+2Sm9uGtmFwA73H11N9a9xsxWmtnKmpqaFFSXXAMHDgRg69atzJ49+xhri0i227S7nmt/Wc7aHXUJ/+yUBb+ZFQG3Ard1Z313X+zuZe5eVlLyni+e9dyaR+GHs+GOobGfax7t+2eKiCRJ5f4mAMYPL0r4Z6fyiH8qMBlYbWZbgVKg3MzGJH3Lax6FJ6+HukrAYz+fvL5P4X/TTTfx4x//+NDzO+64g+985zssWLCA+fPnc9JJJ7F8+fKjfkYkEuGGG27g1FNPZc6cOfzsZz8D4PLLL3/Xey+77DJWrFjR61pFJPNU7m8GYNzQAQn/7JQFv7uvdfdR7j7J3ScBVcB8d9+d9I3/7rsQbn73snBzbHkvLVq0iEceeeTQ80cffZTPf/7zPP7445SXl/Piiy/yjW98g6Pd2vK+++5jyJAhvP7667z++uv8/Oc/5+233+bqq6/mgQceAKCuro7XXnuNc889t9e1ikjmqaxtYvTgAgrzchL+2ckczvkwcCYw0syqgNvd/b5kbe+o6qp6trwb5s2bR3V1NTt37qSmpoZhw4YxduxYvv71r/Pyyy8TCoXYsWMHe/bsYcyYrk9qnn32WdasWcPSpUtj5dTVUVFRwdlnn81XvvIVqqurWbZsGRdffDG5uRn7lQsR6YXK/U2MH5b4bh5IYvC7+2eP8fqkZG37PYaUxrt5uljeB5dccglLly5l9+7dLFq0iIceeoiamhpWrVpFXl4ekyZNOupwS3fn3nvvZeHChe957fLLL+ehhx5iyZIl3H///X2qU0QyT1VtM++bPDwpn50dUzYsuA3yDusnyxsQW94HixYtYsmSJSxdupRLLrmEuro6Ro0aRV5eHi+++CLbtnU5Md4hCxcu5Cc/+QnhcBiAzZs309jYCMCVV17J3XffDcCsWbP6VKeIZJZwJMquumbGD0t8/z5k8JQNPTLn0tjP33031r0zpDQW+h3Le2nWrFnU19czbtw4xo4dy2WXXcb5559PWVkZc+fOZcaMGUd9/9VXX83WrVuZP38+7k5JSQlPPPEEAKNHj2bmzJlcdNFFfapRRDLPzgPNRB1KkzCiB8COdvExXZSVlfnh8/Fv2LCBmTNnBlRR8jU1NXHSSSdRXl7OkCFDjrhef//vIJKNXq3Yy+fu+zMPf/E0PjB1RK8/x8xWuXvZ4cuzo6snwzz//PPMmDGD66677qihLyL9U2VtbAx/qbp6ssdZZ53F9u19mspIRDJY5f4mckLG2CGFSfl8HfGLiKSZytpmjhtaSG5OciI6o4M/E65PJFO2t1+kv6qqTd4Yfsjg4C8sLGTfvn1ZG34d8/EXFibnVFBEglO5vzmpwZ+xffylpaVUVVXRH2bu7K2OO3CJSP/R3BZhb0Mr44cn58IuZHDw5+Xl6c5TItLvVNUmb1bODhnb1SMi0h+9M5RTwS8ikhU6pmNOZlePgl9EJI1U7m+iMC9EycCCpG1DwS8ikkYqa5soHVaU1HtqK/hFRNJIbChn8rp5QMEvIpJWOo74k0nBLyKSJuqawtS3tCf1wi4o+EVE0kbHUM5kfmsXFPwiImkjFV/eAgW/iEjaODSGX0f8IiLZobK2iUGFuQwpykvqdhT8IiJponJ/cqdj7qDgFxFJE5W1zUkf0QMKfhGRtODuSb8BSwcFv4hIGqhpaKUlHE36iB5Q8IuIpIXK/R1DOdXVIyKSFSr2NAAwrWRQ0rel4BcRSQMV1Q0U5oUoTfIEbaDgFxFJC5v31DNt1EBCoeRNx9xBwS8ikgYq9jRwwqjkd/OAgl9EJHAHW8LsPtjCtNEDU7K9pAW/md1vZtVmtq7Tsu+Z2UYzW2Nmj5vZ0GRtX0QkU3Rc2O0PR/wPAucctuw5YLa7zwE2A7ckcfsiIhmhYk89ACeMzvDgd/eXgf2HLXvW3dvjT/8ElCZr+yIimWLzntSN6IFg+/i/APzmSC+a2TVmttLMVtbU1KSwLBGR1KqoTt2IHggo+M3sVqAdeOhI67j7Yncvc/eykpKS1BUnIpJiqRzRA5Cbsi3FmdkVwHnAAnf3VG9fRCSd1DXHRvQcn6L+fUhx8JvZOcBNwEfcvSmV2xYRSUdbqmMjeo4flZqhnJDc4ZwPA38EpptZlZldBfwnMAh4zszeMLOfJmv7IiKZINUjeiCJR/zu/tkuFt+XrO2JiGSiVI/oAX1zV0QkUKke0QMKfhGRQKV6RA8o+EVEAhPEiB5Q8IuIBGZLdezCbipH9ICCX0QkMIcmZ9MRv4hIdti8p4EBeTkpHdEDCn4RkcAEMaIHFPwiIoGp2NOQ8v59UPCLiAQiqBE9oOAXEQlEEHP0dFDwi4gEoGMoZ6pH9ICCX0QkEBV7GijIDTEuxSN6QMEvIhKILTUNTC0ZSE6KR/SAgl9EJBAVexqYFkD/Pij4RURSrrG1nR0HmgO5sAsKfhGRlHurphGA40cr+EVEskJFfESPunpERLLEluoGckPGxBHFgWxfwS8ikmIV1Q1MHllMXk4wEazgFxFJsS3VwY3oAQW/iEhKtbZH2LavMbARPaDgFxFJqbf3NhJ1mKrgFxHJDu9Mzpb6OXo6KPhFRFKoYk8DIYMpJcGM6AEFv4hISm2pbmD88CIK83ICq0HBLyKSQluqg7nrVmcKfhGRFGmPRHlrb0OgF3ZBwS8ikjLb9zcRjnigF3ZBwS8ikjIVAd5usTMFv4hIinQM5VRXj4hIlthS3cBxQwoZWJAbaB1JC34zu9/Mqs1sXadlw83sOTOriP8clqzti4ikm4rqeqYFcHP1wyXziP9B4JzDlt0M/M7djwd+F38uItLvRaMem5ytJNhuHkhi8Lv7y8D+wxZfCPwi/vsvgIuStX0RkXSy40AzLeFoYHfd6izVffyj3X0XQPznqCOtaGbXmNlKM1tZU1OTsgJFRJJh/a6DAJzQz7t6+sTdF7t7mbuXlZSUBF2OiEiflG+rJT8nxKzjBgddSsqDf4+ZjQWI/6xO8fZFRAKxalsts8cNDnSOng6pDv4VwBXx368Alqd4+yIiKdfaHmHNjjpOmZgeAxmTOZzzYeCPwHQzqzKzq4C7gI+bWQXw8fhzEZF+7c2dB2lrjzJ/QnoEf9K+ReDunz3CSwuStU0RkXRUvq0WgPn9/YhfRERiyrfXUjpsAKMHFwZdCqDgFxFJKndn1bbatOnfBwW/iEhS7TjQzJ6DrQp+EZFsUb79AEDaXNgFBb+ISFKVb6tlQF4OM8YE/43dDt0KfjMrNrNQ/PcTzOwCM8tLbmkiIplv1bZa5o4fSm5O+hxnd7eSl4FCMxtHbFbNzxObfVNERI6gqa2d9bsOplX/PnQ/+M3dm4BPAfe6+yeBE5NXlohI5ltdWUck6syfODToUt6l28FvZh8ALgOeii8L9hYyIiJprnx77Itb88Zn5hH/14BbgMfd/U0zmwK8mLSqRET6gfJttUwtKWZYcX7QpbxLt47a3f33wO8B4hd597r79cksTEQkk7k7q7bXcvaJo4Mu5T26O6rnV2Y22MyKgfXAJjO7IbmliYhkrr/WNHKgKZx2F3ah+109J7r7QWK3SnwamABcnqyiREQy3SsVsTsHvn/yiIArea/uBn9efNz+RcBydw8DnrSqREQy3Asbq5laUsykkcVBl/Ie3Q3+nwFbgWLgZTObCBxMVlEiIpmsobWdP721jwUz069/H7p/cfce4J5Oi7aZ2UeTU5KISGZ7taKGcMT52IxRQZfSpe5e3B1iZj8ws5Xxx/eJHf2LiMhhnt9QzeDCXMrS8MIudL+r536gHrg0/jgIPJCsokREMlU06ry4sZozp49Kq/l5Ouvut2+nuvvFnZ5/x8zeSEI9IiIZbXXVAfY1trFgZnp280D3j/ibzeyDHU/M7AygOTkliYhkrhc2VpMTMj5yQknQpRxRd4/4rwX+n5kNiT+vBa5ITkkiIpnr+Q3VnDJxGEOL0muahs66dcTv7qvd/WRgDjDH3ecBH0tqZSIiGWbngWY27DrIgjQdzdOhR1ce3P1g/Bu8AP+YhHpERDLWCxurAdK6fx/6dutFS1gVIiL9wAsbq5k4ooipJQODLuWo+hL8mrJBRCSuuS3CH7bs5WMzRmGW3sfFR724a2b1dB3wBgxISkUiIhnopU3VtLZHOStNp2no7KjB7+7pc1t4EZE09uvyHYwaVMBpU9JvNs7DpefXykREMsi+hlZe2lTNJ+eNIyeU3t08oOAXEemzFat30h51PjW/NOhSukXBLyLSR78ur2L2uMFMH5MZveOBBL+Zfd3M3jSzdWb2sJkVBlGHiEhfbdpdz7odB/nUvMw42ocAgt/MxgHXA2XuPhvIARalug4RkURYVl5Fbsi4YO5xQZfSbUF19eQCA8wsFygCdgZUh4hIr0WizuN/2cGZ00sYObAg6HK6LeXB7+47gP8AtgO7gDp3fzbVdYiI9NWrW/ZSXd+aMRd1OwTR1TMMuBCYDBwHFJvZ57pY75qOO37V1NSkukwRkWNaVl7F4MLctJ+b53BBdPWcBbzt7jXuHgaWAacfvpK7L3b3MncvKylJ33mtRSQ71beE+e2buzn/5OMoyM0JupweCSL4twOnmVmRxSa0WABsCKAOEZFeu+/Vt2kJR7m0bHzQpfRYEH38fwaWAuXA2ngNi1Ndh4hIb+080MxPf/9X/mbOWE4ePzTocnqsu3fgSih3vx24PYhti4j01V2/2Yg73PKJGUGX0iv65q6ISA+8vnU/K1bv5EsfnkLpsKKgy+kVBb+ISDdFo853n1zPmMGFXHvm1KDL6TUFv4hINy0tr2LtjjpuOXcGRfmB9JQnhIJfRKQb6lvC/Pszmzhl4jAuODlzpmfoioJfRKQbvv/sZvY1tnLbeSem/a0Vj0XBLyJyDG9UHuAXf9zK3502MSOHbx5OwS8ichTtkSi3LFvLqEEFfHPh9KDLSYjMvTohIpIC9//hbTbsOshPPzefQYV5QZeTEDriFxE5gsr9TfzwuQrOmjmahbPGBF1Owij4RUS64O58e/k6zOC7F87K+Au6nSn4RUS6sKx8By9tquEbZ0/nuKEDgi4noRT8IiKH2b6vidtXvMn7Jg3nytMnBV1Owin4RUQ6aY9E+dojf8EMfvCZk8kJ9Z8ung4a1SMi0sm9L2yhfPsB7vnsvIydhO1YdMQvIhK3att+7n2hgk/NG5fx0zIcjYJfRITYXDxfXfIG44YN4DsXzgq6nKRSV4+ICHDHivXsPNDMY9d+oN98UetIdMQvIlnvqTW7+HV5Ff/w0WmcMnF40OUknYJfRLLa7roW/unxtZw8fijXLTg+6HJSQsEvIlkrGnW++dhq2tqj3P2ZueTlZEckZkcrRUS68MBrW3l1y15uO/9EJo8sDrqclFHwi0hWqthTz789s5GzZo5m0anjgy4npRT8IpJ1IlHnxl+voTg/h7suPqlfTcDWHQp+Eck6//PHrfxl+wFuP38WIwcWBF1Oyin4RSSr7DjQzL//dhMfOaGEC+f232/nHo2CX0SyhrvzrcfXAnDnJ2dnXRdPBwW/iGSNFat38uKmGr559vR+OwFbdyj4RSQrVO5v4jtPrmfu+KFc0Q/n2O8JzdUjIv3eU2t2cfOyNeDwbxfP6Zdz7PeEgl9E+q2mtna+++R6lrxeybwJQ7ln0TzGD8/eLp4OCn4R6ZdWVx7gG4+t5q81DXz5zKl8/eMnZM2UDMcSSPCb2VDgv4HZgANfcPc/BlGLiPQvTW3tfP/ZzTzwh7cpGVTAL696P2dMGxl0WWklqCP+HwHPuPslZpYP6NxLRPrslYoa/unxtVTub+ay90/gpk/MYHA/n1u/N1Ie/GY2GPgwcCWAu7cBbamuQ0T6j7qmMP/81HoeW1XFlJHFPHLNabx/yoigy0pbQRzxTwFqgAfM7GRgFfBVd2/svJKZXQNcAzBhwoSUFykimeGZdbv59vJ17G9s48tnTuX6BcdTmJcTdFlpLYgrHbnAfOAn7j4PaARuPnwld1/s7mXuXlZSUpLqGkUkzVXXt/Dlh1Zx7S9XUTKwgOVfOYMbz5mh0O+GII74q4Aqd/9z/PlSugh+EZGuRKPOktcrues3G2gJR7lh4XSu+fAUjdjpgZQHv7vvNrNKM5vu7puABcD6VNchIpmnYk89tyxby8pttZw2ZTj/8smTmFIyMOiyMk5Qo3quAx6Kj+h5C/h8QHWISAY42BLmP1/YwgN/eJvigly+d8kcLjmlNGsnWeurQILf3d8AyoLYtohkjvZIlCWvV/KD5zZT29TGp08p5aZzZjAiC+fQTyR9c1dE0o678+Kmav7tN5vYtKee900ezm3nncjscUOCLq1fUPCLSFr501v7+N5vN7FqWy0Thhfxk8vmc87sMerWSSAFv4ikhU276/nnp9bzSsVeRg8u4M5PzubSsvEarZMECn4RCVR9S5gfPV/BA69tZVBhLreeO5PLPzBR4/GTSMEvIoFwd1as3smdT22gpqGVRadO4MaF0xlWnB90af2egl9EUq66voWbf72WFzZWM6d0CIv/roy544cGXVbWUPCLSEo9vXYXtz6+lqa2CLeddyJXnD4p6++IlWoKfhFJibqmMLevWMcTb+xkTukQfnDpXKaN0rdug6DgF5Gke6WihhseW0NNQytfP+sEvvzRqRqtEyAFv4gkTXNbhLt+s4Ff/HEbU0uKWfx3pzOndGjQZWU9Bb+IJEXl/iaueOB/eaumkS+cMZkbz5muIZppQsEvIgnXEo7wpf9Zxb6GNn519fs5Xfe8TSsKfhFJuNuXv8n6XQd54MpTFfppSFdXRCShHn29kkdWVnLdx6bx0Rmjgi5HuqDgF5GEeXNnHd9evo4PThvJ1846Iehy5AgU/CKSEHXNYb78UDnDivL50aK5+lJWGlMfv4j0mbtz09I17Kht5pEvnaYbpaQ5HfGLSJ/94rWtPPPmbm7+xAxOmTg86HLkGBT8ItIna6oOcOfTGzhr5iiu+uDkoMuRblDwi0iv1TWH+cqvyhk1qJD/+PTJuktWhlAfv4j0Ske//q4DLTx67QcYWqR59DOFjvhFpFc69+vPnzAs6HKkBxT8ItJj63bU8S9Pb2TBDPXrZyIFv4j0SENrO//wq3KGF+fzPfXrZyT18YtIt7k733p8Ldv3N7Hkmg8wXPfHzUg64heRbntsVRVPvLGTr511Au+brPH6mapfB/9Df97G9Q//hb9srw26FJGMFok6K1bv5Pblb3L61BF85aPTgi5J+qBfd/U0trbz4sZqVqzeydzxQ/n8GZNYOGsMreEoB1vCNLS209YeJT83FHvkxH6GzAgZhMzIzbFDr6kvU7JNNOo8vW4XP3q+gorqBmaMGcTdn9E8PJnO3D3oGo6prKzMV65c2av3NrS28+tVVTz42lbe3tvYpzoKckMU5ecwvDifEcUFDC/Op7ggl3AkSlt7lHAkihkMGZDPsKI8hhXnM7w4n5EDCygZFHsMKswlEnHC0SjtEaetPUpLe4SWcJSWcIRI1MkJGbkhIzcnRG7IsPhOKCcUewzIy4k98nMoyO2nO6Q1j8Lvvgt1VTCkFBbcBnMuDbqqrLG3oZWn1uziV3/ezqY99UwbNZCvLjievzlpLCGFfsYws1XuXvae5f09+DtEo87vK2pYU1nHwMJcBhXmMqggl/zcEOFIlNb22CMciRL12PpR91g4H3o9QmNrO7WNYfY1trKvoY2mtgj5uSHy4mcGkSjUNbVR2xSmORxJ0H+BIwsZDBmQ986jKJ9BhbkMLsxjcGEuAwtyKcgLUZAb20nk5oRwd9wh6o4Zh14rzIv9zM+Nrd9xJpSXY+TFd0K5oRARj/23iUYdDArjO6KE3Tx7zaPw5PUQbn5nWd4AOP8ehX8SNba28+z63Sx/YyevVOwlEnVmjBnE3585lfPmHKej/AyU9cEfhJZwhH2Nbeytb2VvQys19a00tLbHjuhzQuSFYoFamJdDYV7sZ27IiESd9qjTHj8rcGKjKaIO4UjszKC5LUJzOEpjazt1zWEONIepiz/qW8IcbG7nYEuYtvZoytqbGzIKckOH2pcTMvI62hrfeXTsYIryY2cshbk50ClPcsy4dfOnGRre857PbxpwHH+68KV3zn7MyIufhRXn51JU8M4OqOP1UMgO7egO/z/dADP65xlTN4UjUV7eXMMTb+zkufW7aQlHGTd0ABfMPY4L5x7HjDGDgy5R+uBIwR9YH7+Z5QArgR3ufl5QdSRTYV4O44YOYNzQAYHV0NYejZ2xhCO0tsd2JGYQCsWuY0Q9tk5LOEJLfJ22+NlP7L0RwhEnHIm9NxJ1QgY5IcPMcKA1/t7mcKzLKhL1Qzuv2PuihKNOOP65zeEIexvaaGprp/WwHVMk6gwOV3fZlsKmXXzhwcQfABTmhRiQl0NRfm78rMjICb2zsxpUmMugjjOo+JlicfxRlJ9zaAeTE78m1LF8YMd6+bkU5qW+S649EuVgSzsH4megdc1t7G1oY0dtM5W1TVTVNrNpdz11zWGGFuVx8fxSLpw7jrKJw9Sd088FeXH3q8AGQIcUSdTRXTOwIIOu4/+wFOoq37O4fdBxPHH1GUTi3XCRaKwrrqmtnca2dhpbY2dCkU6vRaJRzGLXSYzYT4COE92IO63xnVbsLCpyaKcViTqt7RFqG9vYtq/p0JlUW6TnZ1FmUJyfy4D8HIrzcxiQnxv/GTvLy+n0iA0uiO2YzezQ2Z4T657rLBp1WsIRmtpiO9/Gtgj1LWHqW9ppauu6q9EMxgwuZPywIhbOGs3CWWP40PEl5Of260F+0kkgaWBmpcDfAHcC/xhEDZLGFtzWZR9//tl3MHf80MDK6tDWHutia2htP7Sj6NgZhTt2RK3tNLTGrgk1tUXiy2I/O543tUVoaG0/tJOKemyHE+10DSbqTsgMI3aBH3tXzxghs1iXWV4OQ4vyGTskh8EDYtd4BhXmMXhALsOK8hlSlMfQAXkML85n7JABCvksF9Rh4N3AjcCgI61gZtcA1wBMmDAhNVVJeui4gJumo3piZ1H5DNO3ViVDpTz4zew8oNrdV5nZmUdaz90XA4shdnE3NdVJ2phzadoEvUh/E8T53hnABWa2FVgCfMzMfhlAHSIiWSnlwe/ut7h7qbtPAhYBL7j751Jdh4hIttIVHhGRLBPoGD93fwl4KcgaRESyjY74RUSyjIJfRCTLKPhFRLJMRkzSZmZ1QEWnRUOAum7+PhLY28tNd/68nq7T1fLDl/XHdnR+rnb0rMburNOXdnT+Peh2HKvOYz1PZDuOVeexXk9kOyCxf5OJ7l7ynrViMxem9wNYfKTnx/odWJmo7fZkna6WZ0M7DqtZ7UijdnTVpqDa0dO6k9mO7rQlVe1Ixd/E3TOmq+fJozzvzu+J2m5P1ulqeTa0o/NztaP7tXR3nb60o/PvQbejq9eCakd3Pqe/tAPIkK6evjCzld7FfNSZRu1IL2pHeukv7YDUtCVTjvj7YnHQBSSI2pFe1I700l/aASloS78/4hcRkXfLhiN+ERHpRMEvIpJlFPwiIlkmq4PfzD5kZj81s/82s9eCrqe3zCxkZnea2b1mdkXQ9fSWmZ1pZq/E/yZnBl1PX5hZsZmtit94KCOZ2cz432Kpmf190PX0lpldZGY/N7PlZnZ20PX0lplNMbP7zGxpXz8rY4PfzO43s2ozW3fY8nPMbJOZbTGzm4/2Ge7+irtfC/x/4BfJrPdIEtEO4EJgHBAGqpJV69EkqB0ONACFZHY7AG4CHk1OlceWoH8fG+L/Pi4FAhkqmaB2POHuXwSuBD6TxHKPKEHteMvdr0pIQX35tluQD+DDwHxgXadlOcBfgSlAPrAaOBE4iVi4d36M6vS+R4HBmdoO4GbgS/H3Ls3gdoTi7xsNPJTB7TiL2E2GrgTOy9R2xN9zAfAa8LeZ3I74+74PzO8H7ejzv/FA5+PvC3d/2cwmHbb4fcAWd38LwMyWABe6+78CXZ5ym9kEoM7dDyaz3iNJRDvMrApoiz+NJLHcI0rU3yOuFihISqHHkKC/x0eBYmL/iJvN7Gl3jya38ndL1N/D3VcAK8zsKeBXSSy5Swn6exhwF/Abdy9PcsldSvC/jz7L2OA/gnFAZafnVcD7j/Geq4AHklZR7/S0HcuAe83sQ8DLySysh3rUDjP7FLAQGAr8Z1Ir65ketcPdbwUwsyuBvakO/aPo6d/jTOBTxHbCTyezsB7q6b+P64idhQ0xs2nu/tNkFtcDPf17jADuBOaZ2S3xHUSv9Lfgty6WHfUbau5+e5Jq6YsetcPdm4jtwNJNT9uxjNhOLN30+P8rAHd/MPGl9ElP/x4vkZ53yOtpO+4B7kleOb3W03bsA65NxIYz9uLuEVQB4zs9LwV2BlRLX6gd6UXtSC9qRx/1t+B/HTjezCabWT6xC2wrAq6pN9SO9KJ2pBe1o6+CuMKdoKvkDwO7eGcI41Xx5ecCm4ldLb816DrVDrVD7VA70q0dmqRNRCTL9LeuHhEROQYFv4hIllHwi4hkGQW/iEiWUfCLiGQZBb+ISJZR8EvGMrOGFG8vIfdsiN93oM7M/mJmG83sP7rxnovM7MREbF9EwS8SZ2ZHnbvK3U9P4OZecfd5wDzgPDM74xjrX0Rstk+RPutvk7RJljOzqcB/ASVAE/BFd99oZucD3yI27/k+4DJ332NmdwDHAZOAvWa2GZhAbI70CcDdHpvkCzNrcPeB8Vkr7wD2ArOBVcDn3N3N7FzgB/HXyoEp7n7EKXbdvdnM3iA2UyNm9kXgmnidW4DLgbnE5sX/iJl9C7g4/vb3tLO3/90ku+iIX/qbxcB17n4K8E3gx/HlrwKnxY+ylwA3dnrPKcTmQf/b+PMZxKaHfh9wu5nldbGdecDXiB2FTwHOMLNC4GfAJ9z9g8RC+ajMbBhwPO9Mp73M3U9195OBDcS+2v8asTlcbnD3ue7+16O0U+SYdMQv/YaZDQROBx6L3XsDeOeGLqXAI2Y2ltjR9Nud3rrC3Zs7PX/K3VuBVjOrJnZHsMNvBfm/7l4V3+4bxM4YGoC33L3jsx8mdvTelQ+Z2RpgOnCXu++OL59tZv9M7J4EA4Hf9rCdIsek4Jf+JAQccPe5Xbx2L/ADd1/RqaumQ+Nh67Z2+j1C1/9Oulqnq/nVj+QVdz/PzE4AXjWzx939DeBB4CJ3Xx2/kcuZXbz3aO0UOSZ19Ui/4bHbZ75tZp+G2C33zOzk+MtDgB3x369IUgkbgSmdbrF3zBt7u/tm4F+J3ZwdYBCwK969dFmnVevjrx2rnSLHpOCXTFZkZlWdHv9ILCyvMrPVwJvAhfF17yDWNfIKsQuvCRfvLvoy8IyZvQrsAeq68dafAh82s8nAt4E/A88R25F0WALcEB8COpUjt1PkmDQts0gCmdlAd2+I3+D7v4AKd/9h0HWJdKYjfpHE+mL8Yu+bxLqXfhZsOSLvpSN+EZEsoyN+EZEso+AXEckyCn4RkSyj4BcRyTIKfhGRLKPgFxHJMv8H+41oNCwOwGYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#gpt2 ouput is tuple, we need just one val\n",
    "class DropOutput(Callback):\n",
    "    def after_pred(self): self.learn.pred = self.pred[0]\n",
    "        \n",
    "        \n",
    "learn = Learner(dls, model, loss_func=CrossEntropyLossFlat(), cbs=[DropOutput], metrics=Perplexity()).to_fp16()\n",
    "lr=learn.lr_find()\n",
    "print(lr)\n",
    "learn.fit_one_cycle(5, lr)\n",
    "# learn.fine_tune(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "766f2b36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'위치추적 전자장치(전자발찌) 훼손 전후로 여성 2명을 잇달아 살해한 녀석이었다. “야, 조아영. 너 왜 그래?” “아니야...! 그건 니가 잘못했어...” 아영이에게 모욕당한 여자애들은, 그녀의 말을 듣고는 크게 놀라며 고개를 숙였다. “그럼 이제부터 어떻게 할 거냐고 물어봐도 돼요?” “어차피 우리 반 애들한테까지 다 들었으니까 얘기해 줄 수 있을 것 같네요.” 지은이는 말꼬리를 흐렸다. “이제부터는 내가 알아서 해라. 알았지? 그럼 내일 다시 만나자!” 예'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt=\"위치추적 전자장치(전자발찌) 훼손 전후로 여성 2명을 잇달아 살해한 \"\n",
    "prompt_ids = tokenizer.encode(prompt)\n",
    "inp = tensor(prompt_ids)[None].cuda()\n",
    "preds = learn.model.generate(inp,\n",
    "                           max_length=128,\n",
    "                           pad_token_id=tokenizer.pad_token_id,\n",
    "                           eos_token_id=tokenizer.eos_token_id,\n",
    "                           bos_token_id=tokenizer.bos_token_id,\n",
    "                           repetition_penalty=2.0,       \n",
    "                           use_cache=True\n",
    "                          ) \n",
    "tokenizer.decode(preds[0].cpu().numpy())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5aabc1b-2e8b-4a39-9758-d84f44a7fa0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.model.save_pretrained(\"kogpt2novel_backup\")\n",
    "model.push_to_hub(\"kogpt2novel\")\n",
    "tokenizer.push_to_hub(\"kogpt2novel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fd3a99b-93b7-43b4-8dd1-aa2855d295a1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}