TAIL-GPT2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TAIL_GPT2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b47962527cdb494bbd99917e476560dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_99bc3c37681940749c971c5ef5fb3aea",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dcb4af4be2e54df199c3a904555d252e",
              "IPY_MODEL_56006c263e904b61a47ee5a43b288435"
            ]
          }
        },
        "99bc3c37681940749c971c5ef5fb3aea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dcb4af4be2e54df199c3a904555d252e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a2f46d9154644aeb8ff26b91d5bdd1f3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a4ebd9b9c4264fe2959b9acdcfcd613a"
          }
        },
        "56006c263e904b61a47ee5a43b288435": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c83c0124be584f91ac054248ec464279",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2/? [00:00&lt;00:00,  7.41 tables/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_537a1e63fec8476a9710808b0f2a9e03"
          }
        },
        "a2f46d9154644aeb8ff26b91d5bdd1f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a4ebd9b9c4264fe2959b9acdcfcd613a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c83c0124be584f91ac054248ec464279": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "537a1e63fec8476a9710808b0f2a9e03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fdeef5c2f9a943bd85a7d1c1a3a31cc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f220949175ad4958ac5af2067c1cf20d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f45a37dce1e34670b1043bc2fa4afc4a",
              "IPY_MODEL_78c08486f3004933a6a1d7ab55be705b"
            ]
          }
        },
        "f220949175ad4958ac5af2067c1cf20d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f45a37dce1e34670b1043bc2fa4afc4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bb0b995aa7d7402aa7a77918d32f9d3a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6871645c8fc9431d8165c5f3175149e6"
          }
        },
        "78c08486f3004933a6a1d7ab55be705b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4f6a1d170413433fb092be8d0800b5cf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/? [00:00&lt;00:00, 16.19 tables/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8458993b481b40d49b83664154680be0"
          }
        },
        "bb0b995aa7d7402aa7a77918d32f9d3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6871645c8fc9431d8165c5f3175149e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f6a1d170413433fb092be8d0800b5cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8458993b481b40d49b83664154680be0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1762e91f2a364b9aa9fd0a47471b6163": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_07e4db16b0ec45e9bc7d1afd6389acb8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4d2f157318e54677a40154c3259667d6",
              "IPY_MODEL_81888fa5177b4cdcb8661d64ff08099d"
            ]
          }
        },
        "07e4db16b0ec45e9bc7d1afd6389acb8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d2f157318e54677a40154c3259667d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f49edf7792b0497388cf6e5f97d67295",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 19,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 19,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eaaa914f90064b3797f0ceb99dcabc62"
          }
        },
        "81888fa5177b4cdcb8661d64ff08099d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dc1dd525ddb44ce7873aebf1e93fa27b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 19/19 [00:14&lt;00:00,  1.27ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_232ee38d28bb49ea886d9faeaddf3941"
          }
        },
        "f49edf7792b0497388cf6e5f97d67295": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eaaa914f90064b3797f0ceb99dcabc62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc1dd525ddb44ce7873aebf1e93fa27b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "232ee38d28bb49ea886d9faeaddf3941": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2bea72bce2664776bc7b77ce4583e91f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_201743a7a4f848a5b7defd01ceb952d0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2d94c4df94f4460ab5a1d3106b8c54c9",
              "IPY_MODEL_70731ab015f845339d0f5ace6d675529"
            ]
          }
        },
        "201743a7a4f848a5b7defd01ceb952d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d94c4df94f4460ab5a1d3106b8c54c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2956b028c099475b91da62f75ac7746b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_51cd7965d9ed4a4b84f0a14690b467b6"
          }
        },
        "70731ab015f845339d0f5ace6d675529": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c8eb22628b9d4cb79705009becaa1150",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [00:58&lt;00:00, 19.48s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_11b96dba92d64fb59c65fcd215cbfe8e"
          }
        },
        "2956b028c099475b91da62f75ac7746b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "51cd7965d9ed4a4b84f0a14690b467b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8eb22628b9d4cb79705009becaa1150": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "11b96dba92d64fb59c65fcd215cbfe8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c577d667e1fa4062b2a7389949ee4c3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_eb14e6c835d04d10b55e79ba189c3671",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8e4ec7dfbbc2413888fd60a07e1b7d50",
              "IPY_MODEL_4f03c12e232f444793f982bdbb82727b"
            ]
          }
        },
        "eb14e6c835d04d10b55e79ba189c3671": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8e4ec7dfbbc2413888fd60a07e1b7d50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7ac64b5ae9c24542b9fd2e42ad83693c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 762,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 762,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35bad58d7a81400e9d0fce2d1c00a54d"
          }
        },
        "4f03c12e232f444793f982bdbb82727b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b6ee8dc704b5419d9353642211eb9670",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 762/762 [00:00&lt;00:00, 10.4kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_456dde99162f4cdcb365aadc832085d2"
          }
        },
        "7ac64b5ae9c24542b9fd2e42ad83693c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "35bad58d7a81400e9d0fce2d1c00a54d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b6ee8dc704b5419d9353642211eb9670": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "456dde99162f4cdcb365aadc832085d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0c968779981a4b80b2b5d72ee8537701": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5cee65ccecf64a9e94597f547c2d9066",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f5bb25652bba418aa17a9b69cd0339b6",
              "IPY_MODEL_138a013fc4f9410f957f009418efb238"
            ]
          }
        },
        "5cee65ccecf64a9e94597f547c2d9066": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f5bb25652bba418aa17a9b69cd0339b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_262cf76d70324d559ce48448d1a45e73",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 352833716,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 352833716,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_51b2b42dbdf54a9d9ddf9167f9648c49"
          }
        },
        "138a013fc4f9410f957f009418efb238": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_134f37ffe9de43ff8ab9f1938e954d7b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 353M/353M [00:06&lt;00:00, 50.4MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_85c37a912f7b4539829daf634e05db4c"
          }
        },
        "262cf76d70324d559ce48448d1a45e73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "51b2b42dbdf54a9d9ddf9167f9648c49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "134f37ffe9de43ff8ab9f1938e954d7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "85c37a912f7b4539829daf634e05db4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lY6gnf4DyZxZ"
      },
      "source": [
        "#**TAIL-GPT2**\n",
        "##*Diretoria de Processamento de Linguagem Natural*\n",
        "####*Autor: Felipe Honorato*\n",
        "\n",
        "Bom, nesse notebook iremos entender o funcionamento da GPT2 e adaptaremos ela para gerar analises do trip advisor. \n",
        "\n",
        "A GPT-2 é construída em cima dos transformers, um tipo de rede que utiliza um mecanísmo de atenção para guiar a rede em onde focar durante a predição da próxima palavra.\n",
        "\n",
        "No paper attention is all you need voces podem ver mais detalhes sobre esse mecanismo. [Attention is all you need](https://arxiv.org/abs/1706.03762)\n",
        "\n",
        "A GPT-2 é uma rede treinada com corpus bem vasto extraído da internet. O modelo possui versões menores que nos possibilitam treinar pelo colab. Utilizaremos a implementação da biblioteca [Huggingface](https://huggingface.co/). Lá voces tem acesso acesso à documentação e a um forum repleto de tutoriais.\n",
        "\n",
        "**PS: Lembre de usar GPU nesse notebook**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lj-zldkk08-l"
      },
      "source": [
        "***\n",
        "# **Preparando o ambiente**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zopY7hgb13WJ"
      },
      "source": [
        "#### Instalando bibliotecas e baixando dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D29fr-2xyWhR"
      },
      "source": [
        "%%capture\n",
        "!pip install transformers #Huggingface\n",
        "!gdown --id 15_qLjo8_qgTAIWqVWpp_JMKzfpwQT29B #Baixando o dataset (Fiz upload no meu drive)\n",
        "!pip install datasets"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-fJQ4_fM1uHM"
      },
      "source": [
        "#### Importação das bibliotecas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3eW6vfq1tWB"
      },
      "source": [
        "import re\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from datasets import load_dataset\n",
        "from sklearn.model_selection import train_test_split\n",
        "from transformers import GPT2Tokenizer, GPT2LMHeadModel, DataCollatorForLanguageModeling\n",
        "from transformers import Trainer, TrainingArguments, pipeline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c_Bh43Z62d8H"
      },
      "source": [
        "***\n",
        "# **Tratamento do dataset**\n",
        "Nessa etapa, prepararemos o dataset em batches para ser passado ao modelo, adicionaremos tokens especiais e tokenizaremos as sentenças.   \n",
        "Funções de tratamento de texto como remover caracteres indevidos devem ser aplicadas nessa fase"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_k75zApW2Wl1"
      },
      "source": [
        "df = pd.read_csv('/content/tripadvisor_hotel_reviews.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E5qregj226av"
      },
      "source": [
        "# Pegamos a média de palavras por amostra no dataset.\n",
        "def get_num_words(df):\n",
        "  num_words = []\n",
        "  for id, row in df.iterrows():\n",
        "    num_words.append(len(row['Review'].split()))\n",
        "  \n",
        "  num_words = np.array(num_words)\n",
        "  return np.mean(num_words)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ptS6ooHuMW1t"
      },
      "source": [
        "MEAN_WORDS = get_num_words(df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lcdhu-X53FoN"
      },
      "source": [
        "Para controlar nossa geração, utilizaremos os conceitos do paper [CTRL](https://arxiv.org/pdf/1909.05858.pdf). Nele, aprendemos a utilizar tokens de maneira a guiar o estilo de texto a ser gerado pelo modelo.   \n",
        "\n",
        "Nesse caso, quero poder selecionar entre gerar analises boas e ruins.   \n",
        "\n",
        "Nosso dataset possui um desbalanceamento claro entre as classes, o que poderia ser resolvido com técnicas de amostragem ou augmentação, mas como não é o foco desse notebook, apenas aplicarei o treinamento."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "znhrXnxK3D_j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dbaabbb1-91ed-4abe-b8ee-5aa4fe9d3969"
      },
      "source": [
        "df.Rating.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5    9054\n",
              "4    6039\n",
              "3    2184\n",
              "2    1793\n",
              "1    1421\n",
              "Name: Rating, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "43QRE-rN3lv0"
      },
      "source": [
        "def adicionar_tokens(row):\n",
        "  # Os tokens serão inseridos dentro do texto e especificaremos eles ao utilizar o tokenizador.\n",
        "  text = row['Review'] \n",
        "  if row['Rating'] >= 4:\n",
        "    # Repare a necessidade do espaço, visto que caso esse não seja inserido, um token de classificação \n",
        "    # será misturado com uma palavra do texto\n",
        "\n",
        "    # PS: O token de início de texto não é essencial.\n",
        "    row['Review'] = \"<|startoftext|> <BOM> \" + text + ' <|endoftext|> '\n",
        "  else:\n",
        "    row['Review'] = \"<|startoftext|> <RUIM> \" + text + ' <|endoftext|> '\n",
        "  \n",
        "  return row\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hin7Vy1H5EfT"
      },
      "source": [
        "# Aplicando o tratamento:\n",
        "df = df.apply(adicionar_tokens, axis = 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "txg9Ho4A5IbK",
        "outputId": "5c981ae4-ad7d-428d-df8a-82b022cb7b0b"
      },
      "source": [
        "# Um exemplo de como fica nossa amostra: \n",
        "print(df.loc[0]['Review'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<|startoftext|> <BOM> nice hotel expensive parking got good deal stay hotel anniversary, arrived late evening took advice previous reviews did valet parking, check quick easy, little disappointed non-existent view room room clean nice size, bed comfortable woke stiff neck high pillows, not soundproof like heard music room night morning loud bangs doors opening closing hear people talking hallway, maybe just noisy neighbors, aveda bath products nice, did not goldfish stay nice touch taken advantage staying longer, location great walking distance shopping, overall nice experience having pay 40 parking night,   <|endoftext|> \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFIu4pbP5K8B"
      },
      "source": [
        "# Separaremos nosso dataset em treino e validação da seguinte maneira:\n",
        "train_set, val_set = train_test_split(df['Review'], test_size = 0.1, random_state = 42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p4753Syb6aWD"
      },
      "source": [
        "\n",
        "Salvaremos os conjuntos como CSV para carregá-los atraves da função load_dataset, que achei mais pratica utilizando arquivos CSVs,\n",
        "mas que podem ser utilizados arquivos nos formatos JSON, Pandas e txt. [Documentação](https://huggingface.co/docs/datasets/)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w337QV4G6sS7"
      },
      "source": [
        "train_set.to_csv('train.csv')\n",
        "val_set.to_csv('test.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89,
          "referenced_widgets": [
            "b47962527cdb494bbd99917e476560dd",
            "99bc3c37681940749c971c5ef5fb3aea",
            "dcb4af4be2e54df199c3a904555d252e",
            "56006c263e904b61a47ee5a43b288435",
            "a2f46d9154644aeb8ff26b91d5bdd1f3",
            "a4ebd9b9c4264fe2959b9acdcfcd613a",
            "c83c0124be584f91ac054248ec464279",
            "537a1e63fec8476a9710808b0f2a9e03",
            "fdeef5c2f9a943bd85a7d1c1a3a31cc4",
            "f220949175ad4958ac5af2067c1cf20d",
            "f45a37dce1e34670b1043bc2fa4afc4a",
            "78c08486f3004933a6a1d7ab55be705b",
            "bb0b995aa7d7402aa7a77918d32f9d3a",
            "6871645c8fc9431d8165c5f3175149e6",
            "4f6a1d170413433fb092be8d0800b5cf",
            "8458993b481b40d49b83664154680be0"
          ]
        },
        "id": "XlO5ZKvL6uQh",
        "outputId": "df5d039c-b343-489e-bbc9-9ed487f622c3"
      },
      "source": [
        "dataset = load_dataset('csv', data_files =  {'train': 'train.csv',\n",
        "                                             'test' : 'test.csv'});"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using custom data configuration default-de075c62ca943650\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset csv/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/csv/default-de075c62ca943650/0.0.0/2dc6629a9ff6b5697d82c25b73731dd440507a69cbce8b425db50b751e8fcfd0...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b47962527cdb494bbd99917e476560dd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fdeef5c2f9a943bd85a7d1c1a3a31cc4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rDataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-de075c62ca943650/0.0.0/2dc6629a9ff6b5697d82c25b73731dd440507a69cbce8b425db50b751e8fcfd0. Subsequent calls will reuse this data.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QluEz1lB691w",
        "outputId": "2c3e3aee-1e7a-4fa6-abf2-09558aad9b11"
      },
      "source": [
        "dataset"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['Unnamed: 0', 'Review'],\n",
              "        num_rows: 18441\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['Unnamed: 0', 'Review'],\n",
              "        num_rows: 2050\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qC--Ai7E7gfL"
      },
      "source": [
        "Agora já temos nosso DatasetDict. Temos uma coluna que não será utilizada, mas removeremos na próxima sessão, junto à tokenização."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MzfeYi35__9Y"
      },
      "source": [
        "***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-cvgMAz15wLJ"
      },
      "source": [
        "# **Tokenizador**\n",
        "O tokenizador converterá nossas palavras para números e gerará attention weights, que conduzirão o modelo na identificação de qual parte do contexto passado é relevante para a palavra atual.   \n",
        "\n",
        "Utilizaremos o tokenizador pré-treinado da distilGPT2, que é um modelo menor e consequentemente mais rápido disponibilizado pela biblioteca."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "30j2xl958sQH"
      },
      "source": [
        "tokenizer = GPT2Tokenizer.from_pretrained('distilgpt2', bos_token = '<|startoftext|>')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LvXB6lVM9GtI"
      },
      "source": [
        "Adicionaremos nossos tokens especiais ao modelo.   \n",
        "Os tokens especiais são guardados de maneira diferenciada pelo modelo, garantindo assim que esses não sofram cortes.   \n",
        "Além disso, esses podem ser ocultados durante a geração. Mais detalhes na [documentação](https://huggingface.co/transformers/main_classes/tokenizer.html)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qxj1OWNz8zl4",
        "outputId": "8ec19a2e-dcb7-46c1-b9de-b7d902c7ff80"
      },
      "source": [
        "# O valor de retorno 2 é referente à quantidade de tokens adicionados ao tokenizador.\n",
        "tokenizer.add_special_tokens({'additional_special_tokens': ['<BOM>', '<RUIM>']})\n",
        "tokenizer.add_special_tokens({'pad_token': tokenizer.eos_token})"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d09EN9KAV6Uk",
        "outputId": "93893610-7c83-41cd-94cc-35ab83782c09"
      },
      "source": [
        "tokenizer.additional_special_tokens"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['<BOM>', '<RUIM>']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DBBTRXS09yTS",
        "outputId": "d0464a70-658a-4e15-ff62-0198daffd352"
      },
      "source": [
        "print(\"String antes da tokenização: \\n\")\n",
        "print(df.loc[0]['Review'])\n",
        "\n",
        "print('\\nString após tokenização: \\n')\n",
        "print(tokenizer(df.loc[0]['Review']))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "String antes da tokenização: \n",
            "\n",
            "<|startoftext|> <BOM> nice hotel expensive parking got good deal stay hotel anniversary, arrived late evening took advice previous reviews did valet parking, check quick easy, little disappointed non-existent view room room clean nice size, bed comfortable woke stiff neck high pillows, not soundproof like heard music room night morning loud bangs doors opening closing hear people talking hallway, maybe just noisy neighbors, aveda bath products nice, did not goldfish stay nice touch taken advantage staying longer, location great walking distance shopping, overall nice experience having pay 40 parking night,   <|endoftext|> \n",
            "\n",
            "String após tokenização: \n",
            "\n",
            "{'input_ids': [50257, 50258, 44460, 7541, 5789, 7647, 1392, 922, 1730, 2652, 7541, 11162, 11, 5284, 2739, 6180, 1718, 5608, 2180, 8088, 750, 1188, 316, 7647, 11, 2198, 2068, 2562, 11, 1310, 11679, 1729, 12, 32786, 1570, 2119, 2119, 3424, 3621, 2546, 11, 3996, 6792, 19092, 15175, 7393, 1029, 9582, 1666, 11, 407, 2128, 13288, 588, 2982, 2647, 2119, 1755, 3329, 7812, 20188, 82, 8215, 4756, 9605, 3285, 661, 3375, 23959, 11, 3863, 655, 31210, 12020, 11, 257, 1079, 64, 7837, 3186, 3621, 11, 750, 407, 3869, 11084, 2652, 3621, 3638, 2077, 4621, 10589, 2392, 11, 4067, 1049, 6155, 5253, 9735, 11, 4045, 3621, 1998, 1719, 1414, 2319, 7647, 1755, 11, 220, 220, 220, 50256, 220], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lzhBSgc8-LUv",
        "outputId": "4a58b721-99ed-4329-ba16-621e647748cd"
      },
      "source": [
        "tokenizer.bos_token_id"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "50257"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w7sZfGp6-b1Q"
      },
      "source": [
        "## **Aplicando ao dataset**\n",
        "Agora aplicaremos a tokenização linha por linha no nosso dataset de treino e validação."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eCOGOX3kBOGx"
      },
      "source": [
        "# Definiremos a função encode e aplicaremos o map nela, dividindo em batches a tokenização. \n",
        "# Depois removemos as colunas indesejadas"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kJjhDB10Bu2o"
      },
      "source": [
        "def encode(data):\n",
        "  return tokenizer(data['Review'], max_length= int(MEAN_WORDS), truncation = True, add_special_tokens= True, padding= False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "1762e91f2a364b9aa9fd0a47471b6163",
            "07e4db16b0ec45e9bc7d1afd6389acb8",
            "4d2f157318e54677a40154c3259667d6",
            "81888fa5177b4cdcb8661d64ff08099d",
            "f49edf7792b0497388cf6e5f97d67295",
            "eaaa914f90064b3797f0ceb99dcabc62",
            "dc1dd525ddb44ce7873aebf1e93fa27b",
            "232ee38d28bb49ea886d9faeaddf3941",
            "2bea72bce2664776bc7b77ce4583e91f",
            "201743a7a4f848a5b7defd01ceb952d0",
            "2d94c4df94f4460ab5a1d3106b8c54c9",
            "70731ab015f845339d0f5ace6d675529",
            "2956b028c099475b91da62f75ac7746b",
            "51cd7965d9ed4a4b84f0a14690b467b6",
            "c8eb22628b9d4cb79705009becaa1150",
            "11b96dba92d64fb59c65fcd215cbfe8e"
          ]
        },
        "id": "UOdf7nU7_6s5",
        "outputId": "85e12bb3-9772-4e75-95e8-1a1af45f3c14"
      },
      "source": [
        "train_set = dataset['train'].map(encode, batched = True).remove_columns(column_names= ['Unnamed: 0', 'Review']);\n",
        "eval_set = dataset['test'].map(encode, batched = True).remove_columns(column_names= ['Unnamed: 0', 'Review']);"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1762e91f2a364b9aa9fd0a47471b6163",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=19.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2bea72bce2664776bc7b77ce4583e91f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=3.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C-91SN8HCzIt"
      },
      "source": [
        "Utilizaremos o [Data collator](https://huggingface.co/transformers/main_classes/data_collator.html) para formar batches dos nossos itens de treinamento. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RYqeTu5iB0gj"
      },
      "source": [
        "# O Parametro MLM é referente à masking language model e como utilizaremos a GPT2 sem MASKS, deixaremos desabilitado\n",
        "data_collator = DataCollatorForLanguageModeling(tokenizer = tokenizer, mlm = False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PGHAqToTWGUM",
        "outputId": "e16efb4f-7081-4034-fcc8-ff6b65086843"
      },
      "source": [
        "len(data_collator(train_set['input_ids']).get('input_ids'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "18441"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wxi84C1b_7yZ"
      },
      "source": [
        "***\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bEtJOR0w92JS"
      },
      "source": [
        "#**Modelo** \n",
        "Uma vez que nossos dados e nosso tokenizador está pronto, inicializaremos o modelo e aplicaremos nosso fine-tuning."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 133,
          "referenced_widgets": [
            "c577d667e1fa4062b2a7389949ee4c3e",
            "eb14e6c835d04d10b55e79ba189c3671",
            "8e4ec7dfbbc2413888fd60a07e1b7d50",
            "4f03c12e232f444793f982bdbb82727b",
            "7ac64b5ae9c24542b9fd2e42ad83693c",
            "35bad58d7a81400e9d0fce2d1c00a54d",
            "b6ee8dc704b5419d9353642211eb9670",
            "456dde99162f4cdcb365aadc832085d2",
            "0c968779981a4b80b2b5d72ee8537701",
            "5cee65ccecf64a9e94597f547c2d9066",
            "f5bb25652bba418aa17a9b69cd0339b6",
            "138a013fc4f9410f957f009418efb238",
            "262cf76d70324d559ce48448d1a45e73",
            "51b2b42dbdf54a9d9ddf9167f9648c49",
            "134f37ffe9de43ff8ab9f1938e954d7b",
            "85c37a912f7b4539829daf634e05db4c"
          ]
        },
        "id": "mWo-K4t3DUwS",
        "outputId": "6ae4a38e-ede6-41e9-c0b2-4ea01c4a91c4"
      },
      "source": [
        "model = GPT2LMHeadModel.from_pretrained('distilgpt2', bos_token_id = tokenizer.bos_token_id);\n",
        "model.resize_token_embeddings(len(tokenizer))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c577d667e1fa4062b2a7389949ee4c3e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=762.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0c968779981a4b80b2b5d72ee8537701",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=352833716.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Embedding(50260, 768)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r7sCrlg5DW1t"
      },
      "source": [
        "# Especificaremos parametros para o treinamento:\n",
        "training_args = TrainingArguments(output_dir= 'GPTAIL/', overwrite_output_dir= True, \n",
        "                                  num_train_epochs = 5,\n",
        "                                  save_steps= -1, per_device_train_batch_size = 8, \n",
        "                                  prediction_loss_only=True,\n",
        "                                  per_device_eval_batch_size = 16, evaluation_strategy = 'epoch', load_best_model_at_end =True,\n",
        "                                  eval_accumulation_steps = 1)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Ah09OnYEWOp"
      },
      "source": [
        "trainer = Trainer(model = model, args = training_args, \n",
        "                train_dataset= train_set, eval_dataset=eval_set, data_collator = data_collator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pQlHsyWVEaT4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        },
        "outputId": "75fc3e18-f940-438f-d40d-6611243bd197"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='11530' max='11530' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [11530/11530 36:56, Epoch 5/5]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>4.989300</td>\n",
              "      <td>4.835339</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>4.756500</td>\n",
              "      <td>4.715217</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>4.653300</td>\n",
              "      <td>4.663646</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>4.559200</td>\n",
              "      <td>4.640491</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>4.517500</td>\n",
              "      <td>4.636616</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=11530, training_loss=4.7500711895958405, metrics={'train_runtime': 2216.4855, 'train_samples_per_second': 5.202, 'total_flos': 0, 'epoch': 5.0, 'init_mem_cpu_alloc_delta': 0, 'init_mem_gpu_alloc_delta': 0, 'init_mem_cpu_peaked_delta': 0, 'init_mem_gpu_peaked_delta': 0, 'train_mem_cpu_alloc_delta': -351977472, 'train_mem_gpu_alloc_delta': 990287872, 'train_mem_cpu_peaked_delta': 364670976, 'train_mem_gpu_peaked_delta': 1419818496})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V-k2zESEAGy9",
        "outputId": "106bf833-cbd0-4bb5-c453-f367cf42f067"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "102.51406411049346"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XW3VNW68XdX6"
      },
      "source": [
        "*** \n",
        "# **Geração**\n",
        "Para a geração, utilizaremos a classe [Pipeline](https://huggingface.co/transformers/main_classes/pipelines.html), que une as principais tarefas em PLN. Nela selecionaremos a geração de texto e a configuraremos."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oHVG0LbihcMZ"
      },
      "source": [
        "model = model.to('cpu')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3XvXbBMAX-Yp"
      },
      "source": [
        "revisor = pipeline('text-generation', tokenizer= tokenizer, model = model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "STjDsLnYYqap"
      },
      "source": [
        "Usaremos o método de decodificação chamado Greedy Search. Nele, simplesmente pegaremos as palavras mais prováveis a cada etapa. O método influencia diretamente no resultado do modelo, portanto vale a pena dar uma olhada nos estilos de decodificação. [How to generate](https://huggingface.co/blog/how-to-generate)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RwYn76KWYoOH",
        "outputId": "9316da5c-8f27-4491-9aed-76284de20011"
      },
      "source": [
        "print(revisor(\"<BOM>\", top_k = 30)[0].get('generated_text'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "<BOM>good bad, arrived 3pm late october 2007 hotel beautiful clean comfortable good location, room small comfortable clean, view bay view ocean floor fabulous, bathroom pretty tiny clean, location great just blocks ginza beach restaurants shops restaurants,   \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3uUQ2HJOhphm",
        "outputId": "8bbf3dce-a4a1-438c-c7b3-d243a6b8c54e"
      },
      "source": [
        "print(revisor(\"<RUIM>\", top_k = 30)[0].get('generated_text'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "<RUIM>avoidance price paid stay, rooms small bathrooms poorly-maintained, bed hard pillows slippers no pillows, n't stay unless paid hotel,                \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BPOIkgcmZhnO"
      },
      "source": [
        "A geração possui diversos parametros a serem adaptados de acordo com o problema de vocês. Para mais detalhes: [Documentação](https://huggingface.co/transformers/main_classes/model.html?highlight=generate)\n"
      ]
    }
  ]
}