mlp_imdb_hf_dset_and_trainer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "42351838529a4455a40fa4031b415de0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_38f532b995bd4609a5226a8dffe1333b",
              "IPY_MODEL_48d99ab3cbda4f129da4d52422a96bfe",
              "IPY_MODEL_d72494e0f62141da99ba26a8670401ed"
            ],
            "layout": "IPY_MODEL_0ef2ff2ce4f6484eb15aad577db81be2"
          }
        },
        "38f532b995bd4609a5226a8dffe1333b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd898ffa1c0147cca90ddfdfb5217427",
            "placeholder": "​",
            "style": "IPY_MODEL_8dd82d120f1241e8b1fe355cb404699f",
            "value": "Downloading builder script: 100%"
          }
        },
        "48d99ab3cbda4f129da4d52422a96bfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50e148a3895b4259b89111d8061022fc",
            "max": 4314,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_507ce3ce11864c54bee4d1eddb9eeca4",
            "value": 4314
          }
        },
        "d72494e0f62141da99ba26a8670401ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_edd42ef9fcaa480d9cabc01b6121bfb8",
            "placeholder": "​",
            "style": "IPY_MODEL_b2513d1573174490866d7b3063373fed",
            "value": " 4.31k/4.31k [00:00&lt;00:00, 208kB/s]"
          }
        },
        "0ef2ff2ce4f6484eb15aad577db81be2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd898ffa1c0147cca90ddfdfb5217427": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8dd82d120f1241e8b1fe355cb404699f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "50e148a3895b4259b89111d8061022fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "507ce3ce11864c54bee4d1eddb9eeca4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "edd42ef9fcaa480d9cabc01b6121bfb8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2513d1573174490866d7b3063373fed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4630891dd8f4f90b2f70c4ebdd8edd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_13674620d858401aa72ae62d9bf0be65",
              "IPY_MODEL_9a633afccca94f299fb2841cfb033179",
              "IPY_MODEL_ef0e64812c854d9e9a0caa79012ae784"
            ],
            "layout": "IPY_MODEL_44ac40d2653a45bbabfe957ddb3c79fb"
          }
        },
        "13674620d858401aa72ae62d9bf0be65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70fc545a54704b818f27bb00155e3561",
            "placeholder": "​",
            "style": "IPY_MODEL_857fe79f69ba4af0b4cc0b1e3f903649",
            "value": "Downloading metadata: 100%"
          }
        },
        "9a633afccca94f299fb2841cfb033179": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c70229dc99924a59bc5a92d6f2897326",
            "max": 2166,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7030adf144b94eeba875eee604a4e5f3",
            "value": 2166
          }
        },
        "ef0e64812c854d9e9a0caa79012ae784": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10abc3d911434e8990deaca602a03b61",
            "placeholder": "​",
            "style": "IPY_MODEL_4254daf008f14097abf01a304ccef189",
            "value": " 2.17k/2.17k [00:00&lt;00:00, 109kB/s]"
          }
        },
        "44ac40d2653a45bbabfe957ddb3c79fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70fc545a54704b818f27bb00155e3561": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "857fe79f69ba4af0b4cc0b1e3f903649": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c70229dc99924a59bc5a92d6f2897326": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7030adf144b94eeba875eee604a4e5f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "10abc3d911434e8990deaca602a03b61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4254daf008f14097abf01a304ccef189": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7d9dec1f32524fb6b420eb28d86c44d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7cf265ef231146fba8533ce6ea792d8e",
              "IPY_MODEL_ae23a53db68c4e7d98b9cb89779611c2",
              "IPY_MODEL_711cbc31c69c4f76b91e4cd9e16c66ed"
            ],
            "layout": "IPY_MODEL_d628077557e242afb3283a3862ca5099"
          }
        },
        "7cf265ef231146fba8533ce6ea792d8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_19b5b66d197e4e4e92046e8df7aedd82",
            "placeholder": "​",
            "style": "IPY_MODEL_6f7dd8e4d39241e3b7ccfea392426f6f",
            "value": "Downloading readme: 100%"
          }
        },
        "ae23a53db68c4e7d98b9cb89779611c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_26e12abd6a8f49949f091811067b02fd",
            "max": 7590,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_da2d8fe4cf8442a9bceddbf9a6e3a3ae",
            "value": 7590
          }
        },
        "711cbc31c69c4f76b91e4cd9e16c66ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cf1a70429aad44a89f66ec38186b7ba4",
            "placeholder": "​",
            "style": "IPY_MODEL_fadedd0bd442493792f69891892952d8",
            "value": " 7.59k/7.59k [00:00&lt;00:00, 224kB/s]"
          }
        },
        "d628077557e242afb3283a3862ca5099": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "19b5b66d197e4e4e92046e8df7aedd82": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6f7dd8e4d39241e3b7ccfea392426f6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "26e12abd6a8f49949f091811067b02fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da2d8fe4cf8442a9bceddbf9a6e3a3ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cf1a70429aad44a89f66ec38186b7ba4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fadedd0bd442493792f69891892952d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5e29e2bf78224e53ac896c4f63b8f6cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ced5ba6de24144e398ab577f5f2f8374",
              "IPY_MODEL_163789fda33f4e31b60435dd3784ff4d",
              "IPY_MODEL_cff3260942ad4d8b88912aab1c7d269a"
            ],
            "layout": "IPY_MODEL_5d546174b0db40eeba5609da158fadc8"
          }
        },
        "ced5ba6de24144e398ab577f5f2f8374": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77f76e3e26e742d298d4d855051f1656",
            "placeholder": "​",
            "style": "IPY_MODEL_1a6f44e6b1c64df3bcf9eed56d063b8d",
            "value": "Downloading data: 100%"
          }
        },
        "163789fda33f4e31b60435dd3784ff4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8956764a0cc24f7bbe856aa5deecaef2",
            "max": 84125825,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_43a29ffd5d1941678861869b56205b75",
            "value": 84125825
          }
        },
        "cff3260942ad4d8b88912aab1c7d269a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f304d3fd6984797b30ec5946c9cd168",
            "placeholder": "​",
            "style": "IPY_MODEL_a8015739dfef48638594760853e334c6",
            "value": " 84.1M/84.1M [00:03&lt;00:00, 32.1MB/s]"
          }
        },
        "5d546174b0db40eeba5609da158fadc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77f76e3e26e742d298d4d855051f1656": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1a6f44e6b1c64df3bcf9eed56d063b8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8956764a0cc24f7bbe856aa5deecaef2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "43a29ffd5d1941678861869b56205b75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5f304d3fd6984797b30ec5946c9cd168": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8015739dfef48638594760853e334c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f2fefd93b41f48efa0073821dc1e17f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_05fc0c3c0a1f40afa30fc0ee8fd842bc",
              "IPY_MODEL_692dc5f6e92b47558eb619157d32c0fd",
              "IPY_MODEL_e9cdfcdae49f41fe9c08e4369913bb3d"
            ],
            "layout": "IPY_MODEL_0331d681442540ca8bd4c700fc4cefd7"
          }
        },
        "05fc0c3c0a1f40afa30fc0ee8fd842bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_454ae06ee6cc4907a2adf95e18220179",
            "placeholder": "​",
            "style": "IPY_MODEL_ffa0fab5cdaa490d83668f12edaf45fb",
            "value": "Generating train split: 100%"
          }
        },
        "692dc5f6e92b47558eb619157d32c0fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_132a7e8115464b1c85a1193cbaaeb40e",
            "max": 25000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c16eaa8120744b4da9274247eb607551",
            "value": 25000
          }
        },
        "e9cdfcdae49f41fe9c08e4369913bb3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_54dd8991e78d43de907844158db3bfab",
            "placeholder": "​",
            "style": "IPY_MODEL_64417186b95742bb898e93ab4db3a8de",
            "value": " 25000/25000 [00:24&lt;00:00, 3764.86 examples/s]"
          }
        },
        "0331d681442540ca8bd4c700fc4cefd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "454ae06ee6cc4907a2adf95e18220179": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ffa0fab5cdaa490d83668f12edaf45fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "132a7e8115464b1c85a1193cbaaeb40e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c16eaa8120744b4da9274247eb607551": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "54dd8991e78d43de907844158db3bfab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64417186b95742bb898e93ab4db3a8de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "94a0a7da61f540d2a970d3cadaa4741c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d0f3bf849d9543c69ea877704bec1d14",
              "IPY_MODEL_bd289390f6dd4237a8cd37bd4d928bee",
              "IPY_MODEL_374d3fc538fd49c8a872243b3430273f"
            ],
            "layout": "IPY_MODEL_f9b5204267ef4053ac35cd7a9596e66b"
          }
        },
        "d0f3bf849d9543c69ea877704bec1d14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_63b54645353141b3abb7ab1fbfffa798",
            "placeholder": "​",
            "style": "IPY_MODEL_c50de387c1644880a8a820976a9ba1ad",
            "value": "Generating test split:  98%"
          }
        },
        "bd289390f6dd4237a8cd37bd4d928bee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_91d8f88d93994267b8fe5576db40f9db",
            "max": 25000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3af2393ea53c4c8ab9ef712387bd1daf",
            "value": 25000
          }
        },
        "374d3fc538fd49c8a872243b3430273f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f511922a95a04008a564a96357f9da85",
            "placeholder": "​",
            "style": "IPY_MODEL_8eef0408358643d3b39fc2622f536397",
            "value": " 24595/25000 [00:07&lt;00:00, 9613.57 examples/s]"
          }
        },
        "f9b5204267ef4053ac35cd7a9596e66b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "63b54645353141b3abb7ab1fbfffa798": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c50de387c1644880a8a820976a9ba1ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "91d8f88d93994267b8fe5576db40f9db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3af2393ea53c4c8ab9ef712387bd1daf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f511922a95a04008a564a96357f9da85": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8eef0408358643d3b39fc2622f536397": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cc7314f3759a478c8edeaf5655bf6aed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d88d8af722594c2eabd4485046d45d88",
              "IPY_MODEL_305871ccf0bd4f319d7cf0c9dad5d1e8",
              "IPY_MODEL_9c5d383869294a5281f880d9413892c0"
            ],
            "layout": "IPY_MODEL_972a378bb99b4cd7aa11a38e781ff1d6"
          }
        },
        "d88d8af722594c2eabd4485046d45d88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e31d3e17f19b4bd9a76ce6cae18a4954",
            "placeholder": "​",
            "style": "IPY_MODEL_ab13b5f94d514530960dd9a5e04ecfb2",
            "value": "Generating unsupervised split: 100%"
          }
        },
        "305871ccf0bd4f319d7cf0c9dad5d1e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a3d71d0f37e4caf96bc51ca8f5add70",
            "max": 50000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e96bd7b982a04de4b9c1853c555ae80b",
            "value": 50000
          }
        },
        "9c5d383869294a5281f880d9413892c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bbd117f6b1aa4b17ae7b8f7e6165b9b2",
            "placeholder": "​",
            "style": "IPY_MODEL_f7ec90e7c08a4107a3632d33f322db92",
            "value": " 50000/50000 [00:09&lt;00:00, 7947.88 examples/s]"
          }
        },
        "972a378bb99b4cd7aa11a38e781ff1d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "e31d3e17f19b4bd9a76ce6cae18a4954": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab13b5f94d514530960dd9a5e04ecfb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6a3d71d0f37e4caf96bc51ca8f5add70": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e96bd7b982a04de4b9c1853c555ae80b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bbd117f6b1aa4b17ae7b8f7e6165b9b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7ec90e7c08a4107a3632d33f322db92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "19e6fccd17b74a119afcd777c037304e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ceab64d976f64788b3b6e35336191a4b",
              "IPY_MODEL_b897e299e5354065aaf78388c2ce1acf",
              "IPY_MODEL_e9046f1d005841b9a300ddeb6a36b760"
            ],
            "layout": "IPY_MODEL_0e79ae10dc314cdaaa0e79b5094f795e"
          }
        },
        "ceab64d976f64788b3b6e35336191a4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3d547f0cb24b4789b1f89e5909c14dd2",
            "placeholder": "​",
            "style": "IPY_MODEL_7c08383552324582b148603564e3091b",
            "value": "100%"
          }
        },
        "b897e299e5354065aaf78388c2ce1acf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ff74f961019c4ffc97bac985c046fea8",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cb5c2a17b4f946dd9fca69e0312f93b4",
            "value": 3
          }
        },
        "e9046f1d005841b9a300ddeb6a36b760": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_31922779ca7f4da3b50593a1672457c3",
            "placeholder": "​",
            "style": "IPY_MODEL_5e412602b023435b811c7d3eff4a9e16",
            "value": " 3/3 [00:00&lt;00:00, 96.08it/s]"
          }
        },
        "0e79ae10dc314cdaaa0e79b5094f795e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d547f0cb24b4789b1f89e5909c14dd2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c08383552324582b148603564e3091b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ff74f961019c4ffc97bac985c046fea8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb5c2a17b4f946dd9fca69e0312f93b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "31922779ca7f4da3b50593a1672457c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e412602b023435b811c7d3eff4a9e16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d8ef93446e564136a55508d62421338e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1ef1cafd24c64584942057f9dbcaff2b",
              "IPY_MODEL_aeced29b19e644ca9edd286990768114",
              "IPY_MODEL_6b1d74d1073046f498d23531c8232911"
            ],
            "layout": "IPY_MODEL_4d52965555ef4f5a8ca8d8e64afe1a09"
          }
        },
        "1ef1cafd24c64584942057f9dbcaff2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_523717980d5f4c33824e18978eb3bdfd",
            "placeholder": "​",
            "style": "IPY_MODEL_7151c043363641e09ef528893fe5467c",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "aeced29b19e644ca9edd286990768114": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c804cfa6ece040ae99641aec9644fb5f",
            "max": 25000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b15ad5ad9e5c4633b6e3f92283460257",
            "value": 25000
          }
        },
        "6b1d74d1073046f498d23531c8232911": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0349ec6365d446cf93bfc72766bf8aaf",
            "placeholder": "​",
            "style": "IPY_MODEL_f142c3a5677941d09a987a989e0c909e",
            "value": " 24948/25000 [00:12&lt;00:00, 1956.93 examples/s]"
          }
        },
        "4d52965555ef4f5a8ca8d8e64afe1a09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "523717980d5f4c33824e18978eb3bdfd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7151c043363641e09ef528893fe5467c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c804cfa6ece040ae99641aec9644fb5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b15ad5ad9e5c4633b6e3f92283460257": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0349ec6365d446cf93bfc72766bf8aaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f142c3a5677941d09a987a989e0c909e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5456f9791de1471ab97e7d3720ee1606": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_85d9f4b0cc144b6cbcdeed25ff593fae",
              "IPY_MODEL_be395a6543d9416a85fe1adc07b60853",
              "IPY_MODEL_6b86c18e2d8446a6b9550aca77dbed3f"
            ],
            "layout": "IPY_MODEL_ef8e8973cab14258bb175a35305c5e80"
          }
        },
        "85d9f4b0cc144b6cbcdeed25ff593fae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0d37e8895b7c47499fa65cd66b5fedd0",
            "placeholder": "​",
            "style": "IPY_MODEL_38d07bf359814cb984a1ca7e99def0bc",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "be395a6543d9416a85fe1adc07b60853": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81f6a56b0a5a4d36a5980888bb0ebe9f",
            "max": 25000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7a3e852d633d41fba505060ce0ec3866",
            "value": 25000
          }
        },
        "6b86c18e2d8446a6b9550aca77dbed3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc54250faf81467cb88739ccb7dd1adb",
            "placeholder": "​",
            "style": "IPY_MODEL_7c811f393ac74438acae0b06763540f6",
            "value": " 24991/25000 [00:14&lt;00:00, 1143.40 examples/s]"
          }
        },
        "ef8e8973cab14258bb175a35305c5e80": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "0d37e8895b7c47499fa65cd66b5fedd0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "38d07bf359814cb984a1ca7e99def0bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "81f6a56b0a5a4d36a5980888bb0ebe9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a3e852d633d41fba505060ce0ec3866": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bc54250faf81467cb88739ccb7dd1adb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c811f393ac74438acae0b06763540f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c6c8f2512c524fa68f3be20390bc477d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_13be322a2c2843cda28a8113d76304f7",
              "IPY_MODEL_d87155f8a3d547ca8dd289739a7b7c99",
              "IPY_MODEL_7fe5e78a064047209e4c3bf3d4afcc5e"
            ],
            "layout": "IPY_MODEL_341a3230aeed4ed29faa2676ad2f461a"
          }
        },
        "13be322a2c2843cda28a8113d76304f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27863c3874d84bf2891f1980985f7f5d",
            "placeholder": "​",
            "style": "IPY_MODEL_e4ef45b666d6413ba340adafb7601a8e",
            "value": "Downloading builder script: 100%"
          }
        },
        "d87155f8a3d547ca8dd289739a7b7c99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ec865302018453aa29c1509203ef8e3",
            "max": 4203,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_26d0425c0e164aaab72617705818fcb2",
            "value": 4203
          }
        },
        "7fe5e78a064047209e4c3bf3d4afcc5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ce34b1ed9e94404a37667edb76f4efa",
            "placeholder": "​",
            "style": "IPY_MODEL_feb4327aecc54e119d53277f168d53bb",
            "value": " 4.20k/4.20k [00:00&lt;00:00, 229kB/s]"
          }
        },
        "341a3230aeed4ed29faa2676ad2f461a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27863c3874d84bf2891f1980985f7f5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4ef45b666d6413ba340adafb7601a8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ec865302018453aa29c1509203ef8e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "26d0425c0e164aaab72617705818fcb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9ce34b1ed9e94404a37667edb76f4efa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "feb4327aecc54e119d53277f168d53bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setup\n",
        "\n",
        "Before we start running our own Python code, install the required Python packages using [pip](https://en.wikipedia.org/wiki/Pip):\n",
        "\n",
        "* [`transformers`](https://huggingface.co/docs/transformers/index) is a popular deep learning package primarily on top of torch\n",
        "* [`datasets`](https://huggingface.co/docs/datasets/) provides support for loading, creating, and manipulating datasets\n",
        "* evaluate is a library of performance metrics (like accuracy etc)\n",
        "\n",
        "Both of these packages will be used in this course."
      ],
      "metadata": {
        "id": "LDZZUKzfPRWT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install -q transformers datasets evaluate"
      ],
      "metadata": {
        "id": "fKPHBYptQDsK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "926f3c7d-cf78-4e4e-c34d-8adfc52f5737"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m44.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m469.0/469.0 KB\u001b[0m \u001b[31m29.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 KB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m66.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 KB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 KB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 KB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 KB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 KB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 KB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "(Above, the `!` at the start of the line tells the notebook to run the line as an operating system command rather than Python code, and the `-q` argument to `pip` runs the command in \"quiet\" mode, with less output.)"
      ],
      "metadata": {
        "id": "xOeI-LA9RcYL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Get and prepare data\n",
        "\n",
        "*   Let us work with the IMDB dataset of movie review sentiment\n",
        "*   25,000 positive reviews\n",
        "*   25,000 negative reviews\n",
        "*   50,000 unlabeled reviews (which we discard for the time being)\n"
      ],
      "metadata": {
        "id": "_fCdfQfNNzwt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pprint import pprint #pprint => pretty-print, I use it occassionally throughout the notebook\n",
        "import datasets\n",
        "dset=datasets.load_dataset(\"imdb\")\n",
        "pprint(dset)"
      ],
      "metadata": {
        "id": "QxmgHoKDTN2_",
        "outputId": "7ca89a4d-0b70-484f-bef0-54e553420b00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472,
          "referenced_widgets": [
            "42351838529a4455a40fa4031b415de0",
            "38f532b995bd4609a5226a8dffe1333b",
            "48d99ab3cbda4f129da4d52422a96bfe",
            "d72494e0f62141da99ba26a8670401ed",
            "0ef2ff2ce4f6484eb15aad577db81be2",
            "cd898ffa1c0147cca90ddfdfb5217427",
            "8dd82d120f1241e8b1fe355cb404699f",
            "50e148a3895b4259b89111d8061022fc",
            "507ce3ce11864c54bee4d1eddb9eeca4",
            "edd42ef9fcaa480d9cabc01b6121bfb8",
            "b2513d1573174490866d7b3063373fed",
            "c4630891dd8f4f90b2f70c4ebdd8edd6",
            "13674620d858401aa72ae62d9bf0be65",
            "9a633afccca94f299fb2841cfb033179",
            "ef0e64812c854d9e9a0caa79012ae784",
            "44ac40d2653a45bbabfe957ddb3c79fb",
            "70fc545a54704b818f27bb00155e3561",
            "857fe79f69ba4af0b4cc0b1e3f903649",
            "c70229dc99924a59bc5a92d6f2897326",
            "7030adf144b94eeba875eee604a4e5f3",
            "10abc3d911434e8990deaca602a03b61",
            "4254daf008f14097abf01a304ccef189",
            "7d9dec1f32524fb6b420eb28d86c44d3",
            "7cf265ef231146fba8533ce6ea792d8e",
            "ae23a53db68c4e7d98b9cb89779611c2",
            "711cbc31c69c4f76b91e4cd9e16c66ed",
            "d628077557e242afb3283a3862ca5099",
            "19b5b66d197e4e4e92046e8df7aedd82",
            "6f7dd8e4d39241e3b7ccfea392426f6f",
            "26e12abd6a8f49949f091811067b02fd",
            "da2d8fe4cf8442a9bceddbf9a6e3a3ae",
            "cf1a70429aad44a89f66ec38186b7ba4",
            "fadedd0bd442493792f69891892952d8",
            "5e29e2bf78224e53ac896c4f63b8f6cc",
            "ced5ba6de24144e398ab577f5f2f8374",
            "163789fda33f4e31b60435dd3784ff4d",
            "cff3260942ad4d8b88912aab1c7d269a",
            "5d546174b0db40eeba5609da158fadc8",
            "77f76e3e26e742d298d4d855051f1656",
            "1a6f44e6b1c64df3bcf9eed56d063b8d",
            "8956764a0cc24f7bbe856aa5deecaef2",
            "43a29ffd5d1941678861869b56205b75",
            "5f304d3fd6984797b30ec5946c9cd168",
            "a8015739dfef48638594760853e334c6",
            "f2fefd93b41f48efa0073821dc1e17f6",
            "05fc0c3c0a1f40afa30fc0ee8fd842bc",
            "692dc5f6e92b47558eb619157d32c0fd",
            "e9cdfcdae49f41fe9c08e4369913bb3d",
            "0331d681442540ca8bd4c700fc4cefd7",
            "454ae06ee6cc4907a2adf95e18220179",
            "ffa0fab5cdaa490d83668f12edaf45fb",
            "132a7e8115464b1c85a1193cbaaeb40e",
            "c16eaa8120744b4da9274247eb607551",
            "54dd8991e78d43de907844158db3bfab",
            "64417186b95742bb898e93ab4db3a8de",
            "94a0a7da61f540d2a970d3cadaa4741c",
            "d0f3bf849d9543c69ea877704bec1d14",
            "bd289390f6dd4237a8cd37bd4d928bee",
            "374d3fc538fd49c8a872243b3430273f",
            "f9b5204267ef4053ac35cd7a9596e66b",
            "63b54645353141b3abb7ab1fbfffa798",
            "c50de387c1644880a8a820976a9ba1ad",
            "91d8f88d93994267b8fe5576db40f9db",
            "3af2393ea53c4c8ab9ef712387bd1daf",
            "f511922a95a04008a564a96357f9da85",
            "8eef0408358643d3b39fc2622f536397",
            "cc7314f3759a478c8edeaf5655bf6aed",
            "d88d8af722594c2eabd4485046d45d88",
            "305871ccf0bd4f319d7cf0c9dad5d1e8",
            "9c5d383869294a5281f880d9413892c0",
            "972a378bb99b4cd7aa11a38e781ff1d6",
            "e31d3e17f19b4bd9a76ce6cae18a4954",
            "ab13b5f94d514530960dd9a5e04ecfb2",
            "6a3d71d0f37e4caf96bc51ca8f5add70",
            "e96bd7b982a04de4b9c1853c555ae80b",
            "bbd117f6b1aa4b17ae7b8f7e6165b9b2",
            "f7ec90e7c08a4107a3632d33f322db92",
            "19e6fccd17b74a119afcd777c037304e",
            "ceab64d976f64788b3b6e35336191a4b",
            "b897e299e5354065aaf78388c2ce1acf",
            "e9046f1d005841b9a300ddeb6a36b760",
            "0e79ae10dc314cdaaa0e79b5094f795e",
            "3d547f0cb24b4789b1f89e5909c14dd2",
            "7c08383552324582b148603564e3091b",
            "ff74f961019c4ffc97bac985c046fea8",
            "cb5c2a17b4f946dd9fca69e0312f93b4",
            "31922779ca7f4da3b50593a1672457c3",
            "5e412602b023435b811c7d3eff4a9e16"
          ]
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/4.31k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "42351838529a4455a40fa4031b415de0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/2.17k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c4630891dd8f4f90b2f70c4ebdd8edd6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/7.59k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7d9dec1f32524fb6b420eb28d86c44d3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset imdb/plain_text to /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/84.1M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5e29e2bf78224e53ac896c4f63b8f6cc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/25000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f2fefd93b41f48efa0073821dc1e17f6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split:   0%|          | 0/25000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "94a0a7da61f540d2a970d3cadaa4741c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating unsupervised split:   0%|          | 0/50000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cc7314f3759a478c8edeaf5655bf6aed"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset imdb downloaded and prepared to /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "19e6fccd17b74a119afcd777c037304e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'test': Dataset({\n",
            "    features: ['text', 'label'],\n",
            "    num_rows: 25000\n",
            "}),\n",
            " 'train': Dataset({\n",
            "    features: ['text', 'label'],\n",
            "    num_rows: 25000\n",
            "}),\n",
            " 'unsupervised': Dataset({\n",
            "    features: ['text', 'label'],\n",
            "    num_rows: 50000\n",
            "})}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dset=dset.shuffle() #This is never a bad idea, datasets may have ordering to them, which is not what we want\n",
        "del dset[\"unsupervised\"] #Delete the unlabeled part of the dataset, we don't need it for anything"
      ],
      "metadata": {
        "id": "lbmgiOj4pWw-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dset['train'][0]['text'])\n",
        "print(dset['train'][0]['label'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KI3kyD3vry8C",
        "outputId": "ea9dc8e9-ed99-413c-c764-9b753c96425e"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "('When I tell people that I review movies as a hobby, the first thing they say '\n",
            " 'is \"What do you think of such-and-such movie?\" There are a couple of '\n",
            " \"problems here. Firstly, there is the probable chance that I've not seen it \"\n",
            " 'and thus, I ruin my reputation. Secondly, I could trash the movie in '\n",
            " \"question without realising that it's actually their favourite. Lastly, I \"\n",
            " 'could be given DVDs to watch so they can judge my opinion. Thus, I find '\n",
            " 'myself sat before \"The Convent\" which is the sort of film I would ignore '\n",
            " \"completely given the choice but unable to avoid here. More's the pity \"\n",
            " 'because this frankly dreadful \"horror\" is about as scary as a box of '\n",
            " 'kittens.<br /><br />Following well-worn clichés, \"The Convent\" focuses on a '\n",
            " 'bunch of American high school students on a trip to an abandoned convent on '\n",
            " \"a mission to get stoned, laid and mildly spooked. But you'll never guess \"\n",
            " 'what happens next? A group of cannibalistic demonic nuns emerge from the '\n",
            " 'cobwebs who proceed to pick the kids off one by one in classic horror movie '\n",
            " \"tradition. Will any of them survive and more importantly, haven't you got \"\n",
            " 'better things to worry about? <br /><br />The only thing that saves \"The '\n",
            " 'Convent\" from being a total waste of time is the fact that nobody is really '\n",
            " \"taking this tosh seriously with the exception of Coolio's bizarre cameo as a \"\n",
            " \"hyperactive cop with an itchy trigger finger. It's far too amusing to be \"\n",
            " 'properly frightening - the zombie cheerleader who makes chipmunk noises for '\n",
            " 'no reason, the day-glo paint jobs that appears when you become a zombie - '\n",
            " 'but what really kills it as a horror is the fact that you can instantly tell '\n",
            " 'when someone is going to jump out and get messily murdered. But even if they '\n",
            " 'were trying, I still doubt that it would work - demonic nuns wearing the '\n",
            " 'sort of make-up you\\'d see in the \"Buffy The Vampire Slayer\" TV show aren\\'t '\n",
            " 'really that scary. The scariest thing about this movie really is that over '\n",
            " \"10% of voters gave this a maximum score. I mean, I know it's funny but I \"\n",
            " \"hardly split my sides. To be honest, I've had more fun in a dentist.<br \"\n",
            " '/><br />\"The Convent\" isn\\'t really a horror movie as such. It\\'s more of a '\n",
            " 'comedy horror like \"Scary Movie\" or \"Shaun Of The Dead\" but you\\'re laughing '\n",
            " \"at it instead of with it. I honestly can't recommend this to anybody except \"\n",
            " \"the family of the cast and crew but even then, I doubt very much they'd \"\n",
            " 'enjoy it. I don\\'t enjoy the \"so bad that it\\'s good\" genre - I personally '\n",
            " \"feel that if the film-makers can't be bothered to salvage a turkey then I \"\n",
            " \"shouldn't bust a gut trying to watch it. Yes, it's a bad film and yes, they \"\n",
            " \"really should have gone home and done something more constructive. A Rubik's \"\n",
            " 'cube, for example. The DVD box has two price stickers - one for £4.99 and a '\n",
            " 'reduced price of £3. But it was sold for £1 and that should tell you '\n",
            " 'everything you need to know about this poxy, cheap, awesomely bad flick. '\n",
            " 'Sorry if you do like it but \"The Convent\" really is a pile of unholy crap.')\n",
            "0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenize and vectorize data\n",
        "         \n",
        "*   We need to achieve two complementary tasks\n",
        "*   **Tokenize** split the text into units which can be interpreted as features (words in this case most likely)\n",
        "*   **Vectorize** build the feature vector\n",
        "*   Since this is NLP, vectorize here means listing the non-zero elements of the feature vector, or in other words the indices of the rows in the embedding matrix\n",
        "*   A traditional and well-tested way it to use sklearn's feature extraction package\n",
        "*   CountVectorizer is most likely what we want in here, but for other NLP work the TfidfVectorizer is also very handy\n",
        "\n"
      ],
      "metadata": {
        "id": "KMnqQ78cMpk2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.feature_extraction\n",
        "\n",
        "vectorizer=sklearn.feature_extraction.text.CountVectorizer(binary=True,max_features=20000)\n",
        "\n",
        "texts=[ex[\"text\"] for ex in dset[\"train\"]] #get a list of all texts from the training data\n",
        "vectorizer.fit(texts) #\"Trains\" the vectorizer, i.e. builds its vocabulary\n"
      ],
      "metadata": {
        "id": "MYOtKJu7Mohd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "7ce5c41b-31af-4d27-ee74-f7739e7c5f6c"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(binary=True, max_features=20000)"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(binary=True, max_features=20000)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer(binary=True, max_features=20000)</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Building the feature vectors\n",
        "\n",
        "* This is super-easy with the vectorizer\n",
        "* It produces a sparse matrix of the non-zero elements"
      ],
      "metadata": {
        "id": "UDNy3w8reO6C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def vectorize_example(ex):\n",
        "    vectorized=vectorizer.transform([ex[\"text\"]]) # [...] because the vectorizer expects a list/iterable over inputs, not one input\n",
        "    non_zero_features=vectorized.nonzero()[1] #.nonzero gives a pair of (rows,columns), we want the columns\n",
        "    non_zero_features+=1 #feature index 0 will have a special meaning\n",
        "                         # so let us not produce it by adding +1 to everything\n",
        "    return {\"input_ids\":non_zero_features} \n",
        "\n",
        "vectorized=vectorize_example(dset[\"train\"][0])"
      ],
      "metadata": {
        "id": "5H9AQOc2rGIO"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# We can map back to vocabulary and check that everything works\n",
        "# vectorizer.vocabulary_ is a dictionary {key:word, value:idx}\n",
        "\n",
        "idx2word=dict((i,w) for (w,i) in vectorizer.vocabulary_.items()) #inverse the vocab dictionary\n",
        "words=[]\n",
        "for idx in vectorized[\"input_ids\"]:\n",
        "    words.append(idx2word[idx-1]) ## It is easy to forgot we moved all by +1\n",
        "pprint(\", \".join(words)) #This is now the bag of words representation of the document"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yvgd3IrWfCai",
        "outputId": "e1288319-74e5-4911-e9b9-0dc263f28031"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "('10, 99, abandoned, about, actually, american, amusing, an, and, any, '\n",
            " 'anybody, appears, are, aren, as, at, avoid, awesomely, bad, be, because, '\n",
            " 'become, before, being, better, bizarre, bothered, box, br, buffy, bunch, '\n",
            " 'bust, but, by, cameo, can, cannibalistic, cast, chance, cheap, cheerleader, '\n",
            " 'choice, classic, clichés, comedy, completely, constructive, convent, cop, '\n",
            " 'could, couple, crap, crew, cube, day, dead, demonic, dentist, do, don, done, '\n",
            " 'doubt, dreadful, dvd, dvds, emerge, enjoy, even, everything, example, '\n",
            " 'except, exception, fact, family, far, favourite, feel, film, find, finger, '\n",
            " 'first, firstly, flick, focuses, following, for, frankly, frightening, from, '\n",
            " 'fun, funny, gave, genre, get, given, going, gone, good, got, group, guess, '\n",
            " 'gut, had, happens, hardly, has, have, haven, here, high, hobby, home, '\n",
            " 'honest, honestly, horror, hyperactive, if, ignore, importantly, in, '\n",
            " 'instantly, instead, is, isn, it, itchy, jobs, judge, jump, kids, kills, '\n",
            " 'kittens, know, laid, lastly, laughing, like, ll, make, makers, makes, '\n",
            " 'maximum, mean, mildly, mission, more, movie, movies, much, murdered, my, '\n",
            " 'myself, need, never, next, no, nobody, noises, not, nuns, of, off, on, one, '\n",
            " 'only, opinion, or, out, over, paint, people, personally, pick, pile, pity, '\n",
            " 'price, probable, problems, proceed, properly, question, re, realising, '\n",
            " 'really, reason, recommend, reduced, reputation, review, ruin, salvage, sat, '\n",
            " 'saves, say, scariest, scary, school, score, secondly, see, seen, seriously, '\n",
            " 'shaun, should, shouldn, show, sides, slayer, so, sold, someone, something, '\n",
            " 'sorry, sort, split, still, stoned, students, such, survive, taking, tell, '\n",
            " 'that, the, their, them, then, there, they, thing, things, think, this, thus, '\n",
            " 'time, to, too, total, tradition, trash, trigger, trip, trying, turkey, tv, '\n",
            " 'two, unable, unholy, up, vampire, ve, very, voters, was, waste, watch, '\n",
            " 'wearing, well, were, what, when, which, who, will, with, without, work, '\n",
            " 'worn, worry, would, yes, you, zombie')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenizing / vectorizing the whole dataset\n",
        "\n",
        "* The datasets library allows us to efficiently map() a function across the whole dataset\n",
        "* Can run in parallel\n",
        "\n",
        "**Note**: confusingly, and unlike the Python`map` function, [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) function _updates_ its argument dataset, keeping existing values. Here, the call adds the values returned by the function call (here `input_ids`) to each example while also keeping the original `text` and `label` values.\n"
      ],
      "metadata": {
        "id": "i33KIMgkiIn7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Apply the tokenizer to the whole dataset using .map()\n",
        "dset_tokenized = dset.map(vectorize_example,num_proc=4)\n",
        "pprint(dset_tokenized[\"train\"][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "d8ef93446e564136a55508d62421338e",
            "1ef1cafd24c64584942057f9dbcaff2b",
            "aeced29b19e644ca9edd286990768114",
            "6b1d74d1073046f498d23531c8232911",
            "4d52965555ef4f5a8ca8d8e64afe1a09",
            "523717980d5f4c33824e18978eb3bdfd",
            "7151c043363641e09ef528893fe5467c",
            "c804cfa6ece040ae99641aec9644fb5f",
            "b15ad5ad9e5c4633b6e3f92283460257",
            "0349ec6365d446cf93bfc72766bf8aaf",
            "f142c3a5677941d09a987a989e0c909e",
            "5456f9791de1471ab97e7d3720ee1606",
            "85d9f4b0cc144b6cbcdeed25ff593fae",
            "be395a6543d9416a85fe1adc07b60853",
            "6b86c18e2d8446a6b9550aca77dbed3f",
            "ef8e8973cab14258bb175a35305c5e80",
            "0d37e8895b7c47499fa65cd66b5fedd0",
            "38d07bf359814cb984a1ca7e99def0bc",
            "81f6a56b0a5a4d36a5980888bb0ebe9f",
            "7a3e852d633d41fba505060ce0ec3866",
            "bc54250faf81467cb88739ccb7dd1adb",
            "7c811f393ac74438acae0b06763540f6"
          ]
        },
        "id": "33xMYRd0q2B9",
        "outputId": "5ca63229-c5c7-4eeb-dacc-30bb4fea88ab"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/25000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d8ef93446e564136a55508d62421338e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/25000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5456f9791de1471ab97e7d3720ee1606"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [10,\n",
            "               264,\n",
            "               272,\n",
            "               309,\n",
            "               436,\n",
            "               826,\n",
            "               857,\n",
            "               860,\n",
            "               887,\n",
            "               1000,\n",
            "               1001,\n",
            "               1044,\n",
            "               1115,\n",
            "               1118,\n",
            "               1207,\n",
            "               1300,\n",
            "               1435,\n",
            "               1461,\n",
            "               1517,\n",
            "               1702,\n",
            "               1737,\n",
            "               1743,\n",
            "               1764,\n",
            "               1793,\n",
            "               1897,\n",
            "               1995,\n",
            "               2250,\n",
            "               2279,\n",
            "               2295,\n",
            "               2510,\n",
            "               2546,\n",
            "               2596,\n",
            "               2604,\n",
            "               2625,\n",
            "               2689,\n",
            "               2711,\n",
            "               2737,\n",
            "               2878,\n",
            "               3032,\n",
            "               3099,\n",
            "               3124,\n",
            "               3199,\n",
            "               3337,\n",
            "               3387,\n",
            "               3599,\n",
            "               3716,\n",
            "               3935,\n",
            "               4018,\n",
            "               4062,\n",
            "               4130,\n",
            "               4152,\n",
            "               4223,\n",
            "               4292,\n",
            "               4404,\n",
            "               4620,\n",
            "               4630,\n",
            "               4840,\n",
            "               4868,\n",
            "               5372,\n",
            "               5429,\n",
            "               5434,\n",
            "               5477,\n",
            "               5552,\n",
            "               5711,\n",
            "               5712,\n",
            "               5974,\n",
            "               6100,\n",
            "               6302,\n",
            "               6320,\n",
            "               6357,\n",
            "               6370,\n",
            "               6372,\n",
            "               6584,\n",
            "               6640,\n",
            "               6663,\n",
            "               6726,\n",
            "               6765,\n",
            "               6878,\n",
            "               6904,\n",
            "               6914,\n",
            "               6939,\n",
            "               6940,\n",
            "               7013,\n",
            "               7084,\n",
            "               7102,\n",
            "               7127,\n",
            "               7265,\n",
            "               7336,\n",
            "               7346,\n",
            "               7388,\n",
            "               7406,\n",
            "               7536,\n",
            "               7593,\n",
            "               7627,\n",
            "               7693,\n",
            "               7777,\n",
            "               7795,\n",
            "               7801,\n",
            "               7839,\n",
            "               8017,\n",
            "               8047,\n",
            "               8098,\n",
            "               8128,\n",
            "               8228,\n",
            "               8246,\n",
            "               8289,\n",
            "               8322,\n",
            "               8323,\n",
            "               8484,\n",
            "               8535,\n",
            "               8619,\n",
            "               8657,\n",
            "               8678,\n",
            "               8679,\n",
            "               8740,\n",
            "               8871,\n",
            "               8929,\n",
            "               8933,\n",
            "               9047,\n",
            "               9085,\n",
            "               9362,\n",
            "               9363,\n",
            "               9602,\n",
            "               9619,\n",
            "               9630,\n",
            "               9635,\n",
            "               9772,\n",
            "               9846,\n",
            "               9873,\n",
            "               10008,\n",
            "               10021,\n",
            "               10062,\n",
            "               10090,\n",
            "               10174,\n",
            "               10244,\n",
            "               10262,\n",
            "               10475,\n",
            "               10576,\n",
            "               10886,\n",
            "               10889,\n",
            "               10890,\n",
            "               11133,\n",
            "               11181,\n",
            "               11387,\n",
            "               11518,\n",
            "               11681,\n",
            "               11762,\n",
            "               11765,\n",
            "               11778,\n",
            "               11814,\n",
            "               11857,\n",
            "               11865,\n",
            "               11993,\n",
            "               12055,\n",
            "               12077,\n",
            "               12134,\n",
            "               12142,\n",
            "               12153,\n",
            "               12202,\n",
            "               12262,\n",
            "               12363,\n",
            "               12364,\n",
            "               12437,\n",
            "               12439,\n",
            "               12445,\n",
            "               12478,\n",
            "               12504,\n",
            "               12577,\n",
            "               12627,\n",
            "               12739,\n",
            "               13028,\n",
            "               13099,\n",
            "               13199,\n",
            "               13229,\n",
            "               13279,\n",
            "               13727,\n",
            "               13777,\n",
            "               13783,\n",
            "               13786,\n",
            "               13883,\n",
            "               14107,\n",
            "               14310,\n",
            "               14337,\n",
            "               14349,\n",
            "               14355,\n",
            "               14406,\n",
            "               14461,\n",
            "               14738,\n",
            "               14906,\n",
            "               15252,\n",
            "               15365,\n",
            "               15415,\n",
            "               15448,\n",
            "               15461,\n",
            "               15489,\n",
            "               15496,\n",
            "               15531,\n",
            "               15557,\n",
            "               15658,\n",
            "               15682,\n",
            "               15696,\n",
            "               15792,\n",
            "               15912,\n",
            "               16020,\n",
            "               16023,\n",
            "               16033,\n",
            "               16093,\n",
            "               16303,\n",
            "               16478,\n",
            "               16519,\n",
            "               16546,\n",
            "               16550,\n",
            "               16591,\n",
            "               16592,\n",
            "               16756,\n",
            "               17018,\n",
            "               17054,\n",
            "               17189,\n",
            "               17302,\n",
            "               17462,\n",
            "               17634,\n",
            "               17781,\n",
            "               17893,\n",
            "               17897,\n",
            "               17907,\n",
            "               17910,\n",
            "               17917,\n",
            "               17929,\n",
            "               17944,\n",
            "               17951,\n",
            "               17952,\n",
            "               17954,\n",
            "               17968,\n",
            "               18036,\n",
            "               18074,\n",
            "               18115,\n",
            "               18166,\n",
            "               18211,\n",
            "               18267,\n",
            "               18340,\n",
            "               18416,\n",
            "               18423,\n",
            "               18484,\n",
            "               18518,\n",
            "               18531,\n",
            "               18553,\n",
            "               18598,\n",
            "               18770,\n",
            "               18922,\n",
            "               19015,\n",
            "               19051,\n",
            "               19112,\n",
            "               19274,\n",
            "               19398,\n",
            "               19407,\n",
            "               19414,\n",
            "               19461,\n",
            "               19509,\n",
            "               19521,\n",
            "               19540,\n",
            "               19549,\n",
            "               19559,\n",
            "               19590,\n",
            "               19639,\n",
            "               19712,\n",
            "               19716,\n",
            "               19773,\n",
            "               19791,\n",
            "               19795,\n",
            "               19805,\n",
            "               19912,\n",
            "               19927,\n",
            "               19990],\n",
            " 'label': 0,\n",
            " 'text': 'When I tell people that I review movies as a hobby, the first thing '\n",
            "         'they say is \"What do you think of such-and-such movie?\" There are a '\n",
            "         'couple of problems here. Firstly, there is the probable chance that '\n",
            "         \"I've not seen it and thus, I ruin my reputation. Secondly, I could \"\n",
            "         \"trash the movie in question without realising that it's actually \"\n",
            "         'their favourite. Lastly, I could be given DVDs to watch so they can '\n",
            "         'judge my opinion. Thus, I find myself sat before \"The Convent\" which '\n",
            "         'is the sort of film I would ignore completely given the choice but '\n",
            "         \"unable to avoid here. More's the pity because this frankly dreadful \"\n",
            "         '\"horror\" is about as scary as a box of kittens.<br /><br />Following '\n",
            "         'well-worn clichés, \"The Convent\" focuses on a bunch of American high '\n",
            "         'school students on a trip to an abandoned convent on a mission to '\n",
            "         \"get stoned, laid and mildly spooked. But you'll never guess what \"\n",
            "         'happens next? A group of cannibalistic demonic nuns emerge from the '\n",
            "         'cobwebs who proceed to pick the kids off one by one in classic '\n",
            "         'horror movie tradition. Will any of them survive and more '\n",
            "         \"importantly, haven't you got better things to worry about? <br /><br \"\n",
            "         '/>The only thing that saves \"The Convent\" from being a total waste '\n",
            "         'of time is the fact that nobody is really taking this tosh seriously '\n",
            "         \"with the exception of Coolio's bizarre cameo as a hyperactive cop \"\n",
            "         \"with an itchy trigger finger. It's far too amusing to be properly \"\n",
            "         'frightening - the zombie cheerleader who makes chipmunk noises for '\n",
            "         'no reason, the day-glo paint jobs that appears when you become a '\n",
            "         'zombie - but what really kills it as a horror is the fact that you '\n",
            "         'can instantly tell when someone is going to jump out and get messily '\n",
            "         'murdered. But even if they were trying, I still doubt that it would '\n",
            "         \"work - demonic nuns wearing the sort of make-up you'd see in the \"\n",
            "         '\"Buffy The Vampire Slayer\" TV show aren\\'t really that scary. The '\n",
            "         'scariest thing about this movie really is that over 10% of voters '\n",
            "         \"gave this a maximum score. I mean, I know it's funny but I hardly \"\n",
            "         \"split my sides. To be honest, I've had more fun in a dentist.<br \"\n",
            "         '/><br />\"The Convent\" isn\\'t really a horror movie as such. It\\'s '\n",
            "         'more of a comedy horror like \"Scary Movie\" or \"Shaun Of The Dead\" '\n",
            "         \"but you're laughing at it instead of with it. I honestly can't \"\n",
            "         'recommend this to anybody except the family of the cast and crew but '\n",
            "         \"even then, I doubt very much they'd enjoy it. I don't enjoy the \"\n",
            "         '\"so bad that it\\'s good\" genre - I personally feel that if the '\n",
            "         \"film-makers can't be bothered to salvage a turkey then I shouldn't \"\n",
            "         \"bust a gut trying to watch it. Yes, it's a bad film and yes, they \"\n",
            "         'really should have gone home and done something more constructive. A '\n",
            "         \"Rubik's cube, for example. The DVD box has two price stickers - one \"\n",
            "         'for £4.99 and a reduced price of £3. But it was sold for £1 and that '\n",
            "         'should tell you everything you need to know about this poxy, cheap, '\n",
            "         'awesomely bad flick. Sorry if you do like it but \"The Convent\" '\n",
            "         'really is a pile of unholy crap.'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Input encoding for MLP\n",
        "\n",
        "* Our `input_ids` are an array containing the indices of the tokens found in the text\n",
        "* This corresponds to the indices into the row of the embedding matrix in the model\n"
      ],
      "metadata": {
        "id": "XTtyHQpIIJWw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Batching and padding\n",
        "\n",
        "* When working with neural networks, one rarely trains one example at a time\n",
        "* Instead, processing always happens a batch at a time\n",
        "* This has two important reasons:\n",
        "  1. No batching is too slow (GPU parallelization cannot kick in across examples)\n",
        "  2. The gradients are averaged across the whole batch and applied only once, i.e. batching acts as a regularizer\n"
      ],
      "metadata": {
        "id": "JvaP1DpHjI3f"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Padding\n",
        "\n",
        "* In order to build a batch as a 2D array of (example, seq) (see below), we need to fit together examples of different length!\n",
        "* Solution: pad the shorter examples with zeroes to the length of the longest example in the batch\n",
        "* Make sure that zero is understood as padding value rather than a (hypothetical) feature with index 0\n",
        "* This is best shown by example, it is in the end easier than it may sound"
      ],
      "metadata": {
        "id": "hNYxw92nj51B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# I need to define it here, will explain below\n",
        "def collator(list_of_examples):\n",
        "    batch={\"labels\":torch.tensor(list(ex[\"label\"] for ex in list_of_examples))} #this is easy, labels are made into a single tensor\n",
        "    #the worse bit is now to pad the examples, as they are of different length\n",
        "    tensors=[]\n",
        "    max_len=max(len(example[\"input_ids\"]) for example in list_of_examples) #this is the longest example in the batch\n",
        "    #everything needs to be padded to fit in length the longest example\n",
        "    #(so we can build a single tensor out of it)\n",
        "    for example in list_of_examples:\n",
        "        ids=torch.tensor(example[\"input_ids\"]) #pick the input ids\n",
        "        # pad(what,(from_left, from_right)) <- this is how we call the stock pad function\n",
        "        padded=torch.nn.functional.pad(ids,(0,max_len-ids.shape[0])) #pad by max - current length, pads with zero by default\n",
        "        tensors.append(padded) #accumulated the padded ids\n",
        "    batch[\"input_ids\"]=torch.vstack(tensors) #now that we have all of them the same length, a simple vstack() stacks them up\n",
        "    return batch #...and that's all there is to it\n",
        "\n",
        "\n",
        "import torch\n",
        "#Build a batch from 2 examples, with padding\n",
        "batch=collator([dset_tokenized[\"train\"][2],dset_tokenized[\"train\"][7]])\n",
        "print(\"Shape of labels:\",batch[\"labels\"].shape)\n",
        "print(\"Shape of input_ids:\",batch[\"input_ids\"].shape)\n",
        "pprint(batch[\"labels\"])\n",
        "pprint(batch[\"input_ids\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zXXDxNo6kwfA",
        "outputId": "cf143aec-a394-45f7-99b6-fa760f14c33b"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of labels: torch.Size([2])\n",
            "Shape of input_ids: torch.Size([2, 143])\n",
            "tensor([0, 0])\n",
            "tensor([[   10,    62,   183,   320,   417,   436,   587,   594,   727,   814,\n",
            "           860,   887,  1115,  1207,  1231,  1341,  1456,  1495,  1516,  1702,\n",
            "          1756,  2103,  2295,  2604,  2840,  2898,  3062,  3337,  3591,  3760,\n",
            "          3901,  4130,  4272,  4641,  4944,  4967,  5372,  5429,  5838,  5942,\n",
            "          6245,  6399,  6473,  6878,  6939,  7127,  7222,  7346,  7692,  7757,\n",
            "          8128,  8322,  8929,  9085,  9147,  9201,  9400,  9428,  9549,  9602,\n",
            "          9628,  9630,  9956,  9981, 10248, 10263, 10475, 10558, 10561, 10681,\n",
            "         10825, 10829, 10915, 10951, 11176, 11278, 11376, 11419, 11681, 11762,\n",
            "         11778, 11865, 12134, 12168, 12202, 12363, 12364, 12437, 12438, 12439,\n",
            "         12504, 12577, 12731, 12868, 12905, 13215, 13344, 13483, 13655, 13858,\n",
            "         14340, 14349, 14625, 14825, 15462, 15502, 15546, 15661, 15682, 15746,\n",
            "         15844, 16161, 16478, 16542, 16550, 16591, 17292, 17885, 17893, 17897,\n",
            "         17929, 17938, 17944, 17968, 17980, 17982, 18074, 18115, 18474, 18483,\n",
            "         18789, 18796, 19156, 19195, 19521, 19540, 19549, 19712, 19745, 19773,\n",
            "         19801, 19805, 19927],\n",
            "        [  887,  1453,  1495,  1617,  2711,  4622,  5869,  6311,  7406,  7757,\n",
            "          7801,  7932,  8758, 10261, 11762, 11777, 12168, 12363, 12852, 13675,\n",
            "         14349, 15025, 16033, 16478, 17304, 17897, 17968, 18115, 19398, 19446,\n",
            "         19521, 19522, 19549, 19609, 19800,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Collator\n",
        "\n",
        "* This is simply a function which takes a list of examples and builds a training batch out of them\n",
        "* Much like examples are dictionaries with the data, also batches are dictionaries with the data\n",
        "* The only difference is that in a batch, all data tensors have one extra dimension, that's all there is to it"
      ],
      "metadata": {
        "id": "yoVB-7rPmPUL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def collator(list_of_examples):\n",
        "    batch={\"labels\":torch.tensor(list(ex[\"label\"] for ex in list_of_examples))} #this is easy, labels are made into a single tensor\n",
        "    #the worse bit is now to pad the examples, as they are of different length\n",
        "    tensors=[]\n",
        "    max_len=max(len(example[\"input_ids\"]) for example in list_of_examples) #this is the longest example in the batch\n",
        "    #everything needs to be padded to fit in length the longest example\n",
        "    #(so we can build a single tensor out of it)\n",
        "    for example in list_of_examples:\n",
        "        ids=torch.tensor(example[\"input_ids\"]) #pick the input ids\n",
        "        # pad(what,(from_left, from_right)) <- this is how we call the stock pad function\n",
        "        padded=torch.nn.functional.pad(ids,(0,max_len-ids.shape[0])) #pad by max - current length, pads with zero by default\n",
        "        tensors.append(padded) #accumulated the padded ids\n",
        "    batch[\"input_ids\"]=torch.vstack(tensors) #now that we have all of them the same length, a simple vstack() stacks them up\n",
        "    return batch #...and that's all there is to it\n"
      ],
      "metadata": {
        "id": "gMqcyhRS0Hzv"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Build the MLP model\n",
        "\n",
        "* Now that all of our data is in shape, we can build the model\n",
        "* That is luckily quite easy in this case\n",
        "\n",
        "The model class in its simplest form has `__init__()` which instantiates the layers and `forward()` which implements the actual computation. For more information on these, please see the [PyTorch turorial](https://pytorch.org/tutorials/beginner/introyt/modelsyt_tutorial.html)."
      ],
      "metadata": {
        "id": "AOYYF5I1OWG9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import transformers\n",
        "\n",
        "# A model wants a config, I can simply inherit from the base\n",
        "# class for pretrained configs\n",
        "class MLPConfig(transformers.PretrainedConfig):\n",
        "    pass\n",
        "\n",
        "# This is the model\n",
        "class MLP(transformers.PreTrainedModel):\n",
        "\n",
        "    config_class=MLPConfig\n",
        "\n",
        "    # In the initialization method, one instantiates the layers\n",
        "    # these will be, for the most part the trained parameters of the model\n",
        "    def __init__(self,config):\n",
        "        super().__init__(config)\n",
        "        self.vocab_size=config.vocab_size #embedding matrix row count\n",
        "        # Build and initialize embedding of vocab size +1 x hidden size (+1 because of the padding index 0!)\n",
        "        self.embedding=torch.nn.Embedding(num_embeddings=self.vocab_size+1,embedding_dim=config.hidden_size,padding_idx=0)\n",
        "        # Normally you would not initialize these yourself, but I have my reasons here ;)\n",
        "        torch.nn.init.uniform_(self.embedding.weight.data,-0.001,0.001) #initialize the embeddings with small random values\n",
        "        # Note! This is quite clever and keeps the embedding for 0, the padding, pure zeros\n",
        "        # This takes care of the lower half of the network, now the upper half\n",
        "        # Output layer: hidden size x output size\n",
        "        self.output=torch.nn.Linear(in_features=config.hidden_size,out_features=config.nlabels)\n",
        "        # Now we have the parameters of the model\n",
        "\n",
        "        \n",
        "    # The computation of the model is put into the forward() function\n",
        "    # it receives a batch of data and optionally the correct `labels`\n",
        "    #\n",
        "    # If given `labels` it returns (loss,output)\n",
        "    # if not, then it returns (output,)\n",
        "    def forward(self,input_ids,labels=None): #nevermind the attention_mask, its time will come, data collator insists on adding it\n",
        "        #1) sum up the embeddings of the items\n",
        "        embedded=self.embedding(input_ids) #(batch,ids)->(batch,ids,embedding_dim)\n",
        "        # Since the Embedding keeps the first row of the matrix pure zeros, we don't need to worry about the padding\n",
        "        # so next we sum the embeddings across the word dimension\n",
        "        # (batch,ids,embedding_dim) -> (batch,embedding_dim)\n",
        "        embedded_summed=torch.sum(embedded,dim=1)\n",
        "        \n",
        "        #2) apply non-linearity\n",
        "        # (batch,embedding_dim) -> (batch,embedding_dim)\n",
        "        projected=torch.tanh(embedded_summed) #Note how non-linearity is applied here and not when configuring the layer in __init__()\n",
        "\n",
        "        #3) and now apply the upper, output layer of the network\n",
        "        # (batch,embedding_dim) -> (batch, num_of_classes i.e. 2 in our case)\n",
        "        logits=self.output(projected)\n",
        "\n",
        "        # ...and that's all there is to it!\n",
        "\n",
        "        #print(\"input_ids.shape\",input_ids.shape)\n",
        "        #print(\"embedded.shape\",embedded.shape)\n",
        "        #print(\"embedded_summed.shape\",embedded_summed.shape)\n",
        "        #print(\"projected.shape\",projected.shape)\n",
        "        #print(\"logits.shape\",logits.shape)\n",
        "        \n",
        "        # We have labels, so we ought to calculate the loss\n",
        "        if labels is not None:\n",
        "            loss=torch.nn.CrossEntropyLoss() #This loss is meant for classification, so let's use it\n",
        "            # You run it as loss(model_output,correct_labels)\n",
        "            return (loss(logits,labels),logits)\n",
        "        else:\n",
        "            # No labels, so just return the logits\n",
        "            return (logits,)\n",
        "\n",
        "# Configure the model:\n",
        "#   these parameters are used in the model's __init__()\n",
        "mlp_config=MLPConfig(vocab_size=len(vectorizer.vocabulary_),hidden_size=20,nlabels=2)\n"
      ],
      "metadata": {
        "id": "vP4FtrMwCpGi"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# And we can make a model\n",
        "mlp=MLP(mlp_config)\n",
        "fake_batch=collator([dset_tokenized[\"train\"][0],dset_tokenized[\"train\"][1]])\n",
        "mlp(**fake_batch) #** expands input_ids and labels as parameters of the call"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JQjSGUYGwrzh",
        "outputId": "e1a91b99-ef89-40e6-cdc9-d3b87f68162d"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.7052, grad_fn=<NllLossBackward0>), tensor([[0.0380, 0.1135],\n",
              "         [0.0017, 0.0306]], grad_fn=<AddmmBackward0>))"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train the model\n",
        "\n",
        "We will use the Hugging Face [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) class for training\n",
        "\n",
        "* Loads of arguments that control the training\n",
        "* Configurable metrics to evaluate performance\n",
        "* Data collator builds the batches\n",
        "* Early stopping callback stops when eval loss no longer improves\n",
        "* Model load/save\n",
        "* Good foundation for later deep learning course\n",
        "  "
      ],
      "metadata": {
        "id": "tdlcMObzQGGP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, let's create a [`TrainingArguments`](https://huggingface.co/docs/transformers/v4.17.0/en/main_classes/trainer#transformers.TrainingArguments) object to specify hyperparameters and various other settings for training. \n",
        "\n",
        "Printing this simple dataclass object will show not only the values we set, but also the defaults for all other arguments. Don't worry if you don't understand what all of these do! Many are not relevant to us here, and you can find the details in [`Trainer` documentation](https://huggingface.co/docs/transformers/main_classes/trainer) if you are interested."
      ],
      "metadata": {
        "id": "aZgcNi4B76SX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set training arguments\n",
        "# their names are mostly self-explanatory\n",
        "trainer_args = transformers.TrainingArguments(\n",
        "    \"mlp_checkpoints\", #save checkpoints here\n",
        "    evaluation_strategy=\"steps\",\n",
        "    logging_strategy=\"steps\",\n",
        "    eval_steps=500,\n",
        "    logging_steps=500,\n",
        "    learning_rate=1e-4, #learning rate of the gradient descent\n",
        "    max_steps=20000,\n",
        "    load_best_model_at_end=True,\n",
        "    per_device_train_batch_size=128\n",
        ")\n",
        "\n",
        "pprint(trainer_args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IhfdW62z8cCn",
        "outputId": "bae98acb-d036-46b0-8c11-a2649102c7fc"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TrainingArguments(\n",
            "_n_gpu=1,\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "auto_find_batch_size=False,\n",
            "bf16=False,\n",
            "bf16_full_eval=False,\n",
            "data_seed=None,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_pin_memory=True,\n",
            "ddp_bucket_cap_mb=None,\n",
            "ddp_find_unused_parameters=None,\n",
            "ddp_timeout=1800,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "do_eval=True,\n",
            "do_predict=False,\n",
            "do_train=False,\n",
            "eval_accumulation_steps=None,\n",
            "eval_delay=0,\n",
            "eval_steps=500,\n",
            "evaluation_strategy=steps,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "fsdp=[],\n",
            "fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\n",
            "fsdp_min_num_params=0,\n",
            "fsdp_transformer_layer_cls_to_wrap=None,\n",
            "full_determinism=False,\n",
            "gradient_accumulation_steps=1,\n",
            "gradient_checkpointing=False,\n",
            "greater_is_better=False,\n",
            "group_by_length=False,\n",
            "half_precision_backend=auto,\n",
            "hub_model_id=None,\n",
            "hub_private_repo=False,\n",
            "hub_strategy=every_save,\n",
            "hub_token=<HUB_TOKEN>,\n",
            "ignore_data_skip=False,\n",
            "include_inputs_for_metrics=False,\n",
            "jit_mode_eval=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=0.0001,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=True,\n",
            "local_rank=-1,\n",
            "log_level=passive,\n",
            "log_level_replica=warning,\n",
            "log_on_each_node=True,\n",
            "logging_dir=mlp_checkpoints/runs/Mar29_10-25-51_34de3e3c1239,\n",
            "logging_first_step=False,\n",
            "logging_nan_inf_filter=True,\n",
            "logging_steps=500,\n",
            "logging_strategy=steps,\n",
            "lr_scheduler_type=linear,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=20000,\n",
            "metric_for_best_model=loss,\n",
            "mp_parameters=,\n",
            "no_cuda=False,\n",
            "num_train_epochs=3.0,\n",
            "optim=adamw_hf,\n",
            "optim_args=None,\n",
            "output_dir=mlp_checkpoints,\n",
            "overwrite_output_dir=False,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=128,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=None,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
            "ray_scope=last,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=mlp_checkpoints,\n",
            "save_on_each_node=False,\n",
            "save_steps=500,\n",
            "save_strategy=steps,\n",
            "save_total_limit=None,\n",
            "seed=42,\n",
            "sharded_ddp=[],\n",
            "skip_memory_metrics=True,\n",
            "tf32=None,\n",
            "torch_compile=False,\n",
            "torch_compile_backend=None,\n",
            "torch_compile_mode=None,\n",
            "torchdynamo=None,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_ipex=False,\n",
            "use_legacy_prediction_loop=False,\n",
            "use_mps_device=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            "xpu_backend=None,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next, let's create a metric for evaluating performance during and after training. We can use the convenience function [`load_metric`](https://huggingface.co/docs/datasets/about_metrics) to load one of many pre-made metrics and wrap this for use by the trainer.\n",
        "\n",
        "As the task is simple binary classification and our data is even 50:50 balanced, we can comfortably use the basic `accuracy` metric, defined as the proportion of correctly predicted labels out of all labels."
      ],
      "metadata": {
        "id": "4sJwNXPU-dQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import evaluate\n",
        "\n",
        "accuracy = evaluate.load(\"accuracy\")\n",
        "\n",
        "def compute_accuracy(outputs_and_labels):\n",
        "    outputs, labels = outputs_and_labels\n",
        "    predictions = np.argmax(outputs, axis=-1) #pick the index of the \"winning\" label\n",
        "    return accuracy.compute(predictions=predictions, references=labels)"
      ],
      "metadata": {
        "id": "u3jxIItb0BL9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "c6c8f2512c524fa68f3be20390bc477d",
            "13be322a2c2843cda28a8113d76304f7",
            "d87155f8a3d547ca8dd289739a7b7c99",
            "7fe5e78a064047209e4c3bf3d4afcc5e",
            "341a3230aeed4ed29faa2676ad2f461a",
            "27863c3874d84bf2891f1980985f7f5d",
            "e4ef45b666d6413ba340adafb7601a8e",
            "9ec865302018453aa29c1509203ef8e3",
            "26d0425c0e164aaab72617705818fcb2",
            "9ce34b1ed9e94404a37667edb76f4efa",
            "feb4327aecc54e119d53277f168d53bb"
          ]
        },
        "outputId": "eba6a4fe-dbaa-44db-db7a-30d7b5de519f"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/4.20k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c6c8f2512c524fa68f3be20390bc477d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create the `Trainer` and train the model by invoking the [`Trainer.train`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.train) function.\n",
        "\n",
        "In addition to the model, the settings passed in through the `TrainingArguments` object created above (`trainer_args`), the data, and the metric defined above, we create and pass the following to the `Trainer`:\n",
        "\n",
        "* [data collator](https://huggingface.co/docs/transformers/main_classes/data_collator): groups input into batches\n",
        "* [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback): stops training when performance stops improving"
      ],
      "metadata": {
        "id": "S7kbz8uU-zpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Make a new model  \n",
        "mlp = MLP(mlp_config)\n",
        "\n",
        "\n",
        "# Argument gives the number of steps of patience before early stopping\n",
        "# i.e. training is stopped when the evaluation loss fails to improve\n",
        "# certain number of times\n",
        "early_stopping = transformers.EarlyStoppingCallback(5)\n",
        "\n",
        "trainer = transformers.Trainer(\n",
        "    model=mlp,\n",
        "    args=trainer_args,\n",
        "    train_dataset=dset_tokenized[\"train\"],\n",
        "    eval_dataset=dset_tokenized[\"test\"].select(range(1000)), #make a smaller subset to evaluate on\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=collator,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "# FINALLY!\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "AoEoWsj4P_zN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "outputId": "e224bf68-0bea-4956-bb94-b746e3de518f"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='4500' max='20000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 4500/20000 01:14 < 04:18, 60.03 it/s, Epoch 22/103]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.515500</td>\n",
              "      <td>0.386089</td>\n",
              "      <td>0.881000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.300200</td>\n",
              "      <td>0.289900</td>\n",
              "      <td>0.896000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.219100</td>\n",
              "      <td>0.261735</td>\n",
              "      <td>0.900000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.173900</td>\n",
              "      <td>0.253160</td>\n",
              "      <td>0.901000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.144200</td>\n",
              "      <td>0.254479</td>\n",
              "      <td>0.902000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.120400</td>\n",
              "      <td>0.260860</td>\n",
              "      <td>0.903000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.102800</td>\n",
              "      <td>0.270904</td>\n",
              "      <td>0.897000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.086400</td>\n",
              "      <td>0.283028</td>\n",
              "      <td>0.892000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.074500</td>\n",
              "      <td>0.298234</td>\n",
              "      <td>0.893000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=4500, training_loss=0.19297833591037328, metrics={'train_runtime': 77.857, 'train_samples_per_second': 32880.787, 'train_steps_per_second': 256.881, 'total_flos': 61761954240.0, 'train_loss': 0.19297833591037328, 'epoch': 22.96})"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then evaluate the trained model on a given dataset (here our test subset) by calling [`Trainer.evaluate`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.evaluate):"
      ],
      "metadata": {
        "id": "Td03fcIa-6Hj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(dset_tokenized[\"test\"])\n",
        "\n",
        "print(eval_results)"
      ],
      "metadata": {
        "id": "9nlEwpnF2Vow",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "d7274262-1603-428e-f9c1-c420808318a6"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3125/3125 00:05]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'eval_loss': 0.27699774503707886, 'eval_accuracy': 0.8888, 'eval_runtime': 5.9141, 'eval_samples_per_second': 4227.201, 'eval_steps_per_second': 528.4, 'epoch': 22.96}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save the model for later use\n",
        "\n",
        "* You can save it with `trainer.save_model()`\n",
        "* You can load it with `MLP.from_pretrained()`\n"
      ],
      "metadata": {
        "id": "P0OYB3TRp-IH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model(\"mlp-imdb\")"
      ],
      "metadata": {
        "id": "FosHTDw3p9dd"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Check save/load"
      ],
      "metadata": {
        "id": "8D8W9EN-zMAc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "mlp2=MLP.from_pretrained(\"mlp-imdb\")"
      ],
      "metadata": {
        "id": "UKMMNbABqLTg"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = transformers.Trainer(\n",
        "    model=mlp2,\n",
        "    args=trainer_args,\n",
        "    train_dataset=dset_tokenized[\"train\"],\n",
        "    eval_dataset=dset_tokenized[\"test\"],\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=collator,\n",
        "    callbacks=[early_stopping]\n",
        ")"
      ],
      "metadata": {
        "id": "pfLnUENrrCyp"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(dset_tokenized[\"test\"])\n",
        "print(eval_results)\n",
        "print('Accuracy:', eval_results['eval_accuracy'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        },
        "id": "hpOM2ZwErMGf",
        "outputId": "1872b479-98e8-4648-beaa-46c3b7e455e1"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3125/3125 00:06]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'eval_loss': 0.27699774503707886, 'eval_accuracy': 0.8888, 'eval_runtime': 7.0658, 'eval_samples_per_second': 3538.147, 'eval_steps_per_second': 442.268}\n",
            "Accuracy: 0.8888\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extra time left?\n",
        "\n",
        "* Read through the TrainingArguments documentation, try to understand at least some parts of it https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments\n",
        "* Read through Torch tensor operations, try to understand at least some parts of it: https://pytorch.org/docs/stable/tensors.html\n",
        "* Run the model with different parameters (hidden layer width, learning rate, etc), how much do the results change?\n"
      ],
      "metadata": {
        "id": "8UT5MV1LtSBh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# What has the model learned?\n",
        "\n",
        "* The embeddings should have some meaning to them\n",
        "* Similar features should have similar embeddings"
      ],
      "metadata": {
        "id": "13aB7DuqzeFi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Grab the embedding matrix out of the trained model\n",
        "# and drop the first row (padding 0)\n",
        "# then we can treat the embeddings as vectors\n",
        "# and maybe compare them to each other\n",
        "# ha ha this below took some googling\n",
        "weights=mlp.embedding.weight.detach().cpu().numpy()\n",
        "weights=weights[1:,:]"
      ],
      "metadata": {
        "id": "M6TUrVkMCmz7"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qry_idx=vectorizer.vocabulary_[\"great\"] #embedding of \"lousy\"\n",
        "\n",
        "#calculate the distance of the \"lousy\" embedding to all other embeddings\n",
        "distance_to_qry=sklearn.metrics.pairwise.euclidean_distances(weights[qry_idx:qry_idx+1,:],weights)\n",
        "nearest_neighbors=np.argsort(distance_to_qry) #indices of words nearest to \"lousy\"\n",
        "for nearest in nearest_neighbors[0,:20]:\n",
        "    print(idx2word[nearest])\n",
        "# This works great!"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4uTTvRgGDU9D",
        "outputId": "707163e9-f8c4-47ba-c4bb-035868c1b44f"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "great\n",
            "superb\n",
            "wonderfully\n",
            "refreshing\n",
            "amazing\n",
            "perfect\n",
            "gem\n",
            "loved\n",
            "rare\n",
            "wonderful\n",
            "fantastic\n",
            "today\n",
            "favorite\n",
            "enjoyable\n",
            "incredible\n",
            "excellent\n",
            "funniest\n",
            "enjoyed\n",
            "perfectly\n",
            "fun\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* The embeddings indeed seem to reflect the task\n",
        "* There is a meaning to them"
      ],
      "metadata": {
        "id": "cQCmThdu2LDR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Feature weights\n",
        "\n",
        "*   A typical \"old-school\" way to approach the classification would be a simple linear model, like LinearSVM\n",
        "*   Under such model, each feature (word) would have a single one weight\n",
        "*   And the classification would simply be based on the sum of these weights\n",
        "*   In this context of this task, \"positive\" words would get a high weight, \"negative\" words would get a low weight\n",
        "*   It is in fact quite easy to reconfigure the MLP model to work more or less like this and this effect can be replicated\n",
        "*   I will leave that as an exercise for you\n",
        "\n"
      ],
      "metadata": {
        "id": "Jydy3ECK3O2w"
      }
    }
  ]
}