sequence_labeling_mlp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "265ff512aa7f4342bacfc4694c427981": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c15efb3003874dc4aca0d3d37c1c6ae8",
              "IPY_MODEL_7b0e352e001046bba4b49292c598d78c",
              "IPY_MODEL_5aa4c2aff91648bf9e649cdb5a5f5196"
            ],
            "layout": "IPY_MODEL_ac6f02eba38446f2b9b6e1856b71dc5f"
          }
        },
        "c15efb3003874dc4aca0d3d37c1c6ae8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1e07031805c24f989e50ca33bab681e4",
            "placeholder": "​",
            "style": "IPY_MODEL_b833f79dc4514725beac5db318952e1b",
            "value": "100%"
          }
        },
        "7b0e352e001046bba4b49292c598d78c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_68d377f4833347c4b3db7d65564b6768",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_13e542fcab62427eacc4a48307da3e89",
            "value": 3
          }
        },
        "5aa4c2aff91648bf9e649cdb5a5f5196": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_994a35ddd0bf4ae8a0007080f97669cb",
            "placeholder": "​",
            "style": "IPY_MODEL_4f1ba6b0da2d4184b80b7ddf206f8974",
            "value": " 3/3 [00:00&lt;00:00, 34.30it/s]"
          }
        },
        "ac6f02eba38446f2b9b6e1856b71dc5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1e07031805c24f989e50ca33bab681e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b833f79dc4514725beac5db318952e1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "68d377f4833347c4b3db7d65564b6768": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "13e542fcab62427eacc4a48307da3e89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "994a35ddd0bf4ae8a0007080f97669cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4f1ba6b0da2d4184b80b7ddf206f8974": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4adf0bb23f840a68fff35108c90110f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_58e873048a16485c9c6f7ef14ba1cf40",
              "IPY_MODEL_95625193dc7a4970a010cb9451d6dd89",
              "IPY_MODEL_3b5a311c62da49f8b962cfda6c70ba59"
            ],
            "layout": "IPY_MODEL_809e39c6b6f64ebc8d8eed40a57901ba"
          }
        },
        "58e873048a16485c9c6f7ef14ba1cf40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ad783079bd334419bf284eb4d900dad8",
            "placeholder": "​",
            "style": "IPY_MODEL_a3aed8bee1394f8aabbb87195956f299",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "95625193dc7a4970a010cb9451d6dd89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f9dc6a4bc1344a6892d0ab24675c8f9",
            "max": 203621,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fd58b82c26ec435eaa93dfdfd4a8e612",
            "value": 203621
          }
        },
        "3b5a311c62da49f8b962cfda6c70ba59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb3270e1d47543ee8e8b25649cb8d6ac",
            "placeholder": "​",
            "style": "IPY_MODEL_110d8395898e470a8f36f0ae4f7d3a47",
            "value": " 203449/203621 [02:03&lt;00:00, 1115.61 examples/s]"
          }
        },
        "809e39c6b6f64ebc8d8eed40a57901ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "ad783079bd334419bf284eb4d900dad8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a3aed8bee1394f8aabbb87195956f299": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f9dc6a4bc1344a6892d0ab24675c8f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd58b82c26ec435eaa93dfdfd4a8e612": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fb3270e1d47543ee8e8b25649cb8d6ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "110d8395898e470a8f36f0ae4f7d3a47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f6662e21419d4087bf1d1ddc25fbddf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ef58170f2c734379871b849c387a14ab",
              "IPY_MODEL_8b9d390cc22d4f769d7c18936bfc87dd",
              "IPY_MODEL_09e7efd6173a4b0687a5a7d47a13d00f"
            ],
            "layout": "IPY_MODEL_2a1987debd814ca8a28c13430f8cf4a4"
          }
        },
        "ef58170f2c734379871b849c387a14ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a90b7d9fe4e4c94a893e6047d71d6cf",
            "placeholder": "​",
            "style": "IPY_MODEL_812b10b04ab94f3392d5acea1fe1c105",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "8b9d390cc22d4f769d7c18936bfc87dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7aa032c101bc4085ba6282ba687ae52b",
            "max": 51362,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_be63bf96db094684b6d53a84184234f3",
            "value": 51362
          }
        },
        "09e7efd6173a4b0687a5a7d47a13d00f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f16e8b62dcea445c862ebff4a57fa65c",
            "placeholder": "​",
            "style": "IPY_MODEL_1c1645d5c2c7417a8bc86eb36f867f10",
            "value": " 51173/51362 [00:22&lt;00:00, 1694.28 examples/s]"
          }
        },
        "2a1987debd814ca8a28c13430f8cf4a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "8a90b7d9fe4e4c94a893e6047d71d6cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "812b10b04ab94f3392d5acea1fe1c105": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7aa032c101bc4085ba6282ba687ae52b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be63bf96db094684b6d53a84184234f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f16e8b62dcea445c862ebff4a57fa65c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c1645d5c2c7417a8bc86eb36f867f10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f54a5b573af0469698cad6817ac8fcb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_67e8f726507640fb949bbd3970e44916",
              "IPY_MODEL_66fbe41935f84bef82cde2777eb404d7",
              "IPY_MODEL_c4b0edd61e4e418db1c8f12704d93aea"
            ],
            "layout": "IPY_MODEL_bdcf667d3e5f4dddbbaf6485a2cd479a"
          }
        },
        "67e8f726507640fb949bbd3970e44916": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_181d5a0757e143a983adf582121eaac4",
            "placeholder": "​",
            "style": "IPY_MODEL_de30e12b17384e0f820b282c6c8292e5",
            "value": "Map (num_proc=4):  99%"
          }
        },
        "66fbe41935f84bef82cde2777eb404d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_13a167141db840bfb568f548fffeb8ce",
            "max": 46435,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a90562bb20304f14abf6fbe0d3786f79",
            "value": 46435
          }
        },
        "c4b0edd61e4e418db1c8f12704d93aea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0daf8155fd6f447497ad0d0cf50504d1",
            "placeholder": "​",
            "style": "IPY_MODEL_4bc4e23ff3544959bf09a00197649a92",
            "value": " 46183/46435 [00:19&lt;00:00, 2519.36 examples/s]"
          }
        },
        "bdcf667d3e5f4dddbbaf6485a2cd479a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "181d5a0757e143a983adf582121eaac4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de30e12b17384e0f820b282c6c8292e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "13a167141db840bfb568f548fffeb8ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a90562bb20304f14abf6fbe0d3786f79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0daf8155fd6f447497ad0d0cf50504d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4bc4e23ff3544959bf09a00197649a92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/sequence_labeling_mlp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sequence labeling (POS tagging) with MLP\n",
        "\n",
        "This notebook builds upon the [classification with MLP notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb) and shows how to implement a basic sequence labeling method."
      ],
      "metadata": {
        "id": "arPRayYSpzJs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Setup\n",
        "\n",
        "Install the required Python packages using [pip](https://en.wikipedia.org/wiki/Pip):\n",
        "\n",
        "* [`transformers`](https://huggingface.co/docs/transformers/index) is a popular deep learning package primarily on top of torch\n",
        "* [`datasets`](https://huggingface.co/docs/datasets/) provides support for loading, creating, and manipulating datasets\n",
        "* [`evaluate`](https://huggingface.co/docs/evaluate/index) is a library of performance metrics (like accuracy etc)"
      ],
      "metadata": {
        "id": "LDZZUKzfPRWT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --quiet transformers datasets evaluate"
      ],
      "metadata": {
        "id": "fKPHBYptQDsK"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Get and prepare data\n",
        "\n",
        "*   Let us work with the venerable, if somewhat dated [CoNLL'03 shared task](https://aclanthology.org/W03-0419.pdf) English data\n",
        "*   These are English news articles, and have annotation for POS, syntactic chunks, and named entities (in the IOB format)\n",
        "\n",
        "The data as originally distributed for the 2003 shared task has the following format:\n",
        "\n",
        "```\n",
        "Only RB I-NP O\n",
        "France NNP I-NP I-LOC\n",
        "and CC I-NP O\n",
        "Britain NNP I-NP I-LOC\n",
        "backed VBD I-VP O\n",
        "Fischler NNP I-NP I-PER\n",
        "'s POS B-NP O\n",
        "proposal NN I-NP O\n",
        ". . O O\n",
        "```\n",
        "\n",
        "Here, the four space-separated columns are token text, POS tag, chunk tag, and NER tag. The goal of the original task is to predict the NER tags using the other information as features, but the dataset can be used to study predicting the other columns too.\n",
        "\n",
        "The dataset happens to be in the HF datasets collection, so we can grab it from there\n"
      ],
      "metadata": {
        "id": "_fCdfQfNNzwt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import transformers\n",
        "import datasets\n",
        "\n",
        "from pprint import pprint    # pretty-print\n",
        "\n",
        "dataset = datasets.load_dataset(\"conll2003\")\n",
        "\n",
        "print(dataset)"
      ],
      "metadata": {
        "id": "QxmgHoKDTN2_",
        "outputId": "0f2343d1-f028-4f87-b12a-04b936698037",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 325,
          "referenced_widgets": [
            "265ff512aa7f4342bacfc4694c427981",
            "c15efb3003874dc4aca0d3d37c1c6ae8",
            "7b0e352e001046bba4b49292c598d78c",
            "5aa4c2aff91648bf9e649cdb5a5f5196",
            "ac6f02eba38446f2b9b6e1856b71dc5f",
            "1e07031805c24f989e50ca33bab681e4",
            "b833f79dc4514725beac5db318952e1b",
            "68d377f4833347c4b3db7d65564b6768",
            "13e542fcab62427eacc4a48307da3e89",
            "994a35ddd0bf4ae8a0007080f97669cb",
            "4f1ba6b0da2d4184b80b7ddf206f8974"
          ]
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Found cached dataset conll2003 (/root/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/9a4d16a94f8674ba3466315300359b0acd891b68b6c8743ddf60b9c702adce98)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "265ff512aa7f4342bacfc4694c427981"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 14041\n",
            "    })\n",
            "    validation: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 3250\n",
            "    })\n",
            "    test: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 3453\n",
            "    })\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dataset[\"train\"][12])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OEtbYIyLg3r7",
        "outputId": "13b22314-21c9-4e55-f3b9-0405f851a210"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': [11, 12, 12, 12, 21, 11, 11, 12, 0],\n",
            " 'id': '12',\n",
            " 'ner_tags': [0, 5, 0, 5, 0, 1, 0, 0, 0],\n",
            " 'pos_tags': [30, 22, 10, 22, 38, 22, 27, 21, 7],\n",
            " 'tokens': ['Only',\n",
            "            'France',\n",
            "            'and',\n",
            "            'Britain',\n",
            "            'backed',\n",
            "            'Fischler',\n",
            "            \"'s\",\n",
            "            'proposal',\n",
            "            '.']}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see above, the various labels (POS, NER and chunk tags) are converted into IDs in this dataset. We can access the textual labels of these tags through the dataset `features`:"
      ],
      "metadata": {
        "id": "v9TMq-d3qFEJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "POS_TAG_NAMES = dataset['train'].features['pos_tags'].feature.names\n",
        "NER_TAG_NAMES = dataset['train'].features['ner_tags'].feature.names\n",
        "CHUNK_TAG_NAMES = dataset['train'].features['chunk_tags'].feature.names"
      ],
      "metadata": {
        "id": "u0kflH3RpHKY"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create mappings from names to IDs and back as Python dictionaries:"
      ],
      "metadata": {
        "id": "O3-QCWMvrazc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "POS2ID = { n: i for i, n in enumerate(POS_TAG_NAMES) }\n",
        "ID2POS = { i: n for i, n in enumerate(POS_TAG_NAMES) }\n",
        "\n",
        "NER2ID = { n: i for i, n in enumerate(NER_TAG_NAMES) }\n",
        "ID2NER = { i: n for i, n in enumerate(NER_TAG_NAMES) }\n",
        "\n",
        "CHUNK2ID = { n: i for i, n in enumerate(CHUNK_TAG_NAMES) }\n",
        "ID2CHUNK = { i: n for i, n in enumerate(CHUNK_TAG_NAMES) }"
      ],
      "metadata": {
        "id": "tqi_FN2yq7MN"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is what these mappings look like:"
      ],
      "metadata": {
        "id": "sUI-FiAssDLL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(NER2ID)"
      ],
      "metadata": {
        "id": "XAYSIMWjsFiY",
        "outputId": "ec2f9653-f5fe-4bc9-acdc-406413e92bb7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'O': 0, 'B-PER': 1, 'I-PER': 2, 'B-ORG': 3, 'I-ORG': 4, 'B-LOC': 5, 'I-LOC': 6, 'B-MISC': 7, 'I-MISC': 8}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(ID2NER)"
      ],
      "metadata": {
        "id": "NO4xI0yasHdM",
        "outputId": "19fcf9e1-df5c-4508-e944-37569d889e6e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{0: 'O', 1: 'B-PER', 2: 'I-PER', 3: 'B-ORG', 4: 'I-ORG', 5: 'B-LOC', 6: 'I-LOC', 7: 'B-MISC', 8: 'I-MISC'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's also add in explanations from Penn Treebank for the POS tags:"
      ],
      "metadata": {
        "id": "jUqKBvtpvt3G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# From the documentation page and from here https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html\n",
        "\n",
        "POS2DESCRIPTION = {\n",
        "    \"CC\": \"Coordinating conjunction\",\n",
        "    \"CD\": \"Cardinal number\",\n",
        "    \"DT\": \"Determiner\",\n",
        "    \"EX\": \"Existential there\",\n",
        "    \"FW\": \"Foreign word\",\n",
        "    \"IN\": \"Preposition or subordinating conjunction\",\n",
        "    \"JJ\": \"Adjective\",\n",
        "    \"JJR\": \"Adjective, comparative\",\n",
        "    \"JJS\": \"Adjective, superlative\",\n",
        "    \"LS\": \"List item marker\",\n",
        "    \"MD\": \"Modal\",\n",
        "    \"NN\": \"Noun, singular or mass\",\n",
        "    \"NNS\": \"Noun, plural\",\n",
        "    \"NNP\": \"Proper noun, singular\",\n",
        "    \"NNPS\": \"Proper noun, plural\",\n",
        "    \"PDT\": \"Predeterminer\",\n",
        "    \"POS\": \"Possessive ending\",\n",
        "    \"PRP\": \"Personal pronoun\",\n",
        "    \"PRP$\": \"Possessive pronoun\",\n",
        "    \"RB\": \"Adverb\",\n",
        "    \"RBR\": \"Adverb, comparative\",\n",
        "    \"RBS\": \"Adverb, superlative\",\n",
        "    \"RP\": \"Particle\",\n",
        "    \"SYM\": \"Symbol\",\n",
        "    \"TO\": \"to\",\n",
        "    \"UH\": \"Interjection\",\n",
        "    \"VB\": \"Verb, base form\",\n",
        "    \"VBD\": \"Verb, past tense\",\n",
        "    \"VBG\": \"Verb, gerund or present participle\",\n",
        "    \"VBN\": \"Verb, past participle\",\n",
        "    \"VBP\": \"Verb, non-3rd person singular present\",\n",
        "    \"VBZ\": \"Verb, 3rd person singular present\",\n",
        "    \"WDT\": \"Wh-determiner\",\n",
        "    \"WP\": \"Wh-pronoun\",\n",
        "    \"WP$\": \"Possessive wh-pronoun\",\n",
        "    \"WRB\": \"Wh-adverb\"\n",
        "}"
      ],
      "metadata": {
        "id": "EDIG9IkNf_ji"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can now try to make sense of the tags:"
      ],
      "metadata": {
        "id": "HRcNtMskyTxq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tabulate\n",
        "\n",
        "e = dataset[\"train\"][12]    # work on the same example\n",
        "\n",
        "table = []\n",
        "for token, pos_id, chunk_id, ner_id in zip(e[\"tokens\"], e[\"pos_tags\"], e[\"chunk_tags\"], e[\"ner_tags\"]):\n",
        "    ner_tag = ID2NER[ner_id]\n",
        "    chunk_tag = ID2CHUNK[chunk_id]\n",
        "    pos_tag = ID2POS[pos_id]\n",
        "    pos_def = POS2DESCRIPTION.get(pos_tag,pos_tag)\n",
        "    table.append([token, ner_tag, chunk_tag, pos_tag, pos_def])\n",
        "\n",
        "print(tabulate.tabulate(table,headers=[\"Token\", \"NER\", \"Chunk\", \"POS\", \"POS definition\"]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JzQXQI0YiD0_",
        "outputId": "bf688ac6-ca7a-4e25-ffa9-0adc6d420d7f"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Token     NER    Chunk    POS    POS definition\n",
            "--------  -----  -------  -----  ------------------------\n",
            "Only      O      B-NP     RB     Adverb\n",
            "France    B-LOC  I-NP     NNP    Proper noun, singular\n",
            "and       O      I-NP     CC     Coordinating conjunction\n",
            "Britain   B-LOC  I-NP     NNP    Proper noun, singular\n",
            "backed    O      B-VP     VBD    Verb, past tense\n",
            "Fischler  B-PER  B-NP     NNP    Proper noun, singular\n",
            "'s        O      B-NP     POS    Possessive ending\n",
            "proposal  O      I-NP     NN     Noun, singular or mass\n",
            ".         O      O        .      .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note that the data is organized into sentences."
      ],
      "metadata": {
        "id": "VXm-E32AdVpo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Create features\n",
        "\n",
        "We'll define a simple function that takes a token sequence, the index of the focus token, and a window size and generates a few basic explicit features relevant to the task.\n",
        "\n",
        "(Note that as we'll be predicting the POS tag, we won't look at the chunk or NER tags, which would typically only be predicted _after_ predicting POS in a \"traditional\" NLP pipeline)"
      ],
      "metadata": {
        "id": "d9WvQiwluhBO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def token_features(tokens, index, window_size):\n",
        "    # Generate features for token in position `index` in given list of tokens\n",
        "    features = []\n",
        "\n",
        "    # Context window start and end\n",
        "    window_start = max(0, index-window_size)\n",
        "    window_end = min(index+window_size+1, len(tokens))    # note +1 for range\n",
        "\n",
        "    for i in range(window_start, window_end):\n",
        "          offset = i - index    # relative position\n",
        "          features.append(f\"token[{offset}]={tokens[i]}\")\n",
        "\n",
        "    # Example custom feature: does focus token start with an upper-case letter?\n",
        "    if tokens[index][0].isupper():\n",
        "        features.append(\"first-letter-capitalized\")\n",
        "\n",
        "    return features"
      ],
      "metadata": {
        "id": "6NjCVYHWe32j"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can call this function for all tokens in a sentence like so:"
      ],
      "metadata": {
        "id": "k1aEFA2Yg8xO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def add_features_to_sentence(sentence):\n",
        "    # Collect lists of features for all tokens here\n",
        "    all_features = []\n",
        "\n",
        "    tokens = sentence[\"tokens\"]\n",
        "    for index in range(len(tokens)):\n",
        "        all_features.append(token_features(tokens, index, window_size=3))\n",
        "\n",
        "    return { \"features\": all_features }"
      ],
      "metadata": {
        "id": "tgTZ5h4hhGFd"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for feats in add_features_to_sentence(dataset[\"train\"][12])[\"features\"]:\n",
        "    print(feats)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VMoD24fQhnRK",
        "outputId": "be455337-fb34-4015-afa7-f28c7b800646"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['token[0]=Only', 'token[1]=France', 'token[2]=and', 'token[3]=Britain', 'first-letter-capitalized']\n",
            "['token[-1]=Only', 'token[0]=France', 'token[1]=and', 'token[2]=Britain', 'token[3]=backed', 'first-letter-capitalized']\n",
            "['token[-2]=Only', 'token[-1]=France', 'token[0]=and', 'token[1]=Britain', 'token[2]=backed', 'token[3]=Fischler']\n",
            "['token[-3]=Only', 'token[-2]=France', 'token[-1]=and', 'token[0]=Britain', 'token[1]=backed', 'token[2]=Fischler', \"token[3]='s\", 'first-letter-capitalized']\n",
            "['token[-3]=France', 'token[-2]=and', 'token[-1]=Britain', 'token[0]=backed', 'token[1]=Fischler', \"token[2]='s\", 'token[3]=proposal']\n",
            "['token[-3]=and', 'token[-2]=Britain', 'token[-1]=backed', 'token[0]=Fischler', \"token[1]='s\", 'token[2]=proposal', 'token[3]=.', 'first-letter-capitalized']\n",
            "['token[-3]=Britain', 'token[-2]=backed', 'token[-1]=Fischler', \"token[0]='s\", 'token[1]=proposal', 'token[2]=.']\n",
            "['token[-3]=backed', 'token[-2]=Fischler', \"token[-1]='s\", 'token[0]=proposal', 'token[1]=.']\n",
            "['token[-3]=Fischler', \"token[-2]='s\", 'token[-1]=proposal', 'token[0]=.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The dataset is organized into sentences, so we can use the above function to add features to the entire dataset as follows.\n",
        "\n",
        "**Note**: unlike e.g. the Python`map` function, [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) function _updates_ its argument dataset, keeping existing values."
      ],
      "metadata": {
        "id": "c3SlJhk7i8aV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = dataset.map(add_features_to_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R9dekapGjNcb",
        "outputId": "aa5107b7-655e-4958-b9df-b310aa1aac81"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/9a4d16a94f8674ba3466315300359b0acd891b68b6c8743ddf60b9c702adce98/cache-de08a1937551abbf.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/9a4d16a94f8674ba3466315300359b0acd891b68b6c8743ddf60b9c702adce98/cache-b3477b26feaed1a0.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/9a4d16a94f8674ba3466315300359b0acd891b68b6c8743ddf60b9c702adce98/cache-c54852f6b86d17e2.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's check that one more time:"
      ],
      "metadata": {
        "id": "Ke-vVsAulwMb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dataset[\"train\"][12])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7zfUEZbDlyeZ",
        "outputId": "8c851432-2f77-4e30-de2a-17410ded505c"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': [11, 12, 12, 12, 21, 11, 11, 12, 0],\n",
            " 'features': [['token[0]=Only',\n",
            "               'token[1]=France',\n",
            "               'token[2]=and',\n",
            "               'token[3]=Britain',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-1]=Only',\n",
            "               'token[0]=France',\n",
            "               'token[1]=and',\n",
            "               'token[2]=Britain',\n",
            "               'token[3]=backed',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-2]=Only',\n",
            "               'token[-1]=France',\n",
            "               'token[0]=and',\n",
            "               'token[1]=Britain',\n",
            "               'token[2]=backed',\n",
            "               'token[3]=Fischler'],\n",
            "              ['token[-3]=Only',\n",
            "               'token[-2]=France',\n",
            "               'token[-1]=and',\n",
            "               'token[0]=Britain',\n",
            "               'token[1]=backed',\n",
            "               'token[2]=Fischler',\n",
            "               \"token[3]='s\",\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-3]=France',\n",
            "               'token[-2]=and',\n",
            "               'token[-1]=Britain',\n",
            "               'token[0]=backed',\n",
            "               'token[1]=Fischler',\n",
            "               \"token[2]='s\",\n",
            "               'token[3]=proposal'],\n",
            "              ['token[-3]=and',\n",
            "               'token[-2]=Britain',\n",
            "               'token[-1]=backed',\n",
            "               'token[0]=Fischler',\n",
            "               \"token[1]='s\",\n",
            "               'token[2]=proposal',\n",
            "               'token[3]=.',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-3]=Britain',\n",
            "               'token[-2]=backed',\n",
            "               'token[-1]=Fischler',\n",
            "               \"token[0]='s\",\n",
            "               'token[1]=proposal',\n",
            "               'token[2]=.'],\n",
            "              ['token[-3]=backed',\n",
            "               'token[-2]=Fischler',\n",
            "               \"token[-1]='s\",\n",
            "               'token[0]=proposal',\n",
            "               'token[1]=.'],\n",
            "              ['token[-3]=Fischler',\n",
            "               \"token[-2]='s\",\n",
            "               'token[-1]=proposal',\n",
            "               'token[0]=.']],\n",
            " 'id': '12',\n",
            " 'ner_tags': [0, 5, 0, 5, 0, 1, 0, 0, 0],\n",
            " 'pos_tags': [30, 22, 10, 22, 38, 22, 27, 21, 7],\n",
            " 'tokens': ['Only',\n",
            "            'France',\n",
            "            'and',\n",
            "            'Britain',\n",
            "            'backed',\n",
            "            'Fischler',\n",
            "            \"'s\",\n",
            "            'proposal',\n",
            "            '.']}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Flatten dataset\n",
        "\n",
        "The MLP code that we introduced previously expects each of the `train`, `validation` and `test` subsets of the data to consist of simple sequences of examples.\n",
        "\n",
        "Now that we have run the feature generation, we no longer need the sentence structure and can \"flatten\" the data into such sequences."
      ],
      "metadata": {
        "id": "N8sZzkiOl9aZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def flatten(subset):\n",
        "    # Keys for values to flatten\n",
        "    keys = [\"tokens\", \"pos_tags\", \"chunk_tags\", \"ner_tags\", \"features\"]\n",
        "\n",
        "    # Initialize to empty lists of tokens etc.\n",
        "    flattened = { k: [] for k in keys }\n",
        "\n",
        "    # Concatenate per-sentence lists of tokens etc.\n",
        "    for sentence in subset:\n",
        "        for key in keys:\n",
        "            flattened[key].extend(sentence[key])\n",
        "\n",
        "    # Return as Dataset object\n",
        "    return datasets.Dataset.from_dict(flattened)"
      ],
      "metadata": {
        "id": "Y4COBECNjPNb"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Call `flatten` for each of the subsets and make a new `DatasetDict` containing the flattened subsets: "
      ],
      "metadata": {
        "id": "wUuPc9uwltTn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "flattened_dict = {\n",
        "    \"train\": flatten(dataset[\"train\"]),\n",
        "    \"validation\": flatten(dataset[\"validation\"]),\n",
        "    \"test\": flatten(dataset[\"test\"]),\n",
        "}\n",
        "\n",
        "flat_dataset = datasets.DatasetDict(flattened_dict)"
      ],
      "metadata": {
        "id": "jcQSwn2Ll6Z0"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check that the new dataset looks OK:"
      ],
      "metadata": {
        "id": "QR4PZp8AmWZ_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "flat_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H9xBRcJKkB7Q",
        "outputId": "0fc72e9f-1412-40b6-d8ef-795595cb4660"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 203621\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 51362\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 46435\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(10):\n",
        "    token = flat_dataset[\"train\"][\"tokens\"][i]\n",
        "    pos_tag = ID2POS[flat_dataset[\"train\"][\"pos_tags\"][i]]\n",
        "    description = POS2DESCRIPTION.get(pos_tag, pos_tag)\n",
        "    print(f\"{token}\\t{pos_tag}\\t{description}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vs55SS3bmrqz",
        "outputId": "281558f4-bde9-40fc-d40d-976c3b775aa5"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "EU\tNNP\tProper noun, singular\n",
            "rejects\tVBZ\tVerb, 3rd person singular present\n",
            "German\tJJ\tAdjective\n",
            "call\tNN\tNoun, singular or mass\n",
            "to\tTO\tto\n",
            "boycott\tVB\tVerb, base form\n",
            "British\tJJ\tAdjective\n",
            "lamb\tNN\tNoun, singular or mass\n",
            ".\t.\t.\n",
            "Peter\tNNP\tProper noun, singular\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note that this is now a single long sequence of tokens without sentence boundaries."
      ],
      "metadata": {
        "id": "lI0nCoZhnode"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "## Vectorize data\n",
        "\n",
        "We'll next follow the steps that you should already be familiar with from the [text classification notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb), with a few changes:\n",
        "\n",
        "* Since the data is already tokenized, we only need to **vectorize** it, i.e. get the non-zero elements of the feature vector\n",
        "* Unlike in the text classification notebook, here we are **vectorizing token features**\n",
        "* We'll again use sklearn's feature extraction package, in particular `CountVectorizer`\n",
        "* Since our features are now lists of strings, we can skip tokenization and use these as-is"
      ],
      "metadata": {
        "id": "KMnqQ78cMpk2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.feature_extraction\n",
        "\n",
        "\n",
        "# Dummy function for tokenization and preprocessing\n",
        "def do_nothing(features):\n",
        "    return features\n",
        "\n",
        "vectorizer = sklearn.feature_extraction.text.CountVectorizer(\n",
        "    binary=True,\n",
        "    max_features=30000,\n",
        "    tokenizer=do_nothing,\n",
        "    preprocessor=do_nothing,\n",
        ")\n",
        "\n",
        "# Get a list of all feature strings from the training data\n",
        "features = [e[\"features\"] for e in flat_dataset[\"train\"]]\n",
        "\n",
        "# \"Train\" the vectorizer, i.e. build its vocabulary\n",
        "vectorizer.fit(features)"
      ],
      "metadata": {
        "id": "MYOtKJu7Mohd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        },
        "outputId": "3332b14d-92d0-4ffa-cf88-bc467c459e21"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/sklearn/feature_extraction/text.py:528: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=<function do_nothing at 0x7fe012dbcca0>,\n",
              "                tokenizer=<function do_nothing at 0x7fe012dbcca0>)"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=&lt;function do_nothing at 0x7fe012dbcca0&gt;,\n",
              "                tokenizer=&lt;function do_nothing at 0x7fe012dbcca0&gt;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=&lt;function do_nothing at 0x7fe012dbcca0&gt;,\n",
              "                tokenizer=&lt;function do_nothing at 0x7fe012dbcca0&gt;)</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As in the text classification notebook, we then invoke the vectorizer and get non-zero elements as a sparse matrix:"
      ],
      "metadata": {
        "id": "UDNy3w8reO6C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def vectorize_example(e):\n",
        "    vectorized = vectorizer.transform([e[\"features\"]])\n",
        "\n",
        "    # nonzero() gives a pair of (rows,columns), we want the columns\n",
        "    non_zero_features = vectorized.nonzero()[1]\n",
        "\n",
        "    # Feature index 0 will have a special meaning, so let us not produce\n",
        "    # it by adding +1 to everything\n",
        "    non_zero_features += 1\n",
        "\n",
        "    return {\n",
        "        \"input_ids\": non_zero_features,\n",
        "        \"label\": e[\"pos_tags\"]\n",
        "    } "
      ],
      "metadata": {
        "id": "5H9AQOc2rGIO"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check one example:"
      ],
      "metadata": {
        "id": "kqKjaSmBqWjm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorized = vectorize_example(flat_dataset[\"train\"][10])\n",
        "\n",
        "print(flat_dataset[\"train\"][10])\n",
        "print(vectorized)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOTjsFCBqOjX",
        "outputId": "505daf8c-a062-4b5e-dce4-63861756f701"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'tokens': 'Blackburn', 'pos_tags': 22, 'chunk_tags': 12, 'ner_tags': 2, 'features': ['token[-1]=Peter', 'token[0]=Blackburn', 'first-letter-capitalized']}\n",
            "{'input_ids': array([    1,  1538, 13814], dtype=int32), 'label': 22}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Map `input_ids` back to the original feature names to confirm that everything works:"
      ],
      "metadata": {
        "id": "WiScO867qrjK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Invert the feature dictionary\n",
        "idx2feat = { i: w for w, i in vectorizer.vocabulary_.items() }\n",
        "\n",
        "feats = []\n",
        "for idx in vectorized[\"input_ids\"]:\n",
        "    feats.append(idx2feat[idx-1])    # It is easy to forget we moved all by +1\n",
        "\n",
        "# This is now the bag of features representation of the token in context\n",
        "pprint(\", \".join(feats))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yvgd3IrWfCai",
        "outputId": "f8654963-2156-4cbe-90b1-110b64eea3ca"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "'first-letter-capitalized, token[-1]=Peter, token[0]=Blackburn'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Vectorizing the whole dataset\n",
        "\n",
        "We'll again use [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) to process the whole dataset:"
      ],
      "metadata": {
        "id": "i33KIMgkiIn7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorized_dataset = flat_dataset.map(vectorize_example, num_proc=4)\n",
        "\n",
        "pprint(vectorized_dataset[\"train\"][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219,
          "referenced_widgets": [
            "c4adf0bb23f840a68fff35108c90110f",
            "58e873048a16485c9c6f7ef14ba1cf40",
            "95625193dc7a4970a010cb9451d6dd89",
            "3b5a311c62da49f8b962cfda6c70ba59",
            "809e39c6b6f64ebc8d8eed40a57901ba",
            "ad783079bd334419bf284eb4d900dad8",
            "a3aed8bee1394f8aabbb87195956f299",
            "7f9dc6a4bc1344a6892d0ab24675c8f9",
            "fd58b82c26ec435eaa93dfdfd4a8e612",
            "fb3270e1d47543ee8e8b25649cb8d6ac",
            "110d8395898e470a8f36f0ae4f7d3a47",
            "f6662e21419d4087bf1d1ddc25fbddf0",
            "ef58170f2c734379871b849c387a14ab",
            "8b9d390cc22d4f769d7c18936bfc87dd",
            "09e7efd6173a4b0687a5a7d47a13d00f",
            "2a1987debd814ca8a28c13430f8cf4a4",
            "8a90b7d9fe4e4c94a893e6047d71d6cf",
            "812b10b04ab94f3392d5acea1fe1c105",
            "7aa032c101bc4085ba6282ba687ae52b",
            "be63bf96db094684b6d53a84184234f3",
            "f16e8b62dcea445c862ebff4a57fa65c",
            "1c1645d5c2c7417a8bc86eb36f867f10",
            "f54a5b573af0469698cad6817ac8fcb8",
            "67e8f726507640fb949bbd3970e44916",
            "66fbe41935f84bef82cde2777eb404d7",
            "c4b0edd61e4e418db1c8f12704d93aea",
            "bdcf667d3e5f4dddbbaf6485a2cd479a",
            "181d5a0757e143a983adf582121eaac4",
            "de30e12b17384e0f820b282c6c8292e5",
            "13a167141db840bfb568f548fffeb8ce",
            "a90562bb20304f14abf6fbe0d3786f79",
            "0daf8155fd6f447497ad0d0cf50504d1",
            "4bc4e23ff3544959bf09a00197649a92"
          ]
        },
        "id": "33xMYRd0q2B9",
        "outputId": "1d44072a-0723-4670-e102-a7da3aa4e0b9"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/203621 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c4adf0bb23f840a68fff35108c90110f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/51362 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f6662e21419d4087bf1d1ddc25fbddf0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/46435 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f54a5b573af0469698cad6817ac8fcb8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': 11,\n",
            " 'features': ['token[0]=EU',\n",
            "              'token[1]=rejects',\n",
            "              'token[2]=German',\n",
            "              'token[3]=call',\n",
            "              'first-letter-capitalized'],\n",
            " 'input_ids': [1, 14116, 23025, 27969],\n",
            " 'label': 22,\n",
            " 'ner_tags': 3,\n",
            " 'pos_tags': 22,\n",
            " 'tokens': 'EU'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* Our `input_ids` are an array containing the indices of the features\n",
        "* This corresponds to the indices into the row of the embedding matrix in the model\n"
      ],
      "metadata": {
        "id": "XTtyHQpIIJWw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Batching and padding\n",
        "\n",
        "As detailed in the [text classification notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb), we typically train neural networks on _batches_ of multiple examples rather than a single example at a time (efficiency and regularization).\n",
        "\n",
        "As examples in a batch need to have identical length, we _pad_ shorter examples to the maximum example length in each batch with the \"dummy\" feature with index 0.\n",
        "\n",
        "(This code is basically unchanged from the previous notebook.)"
      ],
      "metadata": {
        "id": "JvaP1DpHjI3f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def collator(list_of_examples):\n",
        "    # Labels are simply converted into a tensor\n",
        "    batch={\n",
        "        \"labels\": torch.tensor([e[\"label\"] for e in list_of_examples])\n",
        "    }\n",
        "\n",
        "    # Examples need to be padded\n",
        "    tensors = []\n",
        "\n",
        "    # Find length of longest example\n",
        "    max_len = max(len(e[\"input_ids\"]) for e in list_of_examples) \n",
        "    max_len = max(1,max_len)\n",
        "\n",
        "    # Pad everything with zeros to length of longest example\n",
        "    for example in list_of_examples:\n",
        "        ids = torch.LongTensor(example[\"input_ids\"])\n",
        "        # pad(what,(from_left, from_right)) <- this is how we call the stock pad function\n",
        "         #pad by max - current length, pads with zero by default\n",
        "        padded = torch.nn.functional.pad(ids, (0, max_len-ids.shape[0]))\n",
        "        tensors.append(padded)\n",
        "\n",
        "    # Now that all examples are of the same length, vstack() can be used\n",
        "    # to vertically stack these into a tensor \n",
        "    batch[\"input_ids\"]=torch.vstack(tensors)\n",
        "\n",
        "    return batch"
      ],
      "metadata": {
        "id": "zXXDxNo6kwfA"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Test that out with a minimal batch of two examples, one requiring padding:"
      ],
      "metadata": {
        "id": "e7Mre52QCTtI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch=collator([vectorized_dataset[\"train\"][2], vectorized_dataset[\"train\"][7]])\n",
        "\n",
        "print(\"Shape of labels:\",batch[\"labels\"].shape)\n",
        "print(\"Shape of input_ids:\",batch[\"input_ids\"].shape)\n",
        "print(\"labels:\",batch[\"labels\"])\n",
        "print(\"input_ids:\",batch[\"input_ids\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JJnW4gwqBpF6",
        "outputId": "6ea647a1-a319-4919-9221-d649c49fcde8"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of labels: torch.Size([2])\n",
            "Shape of input_ids: torch.Size([2, 6])\n",
            "labels: tensor([16, 21])\n",
            "input_ids: tensor([[    1,  5296, 14260, 20066, 26120, 27943],\n",
            "        [  567,  6775, 13008, 18099,     0,     0]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# MLP model\n",
        "\n",
        "With the data now ready, we'll build the MLP model. Note that this is _identical_ to the MLP model we used for text classification: the only difference between the two applications is in the data.\n",
        "\n",
        "The model class in its simplest form has `__init__()` which instantiates the layers and `forward()` which implements the actual computation. For more information on these, please see the [PyTorch turorial](https://pytorch.org/tutorials/beginner/introyt/modelsyt_tutorial.html)."
      ],
      "metadata": {
        "id": "AOYYF5I1OWG9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# A model wants a config, I can simply inherit from the base\n",
        "# class for pretrained configs\n",
        "class MLPConfig(transformers.PretrainedConfig):\n",
        "    pass\n",
        "\n",
        "# This is the model\n",
        "class MLP(transformers.PreTrainedModel):\n",
        "\n",
        "    config_class=MLPConfig\n",
        "\n",
        "    # In the initialization method, one instantiates the layers\n",
        "    # these will be, for the most part the trained parameters of the model\n",
        "    def __init__(self, config):\n",
        "        super().__init__(config)\n",
        "\n",
        "        self.vocab_size=config.vocab_size    # embedding matrix row count\n",
        "  \n",
        "        # Build and initialize embedding of vocab size +1 x hidden size\n",
        "        # (+1 because of the padding index 0!)\n",
        "        self.embedding = torch.nn.Embedding(\n",
        "            num_embeddings=self.vocab_size+1,\n",
        "            embedding_dim=config.hidden_size,\n",
        "            padding_idx=0\n",
        "        )\n",
        "\n",
        "        # Initialize the embeddings with small random values\n",
        "        torch.nn.init.uniform_(self.embedding.weight.data, -0.001, 0.001)\n",
        "        # Enforce zero values for padding\n",
        "        torch.nn.init.zeros_(self.embedding.weight.data[0,:])\n",
        "\n",
        "        # Output layer: hidden size x output size\n",
        "        self.output=torch.nn.Linear(\n",
        "            in_features=config.hidden_size,\n",
        "            out_features=config.nlabels\n",
        "        )\n",
        "        \n",
        "    # The computation of the model is put into the forward() function\n",
        "    # it receives a batch of data and optionally the correct `labels`:\n",
        "    # - if given `labels`, returns (loss, output)\n",
        "    # - if not, only returns (output,)\n",
        "    def forward(self, input_ids, labels=None):\n",
        "        # 1) Look up embeddings of features, sum them up\n",
        "        embedded = self.embedding(input_ids)    # (batch, ids) -> (batch, ids, embedding_dim)\n",
        "        embedded_summed = torch.sum(embedded, dim=1)    # (batch, ids, embedding_dim) -> (batch, embedding_dim)\n",
        "        \n",
        "        # NOTE: we're explicitly *not* applying a nonlinearity here to keep\n",
        "        # things linear for later analysis\n",
        "\n",
        "        # 2) Apply output layer\n",
        "        # (batch, embedding_dim) -> (batch, num_classes)\n",
        "        logits = self.output(embedded_summed)\n",
        "        \n",
        "        if labels is not None:\n",
        "            # We have labels, so we ought to calculate the loss\n",
        "            loss_fn = torch.nn.CrossEntropyLoss()    # Classification loss function\n",
        "            loss = loss_fn(logits, labels)\n",
        "            return (loss, logits)\n",
        "        else:\n",
        "            # No labels, so just return the logits\n",
        "            return (logits,)"
      ],
      "metadata": {
        "id": "vP4FtrMwCpGi"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Configure the model"
      ],
      "metadata": {
        "id": "Ix6-Z0XqFGfl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_labels = len(POS2ID)\n",
        "\n",
        "mlp_config = MLPConfig(\n",
        "    vocab_size=len(vectorizer.vocabulary_),\n",
        "    hidden_size=20,\n",
        "    nlabels=num_labels\n",
        ")"
      ],
      "metadata": {
        "id": "I2Lwe8eoFFtw"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Train the model\n",
        "\n",
        "We will use the Hugging Face [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) class for training\n",
        "\n",
        "* Loads of arguments that control the training\n",
        "* Configurable metrics to evaluate performance\n",
        "* Data collator builds the batches\n",
        "* Early stopping callback stops when eval loss no longer improves\n",
        "* Model load/save\n",
        "* Good foundation for later deep learning course\n",
        "  "
      ],
      "metadata": {
        "id": "tdlcMObzQGGP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, let's create a [`TrainingArguments`](https://huggingface.co/docs/transformers/v4.17.0/en/main_classes/trainer#transformers.TrainingArguments) object to specify hyperparameters and various other settings for training. \n",
        "\n",
        "Printing this simple dataclass object will show not only the values we set, but also the defaults for all other arguments. Don't worry if you don't understand what all of these do! Many are not relevant to us here, and you can find the details in [`Trainer` documentation](https://huggingface.co/docs/transformers/main_classes/trainer) if you are interested."
      ],
      "metadata": {
        "id": "aZgcNi4B76SX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer_args = transformers.TrainingArguments(\n",
        "    \"mlp_checkpoints\", #save checkpoints here\n",
        "    evaluation_strategy=\"steps\",\n",
        "    logging_strategy=\"steps\",\n",
        "    eval_steps=500,\n",
        "    logging_steps=500,\n",
        "    learning_rate=1e-4, #learning rate of the gradient descent\n",
        "    max_steps=20000,\n",
        "    load_best_model_at_end=True,\n",
        "    per_device_train_batch_size=128\n",
        ")\n",
        "\n",
        "pprint(trainer_args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IhfdW62z8cCn",
        "outputId": "18020b78-5b13-4a07-c957-c63374cb9778"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TrainingArguments(\n",
            "_n_gpu=0,\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "auto_find_batch_size=False,\n",
            "bf16=False,\n",
            "bf16_full_eval=False,\n",
            "data_seed=None,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_pin_memory=True,\n",
            "ddp_bucket_cap_mb=None,\n",
            "ddp_find_unused_parameters=None,\n",
            "ddp_timeout=1800,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "do_eval=True,\n",
            "do_predict=False,\n",
            "do_train=False,\n",
            "eval_accumulation_steps=None,\n",
            "eval_delay=0,\n",
            "eval_steps=500,\n",
            "evaluation_strategy=steps,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "fsdp=[],\n",
            "fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\n",
            "fsdp_min_num_params=0,\n",
            "fsdp_transformer_layer_cls_to_wrap=None,\n",
            "full_determinism=False,\n",
            "gradient_accumulation_steps=1,\n",
            "gradient_checkpointing=False,\n",
            "greater_is_better=False,\n",
            "group_by_length=False,\n",
            "half_precision_backend=auto,\n",
            "hub_model_id=None,\n",
            "hub_private_repo=False,\n",
            "hub_strategy=every_save,\n",
            "hub_token=<HUB_TOKEN>,\n",
            "ignore_data_skip=False,\n",
            "include_inputs_for_metrics=False,\n",
            "jit_mode_eval=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=0.0001,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=True,\n",
            "local_rank=-1,\n",
            "log_level=passive,\n",
            "log_level_replica=warning,\n",
            "log_on_each_node=True,\n",
            "logging_dir=mlp_checkpoints/runs/Apr04_08-23-32_1830ef2d110e,\n",
            "logging_first_step=False,\n",
            "logging_nan_inf_filter=True,\n",
            "logging_steps=500,\n",
            "logging_strategy=steps,\n",
            "lr_scheduler_type=linear,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=20000,\n",
            "metric_for_best_model=loss,\n",
            "mp_parameters=,\n",
            "no_cuda=False,\n",
            "num_train_epochs=3.0,\n",
            "optim=adamw_hf,\n",
            "optim_args=None,\n",
            "output_dir=mlp_checkpoints,\n",
            "overwrite_output_dir=False,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=128,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=None,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
            "ray_scope=last,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=mlp_checkpoints,\n",
            "save_on_each_node=False,\n",
            "save_steps=500,\n",
            "save_strategy=steps,\n",
            "save_total_limit=None,\n",
            "seed=42,\n",
            "sharded_ddp=[],\n",
            "skip_memory_metrics=True,\n",
            "tf32=None,\n",
            "torch_compile=False,\n",
            "torch_compile_backend=None,\n",
            "torch_compile_mode=None,\n",
            "torchdynamo=None,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_ipex=False,\n",
            "use_legacy_prediction_loop=False,\n",
            "use_mps_device=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            "xpu_backend=None,\n",
            ")\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/cuda/__init__.py:546: UserWarning: Can't initialize NVML\n",
            "  warnings.warn(\"Can't initialize NVML\")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next, let's create a metric for evaluating performance during and after training. We can use the convenience function [`load_metric`](https://huggingface.co/docs/datasets/about_metrics) to load one of many pre-made metrics and wrap this for use by the trainer.\n",
        "\n",
        "We can use the basic `accuracy` metric, defined as the proportion of correctly predicted labels out of all labels. This time, though, the data is not evenly split."
      ],
      "metadata": {
        "id": "4sJwNXPU-dQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import evaluate\n",
        "\n",
        "accuracy = evaluate.load(\"accuracy\")\n",
        "\n",
        "def compute_accuracy(outputs_and_labels):\n",
        "    outputs, labels = outputs_and_labels\n",
        "    predictions = np.argmax(outputs, axis=-1) #pick the index of the \"winning\" label\n",
        "    return accuracy.compute(predictions=predictions, references=labels)"
      ],
      "metadata": {
        "id": "u3jxIItb0BL9"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create the `Trainer` and train the model by invoking the [`Trainer.train`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.train) function.\n",
        "\n",
        "In addition to the model, the settings passed in through the `TrainingArguments` object created above (`trainer_args`), the data, and the metric defined above, we create and pass the following to the `Trainer`:\n",
        "\n",
        "* [data collator](https://huggingface.co/docs/transformers/main_classes/data_collator): groups input into batches\n",
        "* [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback): stops training when performance stops improving"
      ],
      "metadata": {
        "id": "S7kbz8uU-zpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Make a new model  \n",
        "mlp = MLP(mlp_config)\n",
        "\n",
        "\n",
        "# Argument gives the number of steps of patience before early stopping\n",
        "# i.e. training is stopped when the evaluation loss fails to improve\n",
        "# certain number of times\n",
        "early_stopping = transformers.EarlyStoppingCallback(5)\n",
        "\n",
        "trainer = transformers.Trainer(\n",
        "    model=mlp,\n",
        "    args=trainer_args,\n",
        "    train_dataset=vectorized_dataset[\"train\"],\n",
        "    eval_dataset=vectorized_dataset[\"validation\"],\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=collator,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "AoEoWsj4P_zN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "9cb70d2a-1a0f-4878-e514-7e89647c3660"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='20000' max='20000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [20000/20000 15:58, Epoch 12/13]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>3.769200</td>\n",
              "      <td>3.607957</td>\n",
              "      <td>0.299073</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>3.365500</td>\n",
              "      <td>3.110463</td>\n",
              "      <td>0.412114</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>2.853900</td>\n",
              "      <td>2.634109</td>\n",
              "      <td>0.459094</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>2.426600</td>\n",
              "      <td>2.281739</td>\n",
              "      <td>0.540653</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>2.113000</td>\n",
              "      <td>2.014794</td>\n",
              "      <td>0.599840</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>1.873200</td>\n",
              "      <td>1.801088</td>\n",
              "      <td>0.636482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>1.668400</td>\n",
              "      <td>1.627963</td>\n",
              "      <td>0.658736</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>1.513400</td>\n",
              "      <td>1.488023</td>\n",
              "      <td>0.678809</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>1.383200</td>\n",
              "      <td>1.372602</td>\n",
              "      <td>0.697286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>1.272800</td>\n",
              "      <td>1.277531</td>\n",
              "      <td>0.712940</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>1.187700</td>\n",
              "      <td>1.196915</td>\n",
              "      <td>0.728535</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>1.108600</td>\n",
              "      <td>1.128155</td>\n",
              "      <td>0.741268</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>1.045400</td>\n",
              "      <td>1.068840</td>\n",
              "      <td>0.752911</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.983200</td>\n",
              "      <td>1.017556</td>\n",
              "      <td>0.765235</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.936900</td>\n",
              "      <td>0.972865</td>\n",
              "      <td>0.774814</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.895300</td>\n",
              "      <td>0.933326</td>\n",
              "      <td>0.782952</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.860500</td>\n",
              "      <td>0.898915</td>\n",
              "      <td>0.789884</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.816100</td>\n",
              "      <td>0.867727</td>\n",
              "      <td>0.796737</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.788900</td>\n",
              "      <td>0.840373</td>\n",
              "      <td>0.803181</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10000</td>\n",
              "      <td>0.761100</td>\n",
              "      <td>0.815978</td>\n",
              "      <td>0.809334</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10500</td>\n",
              "      <td>0.739600</td>\n",
              "      <td>0.794244</td>\n",
              "      <td>0.813617</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11000</td>\n",
              "      <td>0.719100</td>\n",
              "      <td>0.774859</td>\n",
              "      <td>0.817686</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11500</td>\n",
              "      <td>0.697200</td>\n",
              "      <td>0.757537</td>\n",
              "      <td>0.820821</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12000</td>\n",
              "      <td>0.676500</td>\n",
              "      <td>0.741835</td>\n",
              "      <td>0.824423</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12500</td>\n",
              "      <td>0.670500</td>\n",
              "      <td>0.727902</td>\n",
              "      <td>0.827207</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13000</td>\n",
              "      <td>0.645400</td>\n",
              "      <td>0.715371</td>\n",
              "      <td>0.829855</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13500</td>\n",
              "      <td>0.636700</td>\n",
              "      <td>0.704056</td>\n",
              "      <td>0.832094</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14000</td>\n",
              "      <td>0.622000</td>\n",
              "      <td>0.693996</td>\n",
              "      <td>0.834430</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14500</td>\n",
              "      <td>0.621300</td>\n",
              "      <td>0.685134</td>\n",
              "      <td>0.836533</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15000</td>\n",
              "      <td>0.604100</td>\n",
              "      <td>0.677149</td>\n",
              "      <td>0.838519</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15500</td>\n",
              "      <td>0.598700</td>\n",
              "      <td>0.670169</td>\n",
              "      <td>0.840096</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16000</td>\n",
              "      <td>0.589000</td>\n",
              "      <td>0.664023</td>\n",
              "      <td>0.841478</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16500</td>\n",
              "      <td>0.584900</td>\n",
              "      <td>0.658739</td>\n",
              "      <td>0.842977</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17000</td>\n",
              "      <td>0.581700</td>\n",
              "      <td>0.654183</td>\n",
              "      <td>0.843970</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17500</td>\n",
              "      <td>0.574000</td>\n",
              "      <td>0.650407</td>\n",
              "      <td>0.844749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18000</td>\n",
              "      <td>0.571800</td>\n",
              "      <td>0.647352</td>\n",
              "      <td>0.845061</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18500</td>\n",
              "      <td>0.565200</td>\n",
              "      <td>0.644994</td>\n",
              "      <td>0.845664</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19000</td>\n",
              "      <td>0.565900</td>\n",
              "      <td>0.643310</td>\n",
              "      <td>0.845820</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19500</td>\n",
              "      <td>0.558600</td>\n",
              "      <td>0.642319</td>\n",
              "      <td>0.846112</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20000</td>\n",
              "      <td>0.562300</td>\n",
              "      <td>0.641987</td>\n",
              "      <td>0.846112</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=20000, training_loss=1.1001900787353516, metrics={'train_runtime': 958.1687, 'train_samples_per_second': 2671.763, 'train_steps_per_second': 20.873, 'total_flos': 120791176128.0, 'train_loss': 1.1001900787353516, 'epoch': 12.57})"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then evaluate the trained model on a given dataset (here our test subset) by calling [`Trainer.evaluate`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.evaluate):"
      ],
      "metadata": {
        "id": "Td03fcIa-6Hj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(vectorized_dataset[\"test\"])\n",
        "\n",
        "print(\"Accuracy:\", eval_results[\"eval_accuracy\"])"
      ],
      "metadata": {
        "id": "9nlEwpnF2Vow",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "f5626537-9e13-4728-b88f-5da2574ee05d"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='5805' max='5805' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [5805/5805 00:14]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.8539894476149457\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "That's pretty poor performance for a task as simple as POS tagging where state-of-the-art accuracies are generally 97-99%. (The approach demonstrated in this notebook should be considered more of a teaching tool than a serious tagger implementation.)\n",
        "\n",
        "However, the result is certainly much better than random, so we can conclude that the model is learning something about the task."
      ],
      "metadata": {
        "id": "pEbNfGDhfdXK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Save model for later use\n",
        "\n",
        "* You can save it with `trainer.save_model()`\n",
        "* You can load it with `MLP.from_pretrained()`\n"
      ],
      "metadata": {
        "id": "P0OYB3TRp-IH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model(\"mlp-postagger\")"
      ],
      "metadata": {
        "id": "FosHTDw3p9dd"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# What has the model learned?\n",
        "\n",
        "* The embeddings should have some meaning to them\n",
        "* Similar features should have similar embeddings"
      ],
      "metadata": {
        "id": "13aB7DuqzeFi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Grab the embedding matrix out of the trained model\n",
        "# and drop the first row (padding 0)\n",
        "# then we can treat the embeddings as vectors\n",
        "\n",
        "weights=mlp.embedding.weight.detach().cpu().numpy()\n",
        "weights=weights[1:,:]"
      ],
      "metadata": {
        "id": "M6TUrVkMCmz7"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qry_idx=vectorizer.vocabulary_[\"token[0]=in\"] \n",
        "\n",
        "#calculate the distance of the \"in\" embedding to all other embeddings\n",
        "distance_to_qry=sklearn.metrics.pairwise.euclidean_distances(weights[qry_idx:qry_idx+1,:],weights)\n",
        "nearest_neighbors=np.argsort(distance_to_qry) #indices of words nearest to \"in\"\n",
        "for nearest in nearest_neighbors[0,:20]:\n",
        "    print(idx2feat[nearest])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4uTTvRgGDU9D",
        "outputId": "80e616ed-680f-4474-af05-fe5bc2f6d3f4"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "token[0]=in\n",
            "token[0]=at\n",
            "token[0]=with\n",
            "token[0]=from\n",
            "token[0]=by\n",
            "token[0]=for\n",
            "token[0]=on\n",
            "token[0]=after\n",
            "token[0]=of\n",
            "token[0]=as\n",
            "token[0]=into\n",
            "token[0]=than\n",
            "token[0]=In\n",
            "token[0]=under\n",
            "token[0]=against\n",
            "token[0]=since\n",
            "token[0]=between\n",
            "token[0]=before\n",
            "token[0]=over\n",
            "token[0]=about\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* The embeddings indeed seem to reflect the task and capture aspects of the meaning of words relevant to the task\n",
        "* But now we have many classes, so we should take that into account too\n",
        "* We can take the dot-product of the feature embeddings with the output layer weight of the class we care about\n",
        "* When you think how the information propagates in the network, this will give us a single number reflecting each feature w.r.t. the selected label\n",
        "* Technically speaking, it is the prediction of an example which only has that one feature, with respect to that one class\n",
        "* Here is how we can implement it (here we rely on the fact that the model is linear, since we didn't include a nonlinearity earlier in the model's `forward()`"
      ],
      "metadata": {
        "id": "cQCmThdu2LDR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy\n",
        "\n",
        "embedding_weights=weights    #shape (features, embedding-dim)\n",
        "output_weights=mlp.output.weight.detach().cpu().numpy()    #shape (num-labels, embedding-dim)\n",
        "\n",
        "# We just matrix-multiply these together, since this gives us all the dot-products\n",
        "weights_by_label=numpy.matmul(embedding_weights, output_weights.T)\n",
        "weights_by_label.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5coMII4ABPt",
        "outputId": "a678b107-a907-4179-8445-12c7c8167682"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(30000, 47)"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_most_important_features_for_and_against(label):\n",
        "    label_idx = POS2ID[label]\n",
        "    feature_weights = weights_by_label[:,label_idx] #pick the column that interests us\n",
        "\n",
        "    #The shape of feature_weights is (feature_vocab_size,) i.e. it is a vector\n",
        "    features_weight_idx = numpy.argsort(-feature_weights) #sort in descending order, this will be vector of indices\n",
        "    features_for = [idx2feat[feature_idx] for feature_idx in features_weight_idx[:20]]\n",
        "    features_against = [idx2feat[feature_idx] for feature_idx in features_weight_idx[-20:][::-1]]\n",
        "    return features_for, features_against\n",
        "\n",
        "for label in (\"DT\", \"NN\", \"VB\"):\n",
        "    dt_plus,dt_minus=get_most_important_features_for_and_against(label)\n",
        "    print(f\"{label}: {POS2DESCRIPTION[label]}\")\n",
        "    print(f\"Most important features *for* label {label}:\")\n",
        "    pprint(\"   \".join(dt_plus))\n",
        "    print()\n",
        "    print(f\"Most important features *against* label {label}:\")\n",
        "    pprint(\"   \".join(dt_minus))\n",
        "    print(\"\\n------\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4vnJj7cNBagc",
        "outputId": "024e73ad-6278-4e4d-eb8f-968c9800e57e"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DT: Determiner\n",
            "Most important features *for* label DT:\n",
            "('token[0]=the   token[0]=a   token[0]=The   token[0]=an   token[0]=this   '\n",
            " 'token[0]=A   token[0]=some   token[0]=no   token[0]=any   token[0]=all   '\n",
            " 'token[0]=and   token[0]=both   token[0]=another   token[0]=those   '\n",
            " 'token[0]=This   token[0]=No   token[0]=An   token[0]=.   token[0]=these   '\n",
            " 'token[1]=year')\n",
            "\n",
            "Most important features *against* label DT:\n",
            "('token[0]=\"   token[0]=)   token[0]=which   token[0]=in   token[0]=said   '\n",
            " 'token[0]=of   token[0]=is   token[0]=on   token[0]=was   token[-1]=.   '\n",
            " 'token[0]=has   token[0]=for   token[0]=U.S.   token[0]=at   token[0]=-   '\n",
            " 'token[0]=with   token[0]=by   token[0]=were   token[0]=from   token[0]=as')\n",
            "\n",
            "------\n",
            "\n",
            "NN: Noun, singular or mass\n",
            "Most important features *for* label NN:\n",
            "('token[0]=percent   token[0]=government   token[0]=year   token[0]=week   '\n",
            " 'token[0]=time   token[0]=police   token[0]=state   token[0]=SOCCER   '\n",
            " 'token[0]=market   token[0]=group   token[0]=company   token[-1]=a   '\n",
            " 'token[0]=spokesman   token[0]=home   token[1]=of   token[0]=division   '\n",
            " 'token[0]=president   token[0]=world   token[0]=peace   token[0]=team')\n",
            "\n",
            "Most important features *against* label NN:\n",
            "('token[0]=\"   token[0]=to   token[0]=$   token[0]=when   token[0]=which   '\n",
            " \"token[0]='s   token[0]=who   token[0]=where   token[0]=more   token[0]=his   \"\n",
            " 'token[0]=their   token[0]=its   token[0]=/   token[0]=-   token[-1]=.   '\n",
            " \"token[0]=there   token[0]=earlier   token[0]=:   token[0]='   token[0]=most\")\n",
            "\n",
            "------\n",
            "\n",
            "VB: Verb, base form\n",
            "Most important features *for* label VB:\n",
            "('token[0]=be   token[0]=beat   token[-1]=will   token[-1]=to   '\n",
            " 'token[-1]=would   token[0]=have   token[-1]=could   token[0]=are   '\n",
            " \"token[0]=take   token[-1]=should   token[-2]=did   token[-1]=n't   \"\n",
            " 'token[-1]=can   token[0]=meet   token[0]=go   token[0]=make   token[0]=do   '\n",
            " 'token[-1]=not   token[0]=play   token[0]=get')\n",
            "\n",
            "Most important features *against* label VB:\n",
            "('token[0]=two   token[0]=0   token[0]=1   token[0]=which   token[0]=million   '\n",
            " 'token[-1]=$   token[0]=2   token[0]=\"   token[0]=3   token[1]=percent   '\n",
            " 'token[0]=.   token[0]=three   token[0]=4   token[0]=1996-08-28   '\n",
            " 'token[0]=one   token[1]=million   token[0]=1996-08-22   '\n",
            " 'token[0]=1996-08-29   token[0]=1996-08-27   token[0]=6')\n",
            "\n",
            "------\n",
            "\n"
          ]
        }
      ]
    }
  ]
}