intro_2023_exercise_7_solution.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "353131bbb19c4da8bd997d44c873b12b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_09e35d093771467dab1d6577c4e95bd3",
              "IPY_MODEL_f9af236814024aeda07c914b693f6c74",
              "IPY_MODEL_3bd1b98d2c4c4849810d62d4cd1dd096"
            ],
            "layout": "IPY_MODEL_6615681670d948f786e2c9854866ab9d"
          }
        },
        "09e35d093771467dab1d6577c4e95bd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0bc2d38d7f4445519687e283987c9695",
            "placeholder": "​",
            "style": "IPY_MODEL_5f3928f3bcdf4cd78d44a61d774248d0",
            "value": "100%"
          }
        },
        "f9af236814024aeda07c914b693f6c74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6829c81a6ba14d40896ac8d1d5090672",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e11f61571c8b4623987dfce7df2452d9",
            "value": 3
          }
        },
        "3bd1b98d2c4c4849810d62d4cd1dd096": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_789ab170a33b44b59d0c6b00c5e71911",
            "placeholder": "​",
            "style": "IPY_MODEL_f2c2a2b6005640fda283b846b682c33b",
            "value": " 3/3 [00:00&lt;00:00, 93.85it/s]"
          }
        },
        "6615681670d948f786e2c9854866ab9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0bc2d38d7f4445519687e283987c9695": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f3928f3bcdf4cd78d44a61d774248d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6829c81a6ba14d40896ac8d1d5090672": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e11f61571c8b4623987dfce7df2452d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "789ab170a33b44b59d0c6b00c5e71911": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2c2a2b6005640fda283b846b682c33b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ec121fc8805e4778aefc4b25d90e2fe8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_12fac778edbf45ff98981531795cef5e",
              "IPY_MODEL_366e9e393b8e4a1aa141addb65ebdde7",
              "IPY_MODEL_f2b7b99bffda4e0ea8c028c22be0afa1"
            ],
            "layout": "IPY_MODEL_6f89f9f69dd24c519beb5722e307da4d"
          }
        },
        "12fac778edbf45ff98981531795cef5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90c2ae2800e747c0ae7736497953e5fb",
            "placeholder": "​",
            "style": "IPY_MODEL_a3624555abbb4ae18383d87c5a68916e",
            "value": "Map: 100%"
          }
        },
        "366e9e393b8e4a1aa141addb65ebdde7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_248a3a825d344ed3b74e1c5cdcaba54c",
            "max": 14041,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_838d6357475643b6a344366ce2445372",
            "value": 14041
          }
        },
        "f2b7b99bffda4e0ea8c028c22be0afa1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b011ff64979a463cae48f1bd0c2678ca",
            "placeholder": "​",
            "style": "IPY_MODEL_f5f3a1fdfcd34b36afcee9d3fbf209a2",
            "value": " 14041/14041 [00:06&lt;00:00, 1252.63 examples/s]"
          }
        },
        "6f89f9f69dd24c519beb5722e307da4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "90c2ae2800e747c0ae7736497953e5fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a3624555abbb4ae18383d87c5a68916e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "248a3a825d344ed3b74e1c5cdcaba54c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "838d6357475643b6a344366ce2445372": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b011ff64979a463cae48f1bd0c2678ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5f3a1fdfcd34b36afcee9d3fbf209a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "abbf99a57f5b49daa852139fc2f3aaf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_59e3d2fd586a4dfe82990a1afc0fcfc7",
              "IPY_MODEL_b41a11920fc740acaed950c932dc3090",
              "IPY_MODEL_42e45f5b8bdd4417849b42481ac2a0a0"
            ],
            "layout": "IPY_MODEL_ff8c733f4e5547b59923a212b5c2364a"
          }
        },
        "59e3d2fd586a4dfe82990a1afc0fcfc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdc7fab151c64beea50ec0c78a2db036",
            "placeholder": "​",
            "style": "IPY_MODEL_8c493e91cbc84c6189918e4e528b98c4",
            "value": "Map:  99%"
          }
        },
        "b41a11920fc740acaed950c932dc3090": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c5a170194c0c40a2a33344a2463ab9ae",
            "max": 3250,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_db40c1e7fdc0408e95630cb51aa1738c",
            "value": 3250
          }
        },
        "42e45f5b8bdd4417849b42481ac2a0a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a643ad62a33244178f579121166b46cd",
            "placeholder": "​",
            "style": "IPY_MODEL_5e14e0d3102d48aaaee4692d58156c23",
            "value": " 3212/3250 [00:01&lt;00:00, 1982.00 examples/s]"
          }
        },
        "ff8c733f4e5547b59923a212b5c2364a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "cdc7fab151c64beea50ec0c78a2db036": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c493e91cbc84c6189918e4e528b98c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c5a170194c0c40a2a33344a2463ab9ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db40c1e7fdc0408e95630cb51aa1738c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a643ad62a33244178f579121166b46cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e14e0d3102d48aaaee4692d58156c23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20844ee7ad4a4715a4a841ad606fe377": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1a9a076edfb842d9bec532f867dea979",
              "IPY_MODEL_690494c66c3b417f9a5429ce2c93f3f7",
              "IPY_MODEL_d2b7b495e7ae4bf681014c90f8361858"
            ],
            "layout": "IPY_MODEL_c4cc8f21ee7a4a488f0fc1979aada5e0"
          }
        },
        "1a9a076edfb842d9bec532f867dea979": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c92da5c06564d5ca6ce22cd0dd85a08",
            "placeholder": "​",
            "style": "IPY_MODEL_c8e15916b3e741adbfa8cb064eaf88ab",
            "value": "Map: 100%"
          }
        },
        "690494c66c3b417f9a5429ce2c93f3f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_55c56976485f431f995852fdfa6ac79d",
            "max": 3453,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b34951541d0942548c23cc40594824a3",
            "value": 3453
          }
        },
        "d2b7b495e7ae4bf681014c90f8361858": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_98bd4aa282a64623835ec94101352a95",
            "placeholder": "​",
            "style": "IPY_MODEL_2f3c93b1183343cf9ce3791cfe82a43d",
            "value": " 3453/3453 [00:01&lt;00:00, 3558.40 examples/s]"
          }
        },
        "c4cc8f21ee7a4a488f0fc1979aada5e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "9c92da5c06564d5ca6ce22cd0dd85a08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8e15916b3e741adbfa8cb064eaf88ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "55c56976485f431f995852fdfa6ac79d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b34951541d0942548c23cc40594824a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "98bd4aa282a64623835ec94101352a95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2f3c93b1183343cf9ce3791cfe82a43d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f374b1d535504d9187d713358312e3ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cca15fbf6d3e40768bb9dfa9a7666cb1",
              "IPY_MODEL_ec91ccc507784bfd92f636c0730b3c70",
              "IPY_MODEL_40fcc90e746843c58eef5cbaeb3ee7d6"
            ],
            "layout": "IPY_MODEL_88395b82b52641f39a51a3d03a98acae"
          }
        },
        "cca15fbf6d3e40768bb9dfa9a7666cb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c517fe8841944c69a263555958f92bd9",
            "placeholder": "​",
            "style": "IPY_MODEL_159b7095fb8642c2bbc45a26b3798647",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "ec91ccc507784bfd92f636c0730b3c70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7adbefe082444c2098199ef7ad549322",
            "max": 203621,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e6ef08188ea346fa8b94c7ecad3c32be",
            "value": 203621
          }
        },
        "40fcc90e746843c58eef5cbaeb3ee7d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64ef5f25bacd4420ac9de75a475bb03e",
            "placeholder": "​",
            "style": "IPY_MODEL_8d489d554d664dcca820628ea5048374",
            "value": " 203621/203621 [01:19&lt;00:00, 2077.50 examples/s]"
          }
        },
        "88395b82b52641f39a51a3d03a98acae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "c517fe8841944c69a263555958f92bd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "159b7095fb8642c2bbc45a26b3798647": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7adbefe082444c2098199ef7ad549322": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e6ef08188ea346fa8b94c7ecad3c32be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "64ef5f25bacd4420ac9de75a475bb03e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d489d554d664dcca820628ea5048374": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba07d3f520fe401da45c67330220ba8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_39f7972fcb7b4090b6621d6119034ea1",
              "IPY_MODEL_92bd39909f9f4f54a6b7a191195bc5a3",
              "IPY_MODEL_6f3e0243a29e433199bd6f3785b2141e"
            ],
            "layout": "IPY_MODEL_a38699186f4a4332bf5d3a8a16cef2ad"
          }
        },
        "39f7972fcb7b4090b6621d6119034ea1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_36847fa46101451a9f9859278f15f1a0",
            "placeholder": "​",
            "style": "IPY_MODEL_0f717f92bad346389e7ed91d0de471ba",
            "value": "Map (num_proc=4):  99%"
          }
        },
        "92bd39909f9f4f54a6b7a191195bc5a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9147eb7a2994d4b96984d6a457b365e",
            "max": 51362,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_65c67cbc2e3c4803be912f6521b059db",
            "value": 51362
          }
        },
        "6f3e0243a29e433199bd6f3785b2141e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f35d8ae9dbcf45898f8c055db5c09228",
            "placeholder": "​",
            "style": "IPY_MODEL_3dbb8aff79d04f8882d7a6edd1492892",
            "value": " 51101/51362 [00:18&lt;00:00, 2458.09 examples/s]"
          }
        },
        "a38699186f4a4332bf5d3a8a16cef2ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "36847fa46101451a9f9859278f15f1a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0f717f92bad346389e7ed91d0de471ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c9147eb7a2994d4b96984d6a457b365e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65c67cbc2e3c4803be912f6521b059db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f35d8ae9dbcf45898f8c055db5c09228": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3dbb8aff79d04f8882d7a6edd1492892": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e15e17e667f34f34bf17ab10dd23536b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cb7b8ff71254418abdd89f66d2e52f7d",
              "IPY_MODEL_2028d613ca33430e9e9da712cb6ecb3a",
              "IPY_MODEL_0adab00e5ad648e18a978d6204a56dad"
            ],
            "layout": "IPY_MODEL_6bd14f3f61e242859eb6c7ed63cb9d3e"
          }
        },
        "cb7b8ff71254418abdd89f66d2e52f7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fcfb72a645354af7b9fc15c9e92a5a62",
            "placeholder": "​",
            "style": "IPY_MODEL_02aa31a2b0544961afe9515c706219e5",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "2028d613ca33430e9e9da712cb6ecb3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_103050b69abd42d0b54d56678a652681",
            "max": 46435,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a2229fe0a9aa49869479667f63970959",
            "value": 46435
          }
        },
        "0adab00e5ad648e18a978d6204a56dad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_254458559e564046a4cd6e156b3f7e50",
            "placeholder": "​",
            "style": "IPY_MODEL_77cae6b2fdba4464a3e083142bb6ac80",
            "value": " 46324/46435 [00:18&lt;00:00, 1866.88 examples/s]"
          }
        },
        "6bd14f3f61e242859eb6c7ed63cb9d3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "fcfb72a645354af7b9fc15c9e92a5a62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02aa31a2b0544961afe9515c706219e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "103050b69abd42d0b54d56678a652681": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a2229fe0a9aa49869479667f63970959": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "254458559e564046a4cd6e156b3f7e50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77cae6b2fdba4464a3e083142bb6ac80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/intro_2023_exercise_7_solution.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sequence labeling (POS tagging) with MLP\n",
        "\n",
        "This notebook builds upon the [classification with MLP notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb) and shows how to implement a basic sequence labeling method."
      ],
      "metadata": {
        "id": "arPRayYSpzJs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Setup\n",
        "\n",
        "Install the required Python packages using [pip](https://en.wikipedia.org/wiki/Pip):\n",
        "\n",
        "* [`transformers`](https://huggingface.co/docs/transformers/index) is a popular deep learning package primarily on top of torch\n",
        "* [`datasets`](https://huggingface.co/docs/datasets/) provides support for loading, creating, and manipulating datasets\n",
        "* [`evaluate`](https://huggingface.co/docs/evaluate/index) is a library of performance metrics (like accuracy etc)"
      ],
      "metadata": {
        "id": "LDZZUKzfPRWT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --quiet transformers datasets evaluate"
      ],
      "metadata": {
        "id": "fKPHBYptQDsK"
      },
      "execution_count": 88,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Get and prepare data\n",
        "\n",
        "*   Let us work with the venerable, if somewhat dated [CoNLL'03 shared task](https://aclanthology.org/W03-0419.pdf) English data\n",
        "*   These are English news articles, and have annotation for POS, syntactic chunks, and named entities (in the IOB format)\n",
        "\n",
        "The data as originally distributed for the 2003 shared task has the following format:\n",
        "\n",
        "```\n",
        "Only RB I-NP O\n",
        "France NNP I-NP I-LOC\n",
        "and CC I-NP O\n",
        "Britain NNP I-NP I-LOC\n",
        "backed VBD I-VP O\n",
        "Fischler NNP I-NP I-PER\n",
        "'s POS B-NP O\n",
        "proposal NN I-NP O\n",
        ". . O O\n",
        "```\n",
        "\n",
        "Here, the four space-separated columns are token text, POS tag, chunk tag, and NER tag. The goal of the original task is to predict the NER tags using the other information as features, but the dataset can be used to study predicting the other columns too.\n",
        "\n",
        "The dataset happens to be in the HF datasets collection, so we can grab it from there\n"
      ],
      "metadata": {
        "id": "_fCdfQfNNzwt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import transformers\n",
        "import datasets\n",
        "\n",
        "from pprint import pprint    # pretty-print\n",
        "\n",
        "dataset = datasets.load_dataset(\"conll2003\")\n",
        "\n",
        "print(dataset)"
      ],
      "metadata": {
        "id": "QxmgHoKDTN2_",
        "outputId": "b61568d1-020b-47ab-9e32-9e4c8e49df67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 345,
          "referenced_widgets": [
            "353131bbb19c4da8bd997d44c873b12b",
            "09e35d093771467dab1d6577c4e95bd3",
            "f9af236814024aeda07c914b693f6c74",
            "3bd1b98d2c4c4849810d62d4cd1dd096",
            "6615681670d948f786e2c9854866ab9d",
            "0bc2d38d7f4445519687e283987c9695",
            "5f3928f3bcdf4cd78d44a61d774248d0",
            "6829c81a6ba14d40896ac8d1d5090672",
            "e11f61571c8b4623987dfce7df2452d9",
            "789ab170a33b44b59d0c6b00c5e71911",
            "f2c2a2b6005640fda283b846b682c33b"
          ]
        }
      },
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Found cached dataset conll2003 (/root/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/9a4d16a94f8674ba3466315300359b0acd891b68b6c8743ddf60b9c702adce98)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "353131bbb19c4da8bd997d44c873b12b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 14041\n",
            "    })\n",
            "    validation: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 3250\n",
            "    })\n",
            "    test: Dataset({\n",
            "        features: ['id', 'tokens', 'pos_tags', 'chunk_tags', 'ner_tags'],\n",
            "        num_rows: 3453\n",
            "    })\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dataset[\"train\"][12])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OEtbYIyLg3r7",
        "outputId": "8c2908dc-9139-4d10-e877-c54e398b5356"
      },
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': [11, 12, 12, 12, 21, 11, 11, 12, 0],\n",
            " 'id': '12',\n",
            " 'ner_tags': [0, 5, 0, 5, 0, 1, 0, 0, 0],\n",
            " 'pos_tags': [30, 22, 10, 22, 38, 22, 27, 21, 7],\n",
            " 'tokens': ['Only',\n",
            "            'France',\n",
            "            'and',\n",
            "            'Britain',\n",
            "            'backed',\n",
            "            'Fischler',\n",
            "            \"'s\",\n",
            "            'proposal',\n",
            "            '.']}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see above, the various labels (POS, NER and chunk tags) are converted into IDs in this dataset. We can access the textual labels of these tags through the dataset `features`:"
      ],
      "metadata": {
        "id": "v9TMq-d3qFEJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "POS_TAG_NAMES = dataset['train'].features['pos_tags'].feature.names\n",
        "NER_TAG_NAMES = dataset['train'].features['ner_tags'].feature.names\n",
        "CHUNK_TAG_NAMES = dataset['train'].features['chunk_tags'].feature.names"
      ],
      "metadata": {
        "id": "u0kflH3RpHKY"
      },
      "execution_count": 91,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create mappings from names to IDs and back as Python dictionaries:"
      ],
      "metadata": {
        "id": "O3-QCWMvrazc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "POS2ID = { n: i for i, n in enumerate(POS_TAG_NAMES) }\n",
        "ID2POS = { i: n for i, n in enumerate(POS_TAG_NAMES) }\n",
        "\n",
        "NER2ID = { n: i for i, n in enumerate(NER_TAG_NAMES) }\n",
        "ID2NER = { i: n for i, n in enumerate(NER_TAG_NAMES) }\n",
        "\n",
        "CHUNK2ID = { n: i for i, n in enumerate(CHUNK_TAG_NAMES) }\n",
        "ID2CHUNK = { i: n for i, n in enumerate(CHUNK_TAG_NAMES) }"
      ],
      "metadata": {
        "id": "tqi_FN2yq7MN"
      },
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is what these mappings look like:"
      ],
      "metadata": {
        "id": "sUI-FiAssDLL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(NER2ID)"
      ],
      "metadata": {
        "id": "XAYSIMWjsFiY",
        "outputId": "d3378249-3f9c-4e96-a0dd-4279032dad22",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'O': 0, 'B-PER': 1, 'I-PER': 2, 'B-ORG': 3, 'I-ORG': 4, 'B-LOC': 5, 'I-LOC': 6, 'B-MISC': 7, 'I-MISC': 8}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(ID2NER)"
      ],
      "metadata": {
        "id": "NO4xI0yasHdM",
        "outputId": "ae533588-693d-4867-9d59-4ce991d7ee73",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{0: 'O', 1: 'B-PER', 2: 'I-PER', 3: 'B-ORG', 4: 'I-ORG', 5: 'B-LOC', 6: 'I-LOC', 7: 'B-MISC', 8: 'I-MISC'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's also add in explanations from Penn Treebank for the POS tags:"
      ],
      "metadata": {
        "id": "jUqKBvtpvt3G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# From the documentation page and from here https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html\n",
        "\n",
        "POS2DESCRIPTION = {\n",
        "    \"CC\": \"Coordinating conjunction\",\n",
        "    \"CD\": \"Cardinal number\",\n",
        "    \"DT\": \"Determiner\",\n",
        "    \"EX\": \"Existential there\",\n",
        "    \"FW\": \"Foreign word\",\n",
        "    \"IN\": \"Preposition or subordinating conjunction\",\n",
        "    \"JJ\": \"Adjective\",\n",
        "    \"JJR\": \"Adjective, comparative\",\n",
        "    \"JJS\": \"Adjective, superlative\",\n",
        "    \"LS\": \"List item marker\",\n",
        "    \"MD\": \"Modal\",\n",
        "    \"NN\": \"Noun, singular or mass\",\n",
        "    \"NNS\": \"Noun, plural\",\n",
        "    \"NNP\": \"Proper noun, singular\",\n",
        "    \"NNPS\": \"Proper noun, plural\",\n",
        "    \"PDT\": \"Predeterminer\",\n",
        "    \"POS\": \"Possessive ending\",\n",
        "    \"PRP\": \"Personal pronoun\",\n",
        "    \"PRP$\": \"Possessive pronoun\",\n",
        "    \"RB\": \"Adverb\",\n",
        "    \"RBR\": \"Adverb, comparative\",\n",
        "    \"RBS\": \"Adverb, superlative\",\n",
        "    \"RP\": \"Particle\",\n",
        "    \"SYM\": \"Symbol\",\n",
        "    \"TO\": \"to\",\n",
        "    \"UH\": \"Interjection\",\n",
        "    \"VB\": \"Verb, base form\",\n",
        "    \"VBD\": \"Verb, past tense\",\n",
        "    \"VBG\": \"Verb, gerund or present participle\",\n",
        "    \"VBN\": \"Verb, past participle\",\n",
        "    \"VBP\": \"Verb, non-3rd person singular present\",\n",
        "    \"VBZ\": \"Verb, 3rd person singular present\",\n",
        "    \"WDT\": \"Wh-determiner\",\n",
        "    \"WP\": \"Wh-pronoun\",\n",
        "    \"WP$\": \"Possessive wh-pronoun\",\n",
        "    \"WRB\": \"Wh-adverb\"\n",
        "}"
      ],
      "metadata": {
        "id": "EDIG9IkNf_ji"
      },
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can now try to make sense of the tags:"
      ],
      "metadata": {
        "id": "HRcNtMskyTxq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tabulate\n",
        "\n",
        "e = dataset[\"train\"][12]    # work on the same example\n",
        "\n",
        "table = []\n",
        "for token, pos_id, chunk_id, ner_id in zip(e[\"tokens\"], e[\"pos_tags\"], e[\"chunk_tags\"], e[\"ner_tags\"]):\n",
        "    ner_tag = ID2NER[ner_id]\n",
        "    chunk_tag = ID2CHUNK[chunk_id]\n",
        "    pos_tag = ID2POS[pos_id]\n",
        "    pos_def = POS2DESCRIPTION.get(pos_tag,pos_tag)\n",
        "    table.append([token, ner_tag, chunk_tag, pos_tag, pos_def])\n",
        "\n",
        "print(tabulate.tabulate(table,headers=[\"Token\", \"NER\", \"Chunk\", \"POS\", \"POS definition\"]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JzQXQI0YiD0_",
        "outputId": "74e7b8ac-7a9e-4643-d3c3-f17010393162"
      },
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Token     NER    Chunk    POS    POS definition\n",
            "--------  -----  -------  -----  ------------------------\n",
            "Only      O      B-NP     RB     Adverb\n",
            "France    B-LOC  I-NP     NNP    Proper noun, singular\n",
            "and       O      I-NP     CC     Coordinating conjunction\n",
            "Britain   B-LOC  I-NP     NNP    Proper noun, singular\n",
            "backed    O      B-VP     VBD    Verb, past tense\n",
            "Fischler  B-PER  B-NP     NNP    Proper noun, singular\n",
            "'s        O      B-NP     POS    Possessive ending\n",
            "proposal  O      I-NP     NN     Noun, singular or mass\n",
            ".         O      O        .      .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note that the data is organized into sentences."
      ],
      "metadata": {
        "id": "VXm-E32AdVpo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Create features\n",
        "\n",
        "We'll define a simple function that takes a token, pos and chunk tag sequences, the index of the focus token, and a window size and generates a few basic explicit features relevant to the task."
      ],
      "metadata": {
        "id": "d9WvQiwluhBO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Performance by feature set\n",
        "\n",
        "We started from just token surface form features, then added POS tag features, and finally added chunk tag features:\n",
        "\n",
        "```\n",
        "Initial    : Accuracy: 0.9012598255626144\n",
        "Add POS    : Accuracy: 0.9263917303757941\n",
        "Add chunks : Accuracy: 0.9284160654678583\n",
        "```"
      ],
      "metadata": {
        "id": "e71EzNYRJO-q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def token_features(tokens, pos_tags, chunk_tags, index, window_size):\n",
        "    # Generate features for token in position `index` in given list of tokens\n",
        "    features = []\n",
        "\n",
        "    # Context window start and end\n",
        "    window_start = max(0, index-window_size)\n",
        "    window_end = min(index+window_size+1, len(tokens))    # note +1 for range\n",
        "\n",
        "    for i in range(window_start, window_end):\n",
        "          offset = i - index    # relative position\n",
        "          features.append(f\"token[{offset}]={tokens[i]}\")\n",
        "          features.append(f\"pos_tag[{offset}]={ID2POS[pos_tags[i]]}\")\n",
        "          features.append(f\"chunk_tag[{offset}]={ID2CHUNK[chunk_tags[i]]}\")\n",
        "\n",
        "    # Example custom feature: does focus token start with an upper-case letter?\n",
        "    if tokens[index][0].isupper():\n",
        "        features.append(\"first-letter-capitalized\")\n",
        "\n",
        "    return features"
      ],
      "metadata": {
        "id": "6NjCVYHWe32j"
      },
      "execution_count": 97,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can call this function for all tokens in a sentence like so:"
      ],
      "metadata": {
        "id": "k1aEFA2Yg8xO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def add_features_to_sentence(sentence):\n",
        "    # Collect lists of features for all tokens here\n",
        "    all_features = []\n",
        "\n",
        "    tokens = sentence[\"tokens\"]\n",
        "    pos_tags = sentence[\"pos_tags\"]\n",
        "    chunk_tags = sentence[\"chunk_tags\"]\n",
        "    for index in range(len(tokens)):\n",
        "        all_features.append(token_features(tokens, pos_tags, chunk_tags, index, window_size=3))\n",
        "\n",
        "    return { \"features\": all_features }"
      ],
      "metadata": {
        "id": "tgTZ5h4hhGFd"
      },
      "execution_count": 98,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for feats in add_features_to_sentence(dataset[\"train\"][12])[\"features\"]:\n",
        "    print(feats)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VMoD24fQhnRK",
        "outputId": "f427071d-5258-472e-e754-89d24fb1d08b"
      },
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['token[0]=Only', 'pos_tag[0]=RB', 'chunk_tag[0]=B-NP', 'token[1]=France', 'pos_tag[1]=NNP', 'chunk_tag[1]=I-NP', 'token[2]=and', 'pos_tag[2]=CC', 'chunk_tag[2]=I-NP', 'token[3]=Britain', 'pos_tag[3]=NNP', 'chunk_tag[3]=I-NP', 'first-letter-capitalized']\n",
            "['token[-1]=Only', 'pos_tag[-1]=RB', 'chunk_tag[-1]=B-NP', 'token[0]=France', 'pos_tag[0]=NNP', 'chunk_tag[0]=I-NP', 'token[1]=and', 'pos_tag[1]=CC', 'chunk_tag[1]=I-NP', 'token[2]=Britain', 'pos_tag[2]=NNP', 'chunk_tag[2]=I-NP', 'token[3]=backed', 'pos_tag[3]=VBD', 'chunk_tag[3]=B-VP', 'first-letter-capitalized']\n",
            "['token[-2]=Only', 'pos_tag[-2]=RB', 'chunk_tag[-2]=B-NP', 'token[-1]=France', 'pos_tag[-1]=NNP', 'chunk_tag[-1]=I-NP', 'token[0]=and', 'pos_tag[0]=CC', 'chunk_tag[0]=I-NP', 'token[1]=Britain', 'pos_tag[1]=NNP', 'chunk_tag[1]=I-NP', 'token[2]=backed', 'pos_tag[2]=VBD', 'chunk_tag[2]=B-VP', 'token[3]=Fischler', 'pos_tag[3]=NNP', 'chunk_tag[3]=B-NP']\n",
            "['token[-3]=Only', 'pos_tag[-3]=RB', 'chunk_tag[-3]=B-NP', 'token[-2]=France', 'pos_tag[-2]=NNP', 'chunk_tag[-2]=I-NP', 'token[-1]=and', 'pos_tag[-1]=CC', 'chunk_tag[-1]=I-NP', 'token[0]=Britain', 'pos_tag[0]=NNP', 'chunk_tag[0]=I-NP', 'token[1]=backed', 'pos_tag[1]=VBD', 'chunk_tag[1]=B-VP', 'token[2]=Fischler', 'pos_tag[2]=NNP', 'chunk_tag[2]=B-NP', \"token[3]='s\", 'pos_tag[3]=POS', 'chunk_tag[3]=B-NP', 'first-letter-capitalized']\n",
            "['token[-3]=France', 'pos_tag[-3]=NNP', 'chunk_tag[-3]=I-NP', 'token[-2]=and', 'pos_tag[-2]=CC', 'chunk_tag[-2]=I-NP', 'token[-1]=Britain', 'pos_tag[-1]=NNP', 'chunk_tag[-1]=I-NP', 'token[0]=backed', 'pos_tag[0]=VBD', 'chunk_tag[0]=B-VP', 'token[1]=Fischler', 'pos_tag[1]=NNP', 'chunk_tag[1]=B-NP', \"token[2]='s\", 'pos_tag[2]=POS', 'chunk_tag[2]=B-NP', 'token[3]=proposal', 'pos_tag[3]=NN', 'chunk_tag[3]=I-NP']\n",
            "['token[-3]=and', 'pos_tag[-3]=CC', 'chunk_tag[-3]=I-NP', 'token[-2]=Britain', 'pos_tag[-2]=NNP', 'chunk_tag[-2]=I-NP', 'token[-1]=backed', 'pos_tag[-1]=VBD', 'chunk_tag[-1]=B-VP', 'token[0]=Fischler', 'pos_tag[0]=NNP', 'chunk_tag[0]=B-NP', \"token[1]='s\", 'pos_tag[1]=POS', 'chunk_tag[1]=B-NP', 'token[2]=proposal', 'pos_tag[2]=NN', 'chunk_tag[2]=I-NP', 'token[3]=.', 'pos_tag[3]=.', 'chunk_tag[3]=O', 'first-letter-capitalized']\n",
            "['token[-3]=Britain', 'pos_tag[-3]=NNP', 'chunk_tag[-3]=I-NP', 'token[-2]=backed', 'pos_tag[-2]=VBD', 'chunk_tag[-2]=B-VP', 'token[-1]=Fischler', 'pos_tag[-1]=NNP', 'chunk_tag[-1]=B-NP', \"token[0]='s\", 'pos_tag[0]=POS', 'chunk_tag[0]=B-NP', 'token[1]=proposal', 'pos_tag[1]=NN', 'chunk_tag[1]=I-NP', 'token[2]=.', 'pos_tag[2]=.', 'chunk_tag[2]=O']\n",
            "['token[-3]=backed', 'pos_tag[-3]=VBD', 'chunk_tag[-3]=B-VP', 'token[-2]=Fischler', 'pos_tag[-2]=NNP', 'chunk_tag[-2]=B-NP', \"token[-1]='s\", 'pos_tag[-1]=POS', 'chunk_tag[-1]=B-NP', 'token[0]=proposal', 'pos_tag[0]=NN', 'chunk_tag[0]=I-NP', 'token[1]=.', 'pos_tag[1]=.', 'chunk_tag[1]=O']\n",
            "['token[-3]=Fischler', 'pos_tag[-3]=NNP', 'chunk_tag[-3]=B-NP', \"token[-2]='s\", 'pos_tag[-2]=POS', 'chunk_tag[-2]=B-NP', 'token[-1]=proposal', 'pos_tag[-1]=NN', 'chunk_tag[-1]=I-NP', 'token[0]=.', 'pos_tag[0]=.', 'chunk_tag[0]=O']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The dataset is organized into sentences, so we can use the above function to add features to the entire dataset as follows.\n",
        "\n",
        "**Note**: unlike e.g. the Python`map` function, [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) function _updates_ its argument dataset, keeping existing values."
      ],
      "metadata": {
        "id": "c3SlJhk7i8aV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = dataset.map(add_features_to_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "ec121fc8805e4778aefc4b25d90e2fe8",
            "12fac778edbf45ff98981531795cef5e",
            "366e9e393b8e4a1aa141addb65ebdde7",
            "f2b7b99bffda4e0ea8c028c22be0afa1",
            "6f89f9f69dd24c519beb5722e307da4d",
            "90c2ae2800e747c0ae7736497953e5fb",
            "a3624555abbb4ae18383d87c5a68916e",
            "248a3a825d344ed3b74e1c5cdcaba54c",
            "838d6357475643b6a344366ce2445372",
            "b011ff64979a463cae48f1bd0c2678ca",
            "f5f3a1fdfcd34b36afcee9d3fbf209a2",
            "abbf99a57f5b49daa852139fc2f3aaf9",
            "59e3d2fd586a4dfe82990a1afc0fcfc7",
            "b41a11920fc740acaed950c932dc3090",
            "42e45f5b8bdd4417849b42481ac2a0a0",
            "ff8c733f4e5547b59923a212b5c2364a",
            "cdc7fab151c64beea50ec0c78a2db036",
            "8c493e91cbc84c6189918e4e528b98c4",
            "c5a170194c0c40a2a33344a2463ab9ae",
            "db40c1e7fdc0408e95630cb51aa1738c",
            "a643ad62a33244178f579121166b46cd",
            "5e14e0d3102d48aaaee4692d58156c23",
            "20844ee7ad4a4715a4a841ad606fe377",
            "1a9a076edfb842d9bec532f867dea979",
            "690494c66c3b417f9a5429ce2c93f3f7",
            "d2b7b495e7ae4bf681014c90f8361858",
            "c4cc8f21ee7a4a488f0fc1979aada5e0",
            "9c92da5c06564d5ca6ce22cd0dd85a08",
            "c8e15916b3e741adbfa8cb064eaf88ab",
            "55c56976485f431f995852fdfa6ac79d",
            "b34951541d0942548c23cc40594824a3",
            "98bd4aa282a64623835ec94101352a95",
            "2f3c93b1183343cf9ce3791cfe82a43d"
          ]
        },
        "id": "R9dekapGjNcb",
        "outputId": "e65840ce-33b3-4ffc-e2c3-941575ebf598"
      },
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/14041 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ec121fc8805e4778aefc4b25d90e2fe8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/3250 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "abbf99a57f5b49daa852139fc2f3aaf9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/3453 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "20844ee7ad4a4715a4a841ad606fe377"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's check that one more time:"
      ],
      "metadata": {
        "id": "Ke-vVsAulwMb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dataset[\"train\"][12])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7zfUEZbDlyeZ",
        "outputId": "5ffc4d70-d10a-4a28-a77c-8afba6ac8b47"
      },
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': [11, 12, 12, 12, 21, 11, 11, 12, 0],\n",
            " 'features': [['token[0]=Only',\n",
            "               'pos_tag[0]=RB',\n",
            "               'chunk_tag[0]=B-NP',\n",
            "               'token[1]=France',\n",
            "               'pos_tag[1]=NNP',\n",
            "               'chunk_tag[1]=I-NP',\n",
            "               'token[2]=and',\n",
            "               'pos_tag[2]=CC',\n",
            "               'chunk_tag[2]=I-NP',\n",
            "               'token[3]=Britain',\n",
            "               'pos_tag[3]=NNP',\n",
            "               'chunk_tag[3]=I-NP',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-1]=Only',\n",
            "               'pos_tag[-1]=RB',\n",
            "               'chunk_tag[-1]=B-NP',\n",
            "               'token[0]=France',\n",
            "               'pos_tag[0]=NNP',\n",
            "               'chunk_tag[0]=I-NP',\n",
            "               'token[1]=and',\n",
            "               'pos_tag[1]=CC',\n",
            "               'chunk_tag[1]=I-NP',\n",
            "               'token[2]=Britain',\n",
            "               'pos_tag[2]=NNP',\n",
            "               'chunk_tag[2]=I-NP',\n",
            "               'token[3]=backed',\n",
            "               'pos_tag[3]=VBD',\n",
            "               'chunk_tag[3]=B-VP',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-2]=Only',\n",
            "               'pos_tag[-2]=RB',\n",
            "               'chunk_tag[-2]=B-NP',\n",
            "               'token[-1]=France',\n",
            "               'pos_tag[-1]=NNP',\n",
            "               'chunk_tag[-1]=I-NP',\n",
            "               'token[0]=and',\n",
            "               'pos_tag[0]=CC',\n",
            "               'chunk_tag[0]=I-NP',\n",
            "               'token[1]=Britain',\n",
            "               'pos_tag[1]=NNP',\n",
            "               'chunk_tag[1]=I-NP',\n",
            "               'token[2]=backed',\n",
            "               'pos_tag[2]=VBD',\n",
            "               'chunk_tag[2]=B-VP',\n",
            "               'token[3]=Fischler',\n",
            "               'pos_tag[3]=NNP',\n",
            "               'chunk_tag[3]=B-NP'],\n",
            "              ['token[-3]=Only',\n",
            "               'pos_tag[-3]=RB',\n",
            "               'chunk_tag[-3]=B-NP',\n",
            "               'token[-2]=France',\n",
            "               'pos_tag[-2]=NNP',\n",
            "               'chunk_tag[-2]=I-NP',\n",
            "               'token[-1]=and',\n",
            "               'pos_tag[-1]=CC',\n",
            "               'chunk_tag[-1]=I-NP',\n",
            "               'token[0]=Britain',\n",
            "               'pos_tag[0]=NNP',\n",
            "               'chunk_tag[0]=I-NP',\n",
            "               'token[1]=backed',\n",
            "               'pos_tag[1]=VBD',\n",
            "               'chunk_tag[1]=B-VP',\n",
            "               'token[2]=Fischler',\n",
            "               'pos_tag[2]=NNP',\n",
            "               'chunk_tag[2]=B-NP',\n",
            "               \"token[3]='s\",\n",
            "               'pos_tag[3]=POS',\n",
            "               'chunk_tag[3]=B-NP',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-3]=France',\n",
            "               'pos_tag[-3]=NNP',\n",
            "               'chunk_tag[-3]=I-NP',\n",
            "               'token[-2]=and',\n",
            "               'pos_tag[-2]=CC',\n",
            "               'chunk_tag[-2]=I-NP',\n",
            "               'token[-1]=Britain',\n",
            "               'pos_tag[-1]=NNP',\n",
            "               'chunk_tag[-1]=I-NP',\n",
            "               'token[0]=backed',\n",
            "               'pos_tag[0]=VBD',\n",
            "               'chunk_tag[0]=B-VP',\n",
            "               'token[1]=Fischler',\n",
            "               'pos_tag[1]=NNP',\n",
            "               'chunk_tag[1]=B-NP',\n",
            "               \"token[2]='s\",\n",
            "               'pos_tag[2]=POS',\n",
            "               'chunk_tag[2]=B-NP',\n",
            "               'token[3]=proposal',\n",
            "               'pos_tag[3]=NN',\n",
            "               'chunk_tag[3]=I-NP'],\n",
            "              ['token[-3]=and',\n",
            "               'pos_tag[-3]=CC',\n",
            "               'chunk_tag[-3]=I-NP',\n",
            "               'token[-2]=Britain',\n",
            "               'pos_tag[-2]=NNP',\n",
            "               'chunk_tag[-2]=I-NP',\n",
            "               'token[-1]=backed',\n",
            "               'pos_tag[-1]=VBD',\n",
            "               'chunk_tag[-1]=B-VP',\n",
            "               'token[0]=Fischler',\n",
            "               'pos_tag[0]=NNP',\n",
            "               'chunk_tag[0]=B-NP',\n",
            "               \"token[1]='s\",\n",
            "               'pos_tag[1]=POS',\n",
            "               'chunk_tag[1]=B-NP',\n",
            "               'token[2]=proposal',\n",
            "               'pos_tag[2]=NN',\n",
            "               'chunk_tag[2]=I-NP',\n",
            "               'token[3]=.',\n",
            "               'pos_tag[3]=.',\n",
            "               'chunk_tag[3]=O',\n",
            "               'first-letter-capitalized'],\n",
            "              ['token[-3]=Britain',\n",
            "               'pos_tag[-3]=NNP',\n",
            "               'chunk_tag[-3]=I-NP',\n",
            "               'token[-2]=backed',\n",
            "               'pos_tag[-2]=VBD',\n",
            "               'chunk_tag[-2]=B-VP',\n",
            "               'token[-1]=Fischler',\n",
            "               'pos_tag[-1]=NNP',\n",
            "               'chunk_tag[-1]=B-NP',\n",
            "               \"token[0]='s\",\n",
            "               'pos_tag[0]=POS',\n",
            "               'chunk_tag[0]=B-NP',\n",
            "               'token[1]=proposal',\n",
            "               'pos_tag[1]=NN',\n",
            "               'chunk_tag[1]=I-NP',\n",
            "               'token[2]=.',\n",
            "               'pos_tag[2]=.',\n",
            "               'chunk_tag[2]=O'],\n",
            "              ['token[-3]=backed',\n",
            "               'pos_tag[-3]=VBD',\n",
            "               'chunk_tag[-3]=B-VP',\n",
            "               'token[-2]=Fischler',\n",
            "               'pos_tag[-2]=NNP',\n",
            "               'chunk_tag[-2]=B-NP',\n",
            "               \"token[-1]='s\",\n",
            "               'pos_tag[-1]=POS',\n",
            "               'chunk_tag[-1]=B-NP',\n",
            "               'token[0]=proposal',\n",
            "               'pos_tag[0]=NN',\n",
            "               'chunk_tag[0]=I-NP',\n",
            "               'token[1]=.',\n",
            "               'pos_tag[1]=.',\n",
            "               'chunk_tag[1]=O'],\n",
            "              ['token[-3]=Fischler',\n",
            "               'pos_tag[-3]=NNP',\n",
            "               'chunk_tag[-3]=B-NP',\n",
            "               \"token[-2]='s\",\n",
            "               'pos_tag[-2]=POS',\n",
            "               'chunk_tag[-2]=B-NP',\n",
            "               'token[-1]=proposal',\n",
            "               'pos_tag[-1]=NN',\n",
            "               'chunk_tag[-1]=I-NP',\n",
            "               'token[0]=.',\n",
            "               'pos_tag[0]=.',\n",
            "               'chunk_tag[0]=O']],\n",
            " 'id': '12',\n",
            " 'ner_tags': [0, 5, 0, 5, 0, 1, 0, 0, 0],\n",
            " 'pos_tags': [30, 22, 10, 22, 38, 22, 27, 21, 7],\n",
            " 'tokens': ['Only',\n",
            "            'France',\n",
            "            'and',\n",
            "            'Britain',\n",
            "            'backed',\n",
            "            'Fischler',\n",
            "            \"'s\",\n",
            "            'proposal',\n",
            "            '.']}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Flatten dataset\n",
        "\n",
        "The MLP code that we introduced previously expects each of the `train`, `validation` and `test` subsets of the data to consist of simple sequences of examples.\n",
        "\n",
        "Now that we have run the feature generation, we no longer need the sentence structure and can \"flatten\" the data into such sequences."
      ],
      "metadata": {
        "id": "N8sZzkiOl9aZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def flatten(subset):\n",
        "    # Keys for values to flatten\n",
        "    keys = [\"tokens\", \"pos_tags\", \"chunk_tags\", \"ner_tags\", \"features\"]\n",
        "\n",
        "    # Initialize to empty lists of tokens etc.\n",
        "    flattened = { k: [] for k in keys }\n",
        "\n",
        "    # Concatenate per-sentence lists of tokens etc.\n",
        "    for sentence in subset:\n",
        "        for key in keys:\n",
        "            flattened[key].extend(sentence[key])\n",
        "\n",
        "    # Return as Dataset object\n",
        "    return datasets.Dataset.from_dict(flattened)"
      ],
      "metadata": {
        "id": "Y4COBECNjPNb"
      },
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Call `flatten` for each of the subsets and make a new `DatasetDict` containing the flattened subsets: "
      ],
      "metadata": {
        "id": "wUuPc9uwltTn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "flattened_dict = {\n",
        "    \"train\": flatten(dataset[\"train\"]),\n",
        "    \"validation\": flatten(dataset[\"validation\"]),\n",
        "    \"test\": flatten(dataset[\"test\"]),\n",
        "}\n",
        "\n",
        "flat_dataset = datasets.DatasetDict(flattened_dict)"
      ],
      "metadata": {
        "id": "jcQSwn2Ll6Z0"
      },
      "execution_count": 103,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check that the new dataset looks OK:"
      ],
      "metadata": {
        "id": "QR4PZp8AmWZ_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "flat_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H9xBRcJKkB7Q",
        "outputId": "f37cfeeb-5cfa-4892-a93a-27b275677d33"
      },
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 203621\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 51362\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['tokens', 'pos_tags', 'chunk_tags', 'ner_tags', 'features'],\n",
              "        num_rows: 46435\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(10):\n",
        "    token = flat_dataset[\"train\"][\"tokens\"][i]\n",
        "    pos_tag = ID2POS[flat_dataset[\"train\"][\"pos_tags\"][i]]\n",
        "    description = POS2DESCRIPTION.get(pos_tag, pos_tag)\n",
        "    print(f\"{token}\\t{pos_tag}\\t{description}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vs55SS3bmrqz",
        "outputId": "e3257802-ac35-4b8e-d641-6302549d4113"
      },
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "EU\tNNP\tProper noun, singular\n",
            "rejects\tVBZ\tVerb, 3rd person singular present\n",
            "German\tJJ\tAdjective\n",
            "call\tNN\tNoun, singular or mass\n",
            "to\tTO\tto\n",
            "boycott\tVB\tVerb, base form\n",
            "British\tJJ\tAdjective\n",
            "lamb\tNN\tNoun, singular or mass\n",
            ".\t.\t.\n",
            "Peter\tNNP\tProper noun, singular\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note that this is now a single long sequence of tokens without sentence boundaries."
      ],
      "metadata": {
        "id": "lI0nCoZhnode"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "## Vectorize data\n",
        "\n",
        "We'll next follow the steps that you should already be familiar with from the [text classification notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb), with a few changes:\n",
        "\n",
        "* Since the data is already tokenized, we only need to **vectorize** it, i.e. get the non-zero elements of the feature vector\n",
        "* Unlike in the text classification notebook, here we are **vectorizing token features**\n",
        "* We'll again use sklearn's feature extraction package, in particular `CountVectorizer`\n",
        "* Since our features are now lists of strings, we can skip tokenization and use these as-is"
      ],
      "metadata": {
        "id": "KMnqQ78cMpk2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.feature_extraction\n",
        "\n",
        "\n",
        "# Dummy function for tokenization and preprocessing\n",
        "def do_nothing(features):\n",
        "    return features\n",
        "\n",
        "vectorizer = sklearn.feature_extraction.text.CountVectorizer(\n",
        "    binary=True,\n",
        "    max_features=30000,\n",
        "    tokenizer=do_nothing,\n",
        "    preprocessor=do_nothing,\n",
        ")\n",
        "\n",
        "# Get a list of all feature strings from the training data\n",
        "features = [e[\"features\"] for e in flat_dataset[\"train\"]]\n",
        "\n",
        "# \"Train\" the vectorizer, i.e. build its vocabulary\n",
        "vectorizer.fit(features)"
      ],
      "metadata": {
        "id": "MYOtKJu7Mohd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        },
        "outputId": "4a616b5c-af36-4056-a8b6-53839b56984e"
      },
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/sklearn/feature_extraction/text.py:528: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=<function do_nothing at 0x7fe8fa269430>,\n",
              "                tokenizer=<function do_nothing at 0x7fe8fa269430>)"
            ],
            "text/html": [
              "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=&lt;function do_nothing at 0x7fe8fa269430&gt;,\n",
              "                tokenizer=&lt;function do_nothing at 0x7fe8fa269430&gt;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer(binary=True, max_features=30000,\n",
              "                preprocessor=&lt;function do_nothing at 0x7fe8fa269430&gt;,\n",
              "                tokenizer=&lt;function do_nothing at 0x7fe8fa269430&gt;)</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As in the text classification notebook, we then invoke the vectorizer and get non-zero elements as a sparse matrix:"
      ],
      "metadata": {
        "id": "UDNy3w8reO6C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def vectorize_example(e):\n",
        "    vectorized = vectorizer.transform([e[\"features\"]])\n",
        "\n",
        "    # nonzero() gives a pair of (rows,columns), we want the columns\n",
        "    non_zero_features = vectorized.nonzero()[1]\n",
        "\n",
        "    # Feature index 0 will have a special meaning, so let us not produce\n",
        "    # it by adding +1 to everything\n",
        "    non_zero_features += 1\n",
        "\n",
        "    return {\n",
        "        \"input_ids\": non_zero_features,\n",
        "        \"label\": e[\"ner_tags\"]\n",
        "    } "
      ],
      "metadata": {
        "id": "5H9AQOc2rGIO"
      },
      "execution_count": 107,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check one example:"
      ],
      "metadata": {
        "id": "kqKjaSmBqWjm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorized = vectorize_example(flat_dataset[\"train\"][10])\n",
        "\n",
        "print(flat_dataset[\"train\"][10])\n",
        "print(vectorized)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOTjsFCBqOjX",
        "outputId": "d5d12eb1-93cf-49f2-80f3-a9cc9e33e81a"
      },
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'tokens': 'Blackburn', 'pos_tags': 22, 'chunk_tags': 12, 'ner_tags': 2, 'features': ['token[-1]=Peter', 'pos_tag[-1]=NNP', 'chunk_tag[-1]=B-NP', 'token[0]=Blackburn', 'pos_tag[0]=NNP', 'chunk_tag[0]=I-NP', 'first-letter-capitalized']}\n",
            "{'input_ids': array([    6,    68,   127,   148,   277,  2001, 13974], dtype=int32), 'label': 2}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Map `input_ids` back to the original feature names to confirm that everything works:"
      ],
      "metadata": {
        "id": "WiScO867qrjK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Invert the feature dictionary\n",
        "idx2feat = { i: w for w, i in vectorizer.vocabulary_.items() }\n",
        "\n",
        "feats = []\n",
        "for idx in vectorized[\"input_ids\"]:\n",
        "    feats.append(idx2feat[idx-1])    # It is easy to forget we moved all by +1\n",
        "\n",
        "# This is now the bag of features representation of the token in context\n",
        "pprint(\", \".join(feats))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yvgd3IrWfCai",
        "outputId": "6e585610-83cd-41e4-d899-620144d15a89"
      },
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "('chunk_tag[-1]=B-NP, chunk_tag[0]=I-NP, first-letter-capitalized, '\n",
            " 'pos_tag[-1]=NNP, pos_tag[0]=NNP, token[-1]=Peter, token[0]=Blackburn')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Vectorizing the whole dataset\n",
        "\n",
        "We'll again use [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) to process the whole dataset:"
      ],
      "metadata": {
        "id": "i33KIMgkiIn7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorized_dataset = flat_dataset.map(vectorize_example, num_proc=4)\n",
        "\n",
        "pprint(vectorized_dataset[\"train\"][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 366,
          "referenced_widgets": [
            "f374b1d535504d9187d713358312e3ee",
            "cca15fbf6d3e40768bb9dfa9a7666cb1",
            "ec91ccc507784bfd92f636c0730b3c70",
            "40fcc90e746843c58eef5cbaeb3ee7d6",
            "88395b82b52641f39a51a3d03a98acae",
            "c517fe8841944c69a263555958f92bd9",
            "159b7095fb8642c2bbc45a26b3798647",
            "7adbefe082444c2098199ef7ad549322",
            "e6ef08188ea346fa8b94c7ecad3c32be",
            "64ef5f25bacd4420ac9de75a475bb03e",
            "8d489d554d664dcca820628ea5048374",
            "ba07d3f520fe401da45c67330220ba8f",
            "39f7972fcb7b4090b6621d6119034ea1",
            "92bd39909f9f4f54a6b7a191195bc5a3",
            "6f3e0243a29e433199bd6f3785b2141e",
            "a38699186f4a4332bf5d3a8a16cef2ad",
            "36847fa46101451a9f9859278f15f1a0",
            "0f717f92bad346389e7ed91d0de471ba",
            "c9147eb7a2994d4b96984d6a457b365e",
            "65c67cbc2e3c4803be912f6521b059db",
            "f35d8ae9dbcf45898f8c055db5c09228",
            "3dbb8aff79d04f8882d7a6edd1492892",
            "e15e17e667f34f34bf17ab10dd23536b",
            "cb7b8ff71254418abdd89f66d2e52f7d",
            "2028d613ca33430e9e9da712cb6ecb3a",
            "0adab00e5ad648e18a978d6204a56dad",
            "6bd14f3f61e242859eb6c7ed63cb9d3e",
            "fcfb72a645354af7b9fc15c9e92a5a62",
            "02aa31a2b0544961afe9515c706219e5",
            "103050b69abd42d0b54d56678a652681",
            "a2229fe0a9aa49869479667f63970959",
            "254458559e564046a4cd6e156b3f7e50",
            "77cae6b2fdba4464a3e083142bb6ac80"
          ]
        },
        "id": "33xMYRd0q2B9",
        "outputId": "99711822-0539-4899-d30d-f2bc7e22dd50"
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/203621 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f374b1d535504d9187d713358312e3ee"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/51362 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ba07d3f520fe401da45c67330220ba8f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/46435 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e15e17e667f34f34bf17ab10dd23536b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'chunk_tags': 11,\n",
            " 'features': ['token[0]=EU',\n",
            "              'pos_tag[0]=NNP',\n",
            "              'chunk_tag[0]=B-NP',\n",
            "              'token[1]=rejects',\n",
            "              'pos_tag[1]=VBZ',\n",
            "              'chunk_tag[1]=B-VP',\n",
            "              'token[2]=German',\n",
            "              'pos_tag[2]=JJ',\n",
            "              'chunk_tag[2]=B-NP',\n",
            "              'token[3]=call',\n",
            "              'pos_tag[3]=NN',\n",
            "              'chunk_tag[3]=I-NP',\n",
            "              'first-letter-capitalized'],\n",
            " 'input_ids': [60, 82, 96, 122, 127, 277, 340, 359, 408, 14276, 23135, 27918],\n",
            " 'label': 3,\n",
            " 'ner_tags': 3,\n",
            " 'pos_tags': 22,\n",
            " 'tokens': 'EU'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* Our `input_ids` are an array containing the indices of the features\n",
        "* This corresponds to the indices into the row of the embedding matrix in the model\n"
      ],
      "metadata": {
        "id": "XTtyHQpIIJWw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Batching and padding\n",
        "\n",
        "As detailed in the [text classification notebook](https://github.com/TurkuNLP/intro-to-nlp/blob/master/mlp_imdb_hf_dset_and_trainer.ipynb), we typically train neural networks on _batches_ of multiple examples rather than a single example at a time (efficiency and regularization).\n",
        "\n",
        "As examples in a batch need to have identical length, we _pad_ shorter examples to the maximum example length in each batch with the \"dummy\" feature with index 0.\n",
        "\n",
        "(This code is basically unchanged from the previous notebook.)"
      ],
      "metadata": {
        "id": "JvaP1DpHjI3f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def collator(list_of_examples):\n",
        "    # Labels are simply converted into a tensor\n",
        "    batch={\n",
        "        \"labels\": torch.tensor([e[\"label\"] for e in list_of_examples])\n",
        "    }\n",
        "\n",
        "    # Examples need to be padded\n",
        "    tensors = []\n",
        "\n",
        "    # Find length of longest example\n",
        "    max_len = max(len(e[\"input_ids\"]) for e in list_of_examples) \n",
        "    max_len = max(1,max_len)\n",
        "\n",
        "    # Pad everything with zeros to length of longest example\n",
        "    for example in list_of_examples:\n",
        "        ids = torch.LongTensor(example[\"input_ids\"])\n",
        "        # pad(what,(from_left, from_right)) <- this is how we call the stock pad function\n",
        "         #pad by max - current length, pads with zero by default\n",
        "        padded = torch.nn.functional.pad(ids, (0, max_len-ids.shape[0]))\n",
        "        tensors.append(padded)\n",
        "\n",
        "    # Now that all examples are of the same length, vstack() can be used\n",
        "    # to vertically stack these into a tensor \n",
        "    batch[\"input_ids\"]=torch.vstack(tensors)\n",
        "\n",
        "    return batch"
      ],
      "metadata": {
        "id": "zXXDxNo6kwfA"
      },
      "execution_count": 111,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Test that out with a minimal batch of two examples, one requiring padding:"
      ],
      "metadata": {
        "id": "e7Mre52QCTtI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch=collator([vectorized_dataset[\"train\"][2], vectorized_dataset[\"train\"][7]])\n",
        "\n",
        "print(\"Shape of labels:\",batch[\"labels\"].shape)\n",
        "print(\"Shape of input_ids:\",batch[\"input_ids\"].shape)\n",
        "print(\"labels:\",batch[\"labels\"])\n",
        "print(\"input_ids:\",batch[\"input_ids\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JJnW4gwqBpF6",
        "outputId": "2d65987c-8be0-4b85-bf42-693f153af726"
      },
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of labels: torch.Size([2])\n",
            "Shape of input_ids: torch.Size([2, 18])\n",
            "labels: tensor([7, 0])\n",
            "input_ids: tensor([[   10,    24,    60,    86,   100,   125,   127,   167,   191,   271,\n",
            "           320,   377,   423,  5782, 14420, 20160, 26120, 27889],\n",
            "        [    6,    35,    46,    68,    90,   142,   205,   245,   276,   307,\n",
            "           990,  7246, 13168, 18170,     0,     0,     0,     0]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# MLP model\n",
        "\n",
        "With the data now ready, we'll build the MLP model. Note that this is _identical_ to the MLP model we used for text classification: the only difference between the two applications is in the data.\n",
        "\n",
        "The model class in its simplest form has `__init__()` which instantiates the layers and `forward()` which implements the actual computation. For more information on these, please see the [PyTorch turorial](https://pytorch.org/tutorials/beginner/introyt/modelsyt_tutorial.html)."
      ],
      "metadata": {
        "id": "AOYYF5I1OWG9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# A model wants a config, I can simply inherit from the base\n",
        "# class for pretrained configs\n",
        "class MLPConfig(transformers.PretrainedConfig):\n",
        "    pass\n",
        "\n",
        "# This is the model\n",
        "class MLP(transformers.PreTrainedModel):\n",
        "\n",
        "    config_class=MLPConfig\n",
        "\n",
        "    # In the initialization method, one instantiates the layers\n",
        "    # these will be, for the most part the trained parameters of the model\n",
        "    def __init__(self, config):\n",
        "        super().__init__(config)\n",
        "\n",
        "        self.vocab_size=config.vocab_size    # embedding matrix row count\n",
        "  \n",
        "        # Build and initialize embedding of vocab size +1 x hidden size\n",
        "        # (+1 because of the padding index 0!)\n",
        "        self.embedding = torch.nn.Embedding(\n",
        "            num_embeddings=self.vocab_size+1,\n",
        "            embedding_dim=config.hidden_size,\n",
        "            padding_idx=0\n",
        "        )\n",
        "\n",
        "        # Initialize the embeddings with small random values\n",
        "        torch.nn.init.uniform_(self.embedding.weight.data, -0.001, 0.001)\n",
        "        # Enforce zero values for padding\n",
        "        torch.nn.init.zeros_(self.embedding.weight.data[0,:])\n",
        "\n",
        "        # Output layer: hidden size x output size\n",
        "        self.output=torch.nn.Linear(\n",
        "            in_features=config.hidden_size,\n",
        "            out_features=config.nlabels\n",
        "        )\n",
        "        \n",
        "    # The computation of the model is put into the forward() function\n",
        "    # it receives a batch of data and optionally the correct `labels`:\n",
        "    # - if given `labels`, returns (loss, output)\n",
        "    # - if not, only returns (output,)\n",
        "    def forward(self, input_ids, labels=None):\n",
        "        # 1) Look up embeddings of features, sum them up\n",
        "        embedded = self.embedding(input_ids)    # (batch, ids) -> (batch, ids, embedding_dim)\n",
        "        embedded_summed = torch.sum(embedded, dim=1)    # (batch, ids, embedding_dim) -> (batch, embedding_dim)\n",
        "        \n",
        "        # NOTE: we're explicitly *not* applying a nonlinearity here to keep\n",
        "        # things linear for later analysis\n",
        "\n",
        "        # 2) Apply output layer\n",
        "        # (batch, embedding_dim) -> (batch, num_classes)\n",
        "        logits = self.output(embedded_summed)\n",
        "        \n",
        "        if labels is not None:\n",
        "            # We have labels, so we ought to calculate the loss\n",
        "            loss_fn = torch.nn.CrossEntropyLoss()    # Classification loss function\n",
        "            loss = loss_fn(logits, labels)\n",
        "            return (loss, logits)\n",
        "        else:\n",
        "            # No labels, so just return the logits\n",
        "            return (logits,)"
      ],
      "metadata": {
        "id": "vP4FtrMwCpGi"
      },
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Configure the model"
      ],
      "metadata": {
        "id": "Ix6-Z0XqFGfl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_labels = len(POS2ID)\n",
        "\n",
        "mlp_config = MLPConfig(\n",
        "    vocab_size=len(vectorizer.vocabulary_),\n",
        "    hidden_size=20,\n",
        "    nlabels=num_labels\n",
        ")"
      ],
      "metadata": {
        "id": "I2Lwe8eoFFtw"
      },
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Train the model\n",
        "\n",
        "We will use the Hugging Face [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) class for training\n",
        "\n",
        "* Loads of arguments that control the training\n",
        "* Configurable metrics to evaluate performance\n",
        "* Data collator builds the batches\n",
        "* Early stopping callback stops when eval loss no longer improves\n",
        "* Model load/save\n",
        "* Good foundation for later deep learning course\n",
        "  "
      ],
      "metadata": {
        "id": "tdlcMObzQGGP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, let's create a [`TrainingArguments`](https://huggingface.co/docs/transformers/v4.17.0/en/main_classes/trainer#transformers.TrainingArguments) object to specify hyperparameters and various other settings for training. \n",
        "\n",
        "Printing this simple dataclass object will show not only the values we set, but also the defaults for all other arguments. Don't worry if you don't understand what all of these do! Many are not relevant to us here, and you can find the details in [`Trainer` documentation](https://huggingface.co/docs/transformers/main_classes/trainer) if you are interested."
      ],
      "metadata": {
        "id": "aZgcNi4B76SX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer_args = transformers.TrainingArguments(\n",
        "    \"mlp_checkpoints\", #save checkpoints here\n",
        "    evaluation_strategy=\"steps\",\n",
        "    logging_strategy=\"steps\",\n",
        "    eval_steps=500,\n",
        "    logging_steps=500,\n",
        "    learning_rate=1e-4, #learning rate of the gradient descent\n",
        "    max_steps=20000,\n",
        "    load_best_model_at_end=True,\n",
        "    per_device_train_batch_size=128\n",
        ")\n",
        "\n",
        "pprint(trainer_args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IhfdW62z8cCn",
        "outputId": "a32f5c17-4611-47f4-c757-08e38ca9b12b"
      },
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TrainingArguments(\n",
            "_n_gpu=1,\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "auto_find_batch_size=False,\n",
            "bf16=False,\n",
            "bf16_full_eval=False,\n",
            "data_seed=None,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_pin_memory=True,\n",
            "ddp_bucket_cap_mb=None,\n",
            "ddp_find_unused_parameters=None,\n",
            "ddp_timeout=1800,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "do_eval=True,\n",
            "do_predict=False,\n",
            "do_train=False,\n",
            "eval_accumulation_steps=None,\n",
            "eval_delay=0,\n",
            "eval_steps=500,\n",
            "evaluation_strategy=steps,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "fsdp=[],\n",
            "fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\n",
            "fsdp_min_num_params=0,\n",
            "fsdp_transformer_layer_cls_to_wrap=None,\n",
            "full_determinism=False,\n",
            "gradient_accumulation_steps=1,\n",
            "gradient_checkpointing=False,\n",
            "greater_is_better=False,\n",
            "group_by_length=False,\n",
            "half_precision_backend=auto,\n",
            "hub_model_id=None,\n",
            "hub_private_repo=False,\n",
            "hub_strategy=every_save,\n",
            "hub_token=<HUB_TOKEN>,\n",
            "ignore_data_skip=False,\n",
            "include_inputs_for_metrics=False,\n",
            "jit_mode_eval=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=0.0001,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=True,\n",
            "local_rank=-1,\n",
            "log_level=passive,\n",
            "log_level_replica=warning,\n",
            "log_on_each_node=True,\n",
            "logging_dir=mlp_checkpoints/runs/Apr12_06-57-34_e97b240eef72,\n",
            "logging_first_step=False,\n",
            "logging_nan_inf_filter=True,\n",
            "logging_steps=500,\n",
            "logging_strategy=steps,\n",
            "lr_scheduler_type=linear,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=20000,\n",
            "metric_for_best_model=loss,\n",
            "mp_parameters=,\n",
            "no_cuda=False,\n",
            "num_train_epochs=3.0,\n",
            "optim=adamw_hf,\n",
            "optim_args=None,\n",
            "output_dir=mlp_checkpoints,\n",
            "overwrite_output_dir=False,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=128,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=None,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
            "ray_scope=last,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=mlp_checkpoints,\n",
            "save_on_each_node=False,\n",
            "save_steps=500,\n",
            "save_strategy=steps,\n",
            "save_total_limit=None,\n",
            "seed=42,\n",
            "sharded_ddp=[],\n",
            "skip_memory_metrics=True,\n",
            "tf32=None,\n",
            "torch_compile=False,\n",
            "torch_compile_backend=None,\n",
            "torch_compile_mode=None,\n",
            "torchdynamo=None,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_ipex=False,\n",
            "use_legacy_prediction_loop=False,\n",
            "use_mps_device=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            "xpu_backend=None,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next, let's create a metric for evaluating performance during and after training. We can use the convenience function [`load_metric`](https://huggingface.co/docs/datasets/about_metrics) to load one of many pre-made metrics and wrap this for use by the trainer.\n",
        "\n",
        "We can use the basic `accuracy` metric, defined as the proportion of correctly predicted labels out of all labels. This time, though, the data is not evenly split."
      ],
      "metadata": {
        "id": "4sJwNXPU-dQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import evaluate\n",
        "\n",
        "accuracy = evaluate.load(\"accuracy\")\n",
        "\n",
        "def compute_accuracy(outputs_and_labels):\n",
        "    outputs, labels = outputs_and_labels\n",
        "    predictions = np.argmax(outputs, axis=-1) #pick the index of the \"winning\" label\n",
        "    return accuracy.compute(predictions=predictions, references=labels)"
      ],
      "metadata": {
        "id": "u3jxIItb0BL9"
      },
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create the `Trainer` and train the model by invoking the [`Trainer.train`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.train) function.\n",
        "\n",
        "In addition to the model, the settings passed in through the `TrainingArguments` object created above (`trainer_args`), the data, and the metric defined above, we create and pass the following to the `Trainer`:\n",
        "\n",
        "* [data collator](https://huggingface.co/docs/transformers/main_classes/data_collator): groups input into batches\n",
        "* [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback): stops training when performance stops improving"
      ],
      "metadata": {
        "id": "S7kbz8uU-zpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Make a new model  \n",
        "mlp = MLP(mlp_config)\n",
        "\n",
        "\n",
        "# Argument gives the number of steps of patience before early stopping\n",
        "# i.e. training is stopped when the evaluation loss fails to improve\n",
        "# certain number of times\n",
        "early_stopping = transformers.EarlyStoppingCallback(5)\n",
        "\n",
        "trainer = transformers.Trainer(\n",
        "    model=mlp,\n",
        "    args=trainer_args,\n",
        "    train_dataset=vectorized_dataset[\"train\"],\n",
        "    eval_dataset=vectorized_dataset[\"validation\"],\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=collator,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "AoEoWsj4P_zN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d0961070-9b06-41ef-df48-ac3607b90987"
      },
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='20000' max='20000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [20000/20000 08:22, Epoch 12/13]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>2.809600</td>\n",
              "      <td>1.534350</td>\n",
              "      <td>0.832542</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.984900</td>\n",
              "      <td>0.763122</td>\n",
              "      <td>0.832873</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.686400</td>\n",
              "      <td>0.632852</td>\n",
              "      <td>0.839414</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.578100</td>\n",
              "      <td>0.546264</td>\n",
              "      <td>0.854581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.497400</td>\n",
              "      <td>0.479024</td>\n",
              "      <td>0.867022</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.445300</td>\n",
              "      <td>0.429364</td>\n",
              "      <td>0.875063</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.389800</td>\n",
              "      <td>0.392266</td>\n",
              "      <td>0.884876</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.366500</td>\n",
              "      <td>0.363565</td>\n",
              "      <td>0.893890</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.333800</td>\n",
              "      <td>0.341260</td>\n",
              "      <td>0.900238</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>0.317600</td>\n",
              "      <td>0.323120</td>\n",
              "      <td>0.904229</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>0.297000</td>\n",
              "      <td>0.307852</td>\n",
              "      <td>0.907909</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>0.284600</td>\n",
              "      <td>0.295078</td>\n",
              "      <td>0.911413</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>0.268300</td>\n",
              "      <td>0.284458</td>\n",
              "      <td>0.913964</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.258100</td>\n",
              "      <td>0.274278</td>\n",
              "      <td>0.917974</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.248900</td>\n",
              "      <td>0.265873</td>\n",
              "      <td>0.920272</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.239700</td>\n",
              "      <td>0.258366</td>\n",
              "      <td>0.922881</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.233900</td>\n",
              "      <td>0.251120</td>\n",
              "      <td>0.924906</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.225100</td>\n",
              "      <td>0.245419</td>\n",
              "      <td>0.926658</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.222400</td>\n",
              "      <td>0.239926</td>\n",
              "      <td>0.928430</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10000</td>\n",
              "      <td>0.211900</td>\n",
              "      <td>0.235002</td>\n",
              "      <td>0.929831</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10500</td>\n",
              "      <td>0.206400</td>\n",
              "      <td>0.230658</td>\n",
              "      <td>0.930805</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11000</td>\n",
              "      <td>0.207900</td>\n",
              "      <td>0.226411</td>\n",
              "      <td>0.932187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11500</td>\n",
              "      <td>0.201200</td>\n",
              "      <td>0.223089</td>\n",
              "      <td>0.933511</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12000</td>\n",
              "      <td>0.198400</td>\n",
              "      <td>0.219577</td>\n",
              "      <td>0.934543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12500</td>\n",
              "      <td>0.194100</td>\n",
              "      <td>0.216752</td>\n",
              "      <td>0.935555</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13000</td>\n",
              "      <td>0.186600</td>\n",
              "      <td>0.214084</td>\n",
              "      <td>0.936568</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13500</td>\n",
              "      <td>0.187800</td>\n",
              "      <td>0.211464</td>\n",
              "      <td>0.937425</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14000</td>\n",
              "      <td>0.183700</td>\n",
              "      <td>0.209088</td>\n",
              "      <td>0.938301</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14500</td>\n",
              "      <td>0.184400</td>\n",
              "      <td>0.207118</td>\n",
              "      <td>0.939021</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15000</td>\n",
              "      <td>0.181500</td>\n",
              "      <td>0.205505</td>\n",
              "      <td>0.939255</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15500</td>\n",
              "      <td>0.176500</td>\n",
              "      <td>0.203744</td>\n",
              "      <td>0.940092</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16000</td>\n",
              "      <td>0.176000</td>\n",
              "      <td>0.202352</td>\n",
              "      <td>0.940501</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16500</td>\n",
              "      <td>0.176000</td>\n",
              "      <td>0.201078</td>\n",
              "      <td>0.940929</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17000</td>\n",
              "      <td>0.173800</td>\n",
              "      <td>0.200072</td>\n",
              "      <td>0.941241</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17500</td>\n",
              "      <td>0.171600</td>\n",
              "      <td>0.199420</td>\n",
              "      <td>0.941299</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18000</td>\n",
              "      <td>0.171900</td>\n",
              "      <td>0.198666</td>\n",
              "      <td>0.941766</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18500</td>\n",
              "      <td>0.172600</td>\n",
              "      <td>0.198084</td>\n",
              "      <td>0.942019</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19000</td>\n",
              "      <td>0.169500</td>\n",
              "      <td>0.197712</td>\n",
              "      <td>0.941961</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19500</td>\n",
              "      <td>0.170900</td>\n",
              "      <td>0.197442</td>\n",
              "      <td>0.942078</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20000</td>\n",
              "      <td>0.170300</td>\n",
              "      <td>0.197373</td>\n",
              "      <td>0.942136</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=20000, training_loss=0.33900676460266116, metrics={'train_runtime': 502.2913, 'train_samples_per_second': 5096.644, 'train_steps_per_second': 39.818, 'total_flos': 332961038928.0, 'train_loss': 0.33900676460266116, 'epoch': 12.57})"
            ]
          },
          "metadata": {},
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then evaluate the trained model on a given dataset (here our test subset) by calling [`Trainer.evaluate`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.evaluate):"
      ],
      "metadata": {
        "id": "Td03fcIa-6Hj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(vectorized_dataset[\"test\"])\n",
        "\n",
        "print(\"Accuracy:\", eval_results[\"eval_accuracy\"])"
      ],
      "metadata": {
        "id": "9nlEwpnF2Vow",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "3537094e-20f5-4e55-ef7d-15aaff100d55"
      },
      "execution_count": 118,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='5805' max='5805' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [5805/5805 00:12]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.9284160654678583\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "That's pretty poor performance for a task as simple as POS tagging where state-of-the-art accuracies are generally 97-99%. (The approach demonstrated in this notebook should be considered more of a teaching tool than a serious tagger implementation.)\n",
        "\n",
        "However, the result is certainly much better than random, so we can conclude that the model is learning something about the task."
      ],
      "metadata": {
        "id": "pEbNfGDhfdXK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Save model for later use\n",
        "\n",
        "* You can save it with `trainer.save_model()`\n",
        "* You can load it with `MLP.from_pretrained()`\n"
      ],
      "metadata": {
        "id": "P0OYB3TRp-IH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model(\"mlp-nertagger\")"
      ],
      "metadata": {
        "id": "FosHTDw3p9dd"
      },
      "execution_count": 119,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# What has the model learned?\n",
        "\n",
        "* The embeddings should have some meaning to them\n",
        "* Similar features should have similar embeddings"
      ],
      "metadata": {
        "id": "13aB7DuqzeFi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Grab the embedding matrix out of the trained model\n",
        "# and drop the first row (padding 0)\n",
        "# then we can treat the embeddings as vectors\n",
        "\n",
        "weights=mlp.embedding.weight.detach().cpu().numpy()\n",
        "weights=weights[1:,:]"
      ],
      "metadata": {
        "id": "M6TUrVkMCmz7"
      },
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qry_idx=vectorizer.vocabulary_[\"token[0]=in\"] \n",
        "\n",
        "#calculate the distance of the \"in\" embedding to all other embeddings\n",
        "distance_to_qry=sklearn.metrics.pairwise.euclidean_distances(weights[qry_idx:qry_idx+1,:],weights)\n",
        "nearest_neighbors=np.argsort(distance_to_qry) #indices of words nearest to \"in\"\n",
        "for nearest in nearest_neighbors[0,:20]:\n",
        "    print(idx2feat[nearest])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4uTTvRgGDU9D",
        "outputId": "caca32bb-760e-4e99-ca8e-8799406471d4"
      },
      "execution_count": 121,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "token[0]=in\n",
            "pos_tag[-1]=PRP\n",
            "token[0]=,\n",
            "pos_tag[0]=,\n",
            "token[0]=at\n",
            "token[-3]=0\n",
            "token[0]=)\n",
            "pos_tag[0]=)\n",
            "pos_tag[0]=(\n",
            "token[0]=(\n",
            "token[0]=the\n",
            "pos_tag[0]=SYM\n",
            "token[0]=:\n",
            "token[1]=the\n",
            "chunk_tag[-1]=B-ADJP\n",
            "pos_tag[1]=VB\n",
            "token[0]=Police\n",
            "pos_tag[1]=PRP\n",
            "token[0]=is\n",
            "token[-1]=this\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* The embeddings indeed seem to reflect the task and capture aspects of the meaning of words relevant to the task\n",
        "* But now we have many classes, so we should take that into account too\n",
        "* We can take the dot-product of the feature embeddings with the output layer weight of the class we care about\n",
        "* When you think how the information propagates in the network, this will give us a single number reflecting each feature w.r.t. the selected label\n",
        "* Technically speaking, it is the prediction of an example which only has that one feature, with respect to that one class\n",
        "* Here is how we can implement it (here we rely on the fact that the model is linear, since we didn't include a nonlinearity earlier in the model's `forward()`"
      ],
      "metadata": {
        "id": "cQCmThdu2LDR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy\n",
        "\n",
        "embedding_weights=weights    #shape (features, embedding-dim)\n",
        "output_weights=mlp.output.weight.detach().cpu().numpy()    #shape (num-labels, embedding-dim)\n",
        "\n",
        "# We just matrix-multiply these together, since this gives us all the dot-products\n",
        "weights_by_label=numpy.matmul(embedding_weights, output_weights.T)\n",
        "weights_by_label.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5coMII4ABPt",
        "outputId": "3ec4efd2-a9aa-401b-bcde-8ea9d7f1bbf3"
      },
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(30000, 47)"
            ]
          },
          "metadata": {},
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_most_important_features_for_and_against(label):\n",
        "    label_idx = NER2ID[label]\n",
        "    feature_weights = weights_by_label[:,label_idx] #pick the column that interests us\n",
        "\n",
        "    #The shape of feature_weights is (feature_vocab_size,) i.e. it is a vector\n",
        "    features_weight_idx = numpy.argsort(-feature_weights) #sort in descending order, this will be vector of indices\n",
        "    features_for = [idx2feat[feature_idx] for feature_idx in features_weight_idx[:20]]\n",
        "    features_against = [idx2feat[feature_idx] for feature_idx in features_weight_idx[-20:][::-1]]\n",
        "    return features_for, features_against\n",
        "\n",
        "for label in (\"I-PER\", \"I-ORG\", \"O\"):\n",
        "    dt_plus,dt_minus=get_most_important_features_for_and_against(label)\n",
        "    print(f\"Most important features *for* label {label}:\")\n",
        "    pprint(\"   \".join(dt_plus))\n",
        "    print()\n",
        "    print(f\"Most important features *against* label {label}:\")\n",
        "    pprint(\"   \".join(dt_minus))\n",
        "    print(\"\\n------\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4vnJj7cNBagc",
        "outputId": "3903a5cf-3f31-4311-a17a-ccc698f6c0bc"
      },
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Most important features *for* label I-PER:\n",
            "('pos_tag[-1]=NNP   token[-2]=President   token[-2]=Minister   token[-1]=M.   '\n",
            " 'token[-1]=Mark   token[-1]=John   token[-2]=beat   token[-3]=Prime   '\n",
            " 'token[-1]=David   token[-1]=Paul   token[-1]=A.   token[-1]=Michael   '\n",
            " 'token[1]=c   token[0]=Akram   token[0]=Ahmed   token[-2]=1.   token[-2]=2.   '\n",
            " 'token[-1]=Bill   token[2]=Spain   token[-2]=champion')\n",
            "\n",
            "Most important features *against* label I-PER:\n",
            "('pos_tag[0]=CD   chunk_tag[0]=B-NP   pos_tag[-1]=CD   pos_tag[-1]=DT   '\n",
            " 'pos_tag[0]=PRP   pos_tag[0]=NNS   token[-1]=the   token[0]=said   '\n",
            " 'token[0]=to   token[0]=AT   pos_tag[0]=\"   token[0]=\"   token[0]=a   '\n",
            " 'token[-1]=on   token[0]=DIVISION   token[0]=0   token[0]=Minister   '\n",
            " 'token[0]=who   pos_tag[1]=NNS   token[0]=-')\n",
            "\n",
            "------\n",
            "\n",
            "Most important features *for* label I-ORG:\n",
            "('pos_tag[-1]=NNP   token[1]=3   token[0]=Inc   token[0]=Newsroom   '\n",
            " 'token[1]=0   token[3]=1   token[3]=0   token[0]=Corp   token[1]=4   '\n",
            " 'token[1]=2   token[0]=Co   token[-1]=St   token[0]=Party   token[1]=Inc   '\n",
            " 'token[2]=0   token[1]=1   pos_tag[0]=NNPS   token[3]=2   token[2]=1   '\n",
            " 'token[-2]=0')\n",
            "\n",
            "Most important features *against* label I-ORG:\n",
            "('token[0]=U.S.   token[-1]=in   token[0]=0   token[-1]=AT   '\n",
            " 'token[0]=Britain   token[0]=to   token[0]=LONDON   token[1]=1996-08-26   '\n",
            " 'token[1]=1996-08-28   token[0]=a   chunk_tag[0]=B-NP   token[0]=England   '\n",
            " 'token[0]=Germany   token[-1]=Minister   token[0]=said   '\n",
            " 'token[1]=1996-08-29   token[0]=4   token[1]=1996-08-27   token[0]=Pakistan   '\n",
            " 'token[1]=1996-08-22')\n",
            "\n",
            "------\n",
            "\n",
            "Most important features *for* label O:\n",
            "('pos_tag[0]=CD   pos_tag[0]=PRP   token[0]=a   token[0]=to   token[0]=\"   '\n",
            " 'pos_tag[0]=\"   token[0]=0   token[0]=said   token[0]=.   token[0]=He   '\n",
            " 'token[0]=Thursday   pos_tag[0]=.   token[0]=AT   token[0]=-   token[0]=It   '\n",
            " 'token[-1]=on   token[0]=who   token[0]=Tuesday   token[0]=The   '\n",
            " 'pos_tag[0]=PRP$')\n",
            "\n",
            "Most important features *against* label O:\n",
            "('first-letter-capitalized   pos_tag[0]=NNPS   pos_tag[0]=NNP   '\n",
            " 'token[0]=Iraqi   token[-1]=South   token[-1]=New   token[0]=Russian   '\n",
            " 'token[0]=&   token[0]=British   token[0]=German   token[0]=European   '\n",
            " 'token[-1]=beat   token[0]=Kurdish   token[0]=Palestinian   token[0]=de   '\n",
            " 'token[3]=0   token[-1]=President   token[0]=French   token[0]=African   '\n",
            " 'token[0]=Israeli')\n",
            "\n",
            "------\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "H9Xg-7PLLcal"
      },
      "execution_count": 123,
      "outputs": []
    }
  ]
}