bert_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0dac681e-6794-412c-bbeb-9cdf42c88578",
   "metadata": {},
   "outputs": [
    {
     "ename": "DatasetNotFoundError",
     "evalue": "Dataset 'dataset/my-own-dataset' doesn't exist on the Hub. If the repo is private or gated, make sure to log in with `huggingface-cli login`.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mDatasetNotFoundError\u001b[0m                      Traceback (most recent call last)",
      "Cell \u001b[1;32mIn [9], line 8\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mseqeval\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m f1_score, precision_score, recall_score, classification_report\n\u001b[0;32m      6\u001b[0m \u001b[38;5;66;03m# 加载数据集\u001b[39;00m\n\u001b[1;32m----> 8\u001b[0m dataset \u001b[38;5;241m=\u001b[39m \u001b[43mload_dataset\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdataset/my-own-dataset\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m#needed to construct a new Dataset class\u001b[39;00m\n\u001b[0;32m      9\u001b[0m num_labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[0;32m     10\u001b[0m \u001b[38;5;66;03m# 分词器和模型\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\ml-project2\\lib\\site-packages\\datasets\\load.py:2548\u001b[0m, in \u001b[0;36mload_dataset\u001b[1;34m(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, verification_mode, ignore_verifications, keep_in_memory, save_infos, revision, token, use_auth_token, task, streaming, num_proc, storage_options, trust_remote_code, **config_kwargs)\u001b[0m\n\u001b[0;32m   2543\u001b[0m verification_mode \u001b[38;5;241m=\u001b[39m VerificationMode(\n\u001b[0;32m   2544\u001b[0m     (verification_mode \u001b[38;5;129;01mor\u001b[39;00m VerificationMode\u001b[38;5;241m.\u001b[39mBASIC_CHECKS) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m save_infos \u001b[38;5;28;01melse\u001b[39;00m VerificationMode\u001b[38;5;241m.\u001b[39mALL_CHECKS\n\u001b[0;32m   2545\u001b[0m )\n\u001b[0;32m   2547\u001b[0m \u001b[38;5;66;03m# Create a dataset builder\u001b[39;00m\n\u001b[1;32m-> 2548\u001b[0m builder_instance \u001b[38;5;241m=\u001b[39m load_dataset_builder(\n\u001b[0;32m   2549\u001b[0m     path\u001b[38;5;241m=\u001b[39mpath,\n\u001b[0;32m   2550\u001b[0m     name\u001b[38;5;241m=\u001b[39mname,\n\u001b[0;32m   2551\u001b[0m     data_dir\u001b[38;5;241m=\u001b[39mdata_dir,\n\u001b[0;32m   2552\u001b[0m     data_files\u001b[38;5;241m=\u001b[39mdata_files,\n\u001b[0;32m   2553\u001b[0m     cache_dir\u001b[38;5;241m=\u001b[39mcache_dir,\n\u001b[0;32m   2554\u001b[0m     features\u001b[38;5;241m=\u001b[39mfeatures,\n\u001b[0;32m   2555\u001b[0m     download_config\u001b[38;5;241m=\u001b[39mdownload_config,\n\u001b[0;32m   2556\u001b[0m     download_mode\u001b[38;5;241m=\u001b[39mdownload_mode,\n\u001b[0;32m   2557\u001b[0m     revision\u001b[38;5;241m=\u001b[39mrevision,\n\u001b[0;32m   2558\u001b[0m     token\u001b[38;5;241m=\u001b[39mtoken,\n\u001b[0;32m   2559\u001b[0m     storage_options\u001b[38;5;241m=\u001b[39mstorage_options,\n\u001b[0;32m   2560\u001b[0m     trust_remote_code\u001b[38;5;241m=\u001b[39mtrust_remote_code,\n\u001b[0;32m   2561\u001b[0m     _require_default_config_name\u001b[38;5;241m=\u001b[39mname \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m   2562\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mconfig_kwargs,\n\u001b[0;32m   2563\u001b[0m )\n\u001b[0;32m   2565\u001b[0m \u001b[38;5;66;03m# Return iterable dataset in case of streaming\u001b[39;00m\n\u001b[0;32m   2566\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m streaming:\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\ml-project2\\lib\\site-packages\\datasets\\load.py:2220\u001b[0m, in \u001b[0;36mload_dataset_builder\u001b[1;34m(path, name, data_dir, data_files, cache_dir, features, download_config, download_mode, revision, token, use_auth_token, storage_options, trust_remote_code, _require_default_config_name, **config_kwargs)\u001b[0m\n\u001b[0;32m   2218\u001b[0m     download_config \u001b[38;5;241m=\u001b[39m download_config\u001b[38;5;241m.\u001b[39mcopy() \u001b[38;5;28;01mif\u001b[39;00m download_config \u001b[38;5;28;01melse\u001b[39;00m DownloadConfig()\n\u001b[0;32m   2219\u001b[0m     download_config\u001b[38;5;241m.\u001b[39mstorage_options\u001b[38;5;241m.\u001b[39mupdate(storage_options)\n\u001b[1;32m-> 2220\u001b[0m dataset_module \u001b[38;5;241m=\u001b[39m \u001b[43mdataset_module_factory\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   2221\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2222\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2223\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2224\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdownload_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2225\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdata_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2226\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdata_files\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata_files\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2227\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2228\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtrust_remote_code\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrust_remote_code\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2229\u001b[0m \u001b[43m    \u001b[49m\u001b[43m_require_default_config_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_require_default_config_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2230\u001b[0m \u001b[43m    \u001b[49m\u001b[43m_require_custom_configs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mbool\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mconfig_kwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   2231\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   2232\u001b[0m \u001b[38;5;66;03m# Get dataset builder class from the processing script\u001b[39;00m\n\u001b[0;32m   2233\u001b[0m builder_kwargs \u001b[38;5;241m=\u001b[39m dataset_module\u001b[38;5;241m.\u001b[39mbuilder_kwargs\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\ml-project2\\lib\\site-packages\\datasets\\load.py:1865\u001b[0m, in \u001b[0;36mdataset_module_factory\u001b[1;34m(path, revision, download_config, download_mode, dynamic_modules_path, data_dir, data_files, cache_dir, trust_remote_code, _require_default_config_name, _require_custom_configs, **download_kwargs)\u001b[0m\n\u001b[0;32m   1863\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCouldn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt reach the Hugging Face Hub for dataset \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me1\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1864\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e1, (DataFilesNotFoundError, DatasetNotFoundError, EmptyDatasetError)):\n\u001b[1;32m-> 1865\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e1 \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1866\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e1, \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m):\n\u001b[0;32m   1867\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m(\n\u001b[0;32m   1868\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCouldn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt find a dataset script at \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrelative_to_absolute_path(combined_path)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m or any data file in the same directory. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1869\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCouldn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt find \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m on the Hugging Face Hub either: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(e1)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me1\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1870\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\ml-project2\\lib\\site-packages\\datasets\\load.py:1812\u001b[0m, in \u001b[0;36mdataset_module_factory\u001b[1;34m(path, revision, download_config, download_mode, dynamic_modules_path, data_dir, data_files, cache_dir, trust_remote_code, _require_default_config_name, _require_custom_configs, **download_kwargs)\u001b[0m\n\u001b[0;32m   1810\u001b[0m     msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDataset \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m doesn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt exist on the Hub\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1811\u001b[0m     msg \u001b[38;5;241m=\u001b[39m msg \u001b[38;5;241m+\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m at revision \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrevision\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m revision \u001b[38;5;28;01melse\u001b[39;00m msg\n\u001b[1;32m-> 1812\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m DatasetNotFoundError(\n\u001b[0;32m   1813\u001b[0m         msg \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m. If the repo is private or gated, make sure to log in with `huggingface-cli login`.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1814\u001b[0m     )\n\u001b[0;32m   1815\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   1816\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n",
      "\u001b[1;31mDatasetNotFoundError\u001b[0m: Dataset 'dataset/my-own-dataset' doesn't exist on the Hub. If the repo is private or gated, make sure to log in with `huggingface-cli login`."
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizerFast, BertForTokenClassification, Trainer, TrainingArguments\n",
    "from datasets import load_dataset, Dataset\n",
    "from seqeval.metrics import f1_score, precision_score, recall_score, classification_report\n",
    "\n",
    "dataset = load_dataset(\"dataset/my-own-dataset\") #needed to construct a new Dataset class, to be done\n",
    "num_of_labels = ...\n",
    "\n",
    "tokenizer = BertTokenizerFast.from_pretrained(\"mshamrai/bert-base-ukr-eng-rus-uncased\")\n",
    "model = BertForTokenClassification.from_pretrained(\"mshamrai/bert-base-ukr-eng-rus-uncased\", num_labels=num_of_labels)  # #number of labels\n",
    "\n",
    "\n",
    "label_list = ...  #Collect all the labels like 'O', 'B-inscription', 'I-inscription'\n",
    "\n",
    "\n",
    "def tokenize_and_align_labels(examples):\n",
    "    tokenized_inputs = tokenizer(examples[\"tokens\"], truncation=True, is_split_into_words=True)\n",
    "    labels = examples[\"ner_tags\"]\n",
    "    aligned_labels = []\n",
    "    for i, label_list in enumerate(labels):\n",
    "        word_ids = tokenized_inputs.word_ids(batch_index=i)\n",
    "        aligned_label = []\n",
    "        previous_word_idx = None\n",
    "        for word_idx in word_ids:\n",
    "            \n",
    "            if word_idx is None:\n",
    "                aligned_label.append(-100)\n",
    "            \n",
    "            elif word_idx != previous_word_idx:\n",
    "                aligned_label.append(label_list[word_idx])\n",
    "            \n",
    "            else:\n",
    "                aligned_label.append(label_list[word_idx])\n",
    "            previous_word_idx = word_idx\n",
    "        aligned_labels.append(aligned_label)\n",
    "    tokenized_inputs[\"labels\"] = aligned_labels\n",
    "    return tokenized_inputs\n",
    "\n",
    "\n",
    "tokenized_dataset = dataset.map(tokenize_and_align_labels, batched=True)\n",
    "\n",
    "\n",
    "train_dataset = tokenized_dataset[\"train\"]\n",
    "val_dataset = tokenized_dataset[\"validation\"]\n",
    "test_dataset = tokenized_dataset[\"test\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6db40190-38f1-43b0-bc3e-7fe17b486334",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data_df = pd.read_csv('data/final/data.tsv', sep='\\t', names=['token', 'label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "073dfd1c-32df-4b1f-855c-ec5b4e01c793",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_set = set(data_df['label'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "f148dcd0-3bbd-4b3a-91bf-f851ed83fe01",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['B-decoration',\n",
       " 'B-inscription_type',\n",
       " 'B-monument',\n",
       " 'I-inscripiton type',\n",
       " 'B-epigraphic_shorthand',\n",
       " 'I-decoration',\n",
       " 'B-material',\n",
       " 'B-object_type',\n",
       " 'I-object_type',\n",
       " 'I-preservation_state',\n",
       " 'B-inscripiton type',\n",
       " 'I-monument',\n",
       " 'B-other',\n",
       " 'I-material',\n",
       " 'O',\n",
       " 'B-preservation_state',\n",
       " 'I-inscription_type',\n",
       " 'B-execution_technique',\n",
       " 'B-dating_criteria',\n",
       " 'B-inscription',\n",
       " 'B-symbol',\n",
       " 'I-other']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_list = list(label_set)\n",
    "label_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "819ba436-fa4c-4e12-9352-3a42cf541ea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_dict = {}\n",
    "for idx in range(len(label_list)):\n",
    "    label_dict[label_list[idx]] = idx\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "edecc936-15e1-4746-86f8-5dea1cf3a4ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'B-decoration': 0,\n",
       " 'B-inscription_type': 1,\n",
       " 'B-monument': 2,\n",
       " 'I-inscripiton type': 3,\n",
       " 'B-epigraphic_shorthand': 4,\n",
       " 'I-decoration': 5,\n",
       " 'B-material': 6,\n",
       " 'B-object_type': 7,\n",
       " 'I-object_type': 8,\n",
       " 'I-preservation_state': 9,\n",
       " 'B-inscripiton type': 10,\n",
       " 'I-monument': 11,\n",
       " 'B-other': 12,\n",
       " 'I-material': 13,\n",
       " 'O': 14,\n",
       " 'B-preservation_state': 15,\n",
       " 'I-inscription_type': 16,\n",
       " 'B-execution_technique': 17,\n",
       " 'B-dating_criteria': 18,\n",
       " 'B-inscription': 19,\n",
       " 'B-symbol': 20,\n",
       " 'I-other': 21}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "fb60bfd2-e818-4921-903e-5e158f326991",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('data/final/splits/dev.csv', 'r', encoding='utf-8') as file:\n",
    "    dev_data = file.read().strip().split('\\n\\n')  # Splitting data by double newlines\n",
    "dev_tokens = []\n",
    "dev_ner_tags = []\n",
    "\n",
    "for record in dev_data:\n",
    "    #print(type(record))\n",
    "    #print(record)\n",
    "    lines = record.split('\\n')\n",
    "    token = []\n",
    "    ner_tag = []\n",
    "    for line in lines:\n",
    "        word = line.split('\\t')\n",
    "        token.append(word[0])\n",
    "        \n",
    "        ner_tag.append(label_dict[word[1]])\n",
    "        \n",
    "    dev_tokens.append(token)\n",
    "    dev_ner_tags.append(ner_tag)\n",
    "with open('data/final/splits/train.csv', 'r', encoding='utf-8') as file:\n",
    "    train_data = file.read().strip().split('\\n\\n')  # Splitting data by double newlinestr\n",
    "train_tokens = []\n",
    "train_ner_tags = []\n",
    "\n",
    "for record in train_data:\n",
    "    #print(type(record))\n",
    "    #print(record)\n",
    "    lines = record.split('\\n')\n",
    "    token = []\n",
    "    ner_tag = []\n",
    "    for line in lines:\n",
    "        word = line.split('\\t')\n",
    "        token.append(word[0])\n",
    "        \n",
    "        ner_tag.append(label_dict[word[1]])\n",
    "        \n",
    "    train_tokens.append(token)\n",
    "    train_ner_tags.append(ner_tag)\n",
    "    \n",
    "with open('data/final/splits/test.csv', 'r', encoding='utf-8') as file:\n",
    "    test_data = file.read().strip().split('\\n\\n')  # Splitting data by double newlinestr\n",
    "test_tokens = []\n",
    "test_ner_tags = []\n",
    "\n",
    "for record in test_data:\n",
    "    #print(type(record))\n",
    "    #print(record)\n",
    "    lines = record.split('\\n')\n",
    "    token = []\n",
    "    ner_tag = []\n",
    "    for line in lines:\n",
    "        word = line.split('\\t')\n",
    "        token.append(word[0])\n",
    "        \n",
    "        ner_tag.append(label_dict[word[1]])\n",
    "        \n",
    "    test_tokens.append(token)\n",
    "    test_ner_tags.append(ner_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "deb6d756-0754-41ef-aa88-836e04f9f05b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "train_json_dict = {\"tokens\": train_tokens, \"ner_tags\": train_ner_tags}\n",
    "dev_json_dict = {\"tokens\": dev_tokens, \"ner_tags\": dev_ner_tags}\n",
    "test_json_dict = {\"tokens\": test_tokens, \"ner_tags\": test_ner_tags}\n",
    "train_path = 'data/bert/train.json'\n",
    "dev_path = 'data/bert/validation.json'\n",
    "test_path = 'data/bert/test.json'\n",
    "with open(train_path, \"w\", encoding=\"utf-8\") as f:\n",
    "    json.dump(train_json_dict, f, ensure_ascii=False, indent=4)\n",
    "with open(dev_path, \"w\", encoding=\"utf-8\") as f:\n",
    "    json.dump(dev_json_dict, f, ensure_ascii=False, indent=4)\n",
    "with open(test_path, \"w\", encoding=\"utf-8\") as f:\n",
    "    json.dump(test_json_dict, f, ensure_ascii=False, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "9fa58638-cfb5-4cd0-acff-9f1cbb06241e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "# 读取 JSON 文件\n",
    "with open(f'{dataset_path}/train.json', 'r', encoding='utf-8') as f:\n",
    "    train_data = json.load(f)\n",
    "\n",
    "# 构建包含 'tokens' 和 'ner_tags' 字段的字典\n",
    "train_dict = {'tokens': train_tokens, 'ner_tags': train_ner_tags}\n",
    "\n",
    "# 创建 Dataset 对象\n",
    "train_dataset = Dataset.from_dict(train_dict)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "e51e04a3-33da-49ed-8791-80e636dc84be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['tokens', 'ner_tags'],\n",
       "    num_rows: 2510\n",
       "})"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f49ad495-fa2a-4ac4-86aa-af7465437ef6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at mshamrai/bert-base-ukr-eng-rus-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3140' max='3140' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3140/3140 1:31:34, Epoch 5/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.196800</td>\n",
       "      <td>0.127810</td>\n",
       "      <td>0.915254</td>\n",
       "      <td>0.698276</td>\n",
       "      <td>0.792176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.121200</td>\n",
       "      <td>0.125853</td>\n",
       "      <td>0.913793</td>\n",
       "      <td>0.685345</td>\n",
       "      <td>0.783251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.081200</td>\n",
       "      <td>0.108655</td>\n",
       "      <td>0.918478</td>\n",
       "      <td>0.728448</td>\n",
       "      <td>0.812500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.065500</td>\n",
       "      <td>0.100946</td>\n",
       "      <td>0.883838</td>\n",
       "      <td>0.754310</td>\n",
       "      <td>0.813953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.062000</td>\n",
       "      <td>0.097996</td>\n",
       "      <td>0.888325</td>\n",
       "      <td>0.754310</td>\n",
       "      <td>0.815851</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory ./results\\checkpoint-500 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
      "Checkpoint destination directory ./results\\checkpoint-1000 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n",
      "Checkpoint destination directory ./results\\checkpoint-1500 already exists and is non-empty. Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                     precision    recall  f1-score   support\n",
      "\n",
      "    dating_criteria       0.91      0.84      0.87        37\n",
      "         decoration       0.29      0.20      0.24        10\n",
      "execution_technique       0.00      0.00      0.00         9\n",
      "        inscription       0.98      0.94      0.96       126\n",
      "           material       0.00      0.00      0.00         1\n",
      "           monument       0.00      0.00      0.00         2\n",
      "        object_type       1.00      0.12      0.22         8\n",
      "              other       0.63      0.47      0.54        36\n",
      "             symbol       0.00      0.00      0.00         1\n",
      "\n",
      "          micro avg       0.89      0.73      0.80       230\n",
      "          macro avg       0.42      0.29      0.31       230\n",
      "       weighted avg       0.83      0.73      0.77       230\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lyh\\anaconda3\\envs\\ml-project2\\lib\\site-packages\\seqeval\\metrics\\v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizerFast, BertForTokenClassification, Trainer, TrainingArguments, DataCollatorForTokenClassification\n",
    "from datasets import Dataset\n",
    "from seqeval.metrics import f1_score, precision_score, recall_score, classification_report\n",
    "import numpy as np\n",
    "import json\n",
    "\n",
    "class CustomDataset:\n",
    "    def __init__(self, dataset_path, tokenizer_name, label_list):\n",
    "        self.tokenizer = BertTokenizerFast.from_pretrained(tokenizer_name, max_length=32)\n",
    "        self.label_list = label_list\n",
    "        self.num_labels = len(label_list)\n",
    "        self.train_data = self.load_data(f'{dataset_path}/train.json')\n",
    "        self.validation_data = self.load_data(f'{dataset_path}/validation.json')\n",
    "        self.test_data = self.load_data(f'{dataset_path}/test.json')\n",
    "        self.tokenized_train_data = self.tokenize_and_align_labels(self.train_data)\n",
    "        self.tokenized_validation_data = self.tokenize_and_align_labels(self.validation_data)\n",
    "        self.tokenized_test_data = self.tokenize_and_align_labels(self.test_data)\n",
    "\n",
    "    def load_data(self, file_path):\n",
    "        with open(file_path, 'r', encoding='utf-8') as f:\n",
    "            data = json.load(f)\n",
    "        return data\n",
    "\n",
    "    def tokenize_and_align_labels(self, data):\n",
    "        tokenized_data = self.tokenizer(data[\"tokens\"], truncation=True, is_split_into_words=True)\n",
    "        labels = data[\"ner_tags\"]\n",
    "        aligned_labels = []\n",
    "        for i, label_list in enumerate(labels):\n",
    "            word_ids = tokenized_data.word_ids(batch_index=i)\n",
    "            aligned_label = []\n",
    "            previous_word_idx = None\n",
    "            for word_idx in word_ids:\n",
    "                if word_idx is None:\n",
    "                    aligned_label.append(-100)\n",
    "                elif word_idx != previous_word_idx:\n",
    "                    aligned_label.append(label_list[word_idx])\n",
    "                else:\n",
    "                    aligned_label.append(label_list[word_idx])\n",
    "                previous_word_idx = word_idx\n",
    "            aligned_labels.append(aligned_label)\n",
    "        tokenized_data[\"labels\"] = aligned_labels\n",
    "        return tokenized_data\n",
    "\n",
    "    def get_train_dataset(self):\n",
    "        return Dataset.from_dict(self.tokenized_train_data)\n",
    "\n",
    "    def get_validation_dataset(self):\n",
    "        return Dataset.from_dict(self.tokenized_validation_data)\n",
    "\n",
    "    def get_test_dataset(self):\n",
    "        return Dataset.from_dict(self.tokenized_test_data)\n",
    "\n",
    "# 参数配置\n",
    "dataset_path = \"data/bert\"\n",
    "tokenizer_name = \"mshamrai/bert-base-ukr-eng-rus-uncased\"\n",
    "\n",
    "# 创建 CustomDataset 对象\n",
    "dataset = CustomDataset(dataset_path, tokenizer_name, label_list)\n",
    "\n",
    "# 加载模型\n",
    "model = BertForTokenClassification.from_pretrained(tokenizer_name, num_labels=len(label_list))\n",
    "\n",
    "# 获取数据集\n",
    "train_dataset = dataset.get_train_dataset()\n",
    "val_dataset = dataset.get_validation_dataset()\n",
    "test_dataset = dataset.get_test_dataset()\n",
    "\n",
    "# 训练参数配置\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./results\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=4,\n",
    "    per_device_eval_batch_size=4,\n",
    "    num_train_epochs=5,\n",
    "    weight_decay=0.01,\n",
    ")\n",
    "\n",
    "# 数据处理器\n",
    "data_collator = DataCollatorForTokenClassification(BertTokenizerFast.from_pretrained(tokenizer_name, max_length=32))\n",
    "\n",
    "# 计算指标\n",
    "def compute_metrics(p):\n",
    "    predictions, labels = p\n",
    "    predictions = np.argmax(predictions, axis=2)\n",
    "\n",
    "    true_labels = [\n",
    "        [label_list[l] for l in label if l != -100]\n",
    "        for label in labels\n",
    "    ]\n",
    "    true_predictions = [\n",
    "        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
    "        for prediction, label in zip(predictions, labels)\n",
    "    ]\n",
    "\n",
    "    results = {\n",
    "        \"precision\": precision_score(true_labels, true_predictions),\n",
    "        \"recall\": recall_score(true_labels, true_predictions),\n",
    "        \"f1\": f1_score(true_labels, true_predictions),\n",
    "    }\n",
    "    return results\n",
    "\n",
    "# 创建 Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    compute_metrics=compute_metrics,\n",
    "    data_collator=data_collator,  # 将数据处理器传递给 Trainer\n",
    ")\n",
    "\n",
    "# 开始训练\n",
    "trainer.train()\n",
    "\n",
    "# 评估模型\n",
    "trainer.evaluate()\n",
    "\n",
    "# 打印测试集的分类报告\n",
    "predictions, labels, _ = trainer.predict(test_dataset)\n",
    "predictions = np.argmax(predictions, axis=2)\n",
    "\n",
    "true_labels = [\n",
    "    [label_list[l] for l in label if l != -100]\n",
    "    for label in labels\n",
    "]\n",
    "true_predictions = [\n",
    "    [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
    "    for prediction, label in zip(predictions, labels)\n",
    "]\n",
    "\n",
    "print(classification_report(true_labels, true_predictions))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6e37cbc4-c667-445e-98c1-68f66c512828",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_model_dir = \"model/bert2\"\n",
    "model.save_pretrained(bert_model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6519a98-020d-4aed-9d61-f645f8d1f73c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}