task-fine-tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12.1\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "print(torch.version.cuda)\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset, DatasetDict\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def load_data(src_file, tgt_file):\n",
    "    \"\"\" Loads the D-wikipedia files (scr = source articles),\n",
    "    (tgt = target articles) and appends the article pairs in a list as dictionaries.\n",
    "    Returns: 40 000 article pairs.\n",
    "    \"\"\"\n",
    "    with open(src_file, 'r', encoding='utf-8') as f_src:\n",
    "        src_lines = f_src.readlines()\n",
    "    \n",
    "    with open(tgt_file, 'r', encoding='utf-8') as f_tgt:\n",
    "        tgt_lines = f_tgt.readlines()\n",
    "\n",
    "    all_data = []\n",
    "    for original_article, simplified_article in zip(src_lines, tgt_lines):\n",
    "        entry = {\n",
    "            \"text\": original_article,\n",
    "            \"summary\": simplified_article,\n",
    "        }\n",
    "        all_data.append(entry)\n",
    "    #print(\"length of all_data\",len(all_data))\n",
    "    return all_data[:40000]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data, test_data = train_test_split(load_data(\"train.src\", \"train.tgt\"), test_size=0.2)        \n",
    "\n",
    "# creating datasets \n",
    "\n",
    "train_dataset = Dataset.from_list(training_data)\n",
    "test_dataset = Dataset.from_list(test_data)\n",
    "#print(train_dataset[1])\n",
    "#print(len(train_dataset))\n",
    "#print(len(test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary: the 1916 summer olympics , officially known as the games of the vi olympiad , were not held.the games were planned for berlin in germany.in 1912 , workers began building the sports facilities for the games . the `` deutsches stadion '' ( `` german stadium '' ) began in 1912 . in june 1913 , the stadium was officially opened with 60,000 people at the ceremonies.the games were cancelled because of world war i . \n",
      "\n",
      "Original: the 1916 summer olympics ( german : `` olympische sommerspiele 1916 '' ) , officially known as the games of the vi olympiad , were scheduled to be held in berlin , german empire , but were eventually cancelled due to the outbreak of world war i. berlin was selected as the host city during the 14th ioc session in stockholm on 4 july 1912 , defeating bids from alexandria , amsterdam , brussels , budapest and cleveland . after the 1916 games were cancelled , berlin would eventually host the 1936 summer olympics , twenty years later . \n",
      "\n",
      "Summary: brendon boyd urie ( born april 12 , 1987 in st. george , utah ) is the lead singer of the rock band panic at the disco . he also plays keyboard , accordion , piano , organ , bass , guitar , cello and drums . married to sarah orzechowski since 2013 . \n",
      "\n",
      "Original: brendon boyd urie ( born april 12 , 1987 ) is an american singer , songwriter , and musician , best known as the lead vocalist of panic ! at the disco , of which he is the only original member remaining . \n",
      "\n",
      "Summary: miroslav klose ( born june 9 1978 in opole , poland ) is a polish born german football player . from 1999 until 2004 he played for kaiserslautern and from 2004 until 2007 for werder bremen . in 2007 changed to bayern munich . in 2011 , he left bayern munich and joined the italian team lazio . on the germany national team he has played in 137 matches since 2001 and scored 71 goals.klose is a great player known for his heading skills and he is currently the all-time top scorer of the germany national football team with 71 goals . he is also the all-time leading goal scorer of the fifa world cup after beating ronaldo 's record in the semi-final against brazil . \n",
      "\n",
      "Original: miroslav josef klose ( , ; born 9 june 1978 ) is a german former professional footballer who played as a striker . he currently is a first-team assistant coach at bayern munich . klose is widely regarded as one of the best strikers of his generation and holds the record for the most goals scored in fifa world cup tournaments.klose is best known for his performances with the german national team . he was part of the german squad that won the 2014 fifa world cup , having previously finished second ( 2002 ) and third ( 2006 , 2010 ) in the competition , and as runner-up in uefa euro 2008 and joint-third place in uefa euro 2012 . klose is the men 's top goalscorer of the fifa world cup with 16 goals , having scored five goals in his debut world cup in 2002 and having won the golden boot at the 2006 world cup in germany by again scoring five times . he also scored four times in the 2010 world cup and twice at the 2014 world cup , in the latter tournament overtaking ronaldo 's then-record of 15 goals to top the all-time list . klose is also the all time top scorer for germany , which never lost a game in which klose scored , and one of the very few players ever to have won gold , silver and bronze medals in the world cup . he retired from the national team on 11 august 2014 , shortly after germany 's victory at the 2014 world cup.at club level , klose has been a less prolific but usually reliable goalscorer . starting his career at fc 08 homburg , he played in the bundesliga for kaiserslautern , werder bremen and bayern munich , and in serie a for lazio . he won two league titles with bayern , along with cup competitions at bayern , werder bremen and lazio . \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    print(\"Summary:\", train_dataset[i]['summary'])\n",
    "    print(\"Original:\", train_dataset[i]['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gusniemian@GU.GU.SE/.local/lib/python3.12/site-packages/transformers/convert_slow_tokenizer.py:558: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "checkpoint = \"google/mt5-small\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint) #legacy=False, use_fast=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = \"simplify the language: \"\n",
    "def preprocess_function(examples):\n",
    "    \"\"\" Preprocesses the data and adds the prefix.\n",
    "        Tokenizes the data with mt5-tokenizer.\n",
    "    \"\"\"\n",
    "    #inputs = []\n",
    "    #for doc in examples[\"text\"]:\n",
    "    #    inputt = prefix + doc\n",
    "    #    inputs.append(inputt)\n",
    "    \n",
    "    # input tokenizing    \n",
    "    encoding = tokenizer(\n",
    "    [prefix + sequence for sequence in examples[\"text\"]],\n",
    "    padding=\"longest\",\n",
    "    max_length=1200,\n",
    "    truncation=True,\n",
    "    return_tensors=\"pt\")\n",
    "\n",
    "    input_ids, attention_mask = encoding.input_ids, encoding.attention_mask\n",
    "\n",
    "    # targets tokenizing\n",
    "    target_encoding = tokenizer(\n",
    "        [sequence for sequence in examples[\"summary\"]],\n",
    "        padding=\"longest\",\n",
    "        max_length=600,\n",
    "        truncation=True,\n",
    "        return_tensors=\"pt\",\n",
    "    )\n",
    "    labels = target_encoding.input_ids\n",
    "\n",
    "    # replace padding token id's of the labels by -100 so it's ignored by the loss\n",
    "    labels[labels == tokenizer.pad_token_id] = -100\n",
    "    \n",
    "    return {\n",
    "        \"input_ids\": input_ids,\n",
    "        \"attention_mask\": attention_mask,\n",
    "        \"labels\": labels\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "009d740faa9a41ba8243e0527781a3f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/32000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_train_dataset = train_dataset.map(preprocess_function, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b7c20df21f94725b8785621466c56ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/8000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_test_dataset = test_dataset.map(preprocess_function, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForSeq2Seq\n",
    "# transforms inputs to tensors, batches\n",
    "data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluate import load\n",
    "import numpy as np\n",
    "\n",
    "sari = load(\"sari\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred  \n",
    "    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
    "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "\n",
    "    sources = [example[\"text\"] for example in test_dataset]  \n",
    "    references = [[example[\"summary\"]] for example in test_dataset] \n",
    "    \n",
    "    print(\"sources\",sources)\n",
    "    print(\"predictions\",decoded_preds)\n",
    "    print(\"references\",references)\n",
    "\n",
    "    # calculating SARI\n",
    "    sari_score = sari.compute(sources=sources, predictions=decoded_preds, references=references)\n",
    "\n",
    "    return {\n",
    "        \"sari\": sari_score,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "import torch\n",
    "from peft import get_peft_model, LoraConfig\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LORA SETTINGS\n",
    "\n",
    "lora_config = LoraConfig(\n",
    "    r=16,  \n",
    "    lora_alpha=32,  \n",
    "    lora_dropout=0.1,  \n",
    "    task_type=\"SEQ_2_SEQ_LM\",\n",
    ")\n",
    "\n",
    "model = get_peft_model(model, lora_config)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.12/site-packages/accelerate/accelerator.py:494: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    }
   ],
   "source": [
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=\"final-task-fine-tuned-model-40k-traindata\",\n",
    "    eval_strategy=\"epoch\",\n",
    "    learning_rate=1e-4,\n",
    "    per_device_train_batch_size=1,\n",
    "    per_device_eval_batch_size=1,\n",
    "    gradient_accumulation_steps=8, # for memory issues\n",
    "    weight_decay=0.01, #to prevent overfitting\n",
    "    save_total_limit=2,\n",
    "    num_train_epochs=5,\n",
    "    overwrite_output_dir=True,\n",
    "    predict_with_generate=True,\n",
    "    logging_strategy=\"steps\",\n",
    "    fp16=True, \n",
    "    warmup_steps=150, \n",
    "    push_to_hub=False,\n",
    ")\n",
    "\n",
    "trainer = Seq2SeqTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_train_dataset,\n",
    "    eval_dataset=tokenized_test_dataset,\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='4007' max='5000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [4007/5000 9:56:36 < 2:27:55, 0.11 it/s, Epoch 4.01/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Sari</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.741000</td>\n",
       "      <td>2.818675</td>\n",
       "      <td>{'sari': 37.15475633080281}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.452000</td>\n",
       "      <td>2.695584</td>\n",
       "      <td>{'sari': 38.797998502753465}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.332800</td>\n",
       "      <td>2.656935</td>\n",
       "      <td>{'sari': 39.71900946529877}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.287000</td>\n",
       "      <td>2.613796</td>\n",
       "      <td>{'sari': 40.07296562727385}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/gusniemian@GU.GU.SE/.local/lib/python3.12/site-packages/transformers/generation/utils.py:1220: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n",
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "IOPub data rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Trainer is attempting to log a value of \"{'sari': 37.15475633080281}\" of type <class 'dict'> for key \"eval/sari\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/gusniemian@GU.GU.SE/.local/lib/python3.12/site-packages/transformers/generation/utils.py:1220: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n",
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "IOPub data rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Trainer is attempting to log a value of \"{'sari': 38.797998502753465}\" of type <class 'dict'> for key \"eval/sari\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/gusniemian@GU.GU.SE/.local/lib/python3.12/site-packages/transformers/generation/utils.py:1220: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n",
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "IOPub data rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Trainer is attempting to log a value of \"{'sari': 39.71900946529877}\" of type <class 'dict'> for key \"eval/sari\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n",
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/gusniemian@GU.GU.SE/.local/lib/python3.12/site-packages/transformers/generation/utils.py:1220: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n",
      "/usr/local/lib64/python3.12/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "IOPub data rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Trainer is attempting to log a value of \"{'sari': 40.07296562727385}\" of type <class 'dict'> for key \"eval/sari\" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[49], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/lib/python3.12/site-packages/transformers/trainer.py:2052\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   2050\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m   2051\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 2052\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2053\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2054\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2055\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2056\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2057\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/lib/python3.12/site-packages/transformers/trainer.py:2393\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2387\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maccelerator\u001b[38;5;241m.\u001b[39maccumulate(model):\n\u001b[1;32m   2388\u001b[0m     tr_loss_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_step(model, inputs)\n\u001b[1;32m   2390\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m   2391\u001b[0m     args\u001b[38;5;241m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   2392\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_xla_available()\n\u001b[0;32m-> 2393\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m (torch\u001b[38;5;241m.\u001b[39misnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43misinf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtr_loss_step\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m   2394\u001b[0m ):\n\u001b[1;32m   2395\u001b[0m     \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   2396\u001b[0m     tr_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m tr_loss \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_globalstep_last_logged)\n\u001b[1;32m   2397\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = \"Mohammed Saleh, the acting director of Al-Awda Hospital in Jabalia, says his hospital is inundated with casualties every day, as Israel carries out raids in the area. He warns the attacks on the local refugee camp are putting the healthcare system 'on the brink of disaster'. 'Medical teams are dealing with complex injuries amid severe shortages of medicines, medical supplies and fuel, in addition to the lack of food for patients and medical staff in the hospital,' Saleh says in a statement to the Palestinian Press Agency. Many doctors at the Kamal Adwan, Al-Awda and the Indonesian hospitals have refused to leave their patients, despite Israel's recent offensive in Jabalia. 'We are talking about more than 300 medical staff working at Kamal Adwan Hospital, and we can't provide even a single meal for them to be able to offer medical services safely,' hospital director Hussam Abu Safiya tells Reuters. Israel says it's targeting Hamas in Jabalia, and earlier said it killed '50 terrorists in close-quarters encounters and aerial strikes over the past day'\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text2 = \"Galileo was an American robotic space program that studied the planet Jupiter and its moons, as well as several other Solar System bodies. Named after the Italian astronomer Galileo Galilei, the Galileo spacecraft consisted of an orbiter and an atmospheric entry probe. It was delivered into Earth orbit on October 18, 1989, by Space Shuttle Atlantis on the STS-34 mission, and arrived at Jupiter on December 7, 1995, after gravity assist flybys of Venus and Earth, and became the first spacecraft to orbit Jupiter. The spacecraft then launched the first probe to directly measure its atmosphere. Despite suffering major antenna problems, Galileo achieved the first asteroid flyby, of 951 Gaspra, and discovered the first asteroid moon, Dactyl, around 243 Ida. In 1994, Galileo observed Comet Shoemaker–Levy 9's collision with Jupiter.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text3 = \"Finland, officially the Republic of Finland, is a Nordic country in Northern Europe. It borders Sweden to the northwest, Norway to the north, and Russia to the east, with the Gulf of Bothnia to the west and the Gulf of Finland to the south, opposite Estonia. Finland covers a total area of 338,145 square kilometres (130,559 sq mi), including a land area of 303,815 square kilometres (117,304 sq mi),and has a population of 5.6 million. Helsinki is the capital and largest city. The vast majority of the population are ethnic Finns. The official languages are Finnish and Swedish; 84.9 percent of the population speak the first as their mother tongue and 5.1 percent the latter. Finland's climate varies from humid continental in the south to boreal in the north. The land cover is predominantly boreal forest biome, with more than 180,000 recorded lakes.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text4 = \"Donald Trump enjoys a huge lead among men, while women tell pollsters they prefer Kamala Harris by a similarly large margin. The political gender gap reflects a decade of social upheaval and could help decide the US election. For the first woman of colour to secure a presidential nomination, and only the second woman to ever get this close, Kamala Harris goes to great lengths not to talk about her identity. “Listen, I am running because I believe that I am the best person to do this job at this moment for all Americans, regardless of race and gender,” the vice-president said in a CNN interview last month. In this piece, the BBC's US special correspondent Katty Kay delves deeper into how this November’s election has turned into a referendum on gender norms, and the social upheavals of recent years.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "text5 = \"Nineteen people died and six others were injured when a bus crashed on a highway in Mexico’s central state of Zacatecas on Saturday, local authorities said. The accident occurred in the early morning hours when the bus carrying the victims collided with the back of a tractor-trailer carrying corn, which had come loose. Zacatecas Governor David Monreal earlier on Saturday had initially reported a preliminary death toll of 24 people, but the state attorney general’s office later revised the tally in a statement. The attorney general’s office said it was “carrying out investigations to arrest the driver” of the tractor-trailer. Efforts were ongoing on Saturday morning to recover some of the bodies that had fallen into a ravine, a local government official who asked not to be named told Reuters. Video footage showed rescue teams and security forces, including military personnel, securing the area while rescuers worked to recover the bodies. The bus was headed for Ciudad Juarez, a city on the US-Mexico border in the state of Chihuahua. The victims did not include migrants, according to the attorney general’s office.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "text6 = \"At least 62 people are known to have died after torrential rain caused devastating flash floods in south-eastern Spain. In the town of Chiva near Valencia more than a year's worth of rain fell in just eight hours and local officials say it is 'impossible' to put a final figure on the number of people who have perished. Footage uploaded to social media shows floodwaters causing chaos in the wider region, knocking down bridges and dragging cars through the streets. Other video appears to show people clinging to trees to avoid being swept away. Much of the country has been badly hit by heavy rain and hailstorms, triggering rapid flooding across multiple areas.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "text7 = \"A manhunt is underway in northern Austria after a hunter allegedly fatally shot two people and fled the scene, local police said Monday. Franz Hofer, mayor of Kirchberg ob der Donau, was killed in the village of Altenfelden in Austria’s rural Muhlviertel region, near the border with Germany and the Czech Republic. A second man was also shot dead a short while later, Upper Austria police spokesperson Ulrike Handelbauer told CNN. A large-scale police operation with helicopters and special forces is underway, she said. Police said Roland Drexler, 56, is suspected of having killed the two men and made a getaway in a Volkswagen Caddy. “The man is believed to be extremely dangerous and armed,” police said. A dispute over hunting rights appeared to have sparked the incident, police added. It was not immediately clear why a long-running feud had escalated. According to Kronen Zeitung, a local outlet, the suspect was known to hunters in the area. “He was a difficult person,” said a hunter from the area who wished not to be named. The shooting shocked officials at the People’s Party (OVP) regional headquarters in Linz. “It’s madness,” said state party leader Florian Hiegelsperger. Herbert Sieghartsleitner, the state hunting master, said the incident was “unbelievable.” “I am deeply shocked by what has happened. I knew Franz Hofer very well personally,” he said, according to Kronen Zeitung.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gusniemian@GU.GU.SE/.local/lib/python3.12/site-packages/transformers/convert_slow_tokenizer.py:558: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "fine_tuned_tokenizer = AutoTokenizer.from_pretrained(\"./final-task-fine-tuned-model-40k-traindata/checkpoint-4000\")\n",
    "inputs = fine_tuned_tokenizer(text7, return_tensors=\"pt\").input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSeq2SeqLM\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(\"./final-task-fine-tuned-model-40k-traindata/checkpoint-4000\")\n",
    "generated_ids = model.generate(\n",
    "    inputs,\n",
    "    do_sample=False,\n",
    "    repetition_penalty=2.0,\n",
    "    max_new_tokens=300,\n",
    "    min_new_tokens=200,   # min length for generation\n",
    "    num_beams=5,     # n beam search\n",
    "    no_repeat_ngram_size=3,  # prevent repetitive n-grams\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<extra_id_0> Hofer, mayor of Kirchberg ob der Donau, was killed in the village of Altenfelden in Austria’s rural Muhlviertel region, near the border with Germany and the Czech Republic. The suspect was fatally shot dead a short while later, police said. It was known to have killed two people and fled the scene. The incident was shocked officials at the People’s Party (OVP) regional headquarters in Linz. The man was shot dead after a hunter who had killed the two people. A second man was attacked by a small-scale police operation with special forces on the front of Austria. He was arrested for a long-running feud that had escalated during the shooting. There was a large scale police operation involving helicopters and Special forces. The first man was named Roland Drexler, 56, where he was murdered.'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tuned_tokenizer.decode(generated_ids[0], skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COMPARING PERFORMANCE WITH THE BASE MODEL (MT5-SMALL)\n",
    "\n",
    "base_tokenizer = AutoTokenizer.from_pretrained(\"google/mt5-small\")\n",
    "base_inputs = base_tokenizer(text7, return_tensors=\"pt\").input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = AutoModelForSeq2SeqLM.from_pretrained(\"google/mt5-small\")\n",
    "#outputs = model.generate(inputs, max_new_tokens=100, do_sample=False)\n",
    "generated_ids = model.generate(\n",
    "   inputs,\n",
    "    do_sample=False,\n",
    "    #temperature=0.2,\n",
    "    #dola_layers=\"low\",\n",
    "    #top_k=40,\n",
    "    repetition_penalty=2.0,\n",
    "    max_new_tokens=300,\n",
    "    min_new_tokens=200,   # min length for generation\n",
    "    num_beams=5,     # n beam search\n",
    "    no_repeat_ngram_size=3,  # prevent repetitive n-grams\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<extra_id_0> Hofer, mayor of Kirchberg ob der Donau, was killed in the village of Altenfelden in the region Muhlviertel region, near the border with Germany and the Czech Republic. “He was a difficult person,” said Herbert Sieghartsleitner. The suspect was known to have killed two people and fled the scene. The man was fatally shot dead a short while later, police said. It was reported that he had killed a long-running feud at the local headquarters in Linz. The incident was shocked officials at the People’s Party (OVP) regional head quarters in the area. He was also known for hunting rights. Police said the shooting was dangerous and armed. A second man was shot dead after a hunter allegedly fatally attacked a small-scale police operation with helicopters and special forces is underway.'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_tokenizer.decode(generated_ids[0], skip_special_tokens=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}