analysis_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install transformers datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from transformers import AutoModel, AutoTokenizer\n",
    "import itertools\n",
    "from collections import Counter\n",
    "import torch\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "import torch.nn.functional as F\n",
    "import pickle\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import numpy as np\n",
    "from numpy import linalg as LA\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "\n",
    "from metrics import self_similarity_and_rogue, identifying_rogue_dimensions, mean_pooling, intra_sim_and_norm, anisotropy_baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset stsb_multi_mt (/home/chenghao/.cache/huggingface/datasets/stsb_multi_mt/en/1.0.0/a5d260e4b7aa82d1ab7379523a005a366d9b124c76a5a5cf0c4c5365458b0ba9)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1379"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = load_dataset(\"stsb_multi_mt\", name=\"en\", split=\"test\")\n",
    "sentences = dataset['sentence1'] \n",
    "len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_NAME = 'sentence-transformers/all-mpnet-base-v2'\n",
    "#MODEL_NAME = 'sentence-transformers/all-MiniLM-L6-v2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
    "model = AutoModel.from_pretrained(MODEL_NAME,\n",
    "                                  output_hidden_states = True # returns all hidden-states, enabling analysis on all layers\n",
    "                                  )\n",
    "encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')# max length should be added here instead of when initializing it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59297\n",
      "1379\n",
      "1379\n",
      "39705\n",
      "455\n"
     ]
    }
   ],
   "source": [
    "tokenized = encoded_input['input_ids'].tolist()\n",
    "token_list = list(itertools.chain.from_iterable(tokenized))\n",
    "counter = Counter(token_list)\n",
    "\n",
    "print(len(token_list)) #59297\n",
    "print(counter[2]) # [sep] token = 1379\n",
    "print(counter[0]) # [cls] token = 1379\n",
    "print(counter[1]) # padding token = 39705\n",
    "\n",
    "# get the words to analyze self similarity - appearing in more than 5 contexts (some contexts) at least\n",
    "index_count = [(index, count) for (index,count) in counter.most_common() if count >=5]\n",
    "token_list = [index for (index,count) in counter.most_common() if count >=5]\n",
    "print(len(token_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def position_list(set_of_token, list_of_tokenized_sentences):\n",
    "\n",
    "    inference_list = {} ## get a dict to store corresponding position (sentence_index, token_index) of words\n",
    "    for n in set_of_token:\n",
    "        position_list = []\n",
    "        for sen_index, sen in enumerate(list_of_tokenized_sentences):\n",
    "            if n in sen:\n",
    "                token_index = sen.index(n)\n",
    "                position_list.append((sen_index, token_index)) # store the corresponding sentence index and token index of that word\n",
    "        inference_list[n] = position_list\n",
    "\n",
    "    return inference_list    \n",
    "\n",
    "inference_list = position_list(token_list, tokenized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDataset(Dataset):\n",
    "    def __init__(self, encoded_inputs):\n",
    "        self.encoded_inputs = encoded_inputs\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.encoded_inputs[\"input_ids\"].shape[0]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return {key: val[idx] for key, val in self.encoded_inputs.items()}\n",
    "\n",
    "\n",
    "dataset = MyDataset(encoded_input)\n",
    "dataloader = DataLoader(dataset, batch_size=32)  \n",
    "\n",
    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
    "model = model.to(device)\n",
    "\n",
    "\n",
    "total_output = []\n",
    "total_hidden_states = []\n",
    "\n",
    "\n",
    "with torch.no_grad(): \n",
    "    for batch in dataloader:\n",
    "        batch = {k: v.to(device) for k, v in batch.items()} \n",
    "        model_output = model(**batch)\n",
    "        total_output.append(model_output.last_hidden_state.detach().cpu())\n",
    "        total_hidden_states.append(tuple(state.detach().cpu() for state in model_output.hidden_states))\n",
    "\n",
    "# Concatenate all outputs and hidden states\n",
    "total_output = torch.cat(total_output, dim=0)\n",
    "total_hidden_states = tuple(torch.cat(state, dim=0) for state in zip(*total_hidden_states))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 77.65it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 76.49it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 78.78it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:06<00:00, 74.71it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 78.66it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 79.72it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 78.78it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 78.49it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 76.69it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 77.27it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 76.01it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 76.55it/s] \n",
      "Token Progress: 100%|██████████| 455/455 [00:05<00:00, 76.99it/s] \n",
      "Layer Progress: 100%|██████████| 13/13 [01:16<00:00,  5.88s/it]\n"
     ]
    }
   ],
   "source": [
    "all_layer_self_sim, self_sim_rogue_dimensions = self_similarity_and_rogue(total_hidden_states, token_list, inference_list, n_layers_start = 0, n_layers = 13, analyze_rogue = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.9921365638766518,\n",
       " 0.8731718061674008,\n",
       " 0.7700726872246696,\n",
       " 0.7106101321585903,\n",
       " 0.6776167400881058,\n",
       " 0.6493039647577092,\n",
       " 0.636011013215859,\n",
       " 0.632352422907489,\n",
       " 0.6328568281938326,\n",
       " 0.6424493392070485,\n",
       " 0.6516850220264317,\n",
       " 0.6550528634361233,\n",
       " 0.35661674008810573]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer = 12\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.049"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_layer_self_sim[12][0]  ## last layer(12), [cls] token (0), self similarity: 0.049"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(self_sim_rogue_dimensions[12][0]) # last layer[12], contribution of each dimension for [cls] token(0)'s self similarity measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3677.43it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3878.87it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 4042.32it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3770.98it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3844.76it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3833.70it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3621.14it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3905.51it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3771.48it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3694.53it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3883.16it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3592.07it/s]\n",
      "sentence progress: 100%|██████████| 1379/1379 [00:00<00:00, 3506.82it/s]\n",
      "layer progress: 100%|██████████| 13/13 [00:05<00:00,  2.39it/s]\n"
     ]
    }
   ],
   "source": [
    "all_layer_intra_sim, all_layer_norm = intra_sim_and_norm(total_hidden_states, sentences, encoded_input, n_layers_start = 0, n_layers = 13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unadjusted intra_sim of layer 0: 0.2868331 \t average norm of layer 0: 5.5752234\n",
      "unadjusted intra_sim of layer 1: 0.33597353 \t average norm of layer 1: 6.760648\n",
      "unadjusted intra_sim of layer 2: 0.3985246 \t average norm of layer 2: 7.3132153\n",
      "unadjusted intra_sim of layer 3: 0.4290589 \t average norm of layer 3: 7.63464\n",
      "unadjusted intra_sim of layer 4: 0.44711876 \t average norm of layer 4: 8.217234\n",
      "unadjusted intra_sim of layer 5: 0.47981295 \t average norm of layer 5: 7.7231436\n",
      "unadjusted intra_sim of layer 6: 0.50441283 \t average norm of layer 6: 8.357803\n",
      "unadjusted intra_sim of layer 7: 0.4990974 \t average norm of layer 7: 8.325736\n",
      "unadjusted intra_sim of layer 8: 0.47839057 \t average norm of layer 8: 8.112216\n",
      "unadjusted intra_sim of layer 9: 0.49515322 \t average norm of layer 9: 7.813351\n",
      "unadjusted intra_sim of layer 10: 0.5039894 \t average norm of layer 10: 7.7475214\n",
      "unadjusted intra_sim of layer 11: 0.53178585 \t average norm of layer 11: 7.194023\n",
      "unadjusted intra_sim of layer 12: 0.8501157 \t average norm of layer 12: 3.3224683\n"
     ]
    }
   ],
   "source": [
    "for layer in range(13):\n",
    "    print('unadjusted intra_sim of layer %s:'%layer, np.mean(all_layer_intra_sim[layer]),'\\t', 'average norm of layer %s:'%layer, np.mean(all_layer_norm[layer]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8501157"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(all_layer_intra_sim[12])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "layer progress: 100%|██████████| 13/13 [00:01<00:00, 10.29it/s]\n"
     ]
    }
   ],
   "source": [
    "anisotropy_matrix, all_layer_anisotropy = anisotropy_baseline(sentences, total_hidden_states, encoded_input, n_samples = 1000, n_layers = 13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.031,\n",
       " 1: 0.041,\n",
       " 2: 0.06,\n",
       " 3: 0.075,\n",
       " 4: 0.076,\n",
       " 5: 0.096,\n",
       " 6: 0.107,\n",
       " 7: 0.104,\n",
       " 8: 0.089,\n",
       " 9: 0.114,\n",
       " 10: 0.1,\n",
       " 11: 0.109,\n",
       " 12: 0.016}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_layer_anisotropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "adjusted intra_sim of layer 0: 0.25583310747146604\n",
      "adjusted intra_sim of layer 1: 0.29497353100776674\n",
      "adjusted intra_sim of layer 2: 0.33852461218833924\n",
      "adjusted intra_sim of layer 3: 0.3540589094161987\n",
      "adjusted intra_sim of layer 4: 0.3711187591552734\n",
      "adjusted intra_sim of layer 5: 0.3838129498958588\n",
      "adjusted intra_sim of layer 6: 0.39741282987594606\n",
      "adjusted intra_sim of layer 7: 0.3950974068641663\n",
      "adjusted intra_sim of layer 8: 0.38939057445526126\n",
      "adjusted intra_sim of layer 9: 0.38115321850776673\n",
      "adjusted intra_sim of layer 10: 0.4039893984794617\n",
      "adjusted intra_sim of layer 11: 0.4227858457565308\n",
      "adjusted intra_sim of layer 12: 0.8341157164573669\n"
     ]
    }
   ],
   "source": [
    "unadjusted_intra_sim = np.array([np.mean(all_layer_intra_sim[layer]) for layer in range(13)])\n",
    "ani_baseline = np.array([ani for layer, ani in all_layer_anisotropy.items()])\n",
    "adjusted_intra_sim = unadjusted_intra_sim - ani_baseline\n",
    "\n",
    "for layer in range(13):\n",
    "    print('adjusted intra_sim of layer %s:'%layer, adjusted_intra_sim[layer])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 (nlp)",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}