success_rate_evalation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Success Rate Evaluation\n",
    "Evaluate success rate using weighted KNN (k-nearest neighbors)\n",
    "\n",
    "## Sections\n",
    "1. visualization of nearest neighbors\n",
    "2. analogy using vector arithmetic: if mit is not in massachusetts but in california, which school would it be?\n",
    "[mit] - [massachusetts] + [california] = ?\n",
    "3. evaluation of category success rate using KNN with successful & unsuccessful categories frequency as weights\n",
    "\n",
    "## Inputs\n",
    "1. Word2Vec model\n",
    "2. successful and unsuccessful company data\n",
    "\n",
    "## Outputs\n",
    "1. scatter plots\n",
    "2. analogy answer\n",
    "3. evaluation of entry success rate\n",
    "\n",
    "v1.0: Liren SONG, Oxford, Dec 17 2021"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<matplotlib.pyplot._IonContext at 0x105b82790>"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from gensim import models\n",
    "from sklearn.manifold import TSNE\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import string\n",
    "import collections\n",
    "import matplotlib.cm as cm\n",
    "# plot with webagg\n",
    "import matplotlib\n",
    "matplotlib.use('WebAgg')\n",
    "import matplotlib.pyplot as plt\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "[('harvard', 0.8769738078117371),\n ('cornell', 0.8724834322929382),\n ('cmu', 0.8669784069061279),\n ('yale', 0.8644719123840332),\n ('carnegie', 0.8591278791427612),\n ('iit', 0.8286830186843872),\n ('mit', 0.8273409008979797),\n ('tsinghua', 0.8253138661384583),\n ('purdue', 0.8185825943946838),\n ('ucsd', 0.8040269017219543)]"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load in model\n",
    "word2vec_path = 'liren_model_better.bin'\n",
    "model = models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)\n",
    "# perform simple test\n",
    "model.most_similar('stanford')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Section 1: visualization of nearest neighbors\n",
    "please note that all the plots are suppressed into two dimension from high dimension, visual clue can be misleading"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "# all the plotting functions\n",
    "def tsne_plot_similar_words(title, labels, embedding_clusters, word_clusters, a, filename=None):\n",
    "    plt.close('all')\n",
    "    colors = cm.rainbow(np.linspace(0, 1, len(labels)))\n",
    "    for label, embeddings, words, color in zip(labels, embedding_clusters, word_clusters, colors):\n",
    "        x = embeddings[:, 0]\n",
    "        y = embeddings[:, 1]\n",
    "        plt.scatter(x, y, c=color, alpha=a, label=label)\n",
    "        for i, word in enumerate(words):\n",
    "            plt.annotate(word, alpha=0.5, xy=(x[i], y[i]), xytext=(5, 2),\n",
    "                         textcoords='offset points', ha='right', va='bottom', size=8)\n",
    "    plt.legend(loc=4)\n",
    "    plt.title(title)\n",
    "    plt.grid(True)\n",
    "    if filename:\n",
    "        plt.savefig(filename, format='png', dpi=150, bbox_inches='tight')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def similar_words_plot(keys):\n",
    "    embedding_clusters = []\n",
    "    word_clusters = []\n",
    "    for word in keys:\n",
    "        embeddings = []\n",
    "        words = []\n",
    "        if word in model:\n",
    "            for similar_word, _ in model.most_similar(word, topn=30):\n",
    "                words.append(similar_word)\n",
    "                embeddings.append(model[similar_word])\n",
    "            embedding_clusters.append(embeddings)\n",
    "            word_clusters.append(words)\n",
    "\n",
    "    embedding_clusters = np.array(embedding_clusters)\n",
    "    n, m, k = embedding_clusters.shape\n",
    "    tsne_model_en_2d = TSNE(perplexity=15, n_components=2, init='pca', n_iter=3500, random_state=32)\n",
    "    embeddings_en_2d = np.array(tsne_model_en_2d.fit_transform(embedding_clusters.reshape(n * m, k))).reshape(n, m, 2)\n",
    "\n",
    "    tsne_plot_similar_words('Similar words plot', keys, embeddings_en_2d, word_clusters, 0.7)\n",
    "\n",
    "def words_scatterplot(plot_model, input_words, label=True):\n",
    "    plt.close('all')\n",
    "    plt.figure(figsize=(6,6))\n",
    "    word_vectors = np.array([model[w] for w in input_words if w in plot_model])\n",
    "    twodim = PCA().fit_transform(word_vectors)[:,:2]\n",
    "    plt.scatter(twodim[:,0], twodim[:,1], edgecolors='k', c='r')\n",
    "    if label:\n",
    "        for word, (x,y) in zip(input_words, twodim):\n",
    "            plt.text(x+0.05, y+0.05, word)\n",
    "    plt.show()\n",
    "\n",
    "def successful_unsuccessful_plot(plot_model, words_successful, words_unsuccessful, label=False):\n",
    "    plt.close('all')\n",
    "    plt.figure(figsize=(6, 6))\n",
    "    for words in words_successful, words_unsuccessful:\n",
    "        word_vectors = np.array([plot_model[w] for w in words if w in plot_model])\n",
    "        twodim = PCA().fit_transform(word_vectors)[:, :2]\n",
    "        if words == words_successful:\n",
    "            plt.scatter(twodim[:, 0], twodim[:, 1], edgecolors='k', c='r')\n",
    "            if label:\n",
    "                for word, (x, y) in zip(words, twodim):\n",
    "                    plt.text(x + 0.05, y + 0.05, f'{word}_successful')\n",
    "        elif words == words_unsuccessful:\n",
    "            plt.scatter(twodim[:, 0], twodim[:, 1], edgecolors='k', c='g')\n",
    "            if label:\n",
    "                for word, (x, y) in zip(words, twodim):\n",
    "                    plt.text(x + 0.05, y + 0.05, f'{word}_unsuccessful')\n",
    "    plt.show()\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/Caskroom/miniforge/base/envs/strategy/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning: The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "  warnings.warn(\n",
      "/opt/homebrew/Caskroom/miniforge/base/envs/strategy/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:982: FutureWarning: The PCA initialization in TSNE will change to have the standard deviation of PC1 equal to 1e-4 in 1.2. This will ensure better convergence.\n",
      "  warnings.warn(\n",
      "*c* argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with *x* & *y*.  Please use the *color* keyword-argument or provide a 2D array with a single row if you intend to specify the same RGB or RGBA value for all points.\n",
      "*c* argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with *x* & *y*.  Please use the *color* keyword-argument or provide a 2D array with a single row if you intend to specify the same RGB or RGBA value for all points.\n"
     ]
    }
   ],
   "source": [
    "similar_words_plot(['stanford', 'hardware'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "words_scatterplot(model,['stanford', 'hardware', 'mit', 'software'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Section 2: analogy using vector arithmetic"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "def analogy(x1, x2, x3):\n",
    "    \"\"\"\n",
    "    with the rule:\n",
    "    [x1] + [x2] - [x3] = result\n",
    "    example:\n",
    "    [king] + [women] - [men] = [queen]\n",
    "    \"\"\"\n",
    "    result = model.most_similar(positive=[x1, x2], negative=[x3])\n",
    "    return result[0][0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "'cmu'"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analogy('mit', 'california', 'massachusetts')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Section 3: evaluation of category success rate using KNN"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# load in successful and unsuccessful data\n",
    "df_successful = pd.read_csv('Moneyball_Successful_Companies.csv')\n",
    "df_unsuccessful = pd.read_csv('Moneyball_UnsuccessfulCompanies.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def clean_sentences(text):\n",
    "    \"\"\"Make text lowercase, remove punctuation and remove words containing numbers.\"\"\"\n",
    "    text = re.sub(r'[^\\w]', ' ', text)  # clear all things except underscore and alphanumeric\n",
    "    text = re.sub(\" \\d+\", \" \", text)  # clear all digits\n",
    "    text = text.lower()  # lower all text\n",
    "\n",
    "    # replace the word 'and'\n",
    "    # todo: this part can be further fine tuned\n",
    "    patterns = ['and']\n",
    "    for pattern in patterns:\n",
    "        text = re.sub(pattern, '', text)\n",
    "    return text\n",
    "\n",
    "def filter_list(df):\n",
    "    df = df[df['country_code'] == 'USA']\n",
    "    category_list = clean_sentences(str(list(df['category_list']))).split()\n",
    "    filtered_category_list = []\n",
    "    for word in category_list:\n",
    "        if len(word) > 1:\n",
    "            filtered_category_list += [word]\n",
    "    return filtered_category_list\n",
    "\n",
    "def generate_frequency_list(df_one, df_two):\n",
    "    idx = 1\n",
    "    for df in df_one, df_two:\n",
    "        filtered_category_list = filter_list(df)\n",
    "        counter = collections.Counter(filtered_category_list)\n",
    "        if idx == 1:\n",
    "            frequency_list_one = pd.DataFrame(counter.most_common())\n",
    "        else:\n",
    "            frequency_list_two = pd.DataFrame(counter.most_common())\n",
    "        idx += 1\n",
    "    return frequency_list_one, frequency_list_two\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# generate frequency(count) dataframe for both successful and unsuccessful categories\n",
    "df_successful_frequency, df_unsuccessful_frequency = generate_frequency_list(df_successful, df_unsuccessful)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_successful_frequency"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_unsuccessful_frequency"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "successful_unsuccessful_plot(model, list(df_successful_frequency[0]), list(df_unsuccessful_frequency[0]))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# initialize weight dataframe with keys\n",
    "keys = list(model.index_to_key)\n",
    "df_keys = pd.DataFrame({\"keys\": keys, 'successful_weights': 0, 'unsuccessful_weights': 0})\n",
    "df_keys"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# find frequency weights\n",
    "def find_weights(keys_dataframe, df_frequency):\n",
    "    key_index = []\n",
    "    weight_list = []\n",
    "    for i in range(0, len(df_frequency[0])):\n",
    "        index = keys_dataframe.index[keys_dataframe['keys'] == df_frequency.iloc[i, 0]].tolist()\n",
    "        key_index += index\n",
    "        # sometimes, depends on the training set, the category might not be in the keys set\n",
    "        # check if the list is empty\n",
    "        if len(index) == 1:\n",
    "            weight = [df_frequency.iloc[i, 1]]\n",
    "            weight_list += weight\n",
    "        elif len(index) == 0:\n",
    "            print(f'{df_frequency.iloc[i, 0]} not in key set')\n",
    "        else:\n",
    "            print(f'{df_frequency.iloc[i, 0]} is repeated in key set')\n",
    "    return key_index, weight_list\n",
    "\n",
    "# found how many keys are in the key set\n",
    "def key_in_set_ratio(key_index, df_frequency):\n",
    "    ratio = len(key_index)/len(df_frequency[0])\n",
    "    print(ratio)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "successful_key_index, successful_weight_list = find_weights(df_keys, df_successful_frequency)\n",
    "unsuccessful_key_index, unsuccessful_weight_list = find_weights(df_keys, df_unsuccessful_frequency)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# check key in set ratio\n",
    "[key_in_set_ratio(successful_key_index, df_successful_frequency), key_in_set_ratio(unsuccessful_key_index, df_unsuccessful_frequency)]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# assign weights to keys dataframe\n",
    "df_keys.iloc[successful_key_index, 1] = successful_weight_list\n",
    "df_keys.iloc[unsuccessful_key_index, 2] = unsuccessful_weight_list\n",
    "df_keys"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def catagory_evaluation(word):\n",
    "    \"\"\"\n",
    "    using knn, in this case k=(the number of successful & unsuccessful weights)\n",
    "    the more positive the result, the more likely the success is\n",
    "    you can enter any words, but the closer the word to the data you train, the most accurate the result\n",
    "    \"\"\"\n",
    "    try:\n",
    "        value_list = list(model.most_similar(word, topn=None))\n",
    "        # score is given by the relative closeness to successful minus closeness to unsuccessful\n",
    "        score = (sum(value_list * df_keys.successful_weights) / sum(df_keys.successful_weights)\n",
    "                 - sum(value_list * df_keys.unsuccessful_weights) / sum(df_keys.unsuccessful_weights))\n",
    "    except:\n",
    "        # if the entry is not in the key set, return 0\n",
    "        score = 0\n",
    "        print(f'{word} not in key set')\n",
    "    return score"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "[catagory_evaluation('saas'), catagory_evaluation('satellite'),catagory_evaluation('cow')]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "[catagory_evaluation('mit'), catagory_evaluation('stanford'), catagory_evaluation('caltech')]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "[catagory_evaluation('california'),catagory_evaluation('hawaii'), catagory_evaluation('newyork')]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def top_n_average_calculation(df_frequency, n):\n",
    "    rate_list = []\n",
    "    for catagory in df_frequency[0][0:n]:\n",
    "        rate = catagory_evaluation(catagory)\n",
    "        rate_list += [rate]\n",
    "    average = sum(rate_list) / len(rate_list)\n",
    "    return average\n",
    "\n",
    "def fact_check(df_frequency, top_entry_number):\n",
    "    \"\"\"check if the mean score of successful is bigger than that of unsuccessful\"\"\"\n",
    "    average = top_n_average_calculation(df_frequency, top_entry_number)\n",
    "    print(f\"top {top_entry_number} category mean score: {average}\")\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "fact_check(df_successful_frequency, 15)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "fact_check(df_unsuccessful_frequency, 15)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}