nn_classifier_features.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    },
    "colab": {
      "name": "nn_classifier_features.ipynb",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/intro-to-nlp/blob/master/nn_classifier_features.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oqnaDSDTUYd4"
      },
      "source": [
        "# What is the network learning?\n",
        "\n",
        "* We can gain some intuition in what the network is learning\n",
        "* Especially fruitful will turn out to look at the weights of the hidden layer\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_J3M2GVVZRy9",
        "outputId": "18f9fa87-f430-4c0c-f348-74ba437fdb45",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YAz55Z0vUYeL",
        "outputId": "07e2d025-553f-45ab-c4bb-0681fa937bfc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import json\n",
        "import pickle\n",
        "from keras.models import Model, model_from_json\n",
        "\n",
        "\n",
        "def load_model(model_name):\n",
        "\n",
        "    with open(model_name+\".model.json\", \"rt\") as f:\n",
        "        model=model_from_json(f.read())\n",
        "    model.load_weights(model_name+\".weights.h5\")\n",
        "    \n",
        "    with open(model_name+\".encoders.pickle\",\"rb\") as f:\n",
        "        label_encoder,vectorizer=pickle.load(f)\n",
        "    \n",
        "    return model,label_encoder,vectorizer\n",
        "\n",
        "#Model saved to drive at the end of the nn_bow_classifier notebook!\n",
        "model,label_encoder,vectorizer=load_model(\"/content/drive/MyDrive/intro-to-nlp/models-2022/imdb_bow\")\n",
        "print(model)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<keras.engine.functional.Functional object at 0x7fed4cbd0fd0>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7NwNMz6iUYeS",
        "outputId": "fc8a1a14-d348-4aac-e80b-f4c9fbe91172",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        }
      },
      "source": [
        "from IPython.display import SVG\n",
        "from keras.utils.vis_utils import model_to_dot\n",
        "\n",
        "SVG(model_to_dot(model,show_shapes=True,show_layer_names=False,dpi=65).create(prog='dot', format='svg'))\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<IPython.core.display.SVG object>"
            ],
            "image/svg+xml": "<svg height=\"200pt\" viewBox=\"0.00 0.00 257.00 221.00\" width=\"232pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g class=\"graph\" id=\"graph0\" transform=\"scale(.9028 .9028) rotate(0) translate(4 217)\">\n<title>G</title>\n<polygon fill=\"#ffffff\" points=\"-4,4 -4,-217 253,-217 253,4 -4,4\" stroke=\"transparent\"/>\n<!-- 140657171407184 -->\n<g class=\"node\" id=\"node1\">\n<title>140657171407184</title>\n<polygon fill=\"none\" points=\"0,-166.5 0,-212.5 249,-212.5 249,-166.5 0,-166.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"40\" y=\"-185.8\">InputLayer</text>\n<polyline fill=\"none\" points=\"80,-166.5 80,-212.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"109\" y=\"-197.3\">input:</text>\n<polyline fill=\"none\" points=\"80,-189.5 138,-189.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"109\" y=\"-174.3\">output:</text>\n<polyline fill=\"none\" points=\"138,-166.5 138,-212.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"193.5\" y=\"-197.3\">[(None, 68546)]</text>\n<polyline fill=\"none\" points=\"138,-189.5 249,-189.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"193.5\" y=\"-174.3\">[(None, 68546)]</text>\n</g>\n<!-- 140657171408400 -->\n<g class=\"node\" id=\"node2\">\n<title>140657171408400</title>\n<polygon fill=\"none\" points=\"18.5,-83.5 18.5,-129.5 230.5,-129.5 230.5,-83.5 18.5,-83.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"44.5\" y=\"-102.8\">Dense</text>\n<polyline fill=\"none\" points=\"70.5,-83.5 70.5,-129.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"99.5\" y=\"-114.3\">input:</text>\n<polyline fill=\"none\" points=\"70.5,-106.5 128.5,-106.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"99.5\" y=\"-91.3\">output:</text>\n<polyline fill=\"none\" points=\"128.5,-83.5 128.5,-129.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179.5\" y=\"-114.3\">(None, 68546)</text>\n<polyline fill=\"none\" points=\"128.5,-106.5 230.5,-106.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179.5\" y=\"-91.3\">(None, 200)</text>\n</g>\n<!-- 140657171407184&#45;&gt;140657171408400 -->\n<g class=\"edge\" id=\"edge1\">\n<title>140657171407184-&gt;140657171408400</title>\n<path d=\"M124.5,-166.3799C124.5,-158.1745 124.5,-148.7679 124.5,-139.8786\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"128.0001,-139.784 124.5,-129.784 121.0001,-139.784 128.0001,-139.784\" stroke=\"#000000\"/>\n</g>\n<!-- 140657171326544 -->\n<g class=\"node\" id=\"node3\">\n<title>140657171326544</title>\n<polygon fill=\"none\" points=\"26,-.5 26,-46.5 223,-46.5 223,-.5 26,-.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"52\" y=\"-19.8\">Dense</text>\n<polyline fill=\"none\" points=\"78,-.5 78,-46.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"107\" y=\"-31.3\">input:</text>\n<polyline fill=\"none\" points=\"78,-23.5 136,-23.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"107\" y=\"-8.3\">output:</text>\n<polyline fill=\"none\" points=\"136,-.5 136,-46.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179.5\" y=\"-31.3\">(None, 200)</text>\n<polyline fill=\"none\" points=\"136,-23.5 223,-23.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179.5\" y=\"-8.3\">(None, 2)</text>\n</g>\n<!-- 140657171408400&#45;&gt;140657171326544 -->\n<g class=\"edge\" id=\"edge2\">\n<title>140657171408400-&gt;140657171326544</title>\n<path d=\"M124.5,-83.3799C124.5,-75.1745 124.5,-65.7679 124.5,-56.8786\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"128.0001,-56.784 124.5,-46.784 121.0001,-56.784 128.0001,-56.784\" stroke=\"#000000\"/>\n</g>\n</g>\n</svg>"
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LJew8Ml2UYeT"
      },
      "source": [
        "* This is our model (watch out, Keras plots models from top to bottom)\n",
        "* Hidden layer has N nodes with 74849 inputs each, one input for one word in the vocabulary\n",
        "* We can also look at it the other way around: each word is assigned one weight for each hidden layer node\n",
        "* So each word is seen by the network as a N-dimensional vector\n",
        "* But what are these vectors? What properties should they have? Let us find out!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BAJDRxWqUYeT",
        "outputId": "eb01318e-3378-4313-de8a-61be133600e9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "learned_weights=model.layers[1].get_weights()[0]\n",
        "learned_weights.shape"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(68546, 200)"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yeX_vtF5UYeU"
      },
      "source": [
        "* A good and easy way to explore the vectors is through their neighborhood\n",
        "* This is easy to implement so let's try"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MKMo1-I7UYeV",
        "outputId": "30761458-10c6-464c-86a1-6e20f8147b2d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import numpy\n",
        "\n",
        "def nearest(word,learned_weights,vectorizer):\n",
        "    inverse_vocab=dict((v,k) for k,v in vectorizer.vocabulary_.items())\n",
        "    word_idx=vectorizer.vocabulary_[word]\n",
        "    word_vector=learned_weights[word_idx]\n",
        "    x=numpy.linalg.norm(word_vector-learned_weights,axis=-1)\n",
        "    nearest=numpy.argsort(x)\n",
        "    for idx in nearest[:30]:\n",
        "        print(inverse_vocab[idx], end=\", \")\n",
        "    print()\n",
        "    \n",
        "nearest(\"bad\",learned_weights,vectorizer)\n",
        "nearest(\"terrible\",learned_weights,vectorizer)\n",
        "\n",
        "nearest(\"great\",learned_weights,vectorizer)\n",
        "nearest(\"enjoyable\",learned_weights,vectorizer)\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "bad, horrible, uninspired, unconvincing, inexplicably, positive, insult, stunk, saxon, dangling, sadly, hoping, devoid, thinner, incoherent, boorish, stereotyped, components, mentions, soulless, horrid, unless, ilk, alleged, blah, wrestling, completist, signal, wrists, supposed, \n",
            "terrible, mst3k, lifeless, laughable, lousy, obnoxious, turkey, fails, lacks, stupidity, unwatchable, skip, unfunny, stinker, save, badly, mildly, junk, completists, dull, yawn, forwarding, pointless, baldwin, uwe, boredom, skate, orca, tripe, disappointing, \n",
            "great, noir, 07, funniest, perfectly, absorbing, enjoyed, excellently, fascinate, testimony, terrific, transcend, explores, liked, definitive, outstanding, connects, taut, swashbuckling, chong, advanced, defending, knockout, wonderful, chavez, mj, loved, baton, reveal, faceted, \n",
            "enjoyable, superb, pleasantly, enthralling, amazing, gem, natali, appreciated, delightfully, demme, favorite, captures, explores, perfect, wonderfully, escalates, philo, sublime, dunk, today, subtitles, rare, magnificently, tight, greene, warms, hooked, fun, demille, savant, \n"
          ]
        }
      ]
    }
  ]
}