char-rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:55:15.861823Z",
     "start_time": "2017-08-04T06:55:15.415373Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Lab 12 Character Sequence RNN\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "\n",
    "torch.manual_seed(777)  # reproducibility\n",
    "\n",
    "sample = \"hello\"\n",
    "idx2char = list(set(sample))  # index -> char\n",
    "char2idx = {c: i for i, c in enumerate(idx2char)}  # char -> index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:55:15.973097Z",
     "start_time": "2017-08-04T06:55:15.951750Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['o', 'h', 'e', 'l']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx2char"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T06:55:19.351320Z",
     "start_time": "2017-08-04T06:55:19.345286Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'e': 2, 'h': 1, 'l': 3, 'o': 0}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T07:29:35.114914Z",
     "start_time": "2017-08-04T07:29:35.086887Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "learning_rate = 0.1\n",
    "num_epochs = 50\n",
    "input_size = len(char2idx)  # RNN input size (one hot size)\n",
    "hidden_size = len(char2idx)  # RNN output size\n",
    "num_classes = len(char2idx)  # final output size (RNN or softmax, etc.)\n",
    "batch_size = 1  # one sample data, one batch\n",
    "sequence_length = len(sample) - 1  # number of lstm rollings (unit #)\n",
    "num_layers = 1  # number of layers in RNN\n",
    "\n",
    "sample_idx = [char2idx[c] for c in sample]  # char to index\n",
    "x_data = [sample_idx[:-1]]  # X data sample (0 ~ n-1) hello: hell\n",
    "y_data = [sample_idx[1:]]   # Y label sample (1 ~ n) hello: ello\n",
    "\n",
    "x_data = torch.Tensor(x_data)\n",
    "y_data = torch.LongTensor(y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T07:29:35.771555Z",
     "start_time": "2017-08-04T07:29:35.764973Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  1  2  3  3\n",
       " [torch.FloatTensor of size 1x4], \n",
       "  2  3  3  0\n",
       " [torch.LongTensor of size 1x4])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_data, y_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T07:29:46.247034Z",
     "start_time": "2017-08-04T07:29:46.234507Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def one_hot(x, num_classes):\n",
    "    idx = x.long()\n",
    "    idx = idx.view(-1, 1)\n",
    "    x_one_hot = torch.zeros(x.size()[0] * x.size()[1], num_classes)\n",
    "    x_one_hot.scatter_(1, idx, 1)\n",
    "    x_one_hot = x_one_hot.view(x.size()[0], x.size()[1], num_classes)\n",
    "    return x_one_hot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-04T07:30:02.954267Z",
     "start_time": "2017-08-04T07:30:02.943243Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(0 ,.,.) = \n",
      "  0  1  0  0\n",
      "  0  0  1  0\n",
      "  0  0  0  1\n",
      "  0  0  0  1\n",
      "[torch.FloatTensor of size 1x4x4]\n",
      "\n",
      "\n",
      " 2  3  3  0\n",
      "[torch.LongTensor of size 1x4]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_one_hot = one_hot(x_data, num_classes)\n",
    "print(x_one_hot)\n",
    "print(y_data)\n",
    "inputs = Variable(x_one_hot)\n",
    "labels = Variable(y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "\n",
    "    def __init__(self, num_classes, input_size, hidden_size, num_layers):\n",
    "        super(LSTM, self).__init__()\n",
    "        self.num_classes = num_classes\n",
    "        self.num_layers = num_layers\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.sequence_length = sequence_length\n",
    "        # Set parameters for RNN block\n",
    "        # Note: batch_first=False by default.\n",
    "        # When true, inputs are (batch_size, sequence_length, input_dimension)\n",
    "        # instead of (sequence_length, batch_size, input_dimension)\n",
    "        self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size,\n",
    "                            num_layers=num_layers, batch_first=True)\n",
    "        # Fully connected layer to obtain outputs corresponding to the number\n",
    "        # of classes\n",
    "        self.fc = nn.Linear(hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # Initialize hidden and cell states\n",
    "        h_0 = Variable(torch.zeros(\n",
    "            self.num_layers, x.size(0), self.hidden_size))\n",
    "        c_0 = Variable(torch.zeros(\n",
    "            self.num_layers, x.size(0), self.hidden_size))\n",
    "\n",
    "        # Reshape input\n",
    "        x.view(x.size(0), self.sequence_length, self.input_size)\n",
    "\n",
    "        # Propagate input through RNN\n",
    "        # Input: (batch, seq_len, input_size)\n",
    "        # h_0: (num_layers * num_directions, batch, hidden_size)\n",
    "        out, _ = self.lstm(x, (h_0, c_0))\n",
    "\n",
    "        # Reshape output from (batch, seq_len, hidden_size) to (batch *\n",
    "        # seq_len, hidden_size)\n",
    "        out = out.view(-1, self.hidden_size)\n",
    "        # Return outputs applied to fully connected layer\n",
    "        out = self.fc(out)\n",
    "        return out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# Instantiate RNN model\n",
    "lstm = LSTM(num_classes, input_size, hidden_size, num_layers)\n",
    "\n",
    "# Set loss and optimizer function\n",
    "criterion = torch.nn.CrossEntropyLoss()    # Softmax is internally computed.\n",
    "optimizer = torch.optim.Adam(lstm.parameters(), lr=learning_rate)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, loss: 1.239\n",
      "Predicted string:  eeee\n",
      "epoch: 2, loss: 1.112\n",
      "Predicted string:  llll\n",
      "epoch: 3, loss: 1.019\n",
      "Predicted string:  llll\n",
      "epoch: 4, loss: 0.924\n",
      "Predicted string:  llll\n",
      "epoch: 5, loss: 0.821\n",
      "Predicted string:  lllo\n",
      "epoch: 6, loss: 0.719\n",
      "Predicted string:  lllo\n",
      "epoch: 7, loss: 0.608\n",
      "Predicted string:  lllo\n",
      "epoch: 8, loss: 0.488\n",
      "Predicted string:  lllo\n",
      "epoch: 9, loss: 0.381\n",
      "Predicted string:  ello\n",
      "epoch: 10, loss: 0.287\n",
      "Predicted string:  ello\n",
      "epoch: 11, loss: 0.218\n",
      "Predicted string:  ello\n",
      "epoch: 12, loss: 0.175\n",
      "Predicted string:  ello\n",
      "epoch: 13, loss: 0.135\n",
      "Predicted string:  ello\n",
      "epoch: 14, loss: 0.109\n",
      "Predicted string:  ello\n",
      "epoch: 15, loss: 0.082\n",
      "Predicted string:  ello\n",
      "epoch: 16, loss: 0.067\n",
      "Predicted string:  ello\n",
      "epoch: 17, loss: 0.052\n",
      "Predicted string:  ello\n",
      "epoch: 18, loss: 0.041\n",
      "Predicted string:  ello\n",
      "epoch: 19, loss: 0.034\n",
      "Predicted string:  ello\n",
      "epoch: 20, loss: 0.026\n",
      "Predicted string:  ello\n",
      "epoch: 21, loss: 0.022\n",
      "Predicted string:  ello\n",
      "epoch: 22, loss: 0.018\n",
      "Predicted string:  ello\n",
      "epoch: 23, loss: 0.017\n",
      "Predicted string:  ello\n",
      "epoch: 24, loss: 0.014\n",
      "Predicted string:  ello\n",
      "epoch: 25, loss: 0.012\n",
      "Predicted string:  ello\n",
      "epoch: 26, loss: 0.011\n",
      "Predicted string:  ello\n",
      "epoch: 27, loss: 0.009\n",
      "Predicted string:  ello\n",
      "epoch: 28, loss: 0.008\n",
      "Predicted string:  ello\n",
      "epoch: 29, loss: 0.008\n",
      "Predicted string:  ello\n",
      "epoch: 30, loss: 0.007\n",
      "Predicted string:  ello\n",
      "epoch: 31, loss: 0.006\n",
      "Predicted string:  ello\n",
      "epoch: 32, loss: 0.005\n",
      "Predicted string:  ello\n",
      "epoch: 33, loss: 0.005\n",
      "Predicted string:  ello\n",
      "epoch: 34, loss: 0.005\n",
      "Predicted string:  ello\n",
      "epoch: 35, loss: 0.004\n",
      "Predicted string:  ello\n",
      "epoch: 36, loss: 0.004\n",
      "Predicted string:  ello\n",
      "epoch: 37, loss: 0.004\n",
      "Predicted string:  ello\n",
      "epoch: 38, loss: 0.004\n",
      "Predicted string:  ello\n",
      "epoch: 39, loss: 0.003\n",
      "Predicted string:  ello\n",
      "epoch: 40, loss: 0.003\n",
      "Predicted string:  ello\n",
      "epoch: 41, loss: 0.003\n",
      "Predicted string:  ello\n",
      "epoch: 42, loss: 0.003\n",
      "Predicted string:  ello\n",
      "epoch: 43, loss: 0.003\n",
      "Predicted string:  ello\n",
      "epoch: 44, loss: 0.003\n",
      "Predicted string:  ello\n",
      "epoch: 45, loss: 0.002\n",
      "Predicted string:  ello\n",
      "epoch: 46, loss: 0.002\n",
      "Predicted string:  ello\n",
      "epoch: 47, loss: 0.002\n",
      "Predicted string:  ello\n",
      "epoch: 48, loss: 0.002\n",
      "Predicted string:  ello\n",
      "epoch: 49, loss: 0.002\n",
      "Predicted string:  ello\n",
      "epoch: 50, loss: 0.002\n",
      "Predicted string:  ello\n",
      "Learning finished!\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "for epoch in range(num_epochs):\n",
    "    outputs = lstm(inputs)\n",
    "    optimizer.zero_grad()\n",
    "    loss = criterion(outputs, labels.view(-1))\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    _, idx = outputs.max(1)\n",
    "    idx = idx.data.numpy()\n",
    "    result_str = [idx2char[c] for c in idx.squeeze()]\n",
    "    print(\"epoch: %d, loss: %1.3f\" % (epoch + 1, loss.data[0]))\n",
    "    print(\"Predicted string: \", ''.join(result_str))\n",
    "\n",
    "print(\"Learning finished!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:lab_conda]",
   "language": "python",
   "name": "conda-env-lab_conda-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}