train_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import config\n",
    "import models\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import multiprocessing\n",
    "from ctypes import c_float # for retrieving test results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_info = {} # log information about the model\n",
    "\n",
    "# main variables\n",
    "model_info['dataset_name'] = \"FB13\"\n",
    "embedding_model            = models.TransD\n",
    "\n",
    "# hyperparameters\n",
    "model_info['batch_size']    = 2048 # mini-batch size (takes precedence over n_batches)\n",
    "model_info['n_batches']     = None # number of batches\n",
    "model_info['n_epochs']      = 1000 # epochs\n",
    "model_info['learning_rate'] = 0.001\n",
    "model_info['margin']        = 1.0\n",
    "model_info['k']             = 100 # embedding dimension\n",
    "model_info['bern']          = 1 # use Bernoulli distribution for generating negative training examples\n",
    "model_info['opt_method']    = 'adam'\n",
    "model_info['score_norm']    = 'l2' # implemented in TransX models so far\n",
    "model_info['shuffle']       = 1 # shuffle training set (each epoch) instead of randomly sampling from it\n",
    "model_info['embedding_initialization'] = 'TransE/1524490825' # initialize parameters from another model\n",
    "model_info['regul_weight']  = None # lambda, the regularization weight in the loss function\n",
    "model_info['ent_neg_rate']  = 1\n",
    "model_info['rel_neg_rate']  = 0\n",
    "\n",
    "# test settings\n",
    "model_info['test_link_prediction'] = True\n",
    "model_info['test_triple_class']    = True\n",
    "\n",
    "# logging settings\n",
    "model_info['log_on']    = 1\n",
    "model_info['log_type']  = 'epoch'\n",
    "model_info['log_print'] = True\n",
    "\n",
    "# GPU settings\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"]= \"0\" # has to be a string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create config instance and set variables/hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "con = config.Config()\n",
    "dataset_path = \"./benchmarks/{}/\".format(model_info['dataset_name'])\n",
    "con.set_in_path(dataset_path)\n",
    "con.set_test_link_prediction(model_info['test_link_prediction'])\n",
    "con.set_test_triple_classification(model_info['test_triple_class'])\n",
    "con.set_log_on(model_info['log_on'], log_type=model_info['log_type'], log_print=model_info['log_print'])\n",
    "con.set_train_times(model_info['n_epochs'])\n",
    "con.set_batch_size(model_info['batch_size']) if model_info['batch_size'] != None else con.set_nbatches(model_info['n_batches'])\n",
    "con.set_alpha(model_info['learning_rate'])\n",
    "con.set_bern(model_info['bern'])\n",
    "con.set_dimension(model_info['k'])\n",
    "con.set_margin(model_info['margin'])\n",
    "if model_info['regul_weight'] != None: con.set_lmbda(model_info['regul_weight'])\n",
    "con.score_norm = model_info['score_norm']\n",
    "con.set_ent_neg_rate(model_info['ent_neg_rate'])\n",
    "con.set_rel_neg_rate(model_info['rel_neg_rate'])\n",
    "con.set_opt_method(model_info['opt_method'])\n",
    "con.shuffle = model_info['shuffle']\n",
    "model_info['work_threads'] = multiprocessing.cpu_count(); con.set_work_threads(model_info['work_threads'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get model name\n",
    "model_info['model_name'] = embedding_model.__name__\n",
    "# define export path\n",
    "export_path = \"./results/{}/{}/{}\".format(\n",
    "    model_info['dataset_name'],\n",
    "    model_info['model_name'],\n",
    "    int(time.time()))\n",
    "# create export_path dir\n",
    "def ensure_dir(file_path):\n",
    "    directory = os.path.dirname(file_path)\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "ensure_dir(export_path + '/') # without the \"/\" in the end the path wasn't being created\n",
    "print(\"Model will be exported to {}. \\n\".format(export_path))\n",
    "\n",
    "# export model parameters\n",
    "con.set_export_files(\"{}/tf_model/model.vec.tf\".format(export_path))\n",
    "# con.set_out_files(\"{}/embedding.vec.json\".format(export_path))\n",
    "\n",
    "con.init()\n",
    "con.set_model(model=embedding_model)\n",
    "if model_info['embedding_initialization'] != None: con.set_parameters_from_json(\"./results/{}/{}/embedding.vec.json\".format(model_info['dataset_name'], model_info['embedding_initialization']))\n",
    "con.run()\n",
    "\n",
    "# logging\n",
    "learning_curve = pd.DataFrame(con.log['training_curve'])\n",
    "learning_curve.to_csv(\"{}/learning_curve.tsv\".format(export_path), sep='\\t')\n",
    "model_info['learning_time'] = con.log['learning_time']\n",
    "print('\\nModel was trained in {} seconds'.format(model_info['learning_time']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot and save the learning curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = learning_curve\n",
    "if model_info['log_type'] == 'epoch':\n",
    "    n_epochs = data.epoch.max()\n",
    "    y_min = data.epoch_loss.min()\n",
    "    y_max = data.epoch_loss.iloc[model_info['n_epochs'] // 10]\n",
    "\n",
    "    fig, ax1 = plt.subplots()\n",
    "\n",
    "    ax1.set(title='Training Loss and Validation Accuracy',\n",
    "            xlabel='Epoch',\n",
    "            ylabel='Loss',\n",
    "            xlim=[0, model_info['n_epochs']],\n",
    "            ylim=[y_min, y_max],\n",
    "           )\n",
    "    ax1.plot(data.epoch, data.epoch_loss, label='Loss')\n",
    "    ax1.legend(loc=2)\n",
    "\n",
    "    ax2 = ax1.twinx()\n",
    "    ax2.plot(data.epoch, data.valid_acc, label='Valid Acc', color='orange')\n",
    "    ax2.set(ylabel='Accuracy')\n",
    "    ax2.legend(loc=1)\n",
    "\n",
    "    fig.set_size_inches(18, 12)\n",
    "    fig.savefig(\"{}/learning_curve.svg\".format(export_path))\n",
    "    \n",
    "elif model_info['log_type'] == 'batch':\n",
    "    fig, ax1 = plt.subplots()\n",
    "\n",
    "    ax1.set(title='Training Loss and Validation Accuracy',\n",
    "            xlabel='Batch (absolute)',\n",
    "            ylabel='Loss',\n",
    "           )\n",
    "    ax1.plot(data.batch_loss, label='Batch Loss')\n",
    "    ax1.legend(loc=2)\n",
    "\n",
    "    ax2 = ax1.twinx()\n",
    "    ax2.plot(data.valid_acc, label='Valid Acc', color='orange')\n",
    "    ax2.set(ylabel='Accuracy')\n",
    "    ax2.legend(loc=1)\n",
    "\n",
    "    fig.set_size_inches(18, 12)\n",
    "    fig.savefig(\"{}/learning_curve.svg\".format(export_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "con.test()\n",
    "\n",
    "if model_info['test_triple_class']: # triple classification\n",
    "    model_info['acc'] = c_float.in_dll(con.lib, 'aveAcc').value\n",
    "    print \"Average accuracy in the test set is {}\".format(model_info['acc'])\n",
    "    \n",
    "if model_info['test_link_prediction']: # link prediction\n",
    "    model_info['mrr_raw'] = c_float.in_dll(con.lib, 'mrr_raw').value\n",
    "    model_info['mr_raw'] = c_float.in_dll(con.lib, 'mr_raw').value\n",
    "    model_info['hits_10_raw'] = c_float.in_dll(con.lib, 'hits_10_raw').value\n",
    "    model_info['hits_3_raw'] = c_float.in_dll(con.lib, 'hits_3_raw').value\n",
    "    model_info['hits_1_raw'] = c_float.in_dll(con.lib, 'hits_1_raw').value\n",
    "    model_info['mrr_filter'] = c_float.in_dll(con.lib, 'mrr_filter').value\n",
    "    model_info['mr_filter'] = c_float.in_dll(con.lib, 'mr_filter').value\n",
    "    model_info['hits_10_filter'] = c_float.in_dll(con.lib, 'hits_10_filter').value\n",
    "    model_info['hits_3_filter'] = c_float.in_dll(con.lib, 'hits_3_filter').value\n",
    "    model_info['hits_1_filter'] = c_float.in_dll(con.lib, 'hits_1_filter').value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving logs and history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model_info DataFrame\n",
    "pd.DataFrame([model_info]).to_csv('{}/model_info.tsv'.format(export_path), sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%notebook history.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bashCommand = \"mv -t {}/ history.ipynb\".format(export_path)\n",
    "import subprocess\n",
    "process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)\n",
    "output, error = process.communicate()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}