notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Bidding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ../MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting ../MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting ../MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ../MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import types\n",
    "from utils import noisy_top_k_gating\n",
    "from utils import SparseDispatcher\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"../MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# FF NN with biases.\n",
    "def expert(i, x, hparams):\n",
    "    with tf.compat.v1.variable_scope(\"expert\"):\n",
    "        sizes = [hparams.n_inputs] + [hparams.e_hidden for _ in range(hparams.e_layers)] + [hparams.n_embedding]\n",
    "        for i in range(len(sizes) - 1):\n",
    "            w = tf.Variable(tf.truncated_normal([sizes[i], sizes[i+1]], stddev=0.1))\n",
    "            b = tf.Variable(tf.constant(0.1, shape=[sizes[i+1]]))\n",
    "            x = tf.matmul(x, w) + b\n",
    "    return x\n",
    "\n",
    "# Cross entropy loss + accuracy.\n",
    "def target_loss(embedding, targets, hparams):\n",
    "    with tf.compat.v1.variable_scope(\"target_loss\"):\n",
    "        w = tf.Variable(tf.truncated_normal([hparams.n_embedding, hparams.n_targets], stddev=0.1))\n",
    "        b = tf.Variable(tf.constant(0.1, shape=[hparams.n_targets])),\n",
    "        logits = tf.add(tf.matmul(embedding, w), b)\n",
    "        target_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=targets, logits=logits))\n",
    "        correct = tf.equal(tf.argmax(logits, 1), tf.argmax(targets, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "        return target_loss, accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Incentive function inputs weights, outputs revenue.\n",
    "# This the most basic, just takes the inloop weight as your score\n",
    "def incentive_fn(weights, hparams):\n",
    "    weights = tf.linalg.normalize(weights)\n",
    "    return tf.slice(weights, [0], [1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn(hparams):    \n",
    "    x_inputs = tf.placeholder(\"float\", [None, hparams.n_inputs], 'inputs')\n",
    "    y_targets = tf.placeholder(\"float\", [None, hparams.n_targets], 'targets')\n",
    "    \n",
    "    \n",
    "    # Sparsely gated mixture of experts with choice k. Produces an importance score \n",
    "    # for each x_input then chooses the topk. These children recieve the outgoing query.\n",
    "    # expert_inputs is a list of tensors, inputs for each expert.\n",
    "    gates, load = noisy_top_k_gating(x_inputs, hparams.n_experts, train = True, k = hparams.k)\n",
    "    dispatcher = SparseDispatcher(hparams.n_experts, gates)\n",
    "    expert_inputs = dispatcher.dispatch(x_inputs)\n",
    "    \n",
    "\n",
    "    # Basic importance scores can attained from the gating network by summing over the importance \n",
    "    # of each example. We choose a 'self-importance' score here which counts as the in loop in our\n",
    "    # incentive function. The network should try to maximize this value.\n",
    "    importance = tf.linalg.normalize(tf.reduce_sum(gates, 0))[0]\n",
    "    self_weight = tf.Variable(tf.constant([1.0]))\n",
    "    weights = tf.linalg.normalize(tf.concat([self_weight, importance], axis=0))[0]\n",
    "    revenue = tf.slice(weights, [0], [1])\n",
    "    \n",
    "    # Dispatch the inputs to the experts. We mask the responses with a faux-bidding system,\n",
    "    # here, we set a mask w.r.t the bids with a hparams.market_shift shifted relu. Bids that\n",
    "    # drop bellow the market shift should zero out.\n",
    "    expert_outputs = []\n",
    "    expert_masks = []\n",
    "    for i in range(hparams.n_experts):\n",
    "        expert_output = expert(i, expert_inputs[i], hparams)\n",
    "        \n",
    "        # Apply mask to the output.\n",
    "        expert_mask = tf.nn.relu(tf.slice(weights, [i], [1]) - hparams.market_shift)\n",
    "        masked_output = expert_mask * expert_output\n",
    "        \n",
    "        expert_masks.append(expert_mask)\n",
    "        expert_outputs.append(masked_output)\n",
    "    expert_masks = tf.concat(expert_masks, axis=0)\n",
    "\n",
    "    \n",
    "    # Combine the expert_inputs.\n",
    "    embedding = dispatcher.combine(expert_outputs)\n",
    "        \n",
    "    # Loss and accuracy stuff.\n",
    "    loss, accuracy = target_loss(embedding, y_targets, hparams)\n",
    "    \n",
    "    # Run the step: optimize for loss and revenue. \n",
    "    train_step = tf.train.AdamOptimizer(hparams.learning_rate).minimize(loss - revenue)\n",
    "    \n",
    "    metrics = {\n",
    "        'loss': loss,\n",
    "        'revenue': revenue,\n",
    "        'accuracy': accuracy,\n",
    "        'importance': importance,\n",
    "        'weights': weights,\n",
    "        'masks': expert_masks,\n",
    "    }\n",
    "    return train_step, metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:  1.8619063\n",
      "revenue:  [0.7074601]\n",
      "accuracy:  0.48046875\n",
      "importance:  [0.53903747 0.57394654 0.61646086]\n",
      "weights:  [0.7074601  0.3809665  0.4056386  0.43568575]\n",
      "masks:  [0.5074601  0.18096651 0.2056386 ]\n",
      "-\n",
      "loss:  0.23665716\n",
      "revenue:  [0.7398423]\n",
      "accuracy:  0.9375\n",
      "importance:  [0.6713292  0.72482896 0.15472582]\n",
      "weights:  [0.7398423  0.4516571  0.48765066 0.10409649]\n",
      "masks:  [0.5398423  0.25165707 0.28765064]\n",
      "-\n",
      "loss:  0.20587385\n",
      "revenue:  [0.765721]\n",
      "accuracy:  0.93359375\n",
      "importance:  [0.5428449  0.80560696 0.2373118 ]\n",
      "weights:  [0.765721   0.34914306 0.5181445  0.15263249]\n",
      "masks:  [0.56572104 0.14914306 0.3181445 ]\n",
      "-\n",
      "loss:  0.1442188\n",
      "revenue:  [0.78689605]\n",
      "accuracy:  0.96484375\n",
      "importance:  [0.6238978  0.73202115 0.27367246]\n",
      "weights:  [0.78689605 0.38499832 0.4517197  0.16887933]\n",
      "masks:  [0.58689606 0.18499832 0.2517197 ]\n",
      "-\n",
      "loss:  0.19727267\n",
      "revenue:  [0.80496234]\n",
      "accuracy:  0.9453125\n",
      "importance:  [0.5529173  0.75980455 0.3420225 ]\n",
      "weights:  [0.80496234 0.32806012 0.45081168 0.2029308 ]\n",
      "masks:  [0.60496235 0.12806012 0.2508117 ]\n",
      "-\n",
      "loss:  0.15476558\n",
      "revenue:  [0.8198568]\n",
      "accuracy:  0.9453125\n",
      "importance:  [0.6401928  0.6977797  0.32133597]\n",
      "weights:  [0.8198568  0.3665543  0.39952675 0.18398689]\n",
      "masks:  [0.61985683 0.16655429 0.19952674]\n",
      "-\n",
      "loss:  0.09529196\n",
      "revenue:  [0.83255327]\n",
      "accuracy:  0.95703125\n",
      "importance:  [0.61228925 0.7087006  0.350493  ]\n",
      "weights:  [0.83255327 0.3391745  0.39258108 0.19415382]\n",
      "masks:  [0.6325533  0.1391745  0.19258107]\n",
      "-\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-ea9066e7be42>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0mbatch_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhparams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0mfeeds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'inputs:0'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'targets:0'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_y\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m     \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mhparams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_print\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workspace/2019/GradientBidding/env/lib/python3.7/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    954\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    955\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 956\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    957\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    958\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workspace/2019/GradientBidding/env/lib/python3.7/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1163\u001b[0m     \u001b[0;31m# Create a fetch handler to take care of the structure of fetches.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1164\u001b[0m     fetch_handler = _FetchHandler(\n\u001b[0;32m-> 1165\u001b[0;31m         self._graph, fetches, feed_dict_tensor, feed_handles=feed_handles)\n\u001b[0m\u001b[1;32m   1166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1167\u001b[0m     \u001b[0;31m# Run request and get response.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workspace/2019/GradientBidding/env/lib/python3.7/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, graph, fetches, feeds, feed_handles)\u001b[0m\n\u001b[1;32m    471\u001b[0m         \u001b[0mdirect\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    472\u001b[0m     \"\"\"\n\u001b[0;32m--> 473\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    474\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_mapper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    110\u001b[0m         \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workspace/2019/GradientBidding/env/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py\u001b[0m in \u001b[0;36mget_controller\u001b[0;34m(self, default)\u001b[0m\n\u001b[1;32m   5477\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5478\u001b[0m       with super(_DefaultGraphStack,\n\u001b[0;32m-> 5479\u001b[0;31m                  self).get_controller(default) as g, context.graph_mode():\n\u001b[0m\u001b[1;32m   5480\u001b[0m         \u001b[0;32myield\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5481\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "hparams = types.SimpleNamespace( \n",
    "    n_inputs = 784,\n",
    "    n_targets = 10,\n",
    "    k = 3,\n",
    "    n_experts = 3,\n",
    "    e_layers = 2,\n",
    "    e_hidden = 256,\n",
    "    n_embedding = 256,\n",
    "    batch_size=256,\n",
    "    learning_rate=1e-3,\n",
    "    n_iterations = 10000,\n",
    "    n_print = 100,\n",
    "    market_shift = 0.2,\n",
    ")\n",
    "\n",
    "graph = tf.Graph()\n",
    "session = tf.Session(graph=graph)\n",
    "with graph.as_default():\n",
    "    train_step, metrics = model_fn(hparams)\n",
    "    session.run(tf.global_variables_initializer())\n",
    "\n",
    "for i in range(hparams.n_iterations):\n",
    "    batch_x, batch_y = mnist.train.next_batch(hparams.batch_size)\n",
    "    feeds = {'inputs:0': batch_x, 'targets:0': batch_y}\n",
    "    session.run(train_step, feeds)\n",
    "\n",
    "    if i % hparams.n_print == 0:\n",
    "        feeds = {'inputs:0': batch_x, 'targets:0': batch_y}\n",
    "        train_metrics = session.run(metrics, feeds)\n",
    "        for key in train_metrics:\n",
    "            print (str(key) + \":  \" + str(train_metrics[key]))\n",
    "        print ('-')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}