examples/Gordo-Workflow-Semi-Low-Level.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a Gordo workflow locally:\n",
    "\n",
    "This demonstrates the basic workflow of gordo, running locally.\n",
    "\n",
    "---\n",
    "\n",
    "### Import and initialize a Gordo dataset\n",
    "In this case we shall be using the `DataLakeBackedDataset` where `InfluxBackedDataset` is also available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dateutil.parser\n",
    "import yaml\n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "from gordo_components.dataset.datasets import TimeSeriesDataset\n",
    "from gordo_components.data_provider.providers import DataLakeProvider\n",
    "from gordo_components import serializer\n",
    "\n",
    "\n",
    "# Parameters used by both DataLakeProvider and TimeSeriesDataset\n",
    "kwargs = dict(\n",
    "    from_ts=dateutil.parser.isoparse('2014-07-01T00:10:00+00:00'),\n",
    "    to_ts=dateutil.parser.isoparse('2015-01-01T00:00:00+00:00'),\n",
    "    tag_list=[\n",
    "        'asgb.19ZT3950%2FY%2FPRIM',\n",
    "        'asgb.19PST3925%2FDispMeasOut%2FPRIM'\n",
    "    ],\n",
    "    data_provider=DataLakeProvider(storename=\"dataplatformdlsprod\", interactive=True),\n",
    ")\n",
    "dataset = TimeSeriesDataset(**kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We'll need to login to Azure to authenticate the ability load data from the Data Lake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = dataset.get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asgb.19ZT3950%2FY%2FPRIM</th>\n",
       "      <th>asgb.19PST3925%2FDispMeasOut%2FPRIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2014-07-01 07:50:00+00:00</th>\n",
       "      <td>98.516167</td>\n",
       "      <td>85.29229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-07-01 08:00:00+00:00</th>\n",
       "      <td>98.516167</td>\n",
       "      <td>85.29229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-07-01 08:10:00+00:00</th>\n",
       "      <td>98.516167</td>\n",
       "      <td>85.29229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-07-01 08:20:00+00:00</th>\n",
       "      <td>98.516167</td>\n",
       "      <td>85.29229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-07-01 08:30:00+00:00</th>\n",
       "      <td>98.516167</td>\n",
       "      <td>85.29229</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           asgb.19ZT3950%2FY%2FPRIM  \\\n",
       "2014-07-01 07:50:00+00:00                 98.516167   \n",
       "2014-07-01 08:00:00+00:00                 98.516167   \n",
       "2014-07-01 08:10:00+00:00                 98.516167   \n",
       "2014-07-01 08:20:00+00:00                 98.516167   \n",
       "2014-07-01 08:30:00+00:00                 98.516167   \n",
       "\n",
       "                           asgb.19PST3925%2FDispMeasOut%2FPRIM  \n",
       "2014-07-01 07:50:00+00:00                             85.29229  \n",
       "2014-07-01 08:00:00+00:00                             85.29229  \n",
       "2014-07-01 08:10:00+00:00                             85.29229  \n",
       "2014-07-01 08:20:00+00:00                             85.29229  \n",
       "2014-07-01 08:30:00+00:00                             85.29229  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a pipeline for model building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('step_0', MinMaxScaler(copy=True, feature_range=(0, 1))), ('step_1', <gordo_components.model.models.KerasAutoEncoder object at 0x7f64bd6de7f0>)])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = yaml.load(\n",
    "    \"\"\" \n",
    "    sklearn.pipeline.Pipeline:\n",
    "        steps:\n",
    "          - sklearn.preprocessing.data.MinMaxScaler\n",
    "          - gordo_components.model.models.KerasAutoEncoder:\n",
    "              kind: feedforward_hourglass\n",
    "    \"\"\"\n",
    ")\n",
    "pipe = serializer.pipeline_from_definition(config)\n",
    "pipe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AutoEncoders were agreed to meet the specifications of a `Transformer`. Therefore, they do not implement a `predict` method.\n",
    "\n",
    "We shall then call `fit_transform` or `fit` -> `transform` if desired to treat datasets separately. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/private/milg/Projects/gordo-components/venv/lib/python3.6/site-packages/sklearn/preprocessing/data.py:323: DataConversionWarning: Data with input dtype float32 were all converted to float64 by MinMaxScaler.\n",
      "  return self.partial_fit(X, y)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "26411/26411 [==============================] - 3s 110us/step - loss: 0.0015 - acc: 0.9846\n"
     ]
    }
   ],
   "source": [
    "pipe.fit(X)\n",
    "xhat = pipe.predict(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### `xhat` is now the auto-encoded result*\n",
    "\n",
    "*where the first half of each resulting sample was the _input_ to the model and secondhalf is the _output_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.32019111, 0.29786357, 0.32511118, 0.30381536],\n",
       "       [0.32019111, 0.29786357, 0.32511118, 0.30381536],\n",
       "       [0.32019111, 0.29786357, 0.32511118, 0.30381536],\n",
       "       ...,\n",
       "       [0.32429197, 0.26762422, 0.32392147, 0.27555713],\n",
       "       [0.32429197, 0.26762422, 0.32392147, 0.27555713],\n",
       "       [0.32450209, 0.26762422, 0.32392147, 0.27555713]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}