tests/demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `read-protobuf` Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import demo_pb2\n",
    "\n",
    "from read_protobuf import read_protobuf "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a protobuf message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# container\n",
    "Collection = demo_pb2.Collection()\n",
    "\n",
    "# individual records\n",
    "Record = demo_pb2.Record()\n",
    "Record.int = 1234\n",
    "Record.float = 43.685\n",
    "Record.nested.data = 1.2\n",
    "Record.rep.extend([1])\n",
    "Record.rep.extend([2])\n",
    "\n",
    "RecordTwo = demo_pb2.Record()\n",
    "RecordTwo.int = 1253135\n",
    "RecordTwo.float = -73.2324\n",
    "\n",
    "# add both types of records\n",
    "for i in range(10_000):\n",
    "    if i % 5 == 0:\n",
    "        Collection.records.extend([RecordTwo])\n",
    "    else:\n",
    "        Collection.records.extend([Record])\n",
    "\n",
    "# serialize\n",
    "msg = Collection.SerializeToString()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpret a protobuf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>int</th>\n",
       "      <th>float</th>\n",
       "      <th>rep</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1253135</td>\n",
       "      <td>-73.232399</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1234</td>\n",
       "      <td>43.685001</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1234</td>\n",
       "      <td>43.685001</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1234</td>\n",
       "      <td>43.685001</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1234</td>\n",
       "      <td>43.685001</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       int      float     rep  data\n",
       "0  1253135 -73.232399     NaN   NaN\n",
       "1     1234  43.685001  [1, 2]   1.2\n",
       "2     1234  43.685001  [1, 2]   1.2\n",
       "3     1234  43.685001  [1, 2]   1.2\n",
       "4     1234  43.685001  [1, 2]   1.2"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CollectionType = demo_pb2.Collection()  # instantiate a new message type\n",
    "df = read_protobuf(msg, CollectionType) # create a dataframe from serialized protobuf bytes\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>int</th>\n",
       "      <th>float</th>\n",
       "      <th>rep</th>\n",
       "      <th>nested.data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1253135</td>\n",
       "      <td>-73.232399</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1234</td>\n",
       "      <td>43.685001</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1234</td>\n",
       "      <td>43.685001</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1234</td>\n",
       "      <td>43.685001</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1234</td>\n",
       "      <td>43.685001</td>\n",
       "      <td>[1, 2]</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       int      float     rep  nested.data\n",
       "0  1253135 -73.232399     NaN          NaN\n",
       "1     1234  43.685001  [1, 2]          1.2\n",
       "2     1234  43.685001  [1, 2]          1.2\n",
       "3     1234  43.685001  [1, 2]          1.2\n",
       "4     1234  43.685001  [1, 2]          1.2"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# options\n",
    "df = read_protobuf(msg, CollectionType, prefix_nested=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Benchmark\n",
    "\n",
    "`Collection` has 10,000 records"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### read_protobuf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 54.6 ms, sys: 3.17 ms, total: 57.7 ms\n",
      "Wall time: 56.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df = read_protobuf(msg, Collection) # create a dataframe from serialized protobuf bytes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### MessageToJson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.protobuf.json_format import MessageToJson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 148 ms, sys: 4.88 ms, total: 153 ms\n",
      "Wall time: 151 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "msg_json = MessageToJson(Collection)\n",
    "df = pd.read_json(msg_json)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### MessageToDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.protobuf.json_format import MessageToDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 99.3 ms, sys: 3.14 ms, total: 102 ms\n",
      "Wall time: 101 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "msg_dict = MessageToDict(Collection)\n",
    "df = pd.DataFrame(msg_dict)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}