Performance_Experiments_v1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPp6YTTvsPnJFyEdBpUc6Q8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/malcolmlett/ml-learning/blob/main/Performance_Experiments_v1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Performance Experiments (v1)\n",
        "This notebook looks into performance considerations. How can we customise models and training algorithms without losing the performance benefits of TensorFlow, GPUs, and the computation graph?"
      ],
      "metadata": {
        "id": "K6q9h9D9ISGM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import time\n",
        "import timeit\n",
        "from datetime import datetime"
      ],
      "metadata": {
        "id": "EISZlD7WIc5G"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Environment specs\n",
        "This is how you can get a quick summary of the environment."
      ],
      "metadata": {
        "id": "0xylXH5URR6A"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from platform import python_version\n",
        "import torch\n",
        "import keras\n",
        "print(\"Python version\", python_version())\n",
        "print(\"Pytorch - version\", torch.__version__)\n",
        "print(\"Pytorch - cuDNN version :\", torch.backends.cudnn.version())\n",
        "print(\"TensorFlow version\", tf.__version__)\n",
        "print(\"Keras version\", keras.__version__)\n",
        "# Python version 3.10.12\n",
        "# Pytorch - version 2.3.0+cu121\n",
        "# Pytorch - cuDNN version : 8906\n",
        "# TensorFlow version 2.15.0\n",
        "# Keras version 2.15.0"
      ],
      "metadata": {
        "id": "jhjPDI1nRQyw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "80f7c186-ae77-42ec-ff3e-ea020cf3be86"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python version 3.10.12\n",
            "Pytorch - version 2.3.0+cu121\n",
            "Pytorch - cuDNN version : 8906\n",
            "TensorFlow version 2.15.0\n",
            "Keras version 2.15.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gpustat"
      ],
      "metadata": {
        "id": "xIt1Rqq0Rigz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "281c41c8-6558-4056-8135-8b502f022883"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting gpustat\n",
            "  Downloading gpustat-1.1.1.tar.gz (98 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/98.1 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.1/98.1 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting nvidia-ml-py>=11.450.129 (from gpustat)\n",
            "  Downloading nvidia_ml_py-12.555.43-py3-none-any.whl (39 kB)\n",
            "Requirement already satisfied: psutil>=5.6.0 in /usr/local/lib/python3.10/dist-packages (from gpustat) (5.9.5)\n",
            "Collecting blessed>=1.17.1 (from gpustat)\n",
            "  Downloading blessed-1.20.0-py2.py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.4/58.4 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wcwidth>=0.1.4 in /usr/local/lib/python3.10/dist-packages (from blessed>=1.17.1->gpustat) (0.2.13)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from blessed>=1.17.1->gpustat) (1.16.0)\n",
            "Building wheels for collected packages: gpustat\n",
            "  Building wheel for gpustat (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gpustat: filename=gpustat-1.1.1-py3-none-any.whl size=26532 sha256=45ebde3cfc22c8ba158b54f2f72f1f116a1651763f4ddcf668e28f1204363066\n",
            "  Stored in directory: /root/.cache/pip/wheels/ec/d7/80/a71ba3540900e1f276bcae685efd8e590c810d2108b95f1e47\n",
            "Successfully built gpustat\n",
            "Installing collected packages: nvidia-ml-py, blessed, gpustat\n",
            "Successfully installed blessed-1.20.0 gpustat-1.1.1 nvidia-ml-py-12.555.43\n",
            "\u001b[1m\u001b[37m17a9d7cefc84\u001b[m  Wed Jun 19 07:07:50 2024  \u001b[1m\u001b[30m535.104.05\u001b[m\n",
            "\u001b[36m[0]\u001b[m \u001b[34mTesla T4\u001b[m |\u001b[1m\u001b[31m 58°C\u001b[m, \u001b[32m  0 %\u001b[m | \u001b[36m\u001b[1m\u001b[33m  129\u001b[m / \u001b[33m15360\u001b[m MB |\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!gpustat"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1tbqVVzrUzrY",
        "outputId": "31360d98-80bd-4964-bcc6-1d02e9f4aa91"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m\u001b[37m17a9d7cefc84\u001b[m  Wed Jun 19 07:08:24 2024  \u001b[1m\u001b[30m535.104.05\u001b[m\n",
            "\u001b[36m[0]\u001b[m \u001b[34mTesla T4\u001b[m |\u001b[1m\u001b[31m 58°C\u001b[m, \u001b[32m  0 %\u001b[m | \u001b[36m\u001b[1m\u001b[33m  129\u001b[m / \u001b[33m15360\u001b[m MB |\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "You can also find out more in Google Colab by clicking on the down-arrow next to the connection info. For example, the following shows that we've got 12.7GB and 15.0GB available for System and GPU RAM, respectively:\n",
        "\n",
        "![Screenshot 2024-06-19 123301.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAbgAAADCCAYAAAA7BNegAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAD0aSURBVHhe7d0PeBNlvi/wr+Le+ODeePCaHlzJRdroUiIrBLWbPojhIk2Xta1CzerWslKK0KW7wvZxrfFP6K6G+qfiuu1ykFK1sVcNVWx75LYop1nkNqI2oCVwlJRHTHfhNB44ZHe55By7ue87M2knbZL+oYUy/X00NDOTTGbeeef9ve87k7yXhBkQQgghCnOp9JcQQghRFApwhBBCFIkCHCGEEEWiAEcIIUSRKMARQghRJApwhBBCFIkCHCGEEEWiAEcIIUSRKMARQghRJApwhBBCFIkCHCGEEEWiAEcIIUSRxkmA88CeXADnCWmSRAnUFyC5wImANE0ukJ4AvHu9CPRI06PlhBMFyXZ2Fow+Ie+Uj8Wa+wvAWZAM+6fSZBz+naWo2B2Ups6TLx2w1fkQkibJxDHCAMcDUjKS5Y95GSj6vXtoJ/+ndvaesTmhhyL0pRO2ZXOk7c7Fujpv4swf9MBRkou0G/m+piLNsg6O/ef5JD0Hgb1VKFoc2d9hHKcLQiwoC+pHFs6FAl2eLyOP0SjkT7pRuWY9GjqlaTIsIXbe/3yHARaTWprDAt5ue3TeLHfBL8ubA48nP/9scB6KnH9xKsdCpUGaf4MFOf71KKr3i8vIhHFOLbhC51EcPSo+Dr+3CemfF2DBBtf4rimdasEjS6uhWr9H3PYPSpD0ai5sLXEC1hl2Ai3LQ8O0EtS3HcbRIwdQv/pKOJblouqQ9JpxLLCDHZM17Uh/4X0cPtJ3nPKe8yi3Rmsqxz4pX/Y+Sg3SwnOgycbmg7tQeIM0TYYu5EHFr3zI/60F2kniLD8LXhnlQSyM5M1mljc7i5DxqxZEnY3y43l4Hzv/gOo7C+AYckVDBcMvS6DdWIGWU9IsMiGMWhelaqoe+aVW6OrehSso1sJz62Q1pk4HspKL8NIGVguzVLMZ1chlNTJ5l0bI1yi1rFgtbVU1vGekBUxwvwPrpFbXnGWsBdXRVzwLtbynWuB5uUBsZbFWmW1n7Npa8LN2dJqLsWq+VIucYkTmohCc+33idD/+HXZUq63YtN4IrVoFTFJBu6iMTYdQ8YYsmEutvDm8lhmrVTjY8q4W2b5Xwfn7BN2SPX64yqV9vTENBS97EIzVIguxFscTblgqNyN/tgYqVrBEjpPq5Uo0RGq9/dcnr0VL3WfOnTbkzmPLk+cg94kW+INeVK9KQyrbn9T0AlTLjoennLXAahply9nx+jKyfGCNu7cLVvisNJS6ANev06L2n9f0C9JTe/OGZ0QNaPGzHXvj57OEx0HeKhjKus54o465rSlxC8LXP417j2kIvnq2LLL/rAXT0iUtEkjLhfeK2+2OmXFC7NhkIHWVs3fdoQ75eWVD4zFx/lD2zx/ZXp5nft8Kf1SGjhbctRXVty9HzlRpBsub1RtY3rSVwRLJmxqWN599BqYmljfjBS+Vmp1/VljzPKje5ZVmDsFkEywP7If9zWG8h1z0xuganAYLl5jg2e3uLaAC+1vhXfIjPLCB1cKchWxOIepZjcx6s7icFWmwv3YaluoDOHqwCcU9dvz8NSkzsuBYkN+A6x8VW1171ifBkZMXXYOrqUBDShn2fHYY+yoMcBWz2lqMQlBtsqKpIpttoSjU5ULDbh3WmvXSHLkgDn3sgT6bBTdpToT+F3/E0d+aWN2Q6fHBsYK18nRW7GE1UbFVmIW816SgOehyL6pWFMFrqsG+wwew53E9vI2slI8pBPfG+2HHKrz/GVvXhzVI/4gFzB0xSrROD1xnLFhoFLayT0oONtWugkGYHYJrQwbsJ+9CXTtbX3sdMnktOqol7kBzwILNbayWvec3mLZjHbIyK6H6Ja95H0bTL4GKtazwk17Nud5oh+7xPWw5b/EGYV9qhztBASiYakHN0X0oN7FK+7P7cLTGIhynkNuO+8uBVe/xFvQ+1BjbkLtupNckE+SzYR0HLtG6/HAW56L1Ftb64K2ThlXAc+y4fRovEapR/clCbOJpvG8zDB8VYf2bYkAMttiQtUWFEr7/Rw+jfqkf6x5yIBIuAzuKxOUfsOPH02d+OwrW9C2P8NcXIW+3CXW/k1pRXU4U3deKdJ7W/Dg+yI5jvh2e3k1MsH+HqrDy137kvMqWfbYHZbO8aHGLiwYKwv1BC7KNBvF84SJ585Z+eXMKayUfbUJ+ijQ9ivRGM7qbPYhdlSVKNHoBLuRHy6uskDOlw8AaR5pFd8HsakarUNtlGdzlgtlsQl/ve396WEvzoZ/Cnk7WIZMFSL+nUyjEPNvtCKwuw9qbxXer5/MWVQD27bLrKnmsVrdIy2qCKmhMmTCjBQcTdGFE+vZTFzyC0z/fhpK5/U40QQin/8L25Yorpek49jthDxSj7EED1LzgYK1C6wslCDznhIfXlAdd3oDK44Ww/twAjUoF1XQTilezUj6WEw3YWmOClbUoxXWxWu/yfLhY6TKgwD9zmhVyKvCGZzQ1dPON0PG0ZuurrWOf92g2dPx1Kh0svBZdV9vXwkM+Vt2vZ9vGFk/Lxk/uDSG4ZDlrFbLjwdJbx9Lb2NUOr6xVZlxZCNN09oZJaujvL0HxVSxIDhrhYgmgYWs1TI+VwMi3l68vj302y1uxWymMqxRpvNXU+5C3GOPns2EdB0GidTlg+3eWJ+/Via2TaWbkr1TD8aG8GiBXiLLHTdDyNNYYUfJYPjxbGoVKg/o2K/Y1WMX9Z7Q8vVmFcb/wQT40v+qCRZ4+91vx4nyWcrIKHr/+tXIDUPaKFYbJ4jzP6zYEHrbCksKPkwpacz4KpzjQ2tv1nuCcbKxEaHVJbx7QLipEoVF4Uww+tDdpcf11srM/bt4cXJC1OmvrtDDfopPmDNE0HYwd7egcUeufXIzOKcBVW2SFSGoWKr7JR/2LYq0bU0z40RIXWj9huSnoxr/sNONHkW7BmDRQXSE9jeg5y/4JwPdlCKbZ0ZlZmzIXoS99YmHCXaHuqx0KQgj1dvEMpMmtEVqDh/c8gyv/kIF1O4eS68Wu1959lrquAl/5ELpdDx0POBHTdZh7xgcfe8GQlt+q6702kVCXj9WrWYs2tW87UgscUlqNAF/f7HTMkgpPwZQU3DjNBZ+sG4wX0hGqy9g/lyUumVSXXS49YybpYchgLYhvRlKy+OFjjShHAe+ek/Y5tYClAHA23vEdcA2uBpZI11jcfDbM4yAYZF0dNmREtpk9Msq8CCWI8VFprNOzSsNpsRXNklroDhS6KNm60ktZHojsfxD+DiP0Qu1EMkkH8/p8oaIpONkA+wPVUPNgNk2aJ51X3rKMvnRNzoCtQ37eJD4n56bI+zXY8Y6XJYIBdCMFmquk6ViEG8/60irqbsx+FZa01c3Q/u51WG9OnAcH0GihY+l1Wt6NTBRt1G4yOXr0AHZtLuw7qVgrwWRm7agPWMvC04bGJT+CSV6InqNQaHSqYappJhSuNKCRtYAGrlGNpGuBzkC3NK2BpUbc333PJqrZM6wkS7iFgy1PZFoJmnrTXXpI3XlRJl8JLSsigzEK1dCpYIIKQAhnT0pPRwUrNEfSeOulRck/99vfqKA1TmVtxoGobWaPEdzsEuBdwrwLsuGAuI62cgyS+6LtVyHnvc2Y9pIdzqhrd0B2pbRO2aPvssEoYa3hAX0gU7Ssfdjd18q82Sp9fj1ry/bTr8JyuK0eZVn9LxoQMtAYXYMTqW+/C9lN7+LpHc5BuicT0UB3gwqujuie8+6uTqhu0A0s1AcR2FmKLGtLX8svQq2KUQFVwXCrGf43XPAmag1ep4Pqj1745K854UfnZB10bAMHXZ6ih4o15boTfEYvVgs1dLXC03szABPvfSkGmCY7WSu6X3TpdCB33jo08ESYpoOpow2H5HeXneqC74wJut7a/rnyw/85C1NXR3IA255vpaeD0kA714/W/f2uKA0lrYZpWMdhEOqpWqh2t8ErT/pB1iuvcIR8XrinXSnkST9rwppWr4KR9xFzLO36VquGdrYbXp/sg4Tv63n6bvpYlAnDdDMeexywl0VuMGHvm65Cyyf9bnYa0r6L5+T+TvkxYS27qBXJqHjvSicC8krTdQZkTmvEu38894qqajI7P/+134cHAuwTtUiSV6oDfvjYfl8pddES5RvTAAe1ET/KakFjkxELb5GFt6k6GFmh1xUYWs3ecI8Vmi0VcHSIJ0Pw0yrYNmlgvWf4tWGNnhVib1Zg615xXfwmk+ptnugL4DLqJVaUf78S6x91wnuCbWwPaxGd8KL1QxZgrxYLIMy1wKqpRMXrXvFuxlMeVG2ogOZhCwy822mw5T/IRLFKXB5gCRI65kLlljg3N0zPwaosL+xPOeDlu9AThPf1AuRujvFdPpURhRuMcJbZ4OwICAVoiG27o9yO0IPF4h1tU3OwPI993ksusUAM+dDIts2VJ7vjbQRcWyrh6hLTy/emHfZThcgRbnbRQmdyw7lDvPNz4L6ygJbMYrDfz/IG3yAtclZmw7vRBof03Sd+DabAUhUdPEbDcI7DIFRGC4qvccBW7hK/cxjyw/VUFtbtHFC1klTD9pR4DEIBNyqedsCwOpu1clgevEoH184GePn5EhCPX9/9HDpkPmCC8+kKuHklRcgPjyB3k5fXz6Josstg7SmFbTsPTCoY7ymG9jUb7LvFbeLngi1nHVribaKMIbsYqsg5yY9xUxWq495kosO8LD+OfCUeP8EkPQpfKETnowWo2OsXexnOBOHf2wwPe33SkGvDLFCu1cHxtB2uY2KvROgYS7/fVkL1ixxE3V913A/37HlIGVlNm1yExjbAsdqS8Q4zYMrEQnlhOW0hlt/bhUfSUlEU94SXSclHjSMdbb9aIPTBL9h4BDlv14zsTqvpbF1v56N7k7iuOZZKYHUTnlkSJ9dP0sLyT3tQpm3F+h+nIvn6VKQtLUXrNSVo2mAWW6WTdMh/pQ7pH63HguuTkXyHHUeW1qPmfum64aDL9Vj7yovQNuYhLXUOFjzlhX5JvCv2aphf2IVnpjagIJ2t6/oFsPkyUf4zFrilV8hpc2uwq1SN1l8tRir77DlLbWi/pQZ1D0cCugqmDbtQ/G2leMv3vDy8e00Zdm2Q7hAdIeMSPbxlC9hnpiLr9SSUVZdId21qYPnNZuha8jCHLVvwcCuSbo/eV8NSK6a9novUeZWssOOVjBexa6MWDYVp4vHf4EPmxkLo423ggJtM2CPeVy7khnUcBsHX9XY9LCfsWMyP+bxc1F5lxWPmeH0OhSi8heUxdkxT04rgWVSDzfeK3XC6lduw+dpm5LHzZU5OBYJ3l8DC9ib0N2ExNHdvRtPqECruYJ/D88PnC1H3Sj6rGvTD8/JjZQha14t3IM9ai/q3LeguXyydC7VIeuwxxN1EOfbebc+yY/LAHCTftAD2rwwwx00qsRxodEd/91J1sxX1TguCL+Wy9GbbfuMCrHy5G5l1dcP6rqH+wXq2/0DlfWliHr+vAv6cOtT/MrrS6v2oEUmZBhY+yURxSZiRno8JT3kqKnV7UJM73M7ECYbX8mU3Gfhey0LG8bLR+YLyeca/B1ep23dxHnMFHYdxJeSBfXEldM4LdO005IYt/S2kf/AizKN4LwAZ38auBXcmiOCXTlS+bsJdiyi4JcRO/qqcgt4u2FCnE/bngii8I9Z388iYoeMwdlQGlLygg+OJvi+Znz8heDbZ4X+0hILbBDNmAc6zJQ1zljqQ9KwV2ZSpEmMnf+ELmfD9drHwyx9z8p3QbhzBbdDk3NBxGFO8S/IPd3vgdMmuxZ0PrKLdoN2Ezbl05+VEM+ZdlIQQQsiFMMY3mRBCCCEXBgU4QgghikQBjhBCiCJRgCOEEKJIFOAIIYQoEgU4QgghikQBjhBCiCKNUYATh7vvG2Ry7ISOeeA5Ntq/ukt69fjR8kQu5vDfcyyXDTBLCCHj3MgDXMCNqqIMseBLnoOMIv5r3tKy88jbWIC8KtfIx1YjCQVbKlD0kQGb24/i6MMX6PcYQwH4Pm2E44kiZKVXCD/ALBf60gmbJU349ZHkeRkoelkcqSAmnm9XSa+9MQ0F5a7on44644MzEtDZ8twnWgb+tFS/wTmFHypeXISqvX0/5yyOGJ8K297YlS9/Xa7wvqiBPWX473meU4XiPKbZkNZ1HtKMkP5GFuBCHlT8tADtP9yE9w+ygu/wHmzLDcH+4wI0nodWm5zhFwdw+FnpV/3JqAudYVWHRZkw8p9bk/0I8fkTgLNoMUrfOAJVEqvQ8CGL5E40omhpNbC6HgeOsLz4nhVJ23PxSKwR2nu8qOL5dlGd+Nr2OmR2FmHltshYgyG4n8tCFVbhfWm55cQ62XK5QtRHBuE8chjvv5CO9jULYHPJty8Ex5stMSpffrjeGcvW8HlMs+Gsa1ynGVGikQW4Q62o7szHqvv10PDBA1VqaBeVoNjkwluuvkEQQ75G2JbNEWplaauq4ZUPFR/0wFEi1ZTn5WJdXWQ8syBaHkpGQX1fzc67+XYkP+GSlrP1umxIznGAn2JRNd0TThQk29GyvxoFwvD+cwbUwIWh//mwMLwm+nsnqgqiP6uXtC7njnVIu7Gv1ujfbZfWLe6TR3YmhjqqUbSY769YO63eL1vYw07Q8gJhXQNrwQO7dIXabGSIl5jbEoKvXtoXYVuq4JbvxhlvVPrammSDU3a19Na4U9P7LZPhaZv2axfwcl/NWdiup5xofIi/397bMohOF9aal40czddTUNOIaqkVkJpegOqOYN+xiHGc+mhg2XoA9RUlsBgH/pZgwP0uXMZiFC/SQsUD8FQT8lm+bPFKBXCAFcA3ZqD6S/b8Kw/aJ7PX3qMTX6vSYaHZCN9HHjGdQ240v6ZD4QNmaKTlloJ8+N5zC3ktrkkqaGbnw/qwDo4dst6EJWaYd9aioX/PRgdLiwBbFnN4GZ4XkpH7MnsqpHtfGgf3O7BOOJ9Y/lq2Do6OfoGr17mlWaCpCKmLq8V9HiTNBk3/eEY1zQiJbWQB7rrrYUILmj+Ql6hqZFceRZ00hhXggv2107BUH8DRg00o7rHj5695xUU9PjhW5KFBZ8UeXuv7oARJr2Yh7zV+UqhhuM0E12eRgOeDp5kVwDvaekfV9h12QRt3XKdqVDTqUNZ6GIf3vQjDH4tQ0SKdPoeqsLLYi4Vb9uFw+x6UzfKiIeF4ltWo/vx/oe6DAyiZy8s/O+4vB1a9d5jVQPehxtiG3HVSEAqyoHGfE1rbHqGGusemhXPZI2gURsoOwbUhA/aTd6GOd/VJteCMDX1Be3DR2xLYUYSsLSqUfMBrw2xb5rejYI2D1XMZFkydxblovaUc+1j6Hm5YBTx3PwtQ/NN8qF6xDv6lYo38gMMC/6P3o6qDvzGaofQo9j1rAh6sF/bJerO0oKYa7XfUYU97CXinJa9wZJQHcZeDHesjB1C3yIeiTBtcsgqN64126B7fg8N8+X1B2B9YgPWfLMSmNrZ9+zbD8JHsOPXHC844NHfX4OjWbFak9wke90F/jdSmv8qIVb8rg5mPTs7HFWxYKw4yy53yornFA1PWQvH9p7pZ+hmhu05YKtLqYOzwx2hRDMG0n2D5z7xwuuSFPWslvlMJzerlWBjzN5wNsLK0rn+QPRXS3SqkMR+FvSC/Adc/KuWv9Ulw5OSJ47rFcg5ppvnhKmy2mcXx5AZJs0HTf7hGlGaExDayADclG7+ps8BnZTVyVhu3veyEh4/eHEUPa2k+9Lxra7IOmUtM8Hs6xWCw3wl7oBhlDxqg5ifOFCOsL5Qg8BxbDwtimrkLod/RDiEcnvCglQ8GuagF7i/4jAC8n3TDfEu8IUxYrbDUBC07EVQaE/tcoOUL8WTxNFbC/6AVa2/WQKVSsVZnMQpZ+R0fX1c2dFPVrHYaQMPWapgeK5G669TQ5y1HvqtZbDmdOY3uMwakG8QTWz2/GH+oXY5ZfOJEA2rrTCh+lK2Ln6C8ZfDsMzDVsZrqkLt05dviQ/OrLljk23K/FS/OZ6nDS2JW07f9uxXWe8Vat2qaGfkr1XB8yFM0iO5OHdtOaVlKDsoc5Ww9Qw+1yGPrztJBM4XvDEuXWgdMv7QiO4VNs5q57t4X8YzJgVrZYLbGlYUwTefLWQXmbguMp/SwrIwcJyMyFwGNnw9S6x8KFgjsr5tRnB0ZTFYDwyIjtLynISJyPWheFpyzarD5bnnxzLZHHhwuY9PS00RCrFXs2OaF6TaWp6V5OB6CfmkxgttaeitnONWC2tdMWL5keMNuerbbEVhdxvJuJH9ZsWl9APbto9Bt1z/NNAaY5rMWmTglGiTNevVfVwJjnWaEjPgmE41xLeo+PoCmZzOh/sKJggVzkGWVdzOxIHKF9DSi56zwJ/CVD6Hb9dDJC5LpOsw944OPl4kpBmRe5YKX1U6DnjZ0ZuZg1W0paP6EFYAhL7wuC9Jni28bSAV1vxIpJJTdAfi+DMGoG86QGfJ1+eFjrT1HAe+G491q7JFaAAdbcpbv89SF+Mm9XhTdlot1TznQ2HEaWiNrDfAA1OWDa3Y6ZvHnEVNScOM0F3yyrrzE5NsShL/DCL0QLSWTdDCvzwePr0L6drBWVWQ72SOjjLWIhXTQI+dxNSqXpqHgiWo49/px5Q+MMEzrl2iJXMGCrPRUTBc90mf1FlGMGil6LVy+vq5P1WWXS88YIWiwNbC/o4oVmKWsFWt49ZnE437dbBWvA7U3wXIoD1mbpZ6FYalGrix9U3Mq0H1/PV6UF/x/OY3Q7GwUairhdIsViMDud9Gy5EcwJdq+AcS8a5odXcBrU+Yi9KVPrDSO1Gim2aDrOp9pRsg5BDgBq43r5ltQUlGPAwdqYNyboJtpMKz07XunHsYlPrR+7ofHvR8WFgx5qw5/9CBwqB3OrHQYhlEejx4tSv6ZneT8RO99REYo1sBsb8KBhjLcNbMb/7IhC3Puq4Y3bsMohLMnpadjIWszDkRtJ3sIo1KroC+ow4EP61B8qwre1/KQllmKllG+OSh0tlt6dp6c8cC+Yh26V28b+vhtU/TI59fY3mCVKWkWF+qtpDHf8iMVi+yGCf5o34XNkR6JKFrkPGCSbpzwo/kNN/Lv5TdF+eH/WHrJCIVCIzzXIkYxzYa2rgufZmRiGVGA48P4Jxc3ygISozZiYUZfd2Aimut0UP3RC5+8IDnhR+dkHXRSZU7/Qwtcuyvw7k4zjN9nM1KMsPx7MyrfcUFnSOnr0hgyDXSzVeg8PtKCVwPtXD9a9/e7IUPah1CXB26+bJoeptwSvOhkAeSEHQ382tY0HUwdbTgkXI+TnOqC74wJOn5tSMCKUVaYDo0a2tlueH2yorcnAO9eD/xslnqqFqrdbdHBNZLWp3xw7/UhOEUHQ1Y+yra+j2e+78Rbstu1h0cLncmLtkPy3BBEVydrcQyrtXwO+DXHhwrgW7kLNYkGtdxfjayc6gG3zGOK1CKdksT2xg3fV8JckZ+l12ztCPJbH3XGT5DPb5yo5zdKFMNiFANASH7TVUIs796ggqsj+tzq7uqE6gYdWzoCo5Vm3FDXNQznnmaEjDDA6cyrYHJVwl7vRYBnuJ4Q/HsrUPGmDoWmIQzvP9cCq6YSFa97xe/LnPKgakMFNA9bei9mq2ayQLGzEY1L0qEX5rECObMTjrogMm8ZWV+8wVwM1ZYKODoCrPbLtnl3JaoT3mQix2qVK7Ph3WiDQyrMgx0OFFiqhECiCrphy7fBKS0L8bvPunRI4iXj1Bwsz3Oh8iWXEIAQ8qGR7a8rbzlyhNYfDxJuOHeI3x8KHWOv3ZJow3TIZDVc59MVcPOg2ROE9/VHkLuJ1alZOaAyWlB8jQO2chcCPH1DfrieysI64ZpYJ95a83NU7JYCGkv7g5+rkPTfe4urYdIgZ3k+XC+x/eHXYVle8O14EhWufCxfMqKid3h44bomA9UpNdjce4OTDAv8nt1u+Hk+naWHobMClW/6xFbaKS8cNQ7oIjcsqYzI/JkP1a+2iOkWFJcbLKwiwpePlLBeL2y/Znl8dbaUnxPTXGcEuroQYPmUb6vhHis0Qt6V8t6nVbBt0sB6zwi+mzhYmgU8cO31iy3XwdJssHWN1AjSjJD+RtZFOTUbNR+UQee24c5bk5F8/RzklvuwcGsdSuYOoaCcpEP+K3VI/2g9FlzP3n+HHUeW1qPmflkxMtWAhbOBbKOht6aov8UM1WSpRTcSs9Zi27NaNDyQhtR5C2A7pB/WbcfqJS9i10b2/sI04RrCgg0+ZG4shJ5vIFt33dZ5aHtogbBsTn4rdFu3ofAG/k4VTBt2ofjbSvG2+Hl5ePeaMuzaYJL2TQPLbzZD15KHOdenYsHDrUi6PfGGae7ejKbVIVTcwdOf7cvnC1H3Sj4LlcwkPda+XQ8La0Eu5uk7Lxe1V1nxmJkFnClmPPNOIbDlTvGLuXdUont1PcrMI2+jqExsX34JVFrmCHkhb2cSyt4rg0l+Y8dY2e9A6W4WVF/OFfen9yF97eKkG1sfsqGFX+tkhWbZeyydd/N05vu+Hm0/qEHdg5FKmQrGh5uwFlvFdEsvQCtbfu4FN1vv0mJ2bMws6A9tXVrTcli+egRpqUVo4V3Z/G5GRzrafiXmrwUbjyDn7Rrkp4ivH5ZB0izw0VYUlbVA6KsYLM0GS/8RG36aEdLfJWFGej4x8Fpob23QB0dOBvxPyG6BJ4QQogjndpPJRSa0vwpZaxzwCt16rNb5ph32U4XIjHtHJiGEkIvVBGvBsaBWb0fpC054TgCamy147LkyZE+XFhNCCFGMiddFSQghZEKYUF2UhBBCJg4KcIQQQhSJAhwhhBBFogBHCCFEkSjAEUIIUSQKcIQQQhSJAhwhhBBFogBHCCFEkSjAEUIIUSQKcIQQQhSJAhwhhBBFogBHCCFEkSjAEUIIUSQKcIQQQhSJAhwhhBBFogBHCCFEkSjAEUIIUSQKcIQQQhSJAhwhhBBFogBHCCFEkSjAEUIIUSQKcIQQQhSJAhwhhBBFogBHCCFEkSjAEUIIUSQKcIQQQhSJAhwhhBBFogBHCCFEkSjAEUIIUSQKcIQQQhSJAhwh5CIXgv9TD/xnpMkEAvUFSC73SFNE6cYuwAXcqCrKwJzkZCQnz0FGkR2uY9Kyc+ApZ+s7bxnUA7uw/X2POYuLYG/ysVOqPz8cy5Jx+++90rTMp3bhvbl1fmlGtNBeG1LZ8oL6gDRnAgl64CjJRdqNPH1TkWZZB8f+oLSQG3gMkudloOj3bgR6xFfEK7QS5RXhPfJ13piG3JJquGMcAvH4FKHxlDSjVwDOAv5eG9wDMwQj5onkZDvbCzIy/Y5/zOPkRePqPFR9KM83hIxVgAt5UPHTArT/cBPeP3gURw/vwbbcEOw/LkDjCek1F5FCJ9uHo+xx5DDefyEdvkez8MjOfifToUZUB7TAFmecwo6dqlsa4ZUK5T4heHY7YwTMCeAMK7yW5aFhWgnq2w6z9D2A+tVXsqCQi6pD0mskvceAPQ6/twnpnxdgwQbXuaWbqRz7Iutsq0ex2om8n1b1O0ZBuLY7kDTtEGp3xq6g4IwDtbtiFK7HXGjYLz0n5yRy/Plxshp8sC0sgLNLWggD1rYfRrlZLU0TIhqbAHeoFdWd+Vh1vx6ayWxapYZ2UQmKTS685fKzmnUqUsvcfYVTyAVb8u2o6pAmO6pRtHiOUGPjLaZqoUYv1uRyX2ZPX86NqhWHOhxYt0x6/TIbGntbivw9BahuqkZBeipbzloIq6rhDfrR8kSu2LqclwtbvIKrv0kqaGbno2RtEhrdnqjC1cuClGb1JpSYHGiOFeGMZphRCWf/ZadaUPuaCeYl0vQE4t9hR7Xaik3rjdCqVUL6aheVsekQKt6IH7xUU/XIL7VCV/cuXKNUaVeptTCtL0F2ZzM8X0kzuVMuvLs7H2XPWBBwuuCTZsuZl5jR8moDa69F8zZWI8CWGaVpcu74cTLklWPT2i7Ynmth1Q9OLBvsnwoTQI94fgu9ArzF90Qj/AMqlhyrXJZnIHWVM85ycrEbmwB33fUwoQXNH8j7EdTIrjyKuntZBr0jH9je2ldT7miDc5oFplnsebAFtvuc0Nr2CDW2PTYtnMseQeMpA6xsuv5B9poH69kyK6u3MV1OFN3XivRn9wktrCa2vCKfBb/e0tEFp0eHsg9YC6G9DstP2pG3cD3aTJuwj71+3+8McBdXoGUYBaVqMqspBkOyAO2Gc0sKLIsMMN+bD2d95MSTUS3E8tV6OFpkgZ3x76yF62fL8ZNp0owJI4hDH3ugz2bBTZoTof/FH3H0tyawkHd+qVS4km0XO7S9hONzfw5MxmwUXlaNFqkSJqfNXY78L5xwdUozOCFPaFB478Lzvx8TgD4jH7qmNtl53se3bSXWnbCgrp21+tg5bznxCO7/w8BLB/76IuTtNqHudxZoJ0kziaKMTYCbko3f1Fngs6YhNb0Atped8HTJcuLcHBRfxVo6UveN192CpHtM0PNMduY0us8YkG4QuxvU84vxh9rl4LEvFs/rNgQetsKSIrUAzPkonOJAa28XlxGFK03Q8pbkFAOy7zEiONOCwkVaqHiLbH4mC8aNaP9SfPVggoccsD/nQ7bZyEK2KPRJM5ymu5A5lZWRRguKP69FQ//rjb4A1EuWw1z3Floi13J6vGjcEkDx0r51TRwhnP4LoLniSml6GEKshv4qa4mb0iFlk3PXE4Cr3A5HCqtofV+ax9pkrnfY8fkxr0ppkfPALFS+E11B4brP6GFZHUT1zr5CNLirFg7TcuTopBlkdF3Bzl12fLoHXBdlaX/SB53RAB2vWah0yLHVoXy+Ouq4hT61Y+UGoOwVVlHmZQNRpDG7yURjXIu6jw+g6dlMqFnttmDBHGRZW8SugEl6mO5JQoubFwheuLYDFpNeeB+mLsRP7vWi6LZcrHvKgcaO09AajdBNERdHC8D3ZQjesgzxArTwyICtg7WuerscWC6/THrKXH4Zm2b/Xy5ND0W1JbLuZMy504GkDU14ZkmkZA2i5U0nLPeaxSAl7FsADXv7dVh1nUZoihnLf+bqu5azvwGVkFquE550w0bkOBY42Zw+8mOQnJqFim/yUf+ihRVy58BVirTIOq9PQ5HXjLr/vVasaHEdjahGIbJni5Pq+T+CaXvzgFZD8EyItUQLoem9/hpA844WmM2mCVhxufD02Vaoq7KQtsqG6no3/FcYYJzLKrTScpxsgP2Baqh5xXjC9ZxMLGMW4AST1NDNt6Ckoh4HDtTAuLcIFVJfoN5kAba74O30oDmqkNfAbG/CgYYy3DWzG/+yIQtz7mO19f7VZpnsygNCd6b8Yb1ZWjgKem9waN+MbFbb087S9Z0sJ5rx7s4QHMv5NT6xsMza5I9zQ4kKxqXFCAjLQnC/54B+dTYrUAPokndvTQhqJF0LdAa6pWkNLDViOu971iTN6yO/yeTo0QPYtbnw3FtvsptMmh5mTa1rboReFjE971XCv9+G2yNBcN46tMS7oWR6DpabpGXHWIv+43wsz2AbeNwPt/QSMor+FmLVCC2SYlR8VbMKWeV6H+pWz4Pqs63Iuy0DpS2y6tJ+FXLe24xpL9llN6oQJRqTAOd7LQvJxY3R16HURizMAFq+kC7TzzaxsObE1pdaEYx0TzKhLg/c+1kLZxprCeWW4EVnHYpP2NEQ49oHLyS101Vo+cQb3W00VheMWQus+GE1Kp7va134d7Nau3BNUF4A70LZFCdc/e4EFMw2o3BKJZw7GoSbS5YvEa9AnR2rbR63VDDcaob/DVbJOdd953nH7e13A0gQXazw0l8ztCio/xmrze+2o/JTKSeF3Gh43YjyNvlxPYrDNfloaXFF522BGuZctuzVBjj5zSWrLTDyWhCryCSom5ER8u5ywJeVDkNvTTMiCN9eN3xBVrm+ORv5v63B+xtT4Hyrta9HYFEmDNPNeOxxwF5GN5go2ZgEOJ15FUyuStjrvQjwL1+yk9y/twIVb+pQGOmKBAtg9wGNTW6YjZF5rNgLumHLt8F5SCxCQl950N6lQ5JUTmmuM4KXXIEQ74ZkLaJ7iqF9zQb7bjH7hrpcsOWwmra8f2sU6e61otBnQ6WLF1v8GpoX+XcIt7vI6FhLTR3zeg1fZrLo4fh1qXBziTlm1+vEoF5iRfn3K7H+USe8J1hKsXwSPOFF64edUF19ZV8reRAa00+Q3WmH/TUpv4UC8NazYOUyYZV5iBfBJptQvCEF1Rud8LECL7j7LThuzcTCqdJyicq4EPk7Y1xjZVTGTOR/YUPpJg0Ks/vyNBk9oSBrEf8+D3lV01D2sHRZoJ/O7QX4+Usu8XuSPUF4PjsUMz9psstg7SmFbfsQ76ImF52x6aKcmo2aD8qgc9tw5638+sYc5Jb7sHBrHUrm9mUz/XwLNJPzkTlXmsHNWou6rfPQ9tACoVtoTn4rdFu3ofAGcbHWtByWrx5BWmoRWk6Kr69/24Lu8sXi6y21SHrsMZjP6eJMAioDikvNcD5ZAU+7C86TFiyUrtHI8a5Z3WsNcMf4dQUtv9kEWuHmkqEW4oo0SQvLP+1BmbYV63+cyvJJKtKWlqL1mhI0bYhdeMXEWtYvflCDeR+tx2J+a3jqYqzfrYb1vRpk9wtQiWjuLkFJkFWWdhyFa2cjDIuMA6/xqYxYmOeBc3eMLwywZZbVrEXOjm/OdGkeGRWRa7Cp6bmo+GohNrfWxLl+xlrSG5tQ+G0l7ryJlz0LUHmqEPWx8hPPf4+VIWhdD8eEu0QwMVwSZqTn5x3/NYkFvmIcLu3fAiKEEELOzdjeZBJPKIjgCTe2bumUbsEmhBBCRtcFCXCBneswJ70I3nv+gMIY3XuEEELIubqgXZSEEELIWLkwXZSEEELIGKMARwghRJEowBFCCFGkMQtwoS+dsFnShIE8hQEqX/YgGO8XA3oCcP++CBnz+Hdd+KCX8iFvEuDD7NxohyfWekMB+D5thOOJImSlV/QOrRMx5O074UQBf02MR+qmOMNY9vjhKo/sjzjkj3139JdJYw24mVVUFXPATeUKwVdvQ67suLck+umkM35hWKKKkjxkFEf/VmV8PjhyCuCMNQ5hTwiBTg8a62woujMNFZHhVnr1+31M6dE7LEt/fJDfVfI81e8XdvoJ7ndgXSQPCgN5OuCJ+omUfoN9ssecxXmw1ccacFe54p+rA9On97GqcfD8caIRBTcObaDhkMuG1HiD6sb6/OQCNEqr9TexPC4M1yWWBVWJTnL5QNEJh/qRDFrWDMzDqelZUQMGKxq/yWTUHW8Ir9AvDj/5wdfhs9/y6dbwk3fMCK9pPC0uj3I23P7C4vDMwtrwwZNs8tvT4YOvrgjPvK0yfJC/N4HTjWvCC146KE3JdYffWnFTeNmvng+/9dKa8IwZT4fbpSWCYW1fDN8eDFfetixc+5U0Lfft1+G3CmeGFz/8Vvjg8bNs+my4+/Na9nkzww+917f+7u0rwjNWvMW2VHT29Nfh5scXhGesbQgPcSsuemc/fJId5yfDzcfZBEunI2+w437H1vARcXE/7eGn9beGVzxeG6797Z1RaZfQ55XhBXHSlB+Dm5Y+FH5+e2V4zYwZ4ac/kRb0OhKuzV4Qfr7tdPj0yb6HkGcG+Dpcu3RGeMW2g+HTfPnJtvDTd8wMP9Ic+2ie/eTp8GL9svDzH34dPs2ySfhvX4dbNywOz7hDnu/ZPsu3S8hLW8MrZiwIV34uzVO6YZ+rp8MNa2eGn/yQJ2oip8PNv5zJygZ2zLYPlpP4Ooee5rxcmrmhjZVsDM9/+hXht3zi9pz+kB33GWvCDbys6+9se/h5lmdWvCrlodMHw7WsLIldxjFDKmt4WSjbR/aa0181h5+8bRjl3UVsTAJc9zus8C5siCqAjrzKCqWNUWFGwg4qOyDPe6RJ7izLxDNYpuAFX1z8wN0ZrvVJk/1FCglWkPQPcMPbvoFOv/dQeGacQlMotPVPhtv6nV8808/Iru0tvPsHOEGMbVWus+HWx2eE73xVFs6+ZUFhxuDHNGbaxdG+kRV2rXEKu3iBpBefP1g+lPhqw3fOeDIs/6iv3/hpeMbjrWJBF0UMhste/1qalggVpxmy7Y21Xf0KLIUb9rnqrQwvHkLl+Gzrk+GbCp8PPz2UtDz+VniF7NxNiB/DO/qC4cC8eiS81RwrrzGe51nZ8XzU+c+3M15eH1pZEzu/tG+cMeTy7mI2Jl2UmrtrcHRrdtTPHAWP++L88K0BJQcPo0T+c12nuuGP80vhvfgvtp+2wJQiTfeXYADD4W1ff340bGvpGx6nH1+HC6G7Fw74EVh11mYcbchH3F9G5GOcNbRA94tMTIxfMQyi+0+A8QZZikyaBp3RC3+MMb4ECY5pTPwHk7dbkCn86nEMg63vhA8+tp0NT4jdY3xsw7jdS6f88Br14MMSRminsyP5p+6BP8wcPIT2/Xrk/LDfUK+T9Fi75yjKTPF/wC3Y0YDmj024a/5Y/Rbd+DK8czUE9/ZKqIUROqRZsYQ8qHjSi5JSS/zzUYb/oHrQYhrSa0NuJyrVfUMsaW5hZcHHzWjtFDuVg2y5sysb82KVW3NLcPhgiTiQsyT4jR+4NmlUy5pQVwsaduqw1qz8kub83GTS6YD9dTOKs4eUReCqsqHrYRa84p/n8O6shvq+zAGjQY/IMLYvtLcadlhROD/2xoVOswx5hXpovzEpH48s9XYUfZGPP/zcMLF+n/I70l/B5RjNnec/mOy8J0f8Vf8R0WJengE5v/xn7Gvfh/rVQGVeERzxrg+zz4kaZzBq32TOnGZBTwPVFdL0IKLGI8zZiqSKZ4b1G5uKkuhcPeZEBavQlOQmLhW8NaVw3VeG/JShjArJf1BdDcuioZQ0fjg3OWFZb+krl6bnY/OzalQslq7B5bmR88aLQ/uR9TMuVG7oQsl9sUe3H05Z4/p1Wm8eSl1QBO/9f0Cx7HeBlWrsAxyrLZSuccDw6jNDOKgheDfnYd2JMmx7MEHtoscL1xuzcNeiUajFDmv7+OCmDpgeyBlyYI2+CG2PvtlFNh7Z0cP70JTZhqziIVwcJ0MQhGvn/nP7KbipRqz9rRWW2Rqop2ig/9mLeGaJB05XjB9aPhf9bmTqf9ODfCy8w62/gerpPFTsn0i3mUgGOVe98mGK4mEB0vaGCWU/G2LrhbWSnD8QR+sfFB8gN1AMi3wDDlUh79dBlLx/WDh+B2qNaLhvXd+o/vGc8aLqgXXwb9iGtUMcEDlRWWN6dl9fHtrXhMyPslC0Q/klzdgGuDMe2FesQ/fqbbDePHhtwV9fhNx3DKj5HasBJehiELoBbv/JuQ81M8ztwyEHKl3SQJZxqK9hJ87xQG+3lKFUKpychdKcOFSsAM1l+7R7K5on0i+b/5f0V3CW13FGx7EG1PoLYR7Vn4JTY9o0VpAeHzganIBtO9uDPlH7JjMliVWQOhHgo2FwUy2oEQqffSgfONZrFNV0Myz3hVAVcygmBRvsXD3Vgq1bBhumKADn03ZMKy2GcbI0K6EQ3O9UwpQ7lJEtWOW3uhKaft2jnpZKhNaWwCL1Xavnl8B6dyNqW6Lvqo7Sw1qCD+WiwVCDzQlaoyMta1QaPSxsn1yvNvcbQ1F5xi7ACQepAL6Vu1AzSJcBF3TbsXKLDjVvW2FImPlC8OxugMV8jkPNDHP7hMzO+/cfLUxYQ9QZMqFteheuwWpoE544orf7S9kp1tMFn1sP7blWXBj/3gZgiNdN4vHVFSDrZXk9OITg3+IMojpFC73bC+lSi8B/zBv7+onKgHlLWP7bzZaTwQ3hXPXv3IqWe1ggSTRM0f5a2FwhNBbPkVo5aSh1Sd13BTG+dhLyoLkxwTVcOVah2ro7RvdoKAT15IHvD7L5MfUE4X5uJapTalD/cOLLFVTWDG5sAhzPkGsyhIO0+d4YGbInwIKUG35prLTQp3bkrvKh8BUrjINVlYIuvLV7VXQ3wHANtn0BD1x7/dE15BMsA8tG4I5rdiE2PdiJR1ZXwH0siFAP279Tfrg/YAVlSuyLxYIzfrg2VaAxJROG66R5iqaC0ZwP3zYHWvh31NiJ7X19KxxzIzcO8UFyXfCMqBeFXzfRDH6sBqGbOw+hlyrhjNwgsLcC9joDi5s8bPbbvhQTLHMdqK3zit/ROuFC9TYP8mNWxNTILi1Hypb1KI0MChwKItDRijafCkn/PU7e5t/b63Cgoqob2cYJcq12sHOVY4HIsSUQe3xF+bnMb+KQuunEh9hiFrrvaiwDxv7j13BdDwzS5Slgle43YneP6udLebxLykMdTtTu0CHzFqnqFVXWsPU8l4uCzkJse9gI9WA3QY2wrAkdc6FiUyN0mYZzqgBeFKS7KUeXcLv7jBgP6Zbr083hR/Q3hSuFrwaIt7HGfH2M21jjf/ctjli33g+yfaebHwnPNFRGvefgSwv6vtsyGP5dvu1PhpcZxe/ZzDAsDv/0V1vDbbI7dYXbh6M++6bw4jWVUa9RvrPhIzydDHz/Z4ZvLZTv/5Hw1jtmhtc0DkyQQb8mkOC7b7HF+5oA+6y2yvAK6TjedMea8FZPZK0xtq+7LVxZeGt4Jj+ehmXhJ7cfSZxfjrPXr1kcvkk4/mz/f7wm/HSj/D3idsnzyUzjENarJIOVJYxwW/zS2nC/L10IYp3LfRJ95WIY33072RBeMyPO92KZrxv7ygKehyplJ3nU9vGvIwzYT/ER82sF3KBlTYzylb1mzUtt4e5BvkqhBBfZaAJ+OJatB16oRz6NmExiCsFdlobmRftQFudOV0IGdcyB3F8Bm97OH507tckFQcPlEEIIUaTz8z04Qggh5DyjAEcIIUSRKMARQghRpDG5BhcMBvHNN9+gp2cijMcw/k2aNAlXX3011OrBvoNx/lAeGV/GYx7hKJ+ML+M1n8QzJgGOf8fke9/7Hi6/fCi/9UbG2tmzZ/HnP/9Z+HLreEF5ZHwZj3mEo3wyvozXfBLPmHRR8toWZcjxgx+L8VYDpjwyvozHPMJRPhlfxms+iYeuwRFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEuCTPS81Fz5MgRXH/99dIUGQ/G2zGhPDL+jMdjwrdpx7Hd0tTInAh+g6nqq6WpaN/87RSuvmKKNDXQkcBXuF5znTR18Th4/AvceM33palofw2dwXdVk6WpgbwnjkA/NX4+uHv6oovm3KUAN0FQgCODGa8BrrX7U2lqZP50+gSuvXKqNBWt+6/fIOm7sYMfd4gV9rMSFPbjVbu/A/O0s6WpaH/9z7/hu//tCmlqoP1dXsydppemBlqYdDMFOCq8xhcKcGQw4zXAUT4ZXy6mY0LX4AghhCgSBThCCCGKRAGOEEKIIlGAI4QQokgU4AghhCgSBThCCCGKRAGOEEKIIo3L78Gd+c//hy8DR6H57tX4tqcHPX//lj168C178L++b45h+lXX4u/hv4Nv/t+FXWB//86m2X8Hjx/B7O/dgEvYf2wXceml/Bn775JLhTmXXsr/ikt7n18izolMX8qnpXnyZdylwnr61iv/DOF9UdPsPewlvc/5+2WfISxhf8XnbEr6y8kPzX/1fAvVZf9Nmhq+8fbdFfp+0/gzHo8J5ZPx52I6JuclwPWwwPPvfzslPE6e+Q/2lz/4NP97UvjbN/+kELD+55Tv4a//eQaXXTIJl026DJNYcJjE/7Lg8C0r7Cd/53Ip4IiBg4eFSOAIfRtiwUAlBkAh8PF/I0FQfC4ugxAUhfnsM4VX8L+yZcKrhXnCM+l1fD3Sa9hr+fv58siyyOsjn89F1iX/fPl8/qqb/+cP8PGxz4T5bAVRwe5/XDFF+Fmh71wqpsd3Lr1M+HuZMP0dfE+dhH/7yze92yn+FbZMWNcrd24cV5mSCq7xhwIcGQoKcCwBNrRV9gas//h/f2EF9D+wx1Xi38n/gKvk06zw5n+vmjwFV7O/VyT4nbSJjB+qb1lrlrfm+F/euv0v3rJl01+f+hOu/YepA1qdPEbyv3/5t/+gAEcSogBHhoICHEuAk9/5KwtYYvDiwYxcWOMtU1LBNf6Mx2NC+WT8uZiOyZjdZJJ23VxcnzSDghshhJALYswCHCGEEHIhUYAjhBCiSBTgCCGEKBIFOEIIIYpEAY4QQogi0YjeE8R4OyZ8e/7Pn/+v8JxnQf69vVh/Ezl++t9wzZX/KE1F++avJ3H1d6+Spgb6ovsoZv5jijTVtw0Rkc+OzJMv77+Mi7V8LHz2p0O46dpZ0lS0v4XOCN8h5dsRaxs+//O/4gffmylNDbTk2vn0NQEyqAn/NYFJkybh7Nmz0hS50Pix4MdkPOHbM0Wl7i2I4/1NqC++DJRomYR/RuQRa5qLtTwiMh1v+fkWibcj2YarLr9y3OURjsqS8WU8liWJjEkLLhgM4ptvvkFPT480h1xIPENeffXVUKvV0pwLj/LI+DIe8whH+WR8Ga/5JJ4xCXCEEELIhUY3mRBCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUiQIcIYQQRaIARwghRJEowBFCCFEkCnCEEEIUCPj/ScJw3/ytMr4AAAAASUVORK5CYII=)"
      ],
      "metadata": {
        "id": "NFRE138ZToDB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Hardware Investigations\n",
        "It turns out that TensorFlow _doesn't_ store everything locally and then transfer to the GPU when it needs to do things. It actually stores the data directly on the GPU.\n",
        "See: https://www.tensorflow.org/tutorials/customization/basics#gpu_acceleration\n",
        "\n",
        "But it depends. And thankfully you can find out where a tensor is stored.\n"
      ],
      "metadata": {
        "id": "AQ4b8jPFIokk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# The output of this depends on what kind of Runtime you're connected to.\n",
        "# For example...\n",
        "# CPU:\n",
        "#  List of physical devices:\n",
        "#  [PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU')]\n",
        "#  Is the Tensor on GPU #0:  False\n",
        "#  Tensor device: /job:localhost/replica:0/task:0/device:CPU:0\n",
        "#\n",
        "# GPU:\n",
        "#  List of physical devices:\n",
        "#  [PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'),\n",
        "#   PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
        "#  Is the Tensor on GPU #0:  True\n",
        "#  Tensor device: /job:localhost/replica:0/task:0/device:GPU:0\n",
        "#\n",
        "# TPU:\n",
        "#  List of physical devices:\n",
        "#  [PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU_SYSTEM:0', device_type='TPU_SYSTEM'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU:0', device_type='TPU'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU:1', device_type='TPU'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU:2', device_type='TPU'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU:3', device_type='TPU'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU:4', device_type='TPU'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU:5', device_type='TPU'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU:6', device_type='TPU'),\n",
        "#   PhysicalDevice(name='/physical_device:TPU:7', device_type='TPU')]\n",
        "#  Is the Tensor on GPU #0:  False\n",
        "#  Tensor device: /job:localhost/replica:0/task:0/device:CPU:0\n",
        "\n",
        "x = tf.random.uniform([3, 3])\n",
        "\n",
        "print(\"List of physical devices: \"),\n",
        "print(tf.config.list_physical_devices())\n",
        "\n",
        "print()\n",
        "print(f\"Is the Tensor on GPU #0:  {x.device.endswith('GPU:0')}\"),\n",
        "print(f\"Tensor device: {x.device}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "79ycFJUTI7GG",
        "outputId": "f0212c88-f017-4f62-f99e-7ef0ce1ccde0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "List of physical devices: \n",
            "[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
            "\n",
            "Is the Tensor on GPU #0:  True\n",
            "Tensor device: /job:localhost/replica:0/task:0/device:GPU:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Even data that starts as a NumPy array and is passed to a tensor gets copied into the GPU\n",
        "ndarray = np.ones([3, 3])\n",
        "tensor = tf.math.multiply(ndarray, 42)\n",
        "print(tensor.device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jDo8q8vBLu7a",
        "outputId": "fd6eda18-156b-41ed-c307-feeb891772d4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/job:localhost/replica:0/task:0/device:GPU:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# However tensors from Datasets may not and you can't force them either\n",
        "ds = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n",
        "for row in ds:\n",
        "  print(row.device)\n",
        "\n",
        "print()\n",
        "print(\"Trying to force it\") # see below\n",
        "with tf.device(\"GPU:0\"):\n",
        "  ds = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n",
        "  for row in ds:\n",
        "    print(row.device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kAGXYkwtOCBN",
        "outputId": "6174a35c-67f3-47c8-a0d9-31dc9d7f2b42"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "\n",
            "Trying to force it\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n",
            "/job:localhost/replica:0/task:0/device:CPU:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# TF Variables also have a device property\n",
        "v = tf.Variable(tf.random.normal((1,2), dtype=tf.float64), name=\"v\")\n",
        "v.device"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "XJnQqLVEQvlj",
        "outputId": "04d1dac3-a215-48f4-86df-360aa0745886"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/job:localhost/replica:0/task:0/device:GPU:0'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "You can also explicitly control where tensors are created via the device manager. For example:"
      ],
      "metadata": {
        "id": "i8tEvWEZLNyg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Typical output\n",
        "#On CPU:\n",
        "#1000 loops: 2983.85ms (sometimes as much as 20s)\n",
        "#On GPU:\n",
        "#1000 loops: 315.55ms\n",
        "#On TPU:\n",
        "#1000 loops: 512.95ms (TPUs have more overhead and so need more data to become efficient)\n",
        "\n",
        "def time_matmul(x):\n",
        "  start = time.time()\n",
        "  for loop in range(1000):\n",
        "    tf.linalg.matmul(x, x)\n",
        "  result = time.time()-start\n",
        "  print(\"1000 loops: {:0.2f}ms\".format(1000*result))\n",
        "\n",
        "# Force execution on CPU\n",
        "print(\"On CPU:\")\n",
        "with tf.device(\"CPU:0\"):\n",
        "  x = tf.random.uniform([1000, 1000])\n",
        "  assert x.device.endswith(\"CPU:0\")\n",
        "  time_matmul(x)\n",
        "\n",
        "# Force execution on GPU #0 if available\n",
        "if tf.config.list_physical_devices(\"GPU\"):\n",
        "  print(\"On GPU:\")\n",
        "  with tf.device(\"GPU:0\"): # Or GPU:1 for the 2nd GPU, GPU:2 for the 3rd etc.\n",
        "    x = tf.random.uniform([1000, 1000])\n",
        "    assert x.device.endswith(\"GPU:0\")\n",
        "    time_matmul(x)\n",
        "\n",
        "# Force execution on TPU #0 if available\n",
        "if tf.config.list_physical_devices(\"TPU\"):\n",
        "  print(\"On TPU:\")\n",
        "  with tf.device(\"TPU:0\"):\n",
        "    x = tf.random.uniform([1000, 1000])\n",
        "    assert x.device.endswith(\"TPU:0\")\n",
        "    time_matmul(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2PONn8yALM2D",
        "outputId": "5580ae51-7652-4c6f-e1f9-a9d49c52f926"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "On CPU:\n",
            "1000 loops: 19039.99ms\n",
            "On GPU:\n",
            "1000 loops: 315.55ms\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "It looks like you can also get log information about where data is stored, but I haven't tried it yet:"
      ],
      "metadata": {
        "id": "bpqJlJ7wbUWe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tf.debugging.set_log_device_placement(True)"
      ],
      "metadata": {
        "id": "pG0GeOMUbd8T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Understanding Auto-diff\n",
        "Auto-diff helps with computations of gradients. It can also be used for any other scenarios when you need to calculate differentials. For example, it has a `jacobian()` method that computes full jacobian matrices, and can be used in so called \"physics informed\" networks.\n",
        "\n",
        "Docs:\n",
        "* https://www.tensorflow.org/guide/autodiff\n",
        "* https://www.tensorflow.org/guide/advanced_autodiff\n",
        "\n",
        "List of ops that have gradients registered:\n",
        "* https://www.tensorflow.org/api_docs/python/tf/raw_ops\n",
        "\n",
        "Some general points (from the autodiff article):\n",
        "* Avoid use of NumPy operations while within GradientTape\n",
        "* Be careful to avoid use of ints and strings, as these aren't differentiable\n",
        "* Be careful not to accidentally convert a tf.Variable to a tf.Tensor when doing things. For example, the following will result in `x` being a Tensor, and then it won't get watched anymore by the gradient tape:\n",
        "\n",
        "```python\n",
        "x = tf.Variable(2.0)\n",
        "with tf.GradientTape() as tape:\n",
        "  y = x+1\n",
        "  x = x + 1   # This should be `x.assign_add(1)`\n",
        "```\n",
        "* Apparently stateful objects, including tf.Variable, block gradients. I don't understand the implications of this yet, but here's an example:\n",
        "\n",
        "```python\n",
        "x0 = tf.Variable(3.0)\n",
        "x1 = tf.Variable(0.0)\n",
        "\n",
        "with tf.GradientTape() as tape:\n",
        "  # Update x1 = x1 + x0.\n",
        "  x1.assign_add(x0)\n",
        "  # The tape starts recording from x1.\n",
        "  y = x1**2   # y = (x1 + x0)**2\n",
        "\n",
        "# This doesn't work.\n",
        "print(tape.gradient(y, x0))   #dy/dx0 = 2*(x1 + x0)\n",
        "```"
      ],
      "metadata": {
        "id": "BJikcfN_XaHA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Understanding Graph Execution\n",
        "Graphs go well beyond auto-diff, and apply to the entire execution of a model and its training.\n",
        "\n",
        "Docs on computation graphs:\n",
        "* https://www.tensorflow.org/guide/intro_to_graphs\n",
        "* https://www.tensorflow.org/guide/function\n",
        "* https://www.tensorflow.org/api_docs/python/tf/function\n",
        "* https://www.tensorflow.org/guide/graph_optimization\n",
        "* https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/index.md\n",
        "\n",
        "General docs on performance optimization and performance troubleshooting:\n",
        "* https://www.tensorflow.org/guide/profiler\n",
        "* https://www.tensorflow.org/guide/gpu_performance_analysis\n",
        "* https://openxla.org/xla\n",
        "\n",
        "Code references:\n",
        "* https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\n",
        "\n",
        "General description:\n",
        "* The \"intro to graphs\" link above provides a really good explanation.\n",
        "* `tf.Graph` graphs are collections of `tf.Operation` operations that are stored as a data structure, in a program language agnostic way (ie: agnostic of Python). They can actually be run on systems that don't even have a Python interpreter, like mobiles.\n",
        "* tf operations are directly captured in the graph. Other python commands are converted via [`tf.AutoGraph`](https://www.tensorflow.org/api_docs/python/tf/autograph).\n",
        "* Relationship between `tf.Function` and `tf.AutoGraph`: `tf.Function` is the high-level API, and uses `tf.AutoGraph` internally. `tf.AutoGraph` is the low-level API, and does not perform some of the high-level functionality such as caching results, or wrapping the result to make it callable.\n",
        "* TensorFlow uses [Grappler](https://www.tensorflow.org/guide/graph_optimization) to optimize the execution graphs in the same way that a compiler tries to optimise code.\n",
        "* And of course this graph is easier for TensorFlow to automatically decide how to parallelise and distribute its computation.\n",
        "* `tf.function` applies to a function and all other functions it calls, including those not marked with `@tf.function`. This works as long as the operations within the called function can be mapped to graph operations.\n",
        "* Graphs are actually \"polymorphic\" - containing separately managed graphs for different kinds of inputs (eg: different shapes) to the original python function. These are then separately optimized too.\n",
        "\n",
        "Notes:\n",
        "* Use TensorFlow APIs like `tf.data`, `tf.print`, `tf.summary`, `tf.Variable.assign`, and `tf.TensorArray` for everything. Any other python/NumPy operations are useful only for debugging.\n",
        "* Construct tf.Variables _outside_ of tf.function, because they are created only during tracing (see [tf.function API docs](https://www.tensorflow.org/api_docs/python/tf/function#variables_may_only_be_created_once)).\n",
        "* Graph execution skips unnecessary operations, so you may not get runtime error checking unless you make it explicit with `tf.debugging`.\n",
        "* In general, only the most basic python control structures like `if`, `for`, `while`, `break`, `continue` and `return` are compiled into the graph reliably. For everything else, it is better to use tf datatypes.\n",
        "* You can supply Python datatypes (eg: lists, arrays) provided that they are treated only as constants - however they can cause re-tracing because even the value of a Python variable is taken as part of its definition. Any variables must be TF types, eg: `tf.TensorArray(..., dynamic_size=True)` for an appendable list. (see [tf.function API docs](https://www.tensorflow.org/api_docs/python/tf/function))\n",
        "* Avoid any references to python variables created outside of the function. Pass everything as arguments and return everything via `return`.\n",
        "* It's possible to customize handling of Python objects via `__tf_tracing_type__` (see [Rules of tracing](https://www.tensorflow.org/guide/function#rules_of_tracing)).\n",
        "* It's also possible to blend in eager-executed python functions via `tf.py_function`, but it's inefficient (see [Executing python side effects](https://www.tensorflow.org/guide/function#executing_python_side_effects)).\n",
        "* It's also possible to force that tf operations only run once during tracing time (as an initialisation), via `tf.init_scope` (see [Changing pypthon variables](https://www.tensorflow.org/guide/function#changing_python_global_and_free_variables)).\n",
        "\n",
        "Debugging notes:\n",
        "* Use python print to check for re-tracing.\n",
        "* Use `tf.print()` to print out intermediate values during execution.\n",
        "* `tf.debugging.enable_check_numerics` is an easy way to track down where NaNs and Inf are created.\n",
        "* Use `tf.config.run_functions_eagerly(True)` to force eager execution.\n",
        "\n",
        "Performance notes:\n",
        "* The initial trace can take some time, so graph-based execution can be slower on toy examples.\n",
        "* Use python `print` statements to test for re-tracing.\n",
        "* It's to be specific about the input signature via `input_signature`, and prefer shapes with `None` to handle variable sizes of input (see [Transformer](https://www.tensorflow.org/text/tutorials/transformer) and [Deep Dream](https://www.tensorflow.org/tutorials/generative/deepdream) tutorials for examples):\n",
        "\n",
        "```python\n",
        "@tf.function(input_signature=(tf.TensorSpec(shape=[None], dtype=tf.int32),))\n",
        "def my_func(x):\n",
        "  ...\n",
        "```\n",
        "* You can also pass `reduce_retracing` to automatically graph to more generic types, but this can lead to inefficiencies.\n",
        "\n",
        "* For faster performance, use `tf.function(jit_compile=True)` which compiles the graph via OpenXLA."
      ],
      "metadata": {
        "id": "dDteqLxMb3om"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Auto-Graph conversions\n",
        "See the AutoGraph reference docs on [Control Flow](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/control_flow.md) for details.\n",
        "\n",
        "`if`:\n",
        "* converted to `tf.cond`, but only if the expression evaluates to a Tensor, otherwise it is executed as Python and only during tracing.\n",
        "* If combined with a python parameter and re-tracing, then you might get the separate branch executions. But you're better off ensuring it's TF compatible.\n",
        "* `tf.cond` represents a split in the graph, with separate sub-graphs.\n",
        "\n",
        "loops in general:\n",
        "* Loops processed as python loops execute only during tracing, unrolling the loop as many iterations as occur.\n",
        "* Loops that are processed as TensorFlow loops (`tf.while_loop`) are represented in the graph as an explicit loop, and thus can handle dynamic iteration counts.\n",
        "\n",
        "`for`:\n",
        "* `for x in y: ...` is mapped to `tf.while_loop` if `y` is a tensor. If it's a Dataset, then it's turned into a series of special Dataset ops.\n",
        "\n",
        "`while`:\n",
        "* `while <condition>: ...` is mapped to `tf.while_loop` only if `<condition>` is a tensor.\n",
        "\n",
        "`iterators`:\n",
        "* Explicit iterator management (like use of `next(iter)`) won't get converted accurately unless the iterator came from a `tf` datatype (see: [Using python iterations and generators](https://www.tensorflow.org/guide/function#using_python_iterators_and_generators)).\n",
        "\n"
      ],
      "metadata": {
        "id": "d5Yw9Ys5LsNh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# A regular python function can be explicitly turned into a graph by passing it to tf.function():\n",
        "\n",
        "# normal python function (using tf ops)\n",
        "def a_regular_function(x, y, b):\n",
        "  x = tf.matmul(x, y)\n",
        "  x = x + b\n",
        "  return x\n",
        "\n",
        "# execute eagerly\n",
        "x1 = tf.constant([[1.0, 2.0]])\n",
        "y1 = tf.constant([[2.0], [3.0]])\n",
        "b1 = tf.constant(4.0)\n",
        "orig_value = a_regular_function(x1, y1, b1).numpy()\n",
        "\n",
        "# convert to PolymorphicFunction and run again\n",
        "a_function_that_uses_a_graph = tf.function(a_regular_function)\n",
        "tf_function_value = a_function_that_uses_a_graph(x1, y1, b1).numpy()\n",
        "\n",
        "assert(orig_value == tf_function_value)"
      ],
      "metadata": {
        "id": "drEeRmLOgpE3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Alternatively, it can be automatically converted to a graph with a decorator:\n",
        "@tf.function\n",
        "def a_second_function(x, y, b):\n",
        "  x = tf.matmul(x, y)\n",
        "  x = x + b\n",
        "  return x\n",
        "\n",
        "tf_function_value2 = a_second_function(x1, y1, b1).numpy()\n",
        "assert(orig_value == tf_function_value2)"
      ],
      "metadata": {
        "id": "jFQrGAlViQhd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "type(a_regular_function)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wrrL4VYciKBc",
        "outputId": "3c010a15-4508-4f30-eb43-f69a880350da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "function"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"{a_regular_function}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5a9rPcrWkmQ9",
        "outputId": "3d8172e5-36bb-4cef-f936-40d337a8f56e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<function a_regular_function at 0x7889bf3ce5f0>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"{a_function_that_uses_a_graph}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VBWyt-olkjiq",
        "outputId": "ca79e7b2-1ab5-4133-b9f3-4148ffcb19d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<tensorflow.python.eager.polymorphic_function.polymorphic_function.Function object at 0x7889c857e0e0>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "type(a_function_that_uses_a_graph)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "id": "u58Q3PGghSmP",
        "outputId": "ea0218c5-5d15-43eb-c416-a0a1b0591627"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensorflow.python.eager.polymorphic_function.polymorphic_function.Function"
            ],
            "text/html": [
              "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
              "      pre.function-repr-contents {\n",
              "        overflow-x: auto;\n",
              "        padding: 8px 12px;\n",
              "        max-height: 500px;\n",
              "      }\n",
              "\n",
              "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
              "        cursor: pointer;\n",
              "        max-height: 100px;\n",
              "      }\n",
              "    </style>\n",
              "    <pre style=\"white-space: initial; background:\n",
              "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
              "         border-bottom: 1px solid var(--colab-border-color);\"><b>tensorflow.python.eager.polymorphic_function.polymorphic_function.Function</b><br/>def error_handler(*args, **kwargs)</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py</a>A `tf.types.experimental.PolymorphicFunction` created by `tf.function`.\n",
              "\n",
              "Currently, individual methods/attributes under this class are not guaranteed\n",
              "by the TF API contract, and are subject to future changes.\n",
              "\n",
              "(Previously also known as `tf.types.experimental.GenericFunction`)</pre>\n",
              "      <script>\n",
              "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
              "        for (const element of document.querySelectorAll('.filepath')) {\n",
              "          element.style.display = 'block'\n",
              "          element.onclick = (event) => {\n",
              "            event.preventDefault();\n",
              "            event.stopPropagation();\n",
              "            google.colab.files.view(element.textContent, 451);\n",
              "          };\n",
              "        }\n",
              "      }\n",
              "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
              "        element.onclick = (event) => {\n",
              "          event.preventDefault();\n",
              "          event.stopPropagation();\n",
              "          element.classList.toggle('function-repr-contents-collapsed');\n",
              "        };\n",
              "      }\n",
              "      </script>\n",
              "      </div>"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "type(a_second_function)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "id": "sP739_Soig1w",
        "outputId": "ad2db854-7cb7-4ddc-8e3e-05998e5afd12"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensorflow.python.eager.polymorphic_function.polymorphic_function.Function"
            ],
            "text/html": [
              "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
              "      pre.function-repr-contents {\n",
              "        overflow-x: auto;\n",
              "        padding: 8px 12px;\n",
              "        max-height: 500px;\n",
              "      }\n",
              "\n",
              "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
              "        cursor: pointer;\n",
              "        max-height: 100px;\n",
              "      }\n",
              "    </style>\n",
              "    <pre style=\"white-space: initial; background:\n",
              "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
              "         border-bottom: 1px solid var(--colab-border-color);\"><b>tensorflow.python.eager.polymorphic_function.polymorphic_function.Function</b><br/>def error_handler(*args, **kwargs)</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py</a>A `tf.types.experimental.PolymorphicFunction` created by `tf.function`.\n",
              "\n",
              "Currently, individual methods/attributes under this class are not guaranteed\n",
              "by the TF API contract, and are subject to future changes.\n",
              "\n",
              "(Previously also known as `tf.types.experimental.GenericFunction`)</pre>\n",
              "      <script>\n",
              "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
              "        for (const element of document.querySelectorAll('.filepath')) {\n",
              "          element.style.display = 'block'\n",
              "          element.onclick = (event) => {\n",
              "            event.preventDefault();\n",
              "            event.stopPropagation();\n",
              "            google.colab.files.view(element.textContent, 451);\n",
              "          };\n",
              "        }\n",
              "      }\n",
              "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
              "        element.onclick = (event) => {\n",
              "          event.preventDefault();\n",
              "          event.stopPropagation();\n",
              "          element.classList.toggle('function-repr-contents-collapsed');\n",
              "        };\n",
              "      }\n",
              "      </script>\n",
              "      </div>"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# You can get some idea of the graphs by viewing the \"graph generating code\".\n",
        "# For this you must pass a straight python function (or obtain the original via .python_function)\n",
        "print(tf.autograph.to_code(a_regular_function))\n",
        "# OR\n",
        "# print(tf.autograph.to_code(a_function_that_uses_a_graph.python_function))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S3z-Sxq-iwW5",
        "outputId": "064da87e-0051-4fc0-eab5-a9e8ffe829fe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "def tf__a_regular_function(x, y, b):\n",
            "    with ag__.FunctionScope('a_regular_function', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n",
            "        do_return = False\n",
            "        retval_ = ag__.UndefinedReturnValue()\n",
            "        x = ag__.converted_call(ag__.ld(tf).matmul, (ag__.ld(x), ag__.ld(y)), None, fscope)\n",
            "        x = ag__.ld(x) + ag__.ld(b)\n",
            "        try:\n",
            "            do_return = True\n",
            "            retval_ = ag__.ld(x)\n",
            "        except:\n",
            "            do_return = False\n",
            "            raise\n",
            "        return fscope.ret(retval_, do_return)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# You can get a list of concrete functions that have been generated so far:\n",
        "x1 = tf.constant([[1.0, 2.0, 3.0]])\n",
        "y1 = tf.constant([[2.0], [3.0], [4.0]])\n",
        "b1 = tf.constant([4.0])\n",
        "a_second_function(x1, y1, b1)\n",
        "\n",
        "x1 = tf.constant([[1.0, 2.0, 3.0]])\n",
        "y1 = tf.constant([[2.0], [3.0], [4.0]])\n",
        "b1 = tf.constant([4.0, 5.0, 6.0])\n",
        "a_second_function(x1, y1, b1)\n",
        "\n",
        "print(a_second_function.pretty_printed_concrete_signatures())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kvjBrbJElOYw",
        "outputId": "b9ac4ca2-ba3a-4e5c-af3f-0ae5505f0650"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input Parameters:\n",
            "  x (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(1, 2), dtype=tf.float32, name=None)\n",
            "  y (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(2, 1), dtype=tf.float32, name=None)\n",
            "  b (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(), dtype=tf.float32, name=None)\n",
            "Output Type:\n",
            "  TensorSpec(shape=(1, 1), dtype=tf.float32, name=None)\n",
            "Captures:\n",
            "  None\n",
            "\n",
            "Input Parameters:\n",
            "  x (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(1, 3), dtype=tf.float32, name=None)\n",
            "  y (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(3, 1), dtype=tf.float32, name=None)\n",
            "  b (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(3,), dtype=tf.float32, name=None)\n",
            "Output Type:\n",
            "  TensorSpec(shape=(1, 3), dtype=tf.float32, name=None)\n",
            "Captures:\n",
            "  None\n",
            "\n",
            "Input Parameters:\n",
            "  x (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(1, 3), dtype=tf.float32, name=None)\n",
            "  y (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(3, 1), dtype=tf.float32, name=None)\n",
            "  b (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(1,), dtype=tf.float32, name=None)\n",
            "Output Type:\n",
            "  TensorSpec(shape=(1, 1), dtype=tf.float32, name=None)\n",
            "Captures:\n",
            "  None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# You can identify a specific concrete function by supplying inputs with the right shapes,\n",
        "# or by supplying a TypeSpec.\n",
        "# You can even execute that function.\n",
        "x1 = tf.constant([[1.0, 2.0, 3.0]])\n",
        "y1 = tf.constant([[2.0], [3.0], [4.0]])\n",
        "b1 = tf.constant([4.0, 5.0, 6.0])\n",
        "\n",
        "# The following two lines produce the same result\n",
        "concrete_fn = a_second_function.get_concrete_function(tf.TensorSpec(shape=[1,3], dtype=tf.float32),\n",
        "                                                      tf.TensorSpec(shape=[3,1], dtype=tf.float32),\n",
        "                                                      tf.TensorSpec(shape=[3,], dtype=tf.float32))\n",
        "concrete_fn = a_second_function.get_concrete_function(x1, y1, b1)\n",
        "print(f\"Concrete function: {concrete_fn}\")\n",
        "print(f\"result: {concrete_fn(x1, y1, b1)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pA4c97uRjaHZ",
        "outputId": "a9906c4a-e00c-4ba0-ff08-d95ba7292c4a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Concrete function: ConcreteFunction Input Parameters:\n",
            "  x (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(1, 3), dtype=tf.float32, name=None)\n",
            "  y (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(3, 1), dtype=tf.float32, name=None)\n",
            "  b (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(3,), dtype=tf.float32, name=None)\n",
            "Output Type:\n",
            "  TensorSpec(shape=(1, 3), dtype=tf.float32, name=None)\n",
            "Captures:\n",
            "  None\n",
            "result: [[24. 25. 26.]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The `function_type` attribute will print the same information more explicitly"
      ],
      "metadata": {
        "id": "KSmb1R1RJNMI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "a_second_function.function_type"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z7PgaKu2JCjN",
        "outputId": "d46d4fac-ca7b-4f0d-811c-c39bde058cd1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Input Parameters:\n",
              "  x (POSITIONAL_OR_KEYWORD): None\n",
              "  y (POSITIONAL_OR_KEYWORD): None\n",
              "  b (POSITIONAL_OR_KEYWORD): None\n",
              "Output Type:\n",
              "  None\n",
              "Captures:\n",
              "  None"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "a_second_function.get_concrete_function(x1, y1, b1).function_type"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m9gmhug9JIQS",
        "outputId": "47b497dd-be4f-40ef-91e7-ab5280c945d4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Input Parameters:\n",
              "  x (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(1, 3), dtype=tf.float32, name=None)\n",
              "  y (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(3, 1), dtype=tf.float32, name=None)\n",
              "  b (POSITIONAL_OR_KEYWORD): TensorSpec(shape=(3,), dtype=tf.float32, name=None)\n",
              "Output Type:\n",
              "  TensorSpec(shape=(1, 3), dtype=tf.float32, name=None)\n",
              "Captures:\n",
              "  None"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# And you can then get the actual graph from that concrete function\n",
        "a_second_function.get_concrete_function(x1, y1, b1).graph.as_graph_def()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YBnB6_1fktnq",
        "outputId": "154a82d9-e7df-49d2-d34f-0cd89ffe6bf5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "node {\n",
              "  name: \"x\"\n",
              "  op: \"Placeholder\"\n",
              "  attr {\n",
              "    key: \"_user_specified_name\"\n",
              "    value {\n",
              "      s: \"x\"\n",
              "    }\n",
              "  }\n",
              "  attr {\n",
              "    key: \"dtype\"\n",
              "    value {\n",
              "      type: DT_FLOAT\n",
              "    }\n",
              "  }\n",
              "  attr {\n",
              "    key: \"shape\"\n",
              "    value {\n",
              "      shape {\n",
              "        dim {\n",
              "          size: 1\n",
              "        }\n",
              "        dim {\n",
              "          size: 3\n",
              "        }\n",
              "      }\n",
              "    }\n",
              "  }\n",
              "}\n",
              "node {\n",
              "  name: \"y\"\n",
              "  op: \"Placeholder\"\n",
              "  attr {\n",
              "    key: \"_user_specified_name\"\n",
              "    value {\n",
              "      s: \"y\"\n",
              "    }\n",
              "  }\n",
              "  attr {\n",
              "    key: \"dtype\"\n",
              "    value {\n",
              "      type: DT_FLOAT\n",
              "    }\n",
              "  }\n",
              "  attr {\n",
              "    key: \"shape\"\n",
              "    value {\n",
              "      shape {\n",
              "        dim {\n",
              "          size: 3\n",
              "        }\n",
              "        dim {\n",
              "          size: 1\n",
              "        }\n",
              "      }\n",
              "    }\n",
              "  }\n",
              "}\n",
              "node {\n",
              "  name: \"b\"\n",
              "  op: \"Placeholder\"\n",
              "  attr {\n",
              "    key: \"_user_specified_name\"\n",
              "    value {\n",
              "      s: \"b\"\n",
              "    }\n",
              "  }\n",
              "  attr {\n",
              "    key: \"dtype\"\n",
              "    value {\n",
              "      type: DT_FLOAT\n",
              "    }\n",
              "  }\n",
              "  attr {\n",
              "    key: \"shape\"\n",
              "    value {\n",
              "      shape {\n",
              "        dim {\n",
              "          size: 3\n",
              "        }\n",
              "      }\n",
              "    }\n",
              "  }\n",
              "}\n",
              "node {\n",
              "  name: \"MatMul\"\n",
              "  op: \"MatMul\"\n",
              "  input: \"x\"\n",
              "  input: \"y\"\n",
              "  attr {\n",
              "    key: \"T\"\n",
              "    value {\n",
              "      type: DT_FLOAT\n",
              "    }\n",
              "  }\n",
              "  attr {\n",
              "    key: \"transpose_a\"\n",
              "    value {\n",
              "      b: false\n",
              "    }\n",
              "  }\n",
              "  attr {\n",
              "    key: \"transpose_b\"\n",
              "    value {\n",
              "      b: false\n",
              "    }\n",
              "  }\n",
              "}\n",
              "node {\n",
              "  name: \"add\"\n",
              "  op: \"AddV2\"\n",
              "  input: \"MatMul\"\n",
              "  input: \"b\"\n",
              "  attr {\n",
              "    key: \"T\"\n",
              "    value {\n",
              "      type: DT_FLOAT\n",
              "    }\n",
              "  }\n",
              "}\n",
              "node {\n",
              "  name: \"Identity\"\n",
              "  op: \"Identity\"\n",
              "  input: \"add\"\n",
              "  attr {\n",
              "    key: \"T\"\n",
              "    value {\n",
              "      type: DT_FLOAT\n",
              "    }\n",
              "  }\n",
              "}\n",
              "versions {\n",
              "  producer: 1645\n",
              "}"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Or for a more streamlined output:\n",
        "graph = a_second_function.get_concrete_function(x1, y1, b1).graph\n",
        "for node in graph.as_graph_def().node:\n",
        "  print(f'{node.input} -> {node.name}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gcbyQMOnKF-x",
        "outputId": "2fe1d77a-0108-43d5-88f1-dc26e0b76176"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[] -> x\n",
            "[] -> y\n",
            "[] -> b\n",
            "['x', 'y'] -> MatMul\n",
            "['MatMul', 'b'] -> add\n",
            "['add'] -> Identity\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "It's possible to force eager execution when you need to.\n",
        "For example:"
      ],
      "metadata": {
        "id": "SDD-z8rWmB9b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "@tf.function\n",
        "def get_MSE(y_true, y_pred):\n",
        "  print(\"Calculating MSE!\")\n",
        "  sq_diff = tf.pow(y_true - y_pred, 2)\n",
        "  return tf.reduce_mean(sq_diff)\n",
        "\n",
        "print(\"With default graph execution enabled (runs print only when first traced):\")\n",
        "y_true = tf.random.uniform([5], maxval=10, dtype=tf.int32)\n",
        "y_pred = tf.random.uniform([5], maxval=10, dtype=tf.int32)\n",
        "get_MSE(y_true, y_pred)\n",
        "get_MSE(y_true, y_pred)\n",
        "get_MSE(y_true, y_pred)\n",
        "\n",
        "print(\"With eager execution forced (calls print each time):\")\n",
        "tf.config.run_functions_eagerly(True)\n",
        "y_true = tf.random.uniform([5], maxval=10, dtype=tf.int32)\n",
        "y_pred = tf.random.uniform([5], maxval=10, dtype=tf.int32)\n",
        "get_MSE(y_true, y_pred)\n",
        "get_MSE(y_true, y_pred)\n",
        "get_MSE(y_true, y_pred)\n",
        "tf.config.run_functions_eagerly(False)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AzXmC02GmGNE",
        "outputId": "c9317d56-8ec7-435d-e23e-669d9a21e743"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "With eager execution enabled (runs print only when first traced):\n",
            "Calculating MSE!\n",
            "With eager execution forced (calls print each time):\n",
            "Calculating MSE!\n",
            "Calculating MSE!\n",
            "Calculating MSE!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Notice that python parameters get converted to literal values. This explains why python parameters such as num_epochs can lead to re-tracing:"
      ],
      "metadata": {
        "id": "GuCva5nHJt1I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "@tf.function\n",
        "def pow(a, b):\n",
        "  return a ** b\n",
        "\n",
        "square = pow.get_concrete_function(a=tf.TensorSpec(None, tf.float32), b=2)\n",
        "print(square)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LcmwZeVAJpBO",
        "outputId": "03ee64b1-4df7-4f4b-aac9-daabac50b19f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ConcreteFunction Input Parameters:\n",
            "  a (POSITIONAL_OR_KEYWORD): TensorSpec(shape=<unknown>, dtype=tf.float32, name=None)\n",
            "  b (POSITIONAL_OR_KEYWORD): Literal[2]\n",
            "Output Type:\n",
            "  TensorSpec(shape=<unknown>, dtype=tf.float32, name=None)\n",
            "Captures:\n",
            "  None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Doing the right thing"
      ],
      "metadata": {
        "id": "lHSQE--DOccE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# It can be useful to measure graph size.\n",
        "def print_graph_size(f, *args):\n",
        "  g = f.get_concrete_function(*args).graph\n",
        "  size = len(g.as_graph_def().node)\n",
        "  print(\"{}({}) contains {} nodes in its graph\".format(f.__name__, ', '.join(map(str, args)), size))"
      ],
      "metadata": {
        "id": "vZyyH8V2NBBh"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "For example, when writing loops, it can be useful to compare the graph size with different types of arguments. And to use that to avoid problems caused by using Python data types."
      ],
      "metadata": {
        "id": "oV8837OQORVo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "@tf.function\n",
        "def train(dataset):\n",
        "  loss = tf.constant(0)\n",
        "  for x, y in dataset:\n",
        "    loss += tf.abs(y - x) # Some dummy computation.\n",
        "  return loss\n",
        "\n",
        "small_data = [(1, 1)] * 3\n",
        "big_data = [(1, 1)] * 10\n",
        "\n",
        "print_graph_size(train, small_data)\n",
        "print_graph_size(train, big_data)\n",
        "print_graph_size(train, tf.data.Dataset.from_generator(lambda: small_data, (tf.int32, tf.int32)))\n",
        "print_graph_size(train, tf.data.Dataset.from_generator(lambda: big_data, (tf.int32, tf.int32)))"
      ],
      "metadata": {
        "id": "2jBQBGssOeNx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "71b34998-fd95-4328-d83d-8540f0a92844"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train([(1, 1), (1, 1), (1, 1)]) contains 11 nodes in its graph\n",
            "train([(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)]) contains 32 nodes in its graph\n",
            "train(<_FlatMapDataset element_spec=(TensorSpec(shape=<unknown>, dtype=tf.int32, name=None), TensorSpec(shape=<unknown>, dtype=tf.int32, name=None))>) contains 6 nodes in its graph\n",
            "train(<_FlatMapDataset element_spec=(TensorSpec(shape=<unknown>, dtype=tf.int32, name=None), TensorSpec(shape=<unknown>, dtype=tf.int32, name=None))>) contains 6 nodes in its graph\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "When accumulating data in a list data structure, use `tf.TensorArray`."
      ],
      "metadata": {
        "id": "OTf4y1YCOOXF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "@tf.function\n",
        "def train(rnn_step, input_data, initial_state):\n",
        "  max_seq_len = input_data.shape[0]\n",
        "\n",
        "  states = tf.TensorArray(tf.float32, size=max_seq_len)\n",
        "  state = initial_state\n",
        "  for i in tf.range(max_seq_len):\n",
        "    state = train_step(input_data[i], state)\n",
        "    states = states.write(i, state)\n",
        "  return states.stack()"
      ],
      "metadata": {
        "id": "qFTbSyWfOm-f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "CV6NpPXEPPHz"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data source Performance Concerns\n",
        "* When wrapping Python/NumPy data in a Dataset, be mindful of tf.data.Dataset.from_generator versus tf.data.Dataset.from_tensor_slices. The former will keep the data in Python and fetch it via tf.py_function which can have performance implications, whereas the latter will bundle a copy of the data as one large tf.constant() node in the graph, which can have memory implications.\n",
        "* Reading data from files via TFRecordDataset, CsvDataset, etc. is the most effective way to consume data, as then TensorFlow itself can manage the asynchronous loading and prefetching of data, without having to involve Python. To learn more, see the tf.data: Build TensorFlow input pipelines guide.\n",
        "\n",
        "For more details, see:\n",
        "* https://www.tensorflow.org/guide/data"
      ],
      "metadata": {
        "id": "bqfibPRRNn4Z"
      }
    }
  ]
}