custom_dense_layer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "custom_dense_layer",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMvK9OINvzdpayiCA8Mm22g",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/armeh429/A-journey-for-learning-advanced-tensorflow/blob/main/custom_dense_layer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XeBBgbYZ8Urd"
      },
      "source": [
        "### imports\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HzqATEoQ7ur4"
      },
      "source": [
        "try: \n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass \n",
        "\n",
        "import tensorflow as tf\n",
        "import numpy as np\n"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1jEFBc5788RZ"
      },
      "source": [
        "###**Custom Layer with weights**\n",
        "\n",
        "To make custom layer that is trainable, we need to define a class that inherits the Layer base class from Keras. The Python syntax is shown below in the class declaration. This class requires three functions: __init__(), build() and call(). These ensure that our custom layer has a state and computation that can be accessed during training or inference."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i13zFC4z8jdb"
      },
      "source": [
        "# inherit from this base class\n",
        "from tensorflow.keras.layers import Layer\n",
        "\n",
        "class SimpleDense(Layer):\n",
        "\n",
        "    def __init__(self, units=32):\n",
        "        '''Initializes the instance attributes'''\n",
        "        super(SimpleDense, self).__init__()\n",
        "        self.units = units\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        '''Create the state of the layer (weights)'''\n",
        "        # initialize the weights\n",
        "        w_init = tf.random_normal_initializer()\n",
        "        self.w = tf.Variable(name=\"kernel\",\n",
        "            initial_value=w_init(shape=(input_shape[-1], self.units),\n",
        "                                 dtype='float32'),\n",
        "            trainable=True)\n",
        "\n",
        "        # initialize the biases\n",
        "        b_init = tf.zeros_initializer()\n",
        "        self.b = tf.Variable(name=\"bias\",\n",
        "            initial_value=b_init(shape=(self.units), dtype='float32'),\n",
        "            trainable=True)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        '''Defines the computation from inputs to outputs'''\n",
        "        return tf.matmul(inputs, self.w) + self.b"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0r_J74lsBTKG"
      },
      "source": [
        "now we can use our custom layer like below:\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q_5eyI79BQd0",
        "outputId": "54cceeb7-e21f-42fc-fdfb-b42696df516f"
      },
      "source": [
        "# declare an instance of the class\n",
        "my_dense = SimpleDense(units=1)\n",
        "\n",
        "# define an input and feed into the layer\n",
        "x = tf.ones((1, 1))\n",
        "y = my_dense(x)\n",
        "\n",
        "# parameters of the base Layer class like `variables` can be used\n",
        "print(my_dense.variables)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[<tf.Variable 'simple_dense/kernel:0' shape=(1, 1) dtype=float32, numpy=array([[0.00928995]], dtype=float32)>, <tf.Variable 'simple_dense/bias:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ln1tiXYcDOry"
      },
      "source": [
        "lets using in in simple network "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UrTK80xtCJ0s",
        "outputId": "f62ee443-9357-4c8b-dc59-b62e60e7c982"
      },
      "source": [
        "#define the dataset \n",
        "xs = np.array([-1.0 , 0.0 , 1.0 , 2.0 , 3.0 , 4.0] , dtype=float)\n",
        "ys = np.array([-3.0 , -1.0 , 1.0 , 3.0 , 5.0 , 7.0] , dtype=float)\n",
        "\n",
        "\n",
        "# use the sequential API  to build a model with our custom layer \n",
        "my_layer = SimpleDense(units=1)\n",
        "model = tf.keras.Sequential([my_layer])\n",
        "\n",
        "# configuration and train the model \n",
        "model.compile(optimizer = 'sgd', loss = 'mean_squared_error')\n",
        "model.fit(xs, ys, epochs = 500 , verbose = 0)\n",
        "\n",
        "#perform inference \n",
        "print(model.predict([10.0]))\n",
        "\n",
        "# see the updated state of the variables\n",
        "print(my_layer.variables)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[18.98158]]\n",
            "[<tf.Variable 'simple_dense_1/kernel:0' shape=(1, 1) dtype=float32, numpy=array([[1.9973305]], dtype=float32)>, <tf.Variable 'simple_dense_1/bias:0' shape=(1,) dtype=float32, numpy=array([-0.9917238], dtype=float32)>]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DOCIGScDHVKZ"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}