JanataHack_Machine_Learning_for_Banking.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "JanataHack: Machine Learning for Banking.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "cvUBXHGKRGtu",
        "Qfy1vTPXCa08",
        "GUX_BiyuDTXG",
        "7RiLKeAN8xfM",
        "wZ-xVLaE81Rv",
        "joGjG49r-ds5",
        "fMpGNTzdEpNz",
        "18ulch9g9I2T",
        "cme-SC1F-XMP",
        "VlWhRs_C-b50",
        "SLXYNGLvFMaL",
        "rRCqNfrlHYxF",
        "355fbf-j0gAs",
        "d4uYeX-0DvW0",
        "t0Cx-s2gNoe7",
        "h0D0xbt4bCHG",
        "N9zeAIWnLksH",
        "O6u4Nnj7QCF6",
        "uOh_jcPgFvck",
        "BEA5BfvEnMbG",
        "Y698SvxJoPXI",
        "YIFrJh4YMuf1",
        "EtGeyeI8QDzf"
      ],
      "authorship_tag": "ABX9TyNnjnxB9jlNcGi5JfsyQf7S",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ArpanSM/MachineLearningHackathons/blob/master/JanataHack_Machine_Learning_for_Banking.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cvUBXHGKRGtu",
        "colab_type": "text"
      },
      "source": [
        "## Loading Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ejr0bma0ulJj",
        "colab_type": "code",
        "outputId": "6c236ef6-8ccf-4476-91eb-470e4b0e5d54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JZio3afvvD1v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "path_train = \"/content/drive/My Drive/Work/Datasets/DA_ML_Banking/train_fNxu4vz.csv\"\n",
        "path_test = \"/content/drive/My Drive/Work/Datasets/DA_ML_Banking/test_fjtUOL8.csv\"\n",
        "path_sub = \"/content/drive/My Drive/Work/Datasets/DA_ML_Banking/sample_submission_HSqiq1Q.csv\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2A5KT3_FvPIz",
        "colab_type": "code",
        "outputId": "38ebb8aa-8653-4ada-b5ff-d295cadefd1e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from matplotlib import rcParams\n",
        "rcParams['figure.figsize'] = 15,7"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "96FMJtu9vRw6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv(path_train)\n",
        "dft = pd.read_csv(path_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NKEHahMdCm-8",
        "colab_type": "code",
        "outputId": "51ee92ca-2f0f-4ac8-c817-b1f9d53f4a5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "df.shape,dft.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((164309, 14), (109541, 13))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyQkUtUiyrHR",
        "colab_type": "code",
        "outputId": "32457943-6fcc-46a6-a145-dace6e6e732e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97
        }
      },
      "source": [
        "df.head(1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Loan_ID</th>\n",
              "      <th>Loan_Amount_Requested</th>\n",
              "      <th>Length_Employed</th>\n",
              "      <th>Home_Owner</th>\n",
              "      <th>Annual_Income</th>\n",
              "      <th>Income_Verified</th>\n",
              "      <th>Purpose_Of_Loan</th>\n",
              "      <th>Debt_To_Income</th>\n",
              "      <th>Inquiries_Last_6Mo</th>\n",
              "      <th>Months_Since_Deliquency</th>\n",
              "      <th>Number_Open_Accounts</th>\n",
              "      <th>Total_Accounts</th>\n",
              "      <th>Gender</th>\n",
              "      <th>Interest_Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>10000001</td>\n",
              "      <td>7,000</td>\n",
              "      <td>&lt; 1 year</td>\n",
              "      <td>Rent</td>\n",
              "      <td>68000.0</td>\n",
              "      <td>not verified</td>\n",
              "      <td>car</td>\n",
              "      <td>18.37</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>9</td>\n",
              "      <td>14</td>\n",
              "      <td>Female</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Loan_ID Loan_Amount_Requested  ...  Gender Interest_Rate\n",
              "0  10000001                 7,000  ...  Female             1\n",
              "\n",
              "[1 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3D4C8hGC5cpe",
        "colab_type": "code",
        "outputId": "6873348f-b93f-4f7e-ac3b-de4d497d28a5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 114
        }
      },
      "source": [
        "dft.head(1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Loan_ID</th>\n",
              "      <th>Loan_Amount_Requested</th>\n",
              "      <th>Length_Employed</th>\n",
              "      <th>Home_Owner</th>\n",
              "      <th>Annual_Income</th>\n",
              "      <th>Income_Verified</th>\n",
              "      <th>Purpose_Of_Loan</th>\n",
              "      <th>Debt_To_Income</th>\n",
              "      <th>Inquiries_Last_6Mo</th>\n",
              "      <th>Months_Since_Deliquency</th>\n",
              "      <th>Number_Open_Accounts</th>\n",
              "      <th>Total_Accounts</th>\n",
              "      <th>Gender</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>10164310</td>\n",
              "      <td>27,500</td>\n",
              "      <td>10+ years</td>\n",
              "      <td>Mortgage</td>\n",
              "      <td>129000.0</td>\n",
              "      <td>VERIFIED - income</td>\n",
              "      <td>debt_consolidation</td>\n",
              "      <td>12.87</td>\n",
              "      <td>0</td>\n",
              "      <td>68.0</td>\n",
              "      <td>10</td>\n",
              "      <td>37</td>\n",
              "      <td>Male</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Loan_ID Loan_Amount_Requested  ... Total_Accounts Gender\n",
              "0  10164310                27,500  ...             37   Male\n",
              "\n",
              "[1 rows x 13 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZdJiw70G5iXv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#df.dropna(inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qfy1vTPXCa08",
        "colab_type": "text"
      },
      "source": [
        "## Data - Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5XiNTrCA8w6H",
        "colab_type": "code",
        "outputId": "ce68d89f-a84a-4a68-e2d6-6a818eb5dff1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 535
        }
      },
      "source": [
        "print(df.isnull().sum())\n",
        "print(\"-------------------------------------\")\n",
        "print(\"-------------------------------------\")\n",
        "print(dft.isnull().sum())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loan_ID                        0\n",
            "Loan_Amount_Requested          0\n",
            "Length_Employed             7371\n",
            "Home_Owner                 25349\n",
            "Annual_Income              25102\n",
            "Income_Verified                0\n",
            "Purpose_Of_Loan                0\n",
            "Debt_To_Income                 0\n",
            "Inquiries_Last_6Mo             0\n",
            "Months_Since_Deliquency    88379\n",
            "Number_Open_Accounts           0\n",
            "Total_Accounts                 0\n",
            "Gender                         0\n",
            "Interest_Rate                  0\n",
            "dtype: int64\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "Loan_ID                        0\n",
            "Loan_Amount_Requested          0\n",
            "Length_Employed             4936\n",
            "Home_Owner                 16711\n",
            "Annual_Income              16898\n",
            "Income_Verified                0\n",
            "Purpose_Of_Loan                0\n",
            "Debt_To_Income                 0\n",
            "Inquiries_Last_6Mo             0\n",
            "Months_Since_Deliquency    58859\n",
            "Number_Open_Accounts           0\n",
            "Total_Accounts                 0\n",
            "Gender                         0\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GUX_BiyuDTXG",
        "colab_type": "text"
      },
      "source": [
        "### Loan_Amount_Requested"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JVkDH281D6qp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#train\n",
        "df['Loan_Amount_Requested'] = df['Loan_Amount_Requested'].str.replace(',', '').astype(float)\n",
        "#df['Loan_Amount_Requested'].fillna(14342,inplace=True)\n",
        "\n",
        "#test\n",
        "dft['Loan_Amount_Requested'] = dft['Loan_Amount_Requested'].str.replace(',', '').astype(float)\n",
        "#dft['Loan_Amount_Requested'].fillna(14342,inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7RiLKeAN8xfM",
        "colab_type": "text"
      },
      "source": [
        "### Length Employed"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-4dzXXRPywV7",
        "colab_type": "code",
        "outputId": "bc76b907-f6d3-4cb1-9c80-53565d7274ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 117
        }
      },
      "source": [
        "df['Length_Employed'].unique(),dft['Length_Employed'].unique()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['< 1 year', '4 years', '7 years', '8 years', '2 years',\n",
              "        '10+ years', '1 year', nan, '6 years', '9 years', '3 years',\n",
              "        '5 years'], dtype=object),\n",
              " array(['10+ years', '< 1 year', '6 years', '8 years', '1 year', '3 years',\n",
              "        '2 years', nan, '4 years', '7 years', '5 years', '9 years'],\n",
              "       dtype=object))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p7OyWhuV1j-Z",
        "colab_type": "code",
        "outputId": "b8c1d7ff-8ad9-4061-ef36-bdd5d43bf073",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "df[\"Length_Employed\"].replace({\"< 1 year\": \"0\",\n",
        "                               \"1 year\": \"1\",\n",
        "                               \"2 years\": \"2\",\n",
        "                               \"3 years\": \"3\",\n",
        "                               \"4 years\": \"4\",\n",
        "                               \"5 years\": \"5\",\n",
        "                               \"6 years\": \"6\",\n",
        "                               \"7 years\": \"7\",\n",
        "                               \"8 years\": \"8\",\n",
        "                               \"9 years\": \"9\",\n",
        "                               \"10+ years\": \"10\",\n",
        "                               }, inplace=True)\n",
        "\n",
        "df[\"Length_Employed\"] = pd.to_numeric(df[\"Length_Employed\"])\n",
        "df['Length_Employed'].fillna(7, inplace=True)\n",
        "print(df['Length_Employed'].unique())\n",
        "\n",
        "print(\"-------------------------------------\")\n",
        "print(\"-------------------------------------\")\n",
        "\n",
        "dft[\"Length_Employed\"].replace({\"< 1 year\": \"0\",\n",
        "                               \"1 year\": \"1\",\n",
        "                               \"2 years\": \"2\",\n",
        "                               \"3 years\": \"3\",\n",
        "                               \"4 years\": \"4\",\n",
        "                               \"5 years\": \"5\",\n",
        "                               \"6 years\": \"6\",\n",
        "                               \"7 years\": \"7\",\n",
        "                               \"8 years\": \"8\",\n",
        "                               \"9 years\": \"9\",\n",
        "                               \"10+ years\": \"10\",\n",
        "                               }, inplace=True)\n",
        "\n",
        "dft[\"Length_Employed\"] = pd.to_numeric(dft[\"Length_Employed\"])\n",
        "dft['Length_Employed'].fillna(7, inplace=True)\n",
        "print(dft['Length_Employed'].unique())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[ 0.  4.  7.  8.  2. 10.  1.  6.  9.  3.  5.]\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "[10.  0.  6.  8.  1.  3.  2.  7.  4.  5.  9.]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tf5JVDcE3g0f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#sns.countplot(df['Length_Employed'],hue=df['Interest_Rate'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wZ-xVLaE81Rv",
        "colab_type": "text"
      },
      "source": [
        "### Home_Owner"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qlQUNOPyzZVl",
        "colab_type": "code",
        "outputId": "4473509b-e2ed-48e4-c266-cf0026196e92",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "print(df['Home_Owner'].unique())\n",
        "print(\"-------------------------------------\")\n",
        "print(\"-------------------------------------\")\n",
        "print(dft['Home_Owner'].unique())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Rent' 'Mortgage' nan 'Own' 'Other' 'None']\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "['Mortgage' nan 'Rent' 'Own' 'Other' 'None']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QT2VFpxc-Bsk",
        "colab_type": "code",
        "outputId": "79ca6193-e127-40a9-da9e-6701242363b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 251
        }
      },
      "source": [
        "print(df['Home_Owner'].value_counts())\n",
        "print(\"-------------------------------------\")\n",
        "print(\"-------------------------------------\")\n",
        "print(dft['Home_Owner'].value_counts())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mortgage    70345\n",
            "Rent        56031\n",
            "Own         12525\n",
            "Other          49\n",
            "None           10\n",
            "Name: Home_Owner, dtype: int64\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "Mortgage    46925\n",
            "Rent        37511\n",
            "Own          8354\n",
            "Other          30\n",
            "None           10\n",
            "Name: Home_Owner, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CAJHa-Of7QP2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#sns.countplot(x=df['Home_Owner'], hue=df['Interest_Rate'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RD6v4y08H7OL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[\"Home_Owner\"].replace({\"None\":\"Other\"}, inplace=True)\n",
        "dft[\"Home_Owner\"].replace({\"None\":\"Other\"}, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7dVjFCLAH7Ir",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[\"Home_Owner\"].fillna(\"Mortgage\", inplace=True)\n",
        "dft[\"Home_Owner\"].fillna(\"Mortgage\", inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hljrM2bgJPvr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "one_hot = pd.get_dummies(df['Home_Owner'])\n",
        "df = df.drop('Home_Owner',axis = 1)\n",
        "df = df.join(one_hot)\n",
        "\n",
        "one_hot = pd.get_dummies(dft['Home_Owner'])\n",
        "dft = dft.drop('Home_Owner',axis = 1)\n",
        "dft = dft.join(one_hot)\n",
        "\n",
        "del df['Other']\n",
        "del dft['Other']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "joGjG49r-ds5",
        "colab_type": "text"
      },
      "source": [
        "### Months_Since_Deliquency"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiZI-saTz3gF",
        "colab_type": "code",
        "outputId": "6d3a6906-f157-415b-a153-5e0168cb52c6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "print(df['Months_Since_Deliquency'].mode())\n",
        "print(dft['Months_Since_Deliquency'].mode())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0    9.0\n",
            "dtype: float64\n",
            "0    9.0\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36PsfdXGCSNT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['Months_Since_Deliquency'].fillna(9, inplace=True)\n",
        "dft['Months_Since_Deliquency'].fillna(9, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fMpGNTzdEpNz",
        "colab_type": "text"
      },
      "source": [
        "### Annual_Income"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j6L9tMLYEsug",
        "colab_type": "code",
        "outputId": "2eb6ac31-6312-4270-f538-73c81013c623",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "df['Annual_Income'].mean(),dft['Annual_Income'].mean()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(73331.15943386468, 73485.40712563276)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BUzagtD1EsrO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['Annual_Income'].fillna(73331,inplace=True)\n",
        "dft['Annual_Income'].fillna(73485,inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "18ulch9g9I2T",
        "colab_type": "text"
      },
      "source": [
        "### Income_Verified"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "93prFalvzqre",
        "colab_type": "code",
        "outputId": "05b9e3bd-e615-4a46-eb0f-fbcabe395343",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "df['Income_Verified'].unique(),dft['Income_Verified'].unique()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['not verified', 'VERIFIED - income', 'VERIFIED - income source'],\n",
              "       dtype=object),\n",
              " array(['VERIFIED - income', 'not verified', 'VERIFIED - income source'],\n",
              "       dtype=object))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bqSIgVFW9Ybf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[\"Income_Verified\"].replace({\"not verified\": \"NV\",\n",
        "                               \"VERIFIED - income\": \"VI\",\n",
        "                               \"VERIFIED - income source\": \"VIS\",\n",
        "                               }, inplace=True)\n",
        "\n",
        "dft[\"Income_Verified\"].replace({\"not verified\": \"NV\",\n",
        "                               \"VERIFIED - income\": \"VI\",\n",
        "                               \"VERIFIED - income source\": \"VIS\",\n",
        "                               }, inplace=True)\n",
        "\n",
        "one_hot = pd.get_dummies(df['Income_Verified'])\n",
        "df = df.drop('Income_Verified',axis = 1)\n",
        "df = df.join(one_hot)\n",
        "\n",
        "one_hot = pd.get_dummies(dft['Income_Verified'])\n",
        "dft = dft.drop('Income_Verified',axis = 1)\n",
        "dft = dft.join(one_hot)\n",
        "\n",
        "del df['NV']\n",
        "del dft['NV']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O9hEa8hp9Nxw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#sns.countplot(x=df['Income_Verified'], hue=df['Interest_Rate'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cme-SC1F-XMP",
        "colab_type": "text"
      },
      "source": [
        "### Purpose_Of_Loan"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5XVv859cz2B8",
        "colab_type": "code",
        "outputId": "1ee0b868-0611-4ed0-dd15-bdc2ef8d19da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 150
        }
      },
      "source": [
        "df['Purpose_Of_Loan'].unique(),dft['Purpose_Of_Loan'].unique()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['car', 'debt_consolidation', 'credit_card', 'home_improvement',\n",
              "        'major_purchase', 'other', 'medical', 'small_business', 'moving',\n",
              "        'wedding', 'vacation', 'house', 'educational', 'renewable_energy'],\n",
              "       dtype=object),\n",
              " array(['debt_consolidation', 'credit_card', 'home_improvement', 'other',\n",
              "        'small_business', 'vacation', 'wedding', 'major_purchase', 'car',\n",
              "        'house', 'medical', 'renewable_energy', 'moving', 'educational'],\n",
              "       dtype=object))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JuseJ1me_UAj",
        "colab_type": "code",
        "outputId": "bbbfdbe5-fad5-4344-af7a-0811fa04a692",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "df[\"Purpose_Of_Loan\"].replace({\"debt_consolidation\": \"P1\",\n",
        "                               \"credit_card\": \"P2\",\n",
        "                               \"home_improvement\": \"P3\",\n",
        "                               \"major_purchase\": \"P4\",\n",
        "                               \"other\": \"P3\",\n",
        "                               \"medical\": \"P5\",\n",
        "                               \"small_business\": \"P5\",\n",
        "                               \"moving\": \"P5\",\n",
        "                               \"wedding\": \"P5\",\n",
        "                               \"vacation\": \"P5\",\n",
        "                               \"house\": \"P5\",\n",
        "                               \"educational\": \"P5\",\n",
        "                               \"renewable_energy\": \"P5\",\n",
        "                               \"car\": \"P5\",\n",
        "                               }, inplace=True)\n",
        "\n",
        "dft[\"Purpose_Of_Loan\"].replace({\"debt_consolidation\": \"P1\",\n",
        "                               \"credit_card\": \"P2\",\n",
        "                               \"home_improvement\": \"P3\",\n",
        "                               \"major_purchase\": \"P4\",\n",
        "                               \"other\": \"P3\",\n",
        "                               \"medical\": \"P5\",\n",
        "                               \"small_business\": \"P5\",\n",
        "                               \"moving\": \"P5\",\n",
        "                               \"wedding\": \"P5\",\n",
        "                               \"vacation\": \"P5\",\n",
        "                               \"house\": \"P5\",\n",
        "                               \"educational\": \"P5\",\n",
        "                               \"renewable_energy\": \"P5\",\n",
        "                               \"car\": \"P5\",\n",
        "                               }, inplace=True)\n",
        "\n",
        "df['Purpose_Of_Loan'].unique(),dft['Purpose_Of_Loan'].unique()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['P5', 'P1', 'P2', 'P3', 'P4'], dtype=object),\n",
              " array(['P1', 'P2', 'P3', 'P5', 'P4'], dtype=object))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tu3OH7zl-oep",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#sns.countplot(x=df['Purpose_Of_Loan'], hue=df['Interest_Rate'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y3OSJciN-ocm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "one_hot = pd.get_dummies(df['Purpose_Of_Loan'])\n",
        "df = df.drop('Purpose_Of_Loan',axis = 1)\n",
        "df = df.join(one_hot)\n",
        "\n",
        "one_hot = pd.get_dummies(dft['Purpose_Of_Loan'])\n",
        "dft = dft.drop('Purpose_Of_Loan',axis = 1)\n",
        "dft = dft.join(one_hot)\n",
        "\n",
        "del df['P4']\n",
        "del dft['P4']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VlWhRs_C-b50",
        "colab_type": "text"
      },
      "source": [
        "### Inquiries_Last_6Mo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oCsy-6VWz3ED",
        "colab_type": "code",
        "outputId": "11b26b4b-0015-4dfd-8d3b-88e9282bed86",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "df['Inquiries_Last_6Mo'].unique(),dft['Inquiries_Last_6Mo'].unique()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([0, 3, 1, 2, 4, 5, 6, 7, 8]), array([0, 2, 1, 3, 6, 4, 5, 7, 8]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4sNhIqhQBGSl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#sns.countplot(x=df['Inquiries_Last_6Mo'], hue=df['Interest_Rate'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bZQBNCl_BRiO",
        "colab_type": "code",
        "outputId": "ff3e24c7-45aa-444f-b988-3b8992d2ffcd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "df[\"Inquiries_Last_6Mo\"].replace({0: \"0\",\n",
        "                               1: \"1\",\n",
        "                               2: \"2\",\n",
        "                               3: \"3\",\n",
        "                               4: \"4\",\n",
        "                               5: \"5\",\n",
        "                               6: \"5\",\n",
        "                               7: \"5\",\n",
        "                               8: \"5\",\n",
        "                               }, inplace=True)\n",
        "\n",
        "dft[\"Inquiries_Last_6Mo\"].replace({0: \"0\",\n",
        "                               1: \"1\",\n",
        "                               2: \"2\",\n",
        "                               3: \"3\",\n",
        "                               4: \"4\",\n",
        "                               5: \"5\",\n",
        "                               6: \"5\",\n",
        "                               7: \"5\",\n",
        "                               8: \"5\",\n",
        "                               }, inplace=True)\n",
        "\n",
        "df[\"Inquiries_Last_6Mo\"] = pd.to_numeric(df[\"Inquiries_Last_6Mo\"])\n",
        "dft[\"Inquiries_Last_6Mo\"] = pd.to_numeric(dft[\"Inquiries_Last_6Mo\"])\n",
        "df['Inquiries_Last_6Mo'].unique(),dft['Inquiries_Last_6Mo'].unique()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([0, 3, 1, 2, 4, 5]), array([0, 2, 1, 3, 5, 4]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SLXYNGLvFMaL",
        "colab_type": "text"
      },
      "source": [
        "### Gender"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9DpeZ4ZAFKKX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[\"Gender\"].replace({\"Male\":1,\"Female\":0}, inplace=True)\n",
        "dft[\"Gender\"].replace({\"Male\":1,\"Female\":0}, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rRCqNfrlHYxF",
        "colab_type": "text"
      },
      "source": [
        "### Observation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vae6ZlMT0JgI",
        "colab_type": "code",
        "outputId": "c2915741-000d-4eb0-dff3-12d2cad221c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 736
        }
      },
      "source": [
        "print(df.dtypes)\n",
        "print(\"=================================\")\n",
        "print(\"=================================\")\n",
        "print(dft.dtypes)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loan_ID                      int64\n",
            "Loan_Amount_Requested      float64\n",
            "Length_Employed            float64\n",
            "Annual_Income              float64\n",
            "Debt_To_Income             float64\n",
            "Inquiries_Last_6Mo           int64\n",
            "Months_Since_Deliquency    float64\n",
            "Number_Open_Accounts         int64\n",
            "Total_Accounts               int64\n",
            "Gender                       int64\n",
            "Interest_Rate                int64\n",
            "Mortgage                     uint8\n",
            "Own                          uint8\n",
            "Rent                         uint8\n",
            "VI                           uint8\n",
            "VIS                          uint8\n",
            "P1                           uint8\n",
            "P2                           uint8\n",
            "P3                           uint8\n",
            "P5                           uint8\n",
            "dtype: object\n",
            "=================================\n",
            "=================================\n",
            "Loan_ID                      int64\n",
            "Loan_Amount_Requested      float64\n",
            "Length_Employed            float64\n",
            "Annual_Income              float64\n",
            "Debt_To_Income             float64\n",
            "Inquiries_Last_6Mo           int64\n",
            "Months_Since_Deliquency    float64\n",
            "Number_Open_Accounts         int64\n",
            "Total_Accounts               int64\n",
            "Gender                       int64\n",
            "Mortgage                     uint8\n",
            "Own                          uint8\n",
            "Rent                         uint8\n",
            "VI                           uint8\n",
            "VIS                          uint8\n",
            "P1                           uint8\n",
            "P2                           uint8\n",
            "P3                           uint8\n",
            "P5                           uint8\n",
            "dtype: object\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b2yhqKcu0OZU",
        "colab_type": "code",
        "outputId": "b160b9d0-c4dd-4846-9095-131e790e5cec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "df.describe(include='all')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Loan_ID</th>\n",
              "      <th>Loan_Amount_Requested</th>\n",
              "      <th>Length_Employed</th>\n",
              "      <th>Annual_Income</th>\n",
              "      <th>Debt_To_Income</th>\n",
              "      <th>Inquiries_Last_6Mo</th>\n",
              "      <th>Months_Since_Deliquency</th>\n",
              "      <th>Number_Open_Accounts</th>\n",
              "      <th>Total_Accounts</th>\n",
              "      <th>Gender</th>\n",
              "      <th>Interest_Rate</th>\n",
              "      <th>Mortgage</th>\n",
              "      <th>Own</th>\n",
              "      <th>Rent</th>\n",
              "      <th>VI</th>\n",
              "      <th>VIS</th>\n",
              "      <th>P1</th>\n",
              "      <th>P2</th>\n",
              "      <th>P3</th>\n",
              "      <th>P5</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.643090e+05</td>\n",
              "      <td>164309.00000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>1.643090e+05</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.00000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "      <td>164309.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.008216e+07</td>\n",
              "      <td>14349.33692</td>\n",
              "      <td>6.038367</td>\n",
              "      <td>7.333114e+04</td>\n",
              "      <td>17.207189</td>\n",
              "      <td>0.779270</td>\n",
              "      <td>20.658917</td>\n",
              "      <td>11.193818</td>\n",
              "      <td>25.067665</td>\n",
              "      <td>0.713144</td>\n",
              "      <td>2.158951</td>\n",
              "      <td>0.582403</td>\n",
              "      <td>0.076228</td>\n",
              "      <td>0.34101</td>\n",
              "      <td>0.361642</td>\n",
              "      <td>0.322654</td>\n",
              "      <td>0.590966</td>\n",
              "      <td>0.223262</td>\n",
              "      <td>0.107207</td>\n",
              "      <td>0.057660</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>4.743207e+04</td>\n",
              "      <td>8281.86870</td>\n",
              "      <td>3.552386</td>\n",
              "      <td>5.557439e+04</td>\n",
              "      <td>7.845083</td>\n",
              "      <td>1.023349</td>\n",
              "      <td>19.417804</td>\n",
              "      <td>4.991813</td>\n",
              "      <td>11.583067</td>\n",
              "      <td>0.452295</td>\n",
              "      <td>0.738364</td>\n",
              "      <td>0.493165</td>\n",
              "      <td>0.265364</td>\n",
              "      <td>0.47405</td>\n",
              "      <td>0.480477</td>\n",
              "      <td>0.467493</td>\n",
              "      <td>0.491657</td>\n",
              "      <td>0.416434</td>\n",
              "      <td>0.309377</td>\n",
              "      <td>0.233099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.000000e+07</td>\n",
              "      <td>500.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>4.000000e+03</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>1.004108e+07</td>\n",
              "      <td>8000.00000</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>4.860000e+04</td>\n",
              "      <td>11.370000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>9.000000</td>\n",
              "      <td>8.000000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>1.008216e+07</td>\n",
              "      <td>12075.00000</td>\n",
              "      <td>7.000000</td>\n",
              "      <td>7.000000e+04</td>\n",
              "      <td>16.840000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>9.000000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>23.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>1.012323e+07</td>\n",
              "      <td>20000.00000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>8.200000e+04</td>\n",
              "      <td>22.780000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>28.000000</td>\n",
              "      <td>14.000000</td>\n",
              "      <td>32.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.00000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.016431e+07</td>\n",
              "      <td>35000.00000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>7.500000e+06</td>\n",
              "      <td>39.990000</td>\n",
              "      <td>5.000000</td>\n",
              "      <td>180.000000</td>\n",
              "      <td>76.000000</td>\n",
              "      <td>156.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.00000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            Loan_ID  Loan_Amount_Requested  ...             P3             P5\n",
              "count  1.643090e+05           164309.00000  ...  164309.000000  164309.000000\n",
              "mean   1.008216e+07            14349.33692  ...       0.107207       0.057660\n",
              "std    4.743207e+04             8281.86870  ...       0.309377       0.233099\n",
              "min    1.000000e+07              500.00000  ...       0.000000       0.000000\n",
              "25%    1.004108e+07             8000.00000  ...       0.000000       0.000000\n",
              "50%    1.008216e+07            12075.00000  ...       0.000000       0.000000\n",
              "75%    1.012323e+07            20000.00000  ...       0.000000       0.000000\n",
              "max    1.016431e+07            35000.00000  ...       1.000000       1.000000\n",
              "\n",
              "[8 rows x 20 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "okJb4kOj2UHJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "355fbf-j0gAs",
        "colab_type": "text"
      },
      "source": [
        "# Scalling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kC9hRtIc0ia0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Robust Scaling\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "robust = RobustScaler()\n",
        "#Loan_Amount_Requested\n",
        "df['Robust_LAR']=robust.fit_transform(df.Loan_Amount_Requested.values.reshape(-1, 1))\n",
        "#Length_Employed\n",
        "df['Robust_LE']=robust.fit_transform(df.Length_Employed.values.reshape(-1, 1))\n",
        "#Annual_Income\n",
        "df['Robust_AI']=robust.fit_transform(df.Annual_Income.values.reshape(-1, 1))\n",
        "#Debt_To_Income\n",
        "df['Robust_DTI'] = robust.fit_transform(df.Debt_To_Income.values.reshape(-1, 1))\n",
        "#Inquiries_Last_6Mo\n",
        "df['Robust_IL6'] = robust.fit_transform(df.Inquiries_Last_6Mo.values.reshape(-1, 1))\n",
        "#Months_Since_Deliquency\n",
        "df['Robust_MSD'] = robust.fit_transform(df.Months_Since_Deliquency.values.reshape(-1, 1))\n",
        "#Number_Open_Accounts\n",
        "df['Robust_NOA'] = robust.fit_transform(df.Number_Open_Accounts.values.reshape(-1, 1))\n",
        "#Total_Accounts\n",
        "df['Robust_TA'] = robust.fit_transform(df.Total_Accounts.values.reshape(-1, 1))\n",
        "\n",
        "\n",
        "#MinMaxScaling\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "scaler = MinMaxScaler(feature_range=(-1,1))\n",
        "#Loan_Amount_Requested\n",
        "df['MM_LAR']=scaler.fit_transform(df.Loan_Amount_Requested.values.reshape(-1, 1))\n",
        "#Length_Employed\n",
        "df['MM_LE']=scaler.fit_transform(df.Length_Employed.values.reshape(-1, 1))\n",
        "#Annual_Income\n",
        "df['MM_AI']=scaler.fit_transform(df.Annual_Income.values.reshape(-1, 1))\n",
        "#Debt_To_Income\n",
        "df['MM_DTI'] = scaler.fit_transform(df.Debt_To_Income.values.reshape(-1, 1))\n",
        "#Inquiries_Last_6Mo\n",
        "df['MM_IL6'] = scaler.fit_transform(df.Inquiries_Last_6Mo.values.reshape(-1, 1))\n",
        "#Months_Since_Deliquency\n",
        "df['MM_MSD'] = scaler.fit_transform(df.Months_Since_Deliquency.values.reshape(-1, 1))\n",
        "#Number_Open_Accounts\n",
        "df['MM_NOA'] = scaler.fit_transform(df.Number_Open_Accounts.values.reshape(-1, 1))\n",
        "#Total_Accounts\n",
        "df['MM_TA'] = scaler.fit_transform(df.Total_Accounts.values.reshape(-1, 1))\n",
        "\n",
        "#MaxAbsScaling\n",
        "from sklearn.preprocessing import MaxAbsScaler\n",
        "maxabs = MaxAbsScaler()\n",
        "#Loan_Amount_Requested\n",
        "df['MA_LAR']=maxabs.fit_transform(df.Loan_Amount_Requested.values.reshape(-1, 1))\n",
        "#Length_Employed\n",
        "df['MA_LE']=maxabs.fit_transform(df.Length_Employed.values.reshape(-1, 1))\n",
        "#Annual_Income\n",
        "df['MA_AI']=maxabs.fit_transform(df.Annual_Income.values.reshape(-1, 1))\n",
        "#Debt_To_Income\n",
        "df['MA_DTI'] = maxabs.fit_transform(df.Debt_To_Income.values.reshape(-1, 1))\n",
        "#Inquiries_Last_6Mo\n",
        "df['MA_IL6'] = maxabs.fit_transform(df.Inquiries_Last_6Mo.values.reshape(-1, 1))\n",
        "#Months_Since_Deliquency\n",
        "df['MA_MSD'] = maxabs.fit_transform(df.Months_Since_Deliquency.values.reshape(-1, 1))\n",
        "#Number_Open_Accounts\n",
        "df['MA_NOA'] = maxabs.fit_transform(df.Number_Open_Accounts.values.reshape(-1, 1))\n",
        "#Total_Accounts\n",
        "df['MA_TA'] = maxabs.fit_transform(df.Total_Accounts.values.reshape(-1, 1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3FKgoeU852Ft",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Robust Scaling\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "robust = RobustScaler()\n",
        "#Loan_Amount_Requested\n",
        "dft['Robust_LAR']=robust.fit_transform(dft.Loan_Amount_Requested.values.reshape(-1, 1))\n",
        "#Length_Employed\n",
        "dft['Robust_LE']=robust.fit_transform(dft.Length_Employed.values.reshape(-1, 1))\n",
        "#Annual_Income\n",
        "dft['Robust_AI']=robust.fit_transform(dft.Annual_Income.values.reshape(-1, 1))\n",
        "#Debt_To_Income\n",
        "dft['Robust_DTI'] = robust.fit_transform(dft.Debt_To_Income.values.reshape(-1, 1))\n",
        "#Inquiries_Last_6Mo\n",
        "dft['Robust_IL6'] = robust.fit_transform(dft.Inquiries_Last_6Mo.values.reshape(-1, 1))\n",
        "#Months_Since_Deliquency\n",
        "dft['Robust_MSD'] = robust.fit_transform(dft.Months_Since_Deliquency.values.reshape(-1, 1))\n",
        "#Number_Open_Accounts\n",
        "dft['Robust_NOA'] = robust.fit_transform(dft.Number_Open_Accounts.values.reshape(-1, 1))\n",
        "#Total_Accounts\n",
        "dft['Robust_TA'] = robust.fit_transform(dft.Total_Accounts.values.reshape(-1, 1))\n",
        "\n",
        "\n",
        "#MinMaxScaling\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "scaler = MinMaxScaler(feature_range=(-1,1))\n",
        "#Loan_Amount_Requested\n",
        "dft['MM_LAR']=scaler.fit_transform(dft.Loan_Amount_Requested.values.reshape(-1, 1))\n",
        "#Length_Employed\n",
        "dft['MM_LE']=scaler.fit_transform(dft.Length_Employed.values.reshape(-1, 1))\n",
        "#Annual_Income\n",
        "dft['MM_AI']=scaler.fit_transform(dft.Annual_Income.values.reshape(-1, 1))\n",
        "#Debt_To_Income\n",
        "dft['MM_DTI'] = scaler.fit_transform(dft.Debt_To_Income.values.reshape(-1, 1))\n",
        "#Inquiries_Last_6Mo\n",
        "dft['MM_IL6'] = scaler.fit_transform(dft.Inquiries_Last_6Mo.values.reshape(-1, 1))\n",
        "#Months_Since_Deliquency\n",
        "dft['MM_MSD'] = scaler.fit_transform(dft.Months_Since_Deliquency.values.reshape(-1, 1))\n",
        "#Number_Open_Accounts\n",
        "dft['MM_NOA'] = scaler.fit_transform(dft.Number_Open_Accounts.values.reshape(-1, 1))\n",
        "#Total_Accounts\n",
        "dft['MM_TA'] = scaler.fit_transform(dft.Total_Accounts.values.reshape(-1, 1))\n",
        "\n",
        "#MaxAbsScaling\n",
        "from sklearn.preprocessing import MaxAbsScaler\n",
        "maxabs = MaxAbsScaler()\n",
        "#Loan_Amount_Requested\n",
        "dft['MA_LAR']=maxabs.fit_transform(dft.Loan_Amount_Requested.values.reshape(-1, 1))\n",
        "#Length_Employed\n",
        "dft['MA_LE']=maxabs.fit_transform(dft.Length_Employed.values.reshape(-1, 1))\n",
        "#Annual_Income\n",
        "dft['MA_AI']=maxabs.fit_transform(dft.Annual_Income.values.reshape(-1, 1))\n",
        "#Debt_To_Income\n",
        "dft['MA_DTI'] = maxabs.fit_transform(dft.Debt_To_Income.values.reshape(-1, 1))\n",
        "#Inquiries_Last_6Mo\n",
        "dft['MA_IL6'] = maxabs.fit_transform(dft.Inquiries_Last_6Mo.values.reshape(-1, 1))\n",
        "#Months_Since_Deliquency\n",
        "dft['MA_MSD'] = maxabs.fit_transform(dft.Months_Since_Deliquency.values.reshape(-1, 1))\n",
        "#Number_Open_Accounts\n",
        "dft['MA_NOA'] = maxabs.fit_transform(dft.Number_Open_Accounts.values.reshape(-1, 1))\n",
        "#Total_Accounts\n",
        "dft['MA_TA'] = maxabs.fit_transform(dft.Total_Accounts.values.reshape(-1, 1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QMYHtW5NAMBP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "h2o_df = df.drop(['Loan_ID','Loan_Amount_Requested','Length_Employed','Annual_Income','Debt_To_Income','Inquiries_Last_6Mo','Months_Since_Deliquency'\t,'Number_Open_Accounts','Total_Accounts'], axis = 1)\n",
        "h2o_df_test = dft.drop(['Loan_ID','Loan_Amount_Requested','Length_Employed','Annual_Income','Debt_To_Income','Inquiries_Last_6Mo','Months_Since_Deliquency'\t,'Number_Open_Accounts','Total_Accounts'], axis = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kyqHoupy4Aaf",
        "colab_type": "code",
        "outputId": "ddf2656c-9ee3-477c-8d1a-d7de8c39b189",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 216
        }
      },
      "source": [
        "h2o_df_test.to_csv(\"h2o_df_test.csv\",index=False)\n",
        "output = pd.read_csv(\"h2o_df_test.csv\")\n",
        "output.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Gender</th>\n",
              "      <th>Mortgage</th>\n",
              "      <th>Own</th>\n",
              "      <th>Rent</th>\n",
              "      <th>VI</th>\n",
              "      <th>VIS</th>\n",
              "      <th>P1</th>\n",
              "      <th>P2</th>\n",
              "      <th>P3</th>\n",
              "      <th>P5</th>\n",
              "      <th>Robust_LAR</th>\n",
              "      <th>Robust_LE</th>\n",
              "      <th>Robust_AI</th>\n",
              "      <th>Robust_DTI</th>\n",
              "      <th>Robust_IL6</th>\n",
              "      <th>Robust_MSD</th>\n",
              "      <th>Robust_NOA</th>\n",
              "      <th>Robust_TA</th>\n",
              "      <th>MM_LAR</th>\n",
              "      <th>MM_LE</th>\n",
              "      <th>MM_AI</th>\n",
              "      <th>MM_DTI</th>\n",
              "      <th>MM_IL6</th>\n",
              "      <th>MM_MSD</th>\n",
              "      <th>MM_NOA</th>\n",
              "      <th>MM_TA</th>\n",
              "      <th>MA_LAR</th>\n",
              "      <th>MA_LE</th>\n",
              "      <th>MA_AI</th>\n",
              "      <th>MA_DTI</th>\n",
              "      <th>MA_IL6</th>\n",
              "      <th>MA_MSD</th>\n",
              "      <th>MA_NOA</th>\n",
              "      <th>MA_TA</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.291667</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>1.770230</td>\n",
              "      <td>-0.349081</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.105263</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.866667</td>\n",
              "      <td>0.565217</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-0.966144</td>\n",
              "      <td>-0.356339</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.087248</td>\n",
              "      <td>-0.761905</td>\n",
              "      <td>-0.426230</td>\n",
              "      <td>0.785714</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.017324</td>\n",
              "      <td>0.321830</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.456376</td>\n",
              "      <td>0.119048</td>\n",
              "      <td>0.298387</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.166667</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>1.200156</td>\n",
              "      <td>-0.480315</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-0.666667</td>\n",
              "      <td>-0.066667</td>\n",
              "      <td>0.478261</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-0.971250</td>\n",
              "      <td>-0.431358</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.879195</td>\n",
              "      <td>-0.857143</td>\n",
              "      <td>-0.655738</td>\n",
              "      <td>0.742857</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.014772</td>\n",
              "      <td>0.284321</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.060403</td>\n",
              "      <td>0.071429</td>\n",
              "      <td>0.185484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.493750</td>\n",
              "      <td>-1.000000</td>\n",
              "      <td>0.150020</td>\n",
              "      <td>-0.877515</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-0.833333</td>\n",
              "      <td>-0.266667</td>\n",
              "      <td>-0.676812</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.980654</td>\n",
              "      <td>-0.658415</td>\n",
              "      <td>-0.2</td>\n",
              "      <td>-0.879195</td>\n",
              "      <td>-0.880952</td>\n",
              "      <td>-0.704918</td>\n",
              "      <td>0.173571</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.010072</td>\n",
              "      <td>0.170793</td>\n",
              "      <td>0.4</td>\n",
              "      <td>0.060403</td>\n",
              "      <td>0.059524</td>\n",
              "      <td>0.161290</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>0.090012</td>\n",
              "      <td>-0.796150</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-0.666667</td>\n",
              "      <td>-1.066667</td>\n",
              "      <td>-0.333333</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-0.981191</td>\n",
              "      <td>-0.611903</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.879195</td>\n",
              "      <td>-0.857143</td>\n",
              "      <td>-0.901639</td>\n",
              "      <td>0.342857</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.009803</td>\n",
              "      <td>0.194049</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.060403</td>\n",
              "      <td>0.071429</td>\n",
              "      <td>0.064516</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.916667</td>\n",
              "      <td>-1.000000</td>\n",
              "      <td>2.580335</td>\n",
              "      <td>-0.633421</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.894737</td>\n",
              "      <td>-0.166667</td>\n",
              "      <td>-0.200000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.958890</td>\n",
              "      <td>-0.518880</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.651007</td>\n",
              "      <td>-0.785714</td>\n",
              "      <td>-0.688525</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.020950</td>\n",
              "      <td>0.240560</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.174497</td>\n",
              "      <td>0.107143</td>\n",
              "      <td>0.169355</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Gender  Mortgage  Own  Rent  ...  MA_IL6    MA_MSD    MA_NOA     MA_TA\n",
              "0       1         1    0     0  ...     0.0  0.456376  0.119048  0.298387\n",
              "1       1         1    0     0  ...     0.0  0.060403  0.071429  0.185484\n",
              "2       1         0    0     1  ...     0.4  0.060403  0.059524  0.161290\n",
              "3       1         1    0     0  ...     0.0  0.060403  0.071429  0.064516\n",
              "4       1         1    0     0  ...     0.0  0.174497  0.107143  0.169355\n",
              "\n",
              "[5 rows x 34 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZIXB8ddZAdmz",
        "colab_type": "code",
        "outputId": "d6f3bb41-d85d-4a2c-f0f6-28a205133680",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 216
        }
      },
      "source": [
        "h2o_df.to_csv(\"h2o_df.csv\",index=False)\n",
        "output = pd.read_csv(\"h2o_df.csv\")\n",
        "output.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Gender</th>\n",
              "      <th>Interest_Rate</th>\n",
              "      <th>Mortgage</th>\n",
              "      <th>Own</th>\n",
              "      <th>Rent</th>\n",
              "      <th>VI</th>\n",
              "      <th>VIS</th>\n",
              "      <th>P1</th>\n",
              "      <th>P2</th>\n",
              "      <th>P3</th>\n",
              "      <th>P5</th>\n",
              "      <th>Robust_LAR</th>\n",
              "      <th>Robust_LE</th>\n",
              "      <th>Robust_AI</th>\n",
              "      <th>Robust_DTI</th>\n",
              "      <th>Robust_IL6</th>\n",
              "      <th>Robust_MSD</th>\n",
              "      <th>Robust_NOA</th>\n",
              "      <th>Robust_TA</th>\n",
              "      <th>MM_LAR</th>\n",
              "      <th>MM_LE</th>\n",
              "      <th>MM_AI</th>\n",
              "      <th>MM_DTI</th>\n",
              "      <th>MM_IL6</th>\n",
              "      <th>MM_MSD</th>\n",
              "      <th>MM_NOA</th>\n",
              "      <th>MM_TA</th>\n",
              "      <th>MA_LAR</th>\n",
              "      <th>MA_LE</th>\n",
              "      <th>MA_AI</th>\n",
              "      <th>MA_DTI</th>\n",
              "      <th>MA_IL6</th>\n",
              "      <th>MA_MSD</th>\n",
              "      <th>MA_NOA</th>\n",
              "      <th>MA_TA</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.422917</td>\n",
              "      <td>-1.000000</td>\n",
              "      <td>-0.059880</td>\n",
              "      <td>0.134093</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-0.166667</td>\n",
              "      <td>-0.600000</td>\n",
              "      <td>-0.623188</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.982924</td>\n",
              "      <td>-0.081270</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.900000</td>\n",
              "      <td>-0.763158</td>\n",
              "      <td>-0.844156</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.009067</td>\n",
              "      <td>0.459365</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.118421</td>\n",
              "      <td>0.089744</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.493750</td>\n",
              "      <td>-0.428571</td>\n",
              "      <td>0.099731</td>\n",
              "      <td>-0.167397</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.421053</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.066667</td>\n",
              "      <td>0.710145</td>\n",
              "      <td>-0.2</td>\n",
              "      <td>-0.981502</td>\n",
              "      <td>-0.253313</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.811111</td>\n",
              "      <td>-0.684211</td>\n",
              "      <td>-0.714286</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>0.4</td>\n",
              "      <td>0.009777</td>\n",
              "      <td>0.373343</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.094444</td>\n",
              "      <td>0.157895</td>\n",
              "      <td>0.153846</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.054167</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.166659</td>\n",
              "      <td>-0.084137</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>-0.466667</td>\n",
              "      <td>0.404348</td>\n",
              "      <td>0.4</td>\n",
              "      <td>-0.980905</td>\n",
              "      <td>-0.205801</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.900000</td>\n",
              "      <td>-0.684211</td>\n",
              "      <td>-0.818182</td>\n",
              "      <td>0.706429</td>\n",
              "      <td>0.7</td>\n",
              "      <td>0.010076</td>\n",
              "      <td>0.397099</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.157895</td>\n",
              "      <td>0.102564</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.327083</td>\n",
              "      <td>-1.000000</td>\n",
              "      <td>-0.414371</td>\n",
              "      <td>-0.219106</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.066667</td>\n",
              "      <td>-0.101449</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.986083</td>\n",
              "      <td>-0.282821</td>\n",
              "      <td>0.2</td>\n",
              "      <td>-0.900000</td>\n",
              "      <td>-0.578947</td>\n",
              "      <td>-0.740260</td>\n",
              "      <td>0.457143</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.007488</td>\n",
              "      <td>0.358590</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.210526</td>\n",
              "      <td>0.141026</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.410417</td>\n",
              "      <td>0.142857</td>\n",
              "      <td>0.778443</td>\n",
              "      <td>0.467134</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.500000</td>\n",
              "      <td>0.466667</td>\n",
              "      <td>-0.043478</td>\n",
              "      <td>0.6</td>\n",
              "      <td>-0.975454</td>\n",
              "      <td>0.108777</td>\n",
              "      <td>-0.6</td>\n",
              "      <td>-0.900000</td>\n",
              "      <td>-0.500000</td>\n",
              "      <td>-0.636364</td>\n",
              "      <td>0.485714</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.012800</td>\n",
              "      <td>0.554389</td>\n",
              "      <td>0.2</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.250000</td>\n",
              "      <td>0.192308</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Gender  Interest_Rate  Mortgage  Own  ...  MA_IL6    MA_MSD    MA_NOA     MA_TA\n",
              "0       0              1         0    0  ...     0.0  0.050000  0.118421  0.089744\n",
              "1       0              3         1    0  ...     0.0  0.094444  0.157895  0.153846\n",
              "2       1              3         1    0  ...     0.0  0.050000  0.157895  0.102564\n",
              "3       1              3         1    0  ...     0.6  0.050000  0.210526  0.141026\n",
              "4       0              1         0    1  ...     0.2  0.050000  0.250000  0.192308\n",
              "\n",
              "[5 rows x 35 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PR2GI3moGFaX",
        "colab_type": "code",
        "outputId": "358c277a-ad53-4e63-b6f5-61366b16d21f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "!wget http://127.0.0.1:54321/3/DownloadDataset?frame_id=combined-prediction-1d5db52e-300e-4cee-97d9-dd874d4cf8bc"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-31 14:17:51--  http://127.0.0.1:54321/3/DownloadDataset?frame_id=combined-prediction-1d5db52e-300e-4cee-97d9-dd874d4cf8bc\n",
            "Connecting to 127.0.0.1:54321... failed: Connection refused.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d4uYeX-0DvW0",
        "colab_type": "text"
      },
      "source": [
        "# Splitting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iFc7-jjlKP78",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = df.drop(['Loan_ID','Interest_Rate'], axis = 1)\n",
        "y = df['Interest_Rate']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UpnFhTIs7Xoh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = df.drop(['Loan_ID','Interest_Rate','Loan_Amount_Requested',\t'Length_Employed',\t'Annual_Income'\t,'Debt_To_Income',\t'Inquiries_Last_6Mo',\t'Months_Since_Deliquency'\t,'Number_Open_Accounts',\t'Total_Accounts'], axis = 1)\n",
        "y = df['Interest_Rate']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ALKyZdABDb9Q",
        "colab_type": "code",
        "outputId": "e9de147c-2639-48d1-dd94-3117964a9d27",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 216
        }
      },
      "source": [
        "X.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Gender</th>\n",
              "      <th>Mortgage</th>\n",
              "      <th>Own</th>\n",
              "      <th>Rent</th>\n",
              "      <th>VI</th>\n",
              "      <th>VIS</th>\n",
              "      <th>P1</th>\n",
              "      <th>P2</th>\n",
              "      <th>P3</th>\n",
              "      <th>P5</th>\n",
              "      <th>Robust_LAR</th>\n",
              "      <th>Robust_LE</th>\n",
              "      <th>Robust_AI</th>\n",
              "      <th>Robust_DTI</th>\n",
              "      <th>Robust_IL6</th>\n",
              "      <th>Robust_MSD</th>\n",
              "      <th>Robust_NOA</th>\n",
              "      <th>Robust_TA</th>\n",
              "      <th>MM_LAR</th>\n",
              "      <th>MM_LE</th>\n",
              "      <th>MM_AI</th>\n",
              "      <th>MM_DTI</th>\n",
              "      <th>MM_IL6</th>\n",
              "      <th>MM_MSD</th>\n",
              "      <th>MM_NOA</th>\n",
              "      <th>MM_TA</th>\n",
              "      <th>MA_LAR</th>\n",
              "      <th>MA_LE</th>\n",
              "      <th>MA_AI</th>\n",
              "      <th>MA_DTI</th>\n",
              "      <th>MA_IL6</th>\n",
              "      <th>MA_MSD</th>\n",
              "      <th>MA_NOA</th>\n",
              "      <th>MA_TA</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.422917</td>\n",
              "      <td>-1.000000</td>\n",
              "      <td>-0.059880</td>\n",
              "      <td>0.134093</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-0.166667</td>\n",
              "      <td>-0.600000</td>\n",
              "      <td>-0.623188</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.982924</td>\n",
              "      <td>-0.081270</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.900000</td>\n",
              "      <td>-0.763158</td>\n",
              "      <td>-0.844156</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.009067</td>\n",
              "      <td>0.459365</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.118421</td>\n",
              "      <td>0.089744</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.493750</td>\n",
              "      <td>-0.428571</td>\n",
              "      <td>0.099731</td>\n",
              "      <td>-0.167397</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.421053</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.066667</td>\n",
              "      <td>0.710145</td>\n",
              "      <td>-0.2</td>\n",
              "      <td>-0.981502</td>\n",
              "      <td>-0.253313</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.811111</td>\n",
              "      <td>-0.684211</td>\n",
              "      <td>-0.714286</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>0.4</td>\n",
              "      <td>0.009777</td>\n",
              "      <td>0.373343</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.094444</td>\n",
              "      <td>0.157895</td>\n",
              "      <td>0.153846</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.054167</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.166659</td>\n",
              "      <td>-0.084137</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>-0.466667</td>\n",
              "      <td>0.404348</td>\n",
              "      <td>0.4</td>\n",
              "      <td>-0.980905</td>\n",
              "      <td>-0.205801</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.900000</td>\n",
              "      <td>-0.684211</td>\n",
              "      <td>-0.818182</td>\n",
              "      <td>0.706429</td>\n",
              "      <td>0.7</td>\n",
              "      <td>0.010076</td>\n",
              "      <td>0.397099</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.157895</td>\n",
              "      <td>0.102564</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.327083</td>\n",
              "      <td>-1.000000</td>\n",
              "      <td>-0.414371</td>\n",
              "      <td>-0.219106</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.066667</td>\n",
              "      <td>-0.101449</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>-0.986083</td>\n",
              "      <td>-0.282821</td>\n",
              "      <td>0.2</td>\n",
              "      <td>-0.900000</td>\n",
              "      <td>-0.578947</td>\n",
              "      <td>-0.740260</td>\n",
              "      <td>0.457143</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.007488</td>\n",
              "      <td>0.358590</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.210526</td>\n",
              "      <td>0.141026</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.410417</td>\n",
              "      <td>0.142857</td>\n",
              "      <td>0.778443</td>\n",
              "      <td>0.467134</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.500000</td>\n",
              "      <td>0.466667</td>\n",
              "      <td>-0.043478</td>\n",
              "      <td>0.6</td>\n",
              "      <td>-0.975454</td>\n",
              "      <td>0.108777</td>\n",
              "      <td>-0.6</td>\n",
              "      <td>-0.900000</td>\n",
              "      <td>-0.500000</td>\n",
              "      <td>-0.636364</td>\n",
              "      <td>0.485714</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.012800</td>\n",
              "      <td>0.554389</td>\n",
              "      <td>0.2</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.250000</td>\n",
              "      <td>0.192308</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Gender  Mortgage  Own  Rent  ...  MA_IL6    MA_MSD    MA_NOA     MA_TA\n",
              "0       0         0    0     1  ...     0.0  0.050000  0.118421  0.089744\n",
              "1       0         1    0     0  ...     0.0  0.094444  0.157895  0.153846\n",
              "2       1         1    0     0  ...     0.0  0.050000  0.157895  0.102564\n",
              "3       1         1    0     0  ...     0.6  0.050000  0.210526  0.141026\n",
              "4       0         0    1     0  ...     0.2  0.050000  0.250000  0.192308\n",
              "\n",
              "[5 rows x 34 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dE2NffNCKfvV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Normalize feature variables\n",
        "#from sklearn.preprocessing import StandardScaler\n",
        "#X_features = X\n",
        "#X = StandardScaler().fit_transform(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dQ88DFJkLdSx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import GridSearchCV, train_test_split, RandomizedSearchCV\n",
        "from sklearn.metrics import classification_report"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W-B_NeGePtfn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.9, random_state=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t0Cx-s2gNoe7",
        "colab_type": "text"
      },
      "source": [
        "# XGBoost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k8XQD9L-NqWL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import xgboost as xgb\n",
        "xgb1 = xgb.XGBClassifier(silent=False)\n",
        "#binary:logistic,multi:softmax\n",
        "parameters = {'learning_rate':[0.01],\n",
        "              'n_estimators':[5000],\n",
        "              'gamma':[0],\n",
        "              'max_depth':[3],\n",
        "              'min_child_weigth':[1],\n",
        "              'gpu_id':[0],\n",
        "              'max_bin':[16],\n",
        "              'tree_method':['gpu_hist'],\n",
        "              'objective':['binary:logistic']\n",
        "              }"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mXxcqFkSPaYc",
        "colab_type": "code",
        "outputId": "3669d259-6265-4057-c391-045c7883cc20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        }
      },
      "source": [
        "rscvj1 = RandomizedSearchCV(xgb1,\n",
        "                            parameters,\n",
        "                            n_jobs=-1,\n",
        "                            cv=2)\n",
        "rscvj1.fit(X_train,y_train)\n",
        "print(\"Best Parameters\")\n",
        "print(rscvj1.best_params_)\n",
        "print(\"--------------------------------------------\")\n",
        "print(\"F1-Score\")\n",
        "print(classification_report(y_test, rscvj1.best_estimator_.predict(X_test)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:281: UserWarning: The total space of parameters 1 is smaller than n_iter=10. Running 1 iterations. For exhaustive searches, use GridSearchCV.\n",
            "  % (grid_size, self.n_iter, grid_size), UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Best Parameters\n",
            "{'tree_method': 'gpu_hist', 'objective': 'binary:logistic', 'n_estimators': 5000, 'min_child_weigth': 1, 'max_depth': 3, 'max_bin': 16, 'learning_rate': 0.01, 'gpu_id': 0, 'gamma': 0}\n",
            "--------------------------------------------\n",
            "F1-Score\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.50      0.26      0.34     30377\n",
            "           2       0.49      0.61      0.55     63565\n",
            "           3       0.57      0.55      0.56     53937\n",
            "\n",
            "    accuracy                           0.52    147879\n",
            "   macro avg       0.52      0.48      0.48    147879\n",
            "weighted avg       0.52      0.52      0.51    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7SO3FDl4bBKl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_test"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h0D0xbt4bCHG",
        "colab_type": "text"
      },
      "source": [
        "### LightGBM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pkDwV-ZzbIdd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import lightgbm as lgb\n",
        "lgb1 = lgb.LGBMClassifier()\n",
        "\n",
        "parameters = {'learning_rate':[0.5],\n",
        "              'n_estimators':[1000],\n",
        "              'num_leaves':[200]}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WkqDVsfqbInR",
        "colab_type": "code",
        "outputId": "db1c1432-41b7-40db-d35b-8df9530623be",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 314
        }
      },
      "source": [
        "rscvj2 = RandomizedSearchCV(lgb1,\n",
        "                            parameters,\n",
        "                            n_jobs=-1,\n",
        "                            cv=2)\n",
        "rscvj2.fit(X_train,y_train)\n",
        "print(\"Best Parameters\")\n",
        "print(rscvj2.best_params_)\n",
        "print(\"--------------------------------------------\")\n",
        "print(\"F1-Score\")\n",
        "print(classification_report(y_test, rscvj2.best_estimator_.predict(X_test)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:281: UserWarning: The total space of parameters 1 is smaller than n_iter=10. Running 1 iterations. For exhaustive searches, use GridSearchCV.\n",
            "  % (grid_size, self.n_iter, grid_size), UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Best Parameters\n",
            "{'num_leaves': 200, 'n_estimators': 1000, 'learning_rate': 0.5}\n",
            "--------------------------------------------\n",
            "F1-Score\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.43      0.29      0.35      8474\n",
            "           2       0.48      0.56      0.52     17549\n",
            "           3       0.55      0.54      0.54     15055\n",
            "\n",
            "    accuracy                           0.50     41078\n",
            "   macro avg       0.49      0.46      0.47     41078\n",
            "weighted avg       0.50      0.50      0.49     41078\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4YMs8XzyguyM",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N9zeAIWnLksH",
        "colab_type": "text"
      },
      "source": [
        "# Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zl2v9vYoGvoB",
        "colab_type": "code",
        "outputId": "d8281cbb-adc4-4db6-9383-492d39f3bb14",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        }
      },
      "source": [
        "h2o_out = pd.read_csv('/content/661876900b99.csv')\n",
        "#h2o_out['predict']=pd.to_numeric(h2o_out['predict'])\n",
        "h2o_out.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>predict</th>\n",
              "      <th>p1</th>\n",
              "      <th>p2</th>\n",
              "      <th>p3</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>0.248626</td>\n",
              "      <td>0.456524</td>\n",
              "      <td>0.294850</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>0.527002</td>\n",
              "      <td>0.386635</td>\n",
              "      <td>0.086364</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0.100525</td>\n",
              "      <td>0.426847</td>\n",
              "      <td>0.472628</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>0.167860</td>\n",
              "      <td>0.509395</td>\n",
              "      <td>0.322745</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>0.155266</td>\n",
              "      <td>0.491937</td>\n",
              "      <td>0.352797</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   predict        p1        p2        p3\n",
              "0        2  0.248626  0.456524  0.294850\n",
              "1        1  0.527002  0.386635  0.086364\n",
              "2        3  0.100525  0.426847  0.472628\n",
              "3        2  0.167860  0.509395  0.322745\n",
              "4        2  0.155266  0.491937  0.352797"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VhkbuE0HqXqD",
        "colab_type": "code",
        "outputId": "faf033dc-c17a-4506-b797-80cf5f89feb1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 100
        }
      },
      "source": [
        "h2o_out.isnull().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "predict    0\n",
              "p1         0\n",
              "p2         1\n",
              "p3         1\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KHDHeqqTHz53",
        "colab_type": "code",
        "outputId": "fb9fbee8-e6b5-4265-c9d3-7cba1cbc6184",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        }
      },
      "source": [
        "sub['Interest_Rate']=h2o_out['predict']\n",
        "sub.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Loan_ID</th>\n",
              "      <th>Interest_Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>10164310</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>10164311</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10164312</td>\n",
              "      <td>3.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>10164313</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>10164314</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Loan_ID  Interest_Rate\n",
              "0  10164310            2.0\n",
              "1  10164311            1.0\n",
              "2  10164312            3.0\n",
              "3  10164313            2.0\n",
              "4  10164314            2.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aVFi6GBQLmpC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#dft['out'] = xgbj1_grid.best_estimator_.predict(jun1t.drop([\"DateTime\",\"ID\"], axis=1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O5m34wftMTsn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submit = dft\n",
        "submit = submit.drop(['Loan_ID'], axis = 1)\n",
        "#submit_features = submit\n",
        "#submit = StandardScaler().fit_transform(submit)\n",
        "submit"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xIYMF2Nb9Alm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submit = dft\n",
        "submit = submit.drop(['Loan_ID','Loan_Amount_Requested','Length_Employed','Annual_Income','Debt_To_Income','Inquiries_Last_6Mo','Months_Since_Deliquency','Number_Open_Accounts','Total_Accounts','out'], axis = 1)\n",
        "submit"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fZGuus6JaEmT",
        "colab_type": "code",
        "outputId": "e1f0ca46-887a-43f1-cda1-f178e809e965",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        }
      },
      "source": [
        "sub = pd.read_csv(path_sub)\n",
        "sub.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Loan_ID</th>\n",
              "      <th>Interest_Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>10164310</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>10164311</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10164312</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>10164313</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>10164314</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Loan_ID  Interest_Rate\n",
              "0  10164310              1\n",
              "1  10164311              1\n",
              "2  10164312              1\n",
              "3  10164313              1\n",
              "4  10164314              1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pDauzKtpMBMQ",
        "colab_type": "code",
        "outputId": "2b1f6b74-99bb-47d1-a8d0-f084813acfb2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        }
      },
      "source": [
        "dft['out'] = stacked_averaged_models.predict(submit)\n",
        "dft['out'] = dft['out'].apply(np.floor)\n",
        "dft['out'] = dft['out'].astype('int32')\n",
        "sub['Interest_Rate']=dft['out']\n",
        "sub.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Loan_ID</th>\n",
              "      <th>Interest_Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>10164310</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>10164311</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10164312</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>10164313</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>10164314</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Loan_ID  Interest_Rate\n",
              "0  10164310              2\n",
              "1  10164311              1\n",
              "2  10164312              2\n",
              "3  10164313              2\n",
              "4  10164314              2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bc6h5yQ_QN6L",
        "colab_type": "code",
        "outputId": "fc912992-4301-4e1e-9b9e-cb53c2ca328a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        }
      },
      "source": [
        "dft['out'] = model.predict(submit)\n",
        "sub['Interest_Rate']=dft['out']\n",
        "sub.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Loan_ID</th>\n",
              "      <th>Interest_Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>10164310</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>10164311</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10164312</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>10164313</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>10164314</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Loan_ID  Interest_Rate\n",
              "0  10164310              2\n",
              "1  10164311              1\n",
              "2  10164312              3\n",
              "3  10164313              2\n",
              "4  10164314              2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rd_Zvxr5OKxU",
        "colab_type": "code",
        "outputId": "89d77298-5d10-4116-b760-74aa2a9fdfec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        }
      },
      "source": [
        "sub.to_csv(\"output1.csv\",index=False)\n",
        "output = pd.read_csv(\"output1.csv\")\n",
        "output.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Loan_ID</th>\n",
              "      <th>Interest_Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>10164310</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>10164311</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10164312</td>\n",
              "      <td>3.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>10164313</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>10164314</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Loan_ID  Interest_Rate\n",
              "0  10164310            2.0\n",
              "1  10164311            1.0\n",
              "2  10164312            3.0\n",
              "3  10164313            2.0\n",
              "4  10164314            2.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mmTf1K5mYO4z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O6u4Nnj7QCF6",
        "colab_type": "text"
      },
      "source": [
        "# **Stacking**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uOh_jcPgFvck",
        "colab_type": "text"
      },
      "source": [
        "##Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nap1JSNO-VmR",
        "colab_type": "code",
        "outputId": "e6b8f2fc-f361-412f-c8cb-33484964e37d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "#Decision Tree\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "model1 = DecisionTreeClassifier(random_state=1)\n",
        "model1.fit(X_train, y_train)\n",
        "y_pred1 = model1.predict(X_test)\n",
        "print(classification_report(y_test, y_pred1))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.29      0.31      0.30     30377\n",
            "           2       0.45      0.45      0.45     63565\n",
            "           3       0.46      0.45      0.46     53937\n",
            "\n",
            "    accuracy                           0.42    147879\n",
            "   macro avg       0.40      0.40      0.40    147879\n",
            "weighted avg       0.42      0.42      0.42    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2hTf3z6_-Vjr",
        "colab_type": "code",
        "outputId": "10a549c4-a33d-4f85-9209-3a0c25ad1300",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "#RandomForestClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "model2 = RandomForestClassifier(random_state=1)\n",
        "model2.fit(X_train, y_train)\n",
        "y_pred2 = model2.predict(X_test)\n",
        "print(classification_report(y_test, y_pred2))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.46      0.24      0.31     30377\n",
            "           2       0.48      0.61      0.54     63565\n",
            "           3       0.54      0.52      0.53     53937\n",
            "\n",
            "    accuracy                           0.50    147879\n",
            "   macro avg       0.49      0.45      0.46    147879\n",
            "weighted avg       0.50      0.50      0.49    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lqbBQuYc-VhI",
        "colab_type": "code",
        "outputId": "18bdf027-e2bd-4580-d78b-3644b19da91b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "#AdaBoostClassifier\n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "model3 = AdaBoostClassifier(random_state=1)\n",
        "model3.fit(X_train, y_train)\n",
        "y_pred3 = model3.predict(X_test)\n",
        "print(classification_report(y_test, y_pred3))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.50      0.25      0.33     30377\n",
            "           2       0.49      0.63      0.55     63565\n",
            "           3       0.58      0.54      0.56     53937\n",
            "\n",
            "    accuracy                           0.52    147879\n",
            "   macro avg       0.52      0.48      0.48    147879\n",
            "weighted avg       0.53      0.52      0.51    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dt-naIcM-Vez",
        "colab_type": "code",
        "outputId": "6fe69952-b449-477e-c6b5-6d4a43468790",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "#GradientBoostingClassifier\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "model4 = GradientBoostingClassifier(random_state=1,n_estimators=1000)\n",
        "model4.fit(X_train, y_train)\n",
        "y_pred4 = model4.predict(X_test)\n",
        "print(classification_report(y_test, y_pred4))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.47      0.29      0.36     30377\n",
            "           2       0.49      0.59      0.54     63565\n",
            "           3       0.57      0.55      0.56     53937\n",
            "\n",
            "    accuracy                           0.52    147879\n",
            "   macro avg       0.51      0.48      0.49    147879\n",
            "weighted avg       0.52      0.52      0.51    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQDwAAFa-Vbz",
        "colab_type": "code",
        "outputId": "de98919c-2f2a-4af9-85e7-93b6e0238eac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "#XGBClassifier\n",
        "import xgboost as xgb\n",
        "model5 = xgb.XGBClassifier(random_state=2)\n",
        "model5.fit(X_train, y_train)\n",
        "y_pred5 = model5.predict(X_test)\n",
        "print(classification_report(y_test, y_pred5))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.55      0.15      0.23     30377\n",
            "           2       0.49      0.68      0.57     63565\n",
            "           3       0.57      0.55      0.56     53937\n",
            "\n",
            "    accuracy                           0.52    147879\n",
            "   macro avg       0.54      0.46      0.45    147879\n",
            "weighted avg       0.53      0.52      0.50    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wVOsxnKR-VJ7",
        "colab_type": "code",
        "outputId": "93a7c9a0-d59d-424e-cc8c-ba2f7dc2afaf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "model6 = ExtraTreesClassifier(random_state=3)\n",
        "model6.fit(X_train, y_train)\n",
        "y_pred6 = model6.predict(X_test)\n",
        "print(classification_report(y_test, y_pred6))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.42      0.23      0.30     30377\n",
            "           2       0.48      0.58      0.52     63565\n",
            "           3       0.53      0.52      0.53     53937\n",
            "\n",
            "    accuracy                           0.49    147879\n",
            "   macro avg       0.47      0.45      0.45    147879\n",
            "weighted avg       0.48      0.49      0.48    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k1MEykREToFe",
        "colab_type": "code",
        "outputId": "d9f53469-3d1f-455d-987a-73447d3dce99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        }
      },
      "source": [
        "import lightgbm \n",
        "model7 = lightgbm.LGBMClassifier(random_state=3)\n",
        "model7.fit(X_train, y_train)\n",
        "y_pred7 = model7.predict(X_test)\n",
        "print(classification_report(y_test, y_pred7))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.50      0.25      0.33     30377\n",
            "           2       0.50      0.63      0.55     63565\n",
            "           3       0.57      0.55      0.56     53937\n",
            "\n",
            "    accuracy                           0.52    147879\n",
            "   macro avg       0.52      0.48      0.48    147879\n",
            "weighted avg       0.53      0.52      0.51    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N2ZUBVOK_5Jk",
        "colab_type": "code",
        "outputId": "298dfe30-5898-4d9c-dc3a-4c614762c962",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 418
        }
      },
      "source": [
        "!pip install catboost\n",
        "from catboost import Pool, CatBoostClassifier\n",
        "model8 = CatBoostClassifier(random_state=3,logging_level='Silent')\n",
        "model8.fit(X_train, y_train)\n",
        "y_pred8 = model8.predict(X_test)\n",
        "print(classification_report(y_test, y_pred8))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: catboost in /usr/local/lib/python3.6/dist-packages (0.23.2)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.18.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from catboost) (1.4.1)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from catboost) (3.2.1)\n",
            "Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.0.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from catboost) (1.12.0)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.6/dist-packages (from catboost) (4.4.1)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (from catboost) (0.10.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (1.2.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->catboost) (2018.9)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (1.3.3)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.48      0.29      0.36     30377\n",
            "           2       0.49      0.60      0.54     63565\n",
            "           3       0.56      0.55      0.56     53937\n",
            "\n",
            "    accuracy                           0.52    147879\n",
            "   macro avg       0.51      0.48      0.48    147879\n",
            "weighted avg       0.51      0.52      0.51    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BEA5BfvEnMbG",
        "colab_type": "text"
      },
      "source": [
        "## Simplest Stacking approach : Averaging base models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JQBecHVAkI9z",
        "colab_type": "text"
      },
      "source": [
        "We begin with this simple approach of averaging base models. We build a new class to extend scikit-learn with our model and also to laverage encapsulation and code reuse "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3pbjO38dkMxh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
        "class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):\n",
        "    def __init__(self, models):\n",
        "        self.models = models\n",
        "        \n",
        "    # we define clones of the original models to fit the data in\n",
        "    def fit(self, X, y):\n",
        "        self.models_ = [clone(x) for x in self.models]\n",
        "        \n",
        "        # Train cloned base models\n",
        "        for model in self.models_:\n",
        "            model.fit(X, y)\n",
        "\n",
        "        return self\n",
        "    \n",
        "    #Now we do the predictions for cloned models and average them\n",
        "    def predict(self, X):\n",
        "        predictions = np.column_stack([\n",
        "            model.predict(X) for model in self.models_\n",
        "        ])\n",
        "        return np.mean(predictions, axis=1) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4WnquQFzkdZi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "averaged_models = AveragingModels(models = (grid_search_gbr.best_estimator_,xgb_grid.best_estimator_, grid_search.best_estimator_))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pTLtTY-pk2HA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "averaged_models.fit(X,Y)\n",
        "from sklearn.metrics import r2_score\n",
        "print(\"R2-Score\")\n",
        "print(round(r2_score(Y_Val, averaged_models.predict(X_Val)),4))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y698SvxJoPXI",
        "colab_type": "text"
      },
      "source": [
        "## Less simple Stacking : Adding a Meta-model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "223aH1ZroXwY",
        "colab_type": "text"
      },
      "source": [
        "In this approach, we add a meta-model on averaged base models and use the out-of-folds predictions of these base models to train our meta-model.\n",
        "\n",
        "The procedure, for the training part, may be described as follows:\n",
        "\n",
        "* Split the total training set into two disjoint sets (here train and .holdout )\n",
        "\n",
        "* Train several base models on the first part (train)\n",
        "\n",
        "* Test these base models on the second part (holdout)\n",
        "\n",
        "* Use the predictions from 3) (called out-of-folds predictions) as the inputs, and the correct responses (target variable) as the outputs to train a higher level learner called meta-model.\n",
        "\n",
        "The first three steps are done iteratively . If we take for example a 5-fold stacking , we first split the training data into 5 folds. Then we will do 5 iterations. In each iteration, we train every base model on 4 folds and predict on the remaining fold (holdout fold).\n",
        "\n",
        "So, we will be sure, after 5 iterations , that the entire data is used to get out-of-folds predictions that we will then use as new feature to train our meta-model in the step 4.\n",
        "\n",
        "For the prediction part , We average the predictions of all base models on the test data and used them as meta-features on which, the final prediction is done with the meta-model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HvwPH-jxFIja",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
        "class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):\n",
        "    def __init__(self, base_models, meta_model, n_folds=5):\n",
        "        self.base_models = base_models\n",
        "        self.meta_model = meta_model\n",
        "        self.n_folds = n_folds\n",
        "   \n",
        "    # We again fit the data on clones of the original models\n",
        "    def fit(self, X, y):\n",
        "        self.base_models_ = [list() for x in self.base_models]\n",
        "        self.meta_model_ = clone(self.meta_model)\n",
        "        kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)\n",
        "        \n",
        "        # Train cloned base models then create out-of-fold predictions\n",
        "        # that are needed to train the cloned meta-model\n",
        "        out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))\n",
        "        for i, model in enumerate(self.base_models):\n",
        "            for train_index, holdout_index in kfold.split(X, y):\n",
        "                instance = clone(model)\n",
        "                self.base_models_[i].append(instance)\n",
        "                instance.fit(X[train_index], y[train_index])\n",
        "                y_pred = instance.predict(X[holdout_index])\n",
        "                out_of_fold_predictions[holdout_index, i] = y_pred\n",
        "                \n",
        "        # Now train the cloned  meta-model using the out-of-fold predictions as new feature\n",
        "        self.meta_model_.fit(out_of_fold_predictions, y)\n",
        "        return self\n",
        "   \n",
        "    #Do the predictions of all base models on the test data and use the averaged predictions as \n",
        "    #meta-features for the final prediction which is done by the meta-model\n",
        "    def predict(self, X):\n",
        "        meta_features = np.column_stack([\n",
        "            np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)\n",
        "            for base_models in self.base_models_ ])\n",
        "        return self.meta_model_.predict(meta_features)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8zJF4naDqgyF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor, ExtraTreesRegressor\n",
        "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
        "from sklearn.kernel_ridge import KernelRidge\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.preprocessing import RobustScaler"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rig1X0lrtMKr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Lasso Regression\n",
        "lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=1))\n",
        "#Elastic Net Regression\n",
        "ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))\n",
        "#Kernel Ridge Regression\n",
        "KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IXhormuKG9TQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "new_ytrain = y_train.values.reshape(16430,1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9-ia6-B0HeTJ",
        "colab_type": "code",
        "outputId": "02b7b0f5-c993-4d6f-ad4a-0ab14aeccc0c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "new_ytrain.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(16430, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJZ5Pa5TRKDI",
        "colab_type": "code",
        "outputId": "38257f5a-d056-4786-a1c3-46a0be308f2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 619
        }
      },
      "source": [
        "stacked_averaged_models = StackingAveragedModels(base_models = (model3,model4),\n",
        "                                                 meta_model = lasso)\n",
        "\n",
        "stacked_averaged_models.fit(X_train.values,y_train.values)\n",
        "#print(\"R2-Score\")\n",
        "#print(round(r2_score(Y_Val.values, stacked_averaged_models.predict(X_Val.values)),4))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "StackingAveragedModels(base_models=(AdaBoostClassifier(algorithm='SAMME.R',\n",
              "                                                       base_estimator=None,\n",
              "                                                       learning_rate=1.0,\n",
              "                                                       n_estimators=50,\n",
              "                                                       random_state=1),\n",
              "                                    GradientBoostingClassifier(ccp_alpha=0.0,\n",
              "                                                               criterion='friedman_mse',\n",
              "                                                               init=None,\n",
              "                                                               learning_rate=0.1,\n",
              "                                                               loss='deviance',\n",
              "                                                               max_depth=3,\n",
              "                                                               max_features=None,\n",
              "                                                               max_leaf_nodes=None,\n",
              "                                                               min_impurity_decrease=0.0,\n",
              "                                                               min_impurity_split=...\n",
              "                       meta_model=Pipeline(memory=None,\n",
              "                                           steps=[('robustscaler',\n",
              "                                                   RobustScaler(copy=True,\n",
              "                                                                quantile_range=(25.0,\n",
              "                                                                                75.0),\n",
              "                                                                with_centering=True,\n",
              "                                                                with_scaling=True)),\n",
              "                                                  ('lasso',\n",
              "                                                   Lasso(alpha=0.0005,\n",
              "                                                         copy_X=True,\n",
              "                                                         fit_intercept=True,\n",
              "                                                         max_iter=1000,\n",
              "                                                         normalize=False,\n",
              "                                                         positive=False,\n",
              "                                                         precompute=False,\n",
              "                                                         random_state=1,\n",
              "                                                         selection='cyclic',\n",
              "                                                         tol=0.0001,\n",
              "                                                         warm_start=False))],\n",
              "                                           verbose=False),\n",
              "                       n_folds=5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kboLV851GICf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"F1-Score\")\n",
        "print(classification_report(y_test,stacked_averaged_models.predict(X_test)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H3Tysq0xGkGP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YIFrJh4YMuf1",
        "colab_type": "text"
      },
      "source": [
        "## Bagging"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PGlYaBnWMvyG",
        "colab_type": "code",
        "outputId": "da7782db-9039-45b3-d85c-f711fbeb7b4a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "from sklearn.ensemble import BaggingClassifier\n",
        "from sklearn import tree\n",
        "model = BaggingClassifier(RandomForestClassifier(random_state=1))\n",
        "model.fit(X_train, y_train)\n",
        "model.score(X_test,y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5080640253179964"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EtGeyeI8QDzf",
        "colab_type": "text"
      },
      "source": [
        "## Voting Classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_DOaBcYIM6Wu",
        "colab_type": "code",
        "outputId": "5f9c200c-d692-460d-b93f-be2e79ec81b8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "from sklearn.ensemble import VotingClassifier\n",
        "model = VotingClassifier(estimators=[('l5', model5), ('l6', model6), ('l4', model4),('l3', model3), ('l2', model2)], voting='hard')\n",
        "model.fit(X_train,y_train)\n",
        "model.score(X_test,y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.524070354817114"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MhpO3_i_OxCy",
        "colab_type": "code",
        "outputId": "24263122-7217-49e8-a863-ba8afe0d6c04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 201
        }
      },
      "source": [
        "print(\"F1-Score\")\n",
        "print(classification_report(y_test,model.predict(X_test)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "F1-Score\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           1       0.53      0.19      0.28     30377\n",
            "           2       0.49      0.67      0.57     63565\n",
            "           3       0.58      0.54      0.56     53937\n",
            "\n",
            "    accuracy                           0.52    147879\n",
            "   macro avg       0.53      0.47      0.47    147879\n",
            "weighted avg       0.53      0.52      0.51    147879\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJEsOU9iQ0Wf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}