SVD_Recommender.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vUkHMQTQtGNW"
      },
      "outputs": [],
      "source": [
        "import pickle\n",
        "import pandas as pd \n",
        "import numpy as np "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pRsdoKioqmX4"
      },
      "outputs": [],
      "source": [
        "dfmerged_orders_with_products = pd.read_pickle(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/dfmerged_orders_with_products.pkl\") "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "niRX_8Ysq6E9"
      },
      "outputs": [],
      "source": [
        "# Ok so I need user_id, product_id, and 'number of times it was purchased'\n",
        "user_item = dfmerged_orders_with_products[['user_id', 'product_id', 'reordered']] # Maybe I can use the reordered flag to increment "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q8dfk5keq89h"
      },
      "outputs": [],
      "source": [
        "del dfmerged_orders_with_products  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4xAp4hTarDQh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "05912169-8023-4cab-8078-f630d809ff86"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          user_id  product_id  reordered\n",
              "0               1         196          0\n",
              "1               1       26088          0\n",
              "2               1       14084          0\n",
              "3               1       26405          0\n",
              "4               1       12427          0\n",
              "...           ...         ...        ...\n",
              "33819101   206209        6846          1\n",
              "33819102   206209       40603          0\n",
              "33819103   206209       37966          0\n",
              "33819104   206209       15655          0\n",
              "33819105   206209       42606          0\n",
              "\n",
              "[33819106 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e5bd4ad5-9e1a-4708-8098-c3bb816b4726\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>product_id</th>\n",
              "      <th>reordered</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>196</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>26088</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>14084</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>26405</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>12427</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33819101</th>\n",
              "      <td>206209</td>\n",
              "      <td>6846</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33819102</th>\n",
              "      <td>206209</td>\n",
              "      <td>40603</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33819103</th>\n",
              "      <td>206209</td>\n",
              "      <td>37966</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33819104</th>\n",
              "      <td>206209</td>\n",
              "      <td>15655</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33819105</th>\n",
              "      <td>206209</td>\n",
              "      <td>42606</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>33819106 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e5bd4ad5-9e1a-4708-8098-c3bb816b4726')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e5bd4ad5-9e1a-4708-8098-c3bb816b4726 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e5bd4ad5-9e1a-4708-8098-c3bb816b4726');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "user_item "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xrlNneGErDEg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c63ba6b2-c11f-4e33-fa42-7f945e57cd6d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "196      11\n",
              "12427    10\n",
              "10258    10\n",
              "25133     9\n",
              "13032     4\n",
              "46149     4\n",
              "26405     3\n",
              "26088     3\n",
              "49235     3\n",
              "13176     2\n",
              "38928     2\n",
              "39657     2\n",
              "14084     1\n",
              "30450     1\n",
              "41787     1\n",
              "17122     1\n",
              "10326     1\n",
              "35951     1\n",
              "27845     1\n",
              "Name: product_id, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "# The value counts of each product id for each user.\n",
        "user_item[user_item['user_id']==1].product_id.value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lfDinRv4rDBC"
      },
      "outputs": [],
      "source": [
        "# dict_for_data_reccomendation = {'user_id':[],'product_id':[],'count':[]}\n",
        "# rec_df = pd.DataFrame(dict_for_data_reccomendation)\n",
        "\n",
        "list_of_rec_df_rows = []\n",
        "i=0\n",
        "for (user,p_id), frame in user_item.groupby(['user_id', 'product_id']):\n",
        "  # rec_df.loc[rec_df.shape[0]] = [user, p_id,len(frame)]\n",
        "  list_of_rec_df_rows.append([user, p_id,len(frame)])\n",
        "  # break\n",
        "  if(i%100000==0):\n",
        "    print(i)\n",
        "  i=i+1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I0hk6XbhrC9w"
      },
      "outputs": [],
      "source": [
        "data_columns = ['user_id','product_id','count']\n",
        "rec_df = pd.DataFrame(list_of_rec_df_rows, columns=data_columns)\n",
        "rec_df "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nc46Eq63rC5A"
      },
      "outputs": [],
      "source": [
        "rec_df.to_pickle('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/Reccomendation_df/rec_df.pkl')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "xO7hS8CWrCsm",
        "outputId": "091ed0d3-4010-4e8b-b2b4-ad3c5c7f4b25"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          user_id  product_id  count\n",
              "0               1         196     11\n",
              "1               1       10258     10\n",
              "2               1       10326      1\n",
              "3               1       12427     10\n",
              "4               1       13032      4\n",
              "...           ...         ...    ...\n",
              "13863741   206209       43961      3\n",
              "13863742   206209       44325      1\n",
              "13863743   206209       48370      1\n",
              "13863744   206209       48697      1\n",
              "13863745   206209       48742      2\n",
              "\n",
              "[13863746 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f4ea6fe1-be37-4f57-b09b-667d407db1ef\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>product_id</th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>196</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>10258</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>10326</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>12427</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>13032</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863741</th>\n",
              "      <td>206209</td>\n",
              "      <td>43961</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863742</th>\n",
              "      <td>206209</td>\n",
              "      <td>44325</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863743</th>\n",
              "      <td>206209</td>\n",
              "      <td>48370</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863744</th>\n",
              "      <td>206209</td>\n",
              "      <td>48697</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863745</th>\n",
              "      <td>206209</td>\n",
              "      <td>48742</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13863746 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f4ea6fe1-be37-4f57-b09b-667d407db1ef')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f4ea6fe1-be37-4f57-b09b-667d407db1ef button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f4ea6fe1-be37-4f57-b09b-667d407db1ef');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "rec_df = pd.read_pickle('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/Reccomendation_df/rec_df.pkl')\n",
        "rec_df "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 645
        },
        "id": "AYIG2D4DrWp2",
        "outputId": "2800f0a7-c422-4a2f-a47e-ce5261a5b7df"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    user_id  product_id  count\n",
              "0         1         196     11\n",
              "1         1       10258     10\n",
              "2         1       10326      1\n",
              "3         1       12427     10\n",
              "4         1       13032      4\n",
              "5         1       13176      2\n",
              "6         1       14084      1\n",
              "7         1       17122      1\n",
              "8         1       25133      9\n",
              "9         1       26088      3\n",
              "10        1       26405      3\n",
              "11        1       27845      1\n",
              "12        1       30450      1\n",
              "13        1       35951      1\n",
              "14        1       38928      2\n",
              "15        1       39657      2\n",
              "16        1       41787      1\n",
              "17        1       46149      4\n",
              "18        1       49235      3"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d597ce77-2739-4e2a-b8db-97caf3cd78b8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>product_id</th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>196</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>10258</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>10326</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>12427</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>13032</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1</td>\n",
              "      <td>13176</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1</td>\n",
              "      <td>14084</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1</td>\n",
              "      <td>17122</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1</td>\n",
              "      <td>25133</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1</td>\n",
              "      <td>26088</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>1</td>\n",
              "      <td>26405</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>1</td>\n",
              "      <td>27845</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>1</td>\n",
              "      <td>30450</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>1</td>\n",
              "      <td>35951</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>1</td>\n",
              "      <td>38928</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>1</td>\n",
              "      <td>39657</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>1</td>\n",
              "      <td>41787</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>1</td>\n",
              "      <td>46149</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>1</td>\n",
              "      <td>49235</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d597ce77-2739-4e2a-b8db-97caf3cd78b8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d597ce77-2739-4e2a-b8db-97caf3cd78b8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d597ce77-2739-4e2a-b8db-97caf3cd78b8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "rec_df[rec_df['user_id']==1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CS-U5aRUrWme",
        "outputId": "ec5777ba-0cc7-483f-ecfa-39937d73a34f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "100"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "rec_df['count'].max()\n",
        "#  No one bought anything more than 100 times."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vCC918E8rWjW",
        "outputId": "7897f5f1-0f58-4b47-f49b-6b0122154011"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "1      8299266\n",
              "2      2296606\n",
              "3      1074345\n",
              "4       612369\n",
              "5       384418\n",
              "        ...   \n",
              "94           5\n",
              "99           4\n",
              "97           4\n",
              "100          3\n",
              "98           2\n",
              "Name: count, Length: 100, dtype: int64"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "rec_df['count'].value_counts()\n",
        "# 8.3 million user/product combinations happened only once "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wzEnwJsMrWgN",
        "outputId": "8b62b02e-ee4b-4cea-e8cc-4e004241fd27"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "206209"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(rec_df.user_id.unique())  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NmImvDDIrWBv"
      },
      "outputs": [],
      "source": [
        "# In case I want a smaller subset of users I'll build users_150K\n",
        "# But I wonder if pulling out items only ordered once or users who only ordered once would be better\n",
        "\n",
        "users_150K = rec_df[rec_df['user_id'] < 150000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 571
        },
        "id": "J8tgQ3SSrV-Z",
        "outputId": "e437c06d-2c6c-4ed3-f75c-14574ae2cf94"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-42e2a0de-3cdb-47bd-b269-71a314557d80\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>num_orders</th>\n",
              "      <th>mode_order_dow</th>\n",
              "      <th>median_order_hour</th>\n",
              "      <th>mean_days_since</th>\n",
              "      <th>aisle_air fresheners candles</th>\n",
              "      <th>aisle_asian foods</th>\n",
              "      <th>aisle_baby accessories</th>\n",
              "      <th>aisle_baby bath body care</th>\n",
              "      <th>aisle_baby food formula</th>\n",
              "      <th>aisle_bakery desserts</th>\n",
              "      <th>...</th>\n",
              "      <th>aisle_spreads</th>\n",
              "      <th>aisle_tea</th>\n",
              "      <th>aisle_tofu meat alternatives</th>\n",
              "      <th>aisle_tortillas flat bread</th>\n",
              "      <th>aisle_trail mix snack mix</th>\n",
              "      <th>aisle_trash bags liners</th>\n",
              "      <th>aisle_vitamins supplements</th>\n",
              "      <th>aisle_water seltzer sparkling water</th>\n",
              "      <th>aisle_white wines</th>\n",
              "      <th>aisle_yogurt</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>user_id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "      <td>13.0</td>\n",
              "      <td>15.357142</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>12.314285</td>\n",
              "      <td>1.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>17.0</td>\n",
              "      <td>7.800000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>22.173914</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>12.0</td>\n",
              "      <td>26.117647</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>28.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>206184</th>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>11.0</td>\n",
              "      <td>24.705883</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>206186</th>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>15.0</td>\n",
              "      <td>16.200001</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>206196</th>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>15.0</td>\n",
              "      <td>25.569767</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>206204</th>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>12.5</td>\n",
              "      <td>15.638298</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>206205</th>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>16.0</td>\n",
              "      <td>14.705882</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>9.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>49404 rows × 138 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-42e2a0de-3cdb-47bd-b269-71a314557d80')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-42e2a0de-3cdb-47bd-b269-71a314557d80 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-42e2a0de-3cdb-47bd-b269-71a314557d80');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "         num_orders  mode_order_dow  median_order_hour  mean_days_since  \\\n",
              "user_id                                                                   \n",
              "4                 5               4               13.0        15.357142   \n",
              "5                 5               0               16.0        12.314285   \n",
              "6                 3               4               17.0         7.800000   \n",
              "8                 4               1                0.0        22.173914   \n",
              "9                 4               5               12.0        26.117647   \n",
              "...             ...             ...                ...              ...   \n",
              "206184            5               2               11.0        24.705883   \n",
              "206186            4               6               15.0        16.200001   \n",
              "206196            5               6               15.0        25.569767   \n",
              "206204            4               5               12.5        15.638298   \n",
              "206205            4               1               16.0        14.705882   \n",
              "\n",
              "         aisle_air fresheners candles  aisle_asian foods  \\\n",
              "user_id                                                    \n",
              "4                                 0.0                0.0   \n",
              "5                                 1.0                4.0   \n",
              "6                                 0.0                0.0   \n",
              "8                                 0.0                1.0   \n",
              "9                                 0.0                0.0   \n",
              "...                               ...                ...   \n",
              "206184                            0.0                1.0   \n",
              "206186                            0.0                0.0   \n",
              "206196                            0.0                0.0   \n",
              "206204                            0.0                2.0   \n",
              "206205                            0.0                0.0   \n",
              "\n",
              "         aisle_baby accessories  aisle_baby bath body care  \\\n",
              "user_id                                                      \n",
              "4                           0.0                        0.0   \n",
              "5                           0.0                        0.0   \n",
              "6                           0.0                        0.0   \n",
              "8                           0.0                        0.0   \n",
              "9                           0.0                        0.0   \n",
              "...                         ...                        ...   \n",
              "206184                      0.0                        0.0   \n",
              "206186                      0.0                        0.0   \n",
              "206196                      0.0                        0.0   \n",
              "206204                      0.0                        0.0   \n",
              "206205                      1.0                        0.0   \n",
              "\n",
              "         aisle_baby food formula  aisle_bakery desserts  ...  aisle_spreads  \\\n",
              "user_id                                                  ...                  \n",
              "4                            0.0                    0.0  ...            0.0   \n",
              "5                            0.0                    0.0  ...            0.0   \n",
              "6                            0.0                    0.0  ...            0.0   \n",
              "8                            0.0                    0.0  ...            0.0   \n",
              "9                            6.0                    0.0  ...            0.0   \n",
              "...                          ...                    ...  ...            ...   \n",
              "206184                       0.0                    0.0  ...            0.0   \n",
              "206186                       0.0                    0.0  ...            1.0   \n",
              "206196                       0.0                    0.0  ...            1.0   \n",
              "206204                       0.0                    0.0  ...            1.0   \n",
              "206205                       0.0                    0.0  ...            0.0   \n",
              "\n",
              "         aisle_tea  aisle_tofu meat alternatives  aisle_tortillas flat bread  \\\n",
              "user_id                                                                        \n",
              "4              0.0                           0.0                         1.0   \n",
              "5              0.0                           0.0                         0.0   \n",
              "6              0.0                           0.0                         0.0   \n",
              "8              0.0                           1.0                         1.0   \n",
              "9              0.0                           0.0                         0.0   \n",
              "...            ...                           ...                         ...   \n",
              "206184         0.0                           0.0                         0.0   \n",
              "206186         0.0                           0.0                         0.0   \n",
              "206196         1.0                           0.0                         0.0   \n",
              "206204         0.0                           0.0                         1.0   \n",
              "206205         0.0                           0.0                         0.0   \n",
              "\n",
              "         aisle_trail mix snack mix  aisle_trash bags liners  \\\n",
              "user_id                                                       \n",
              "4                              0.0                      0.0   \n",
              "5                              0.0                      0.0   \n",
              "6                              0.0                      0.0   \n",
              "8                              0.0                      0.0   \n",
              "9                              0.0                      0.0   \n",
              "...                            ...                      ...   \n",
              "206184                         0.0                      0.0   \n",
              "206186                         0.0                      0.0   \n",
              "206196                         0.0                      0.0   \n",
              "206204                         0.0                      0.0   \n",
              "206205                         0.0                      0.0   \n",
              "\n",
              "         aisle_vitamins supplements  aisle_water seltzer sparkling water  \\\n",
              "user_id                                                                    \n",
              "4                               0.0                                  1.0   \n",
              "5                               0.0                                  0.0   \n",
              "6                               0.0                                  0.0   \n",
              "8                               0.0                                  0.0   \n",
              "9                               0.0                                  3.0   \n",
              "...                             ...                                  ...   \n",
              "206184                          0.0                                  0.0   \n",
              "206186                          0.0                                  4.0   \n",
              "206196                          1.0                                  7.0   \n",
              "206204                          0.0                                  0.0   \n",
              "206205                          0.0                                  0.0   \n",
              "\n",
              "         aisle_white wines  aisle_yogurt  \n",
              "user_id                                   \n",
              "4                      0.0           0.0  \n",
              "5                      0.0           3.0  \n",
              "6                      0.0           0.0  \n",
              "8                      0.0           0.0  \n",
              "9                      0.0          28.0  \n",
              "...                    ...           ...  \n",
              "206184                 0.0           0.0  \n",
              "206186                 0.0           0.0  \n",
              "206196                 0.0           0.0  \n",
              "206204                 0.0           2.0  \n",
              "206205                 0.0           9.0  \n",
              "\n",
              "[49404 rows x 138 columns]"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Might be better by eliminating users with few products ordered instead of few orders\n",
        "# Grouped_users is from the clustering notebook\n",
        "# Need to recalculate this\n",
        "grouped_users = pickle.load(open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/overall_grouped_users/grouped_users.pkl\", \"rb\"))\n",
        "grouped_users[grouped_users['num_orders']<=5] # 49000 users with 5 or fewer orders"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DmNljAFyrV63",
        "outputId": "d66660a9-55fb-400f-cddb-4bbb448c6888"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 206209 entries, 1 to 206209\n",
            "Columns: 138 entries, num_orders to aisle_yogurt\n",
            "dtypes: float32(1), float64(135), int64(1), int8(1)\n",
            "memory usage: 216.5 MB\n"
          ]
        }
      ],
      "source": [
        "grouped_users.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "P9wX2wfXWJZw",
        "outputId": "0fb56d98-7df9-4d67-f440-ec23f1671250"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-9980e66b-f16f-4dfa-bab2-bcd870734ded\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>product_id</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>36233</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11254</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43098</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14485</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7626</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8009</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45085</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6525</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23068</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40766</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2741 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9980e66b-f16f-4dfa-bab2-bcd870734ded')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9980e66b-f16f-4dfa-bab2-bcd870734ded button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9980e66b-f16f-4dfa-bab2-bcd870734ded');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "            count\n",
              "product_id       \n",
              "36233           1\n",
              "11254           1\n",
              "43098           1\n",
              "14485           1\n",
              "7626            1\n",
              "...           ...\n",
              "8009            5\n",
              "45085           5\n",
              "6525            5\n",
              "23068           5\n",
              "40766           5\n",
              "\n",
              "[2741 rows x 1 columns]"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "products_with_count = rec_df.groupby('product_id').sum().drop(columns='user_id').sort_values(by='count')\n",
        "products_with_count[products_with_count['count']<=5]  # 2741 products ordered 5 times or less"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ph1duejFftlt"
      },
      "source": [
        "# Modelling a Recommendation System"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6QviEQWCf-jK",
        "outputId": "2866f532-55ad-4a11-94af-bacad8b1f72b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[31mERROR: Operation cancelled by user\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install surprise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WphK2eHffklK"
      },
      "outputs": [],
      "source": [
        "from surprise import NormalPredictor\n",
        "from surprise import Dataset\n",
        "from surprise import Reader\n",
        "from surprise.model_selection import cross_validate\n",
        "from surprise import SVD \n",
        "from surprise import KNNBaseline \n",
        "from surprise import KNNBasic \n",
        "from surprise.model_selection import train_test_split \n",
        "from surprise.prediction_algorithms import knns \n",
        "from surprise.similarities import cosine, msd, pearson\n",
        "from surprise.prediction_algorithms import SVD, SVDpp\n",
        "from surprise.model_selection import GridSearchCV\n",
        "from surprise import accuracy"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# A reader is still needed but only the rating_scale param is required.\n",
        "reader = Reader(rating_scale=(1, 100))\n",
        "\n",
        "# The columns must correspond to user id, item id and ratings (in that order).\n",
        "rec_data = Dataset.load_from_df(rec_df, reader)"
      ],
      "metadata": {
        "id": "elgL5XzXAD3T",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "acf42a09-16ba-419b-8d91-0f5eecbb67b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-b25daad2bc6f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# The columns must correspond to user id, item id and ratings (in that order).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mrec_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_from_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'rec_df' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w6C4SJIof9AZ"
      },
      "outputs": [],
      "source": [
        "# sample random trainset and testset\n",
        "# test set is made of 25% of the ratings.\n",
        "trainset, testset = train_test_split(rec_data, test_size=.25, random_state=12)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3rQHiSMtgvUv"
      },
      "outputs": [],
      "source": [
        "rec_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nWFkFrw6izNs"
      },
      "outputs": [],
      "source": [
        "import time"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hW6psdlmhL-V"
      },
      "outputs": [],
      "source": [
        "# Start with a memory-based model\n",
        "# These just take up too much memory for the size of my data.  Moving on to model-based.\n",
        "\n",
        "# sim_cos = {'name':'cosine', 'user_based':False}\n",
        "\n",
        "# tic = time.time()\n",
        "\n",
        "# basic = knns.KNNBasic(sim_options=sim_cos)\n",
        "# basic.fit(trainset)\n",
        "\n",
        "# fit_time = time.time()-tic\n",
        "# print('fit ', fit_time)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZLVoDeNrh4xm"
      },
      "outputs": [],
      "source": [
        "# Can't believe I got this to work on the whole data set\n",
        "\n",
        "tic = time.time()\n",
        "\n",
        "svd_algo = SVD()\n",
        "\n",
        "svd_algo.fit(trainset)\n",
        "\n",
        "fit_time = time.time()-tic\n",
        "print('fit ', fit_time)\n",
        "toc = time.time()\n",
        "\n",
        "svd_algo_predictions = svd_algo.test(testset)\n",
        "\n",
        "pred_time = time.time()-toc\n",
        "print('pred ', pred_time)\n",
        "\n",
        "# Then compute RMSE\n",
        "accuracy.rmse(svd_algo_predictions)\n",
        "\n",
        "# 3.46 RMSE on a scale of 1-100 is pretty darn good, but my data is so skewed it may be misleading."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2_nXjiQGxhYV"
      },
      "outputs": [],
      "source": [
        "pickle.dump(svd_algo, open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/svd_algo.p\", \"wb\"))\n",
        "pickle.dump(svd_algo_predictions, open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/svd_algo_predictions.p\", \"wb\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KWXDQR07jJBJ"
      },
      "outputs": [],
      "source": [
        "svd_algo = pickle.load(open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/svd_algo.p\", \"rb\"))\n",
        "svd_algo_predictions = pickle.load(open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/svd_algo_predictions.p\", \"rb\"))\n",
        "accuracy.rmse(svd_algo_predictions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tuJxO6GUj504"
      },
      "outputs": [],
      "source": [
        "# Picking a random user to look at one of their ratings\n",
        "\n",
        "rec_df[rec_df['user_id'] == 196]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "j4tZDEKkkE1g"
      },
      "outputs": [],
      "source": [
        "uid=196\n",
        "iid=13176\n",
        "# actual value was 2, prediction is 1.77\n",
        "pred = svd_algo.predict(uid, iid, r_ui=2, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rNVkNSWnkH6p"
      },
      "outputs": [],
      "source": [
        "# Examining the first record in my test set for its actual value\n",
        "testset[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dIVYvIUJkNhR"
      },
      "outputs": [],
      "source": [
        "# Prediction is 1.15\n",
        "pred = svd_algo.predict(127895, 6101, r_ui=2, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8y0lVSPWkQDd"
      },
      "outputs": [],
      "source": [
        "svd_algo_predictions[0]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "SVD"
      ],
      "metadata": {
        "id": "6rfTQUU0-tHO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7vdnTmkJkVB3"
      },
      "source": [
        "# Perform a grid Search"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Started at 6:45pm, ended at 9:30\n",
        "\n",
        "# param_grid = {'n_factors':[20, 100],'n_epochs': [5, 10], 'lr_all': [0.002, 0.005],\n",
        "#               'reg_all': [0.4, 0.6]} \n",
        "# gs_model = GridSearchCV(SVD, param_grid=param_grid, n_jobs = -1, joblib_verbose=5)\n",
        "# gs_model.fit(rec_data)\n",
        "# pickle.dump(gs_model, open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/gs_model.p\", \"wb\"))"
      ],
      "metadata": {
        "id": "Io9WZzDi_R0w"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating the model from the best params\n",
        "\n",
        "tic = time.time()\n",
        "\n",
        "svd_best_gs = SVD(n_factors = 100, n_epochs = 10, lr_all = 0.005, reg_all = 0.4)\n",
        "\n",
        "svd_best_gs.fit(trainset)\n",
        "\n",
        "fit_time = time.time()-tic\n",
        "print('fit ', fit_time)\n",
        "toc = time.time()\n",
        "\n",
        "svd_best_gs_predictions = svd_best_gs.test(testset)\n",
        "\n",
        "pred_time = time.time()-toc\n",
        "print('pred ', pred_time)\n",
        "\n",
        "# Then compute RMSE\n",
        "accuracy.rmse(svd_best_gs_predictions)\n",
        "\n",
        "# RMSE went down on grid search model from 3.4616 to 3.2483"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "61juk6LNJL5G",
        "outputId": "016d60c6-d0d9-4784-d0c7-5e729281752e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fit  370.9870889186859\n",
            "pred  75.83546209335327\n",
            "RMSE: 3.2391\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3.239057193416181"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Try a different rating scale"
      ],
      "metadata": {
        "id": "ULu6NoU1KFaL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# I am curious how the model predicts one of the items purchased 100 times\n",
        "\n",
        "max(testset,key=lambda item:item[2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AFhUbnenJhdH",
        "outputId": "1c58b68a-5133-48ff-eac8-e5fe30112535"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(17997, 4210, 99.0)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# This one was predicted to be 22.27, that's pretty far off\n",
        "\n",
        "# pred = svd_best_gs.predict(41356,38652, r_ui=100.0, verbose=True)\n",
        "pred = svd_best_gs.predict(17997,4210, r_ui=99.0, verbose=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UTOHiu1SKZuJ",
        "outputId": "af13ad52-7f8e-4c2a-d570-1bef8c60633b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "user: 17997      item: 4210       r_ui = 99.00   est = 49.50   {'was_impossible': False}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# What is my highest prediction?  53.221 on a true rating of 96\n",
        "\n",
        "# It seems like the model gets worse for higher ratings\n",
        "\n",
        "top_est = 0\n",
        "top_pred = [0,0,0,0]\n",
        "for uid, iid, true_r, est, _ in svd_best_gs_predictions:\n",
        "    if est > top_est:\n",
        "        top_est = est\n",
        "        top_pred = [uid,iid,true_r,est]\n",
        "top_pred        "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vEaKw-UZKfI4",
        "outputId": "5ef72a3d-aead-472f-8dc4-779ba72690ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[140440, 19156, 37.0, 74.27454011716559]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Perhaps I can convert into a rating on a 1 to 5 scale\n",
        "rec_df['count'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2HKiNrFpKfFd",
        "outputId": "6b696e96-3a8e-48b5-abb7-be585a333974"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1      8299266\n",
              "2      2296606\n",
              "3      1074345\n",
              "4       612369\n",
              "5       384418\n",
              "        ...   \n",
              "94           5\n",
              "99           4\n",
              "97           4\n",
              "100          3\n",
              "98           2\n",
              "Name: count, Length: 100, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "umJiwM1GNKrS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.hist(rec_df['count']);"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "id": "NsegbKENKfCO",
        "outputId": "6a5fd546-b907-47a2-9907-cafe8808dd38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQOklEQVR4nO3df5BdZ13H8feHpC3yQ1rIymASSNQAZhBoZylVHKhQZ9LiNDqikwwKaCHjDEUURgmDU7D8A+IAMpZChFpgNLUUBjMQqFjq1BFauxUoTULL0laSWsxS2iIyWCJf/7gnet3u5p4kd3eTZ9+vmTu55zzP3vM9eTafOfc5P5KqQpJ08nvEUhcgSRoPA12SGmGgS1IjDHRJaoSBLkmNMNAlqRFLGuhJrkhyMMltPfq+K8mXutcdSR5YjBol6WSRpbwOPcnzge8CH66qZxzFz70GOLOqfnvBipOkk8ySHqFX1Q3At4fXJfnJJJ9JckuSf0zy9Dl+dCuwc1GKlKSTxMqlLmAOO4DfqaqvJXku8F7ghYcbkzwFWA98bonqk6QT0gkV6EkeA/wc8NEkh1efNqvbFuCaqvrvxaxNkk50J1SgM5gCeqCqnn2EPluAVy9SPZJ00jihLlusqu8AdyX5NYAMPOtwezeffgbwhSUqUZJOWEt92eJOBuH8tCQHklwEvBS4KMmXgT3A5qEf2QJcVT4iUpIeZkkvW5Qkjc/II/S+N/8keU6SQ0leMr7yJEl9jTxC73PzT5IVwGeB7wNXVNU1oza8atWqWrdu3VEXLEnL2S233PKtqpqYq23kVS5VdUOSdSO6vQb4GPCcvkWtW7eOqampvt0lSUCSf52v7bhPiiZZDfwKcHmPvtuSTCWZmpmZOd5NS5KGjOMql3cDb6iqH47qWFU7qmqyqiYnJub8xiBJOkbjuLFoEriqu7NzFXBBkkNV9YkxfLYkqafjDvSqWn/4fZIrgU8a5pK0+EYGenfzz7nAqiQHgDcDpwBU1fsWtDpJUm99rnLZ2vfDquoVx1WNJOmYnVDPcpEkHTsDXZIaYaBLUiNOtOeh97Ju+6eWbNt3v+3FS7ZtSToSj9AlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDViZKAnuSLJwSS3zdP+0iS3JvlKks8nedb4y5QkjdLnCP1KYNMR2u8CXlBVPwO8FdgxhrokSUdp5H8SXVU3JFl3hPbPDy3eCKw5/rIkSUdr3HPoFwGfnq8xybYkU0mmZmZmxrxpSVrexhboSX6BQaC/Yb4+VbWjqiaranJiYmJcm5Yk0WPKpY8kzwQ+AJxfVfeN4zMlSUfnuI/QkzwZ+Djwm1V1x/GXJEk6FiOP0JPsBM4FViU5ALwZOAWgqt4HXAI8AXhvEoBDVTW5UAVLkubW5yqXrSPaXwm8cmwVSZKOiXeKSlIjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRowM9CRXJDmY5LZ52pPkPUmmk9ya5KzxlylJGqXPEfqVwKYjtJ8PbOhe24DLj78sSdLRGhnoVXUD8O0jdNkMfLgGbgROT/KkcRUoSepnHHPoq4H9Q8sHunUPk2RbkqkkUzMzM2PYtCTpsEU9KVpVO6pqsqomJyYmFnPTktS8cQT6PcDaoeU13TpJ0iIaR6DvAl7WXe1yDvBgVd07hs+VJB2FlaM6JNkJnAusSnIAeDNwCkBVvQ/YDVwATAPfA35roYqVJM1vZKBX1dYR7QW8emwVSZKOiXeKSlIjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDWiV6An2ZTk9iTTSbbP0f7kJNcn+WKSW5NcMP5SJUlHMjLQk6wALgPOBzYCW5NsnNXtj4Crq+pMYAvw3nEXKkk6sj5H6GcD01V1Z1U9BFwFbJ7Vp4Af7d4/Dvi38ZUoSepjZY8+q4H9Q8sHgOfO6vMW4O+SvAZ4NHDeWKqTJPU2rpOiW4Erq2oNcAHwkSQP++wk25JMJZmamZkZ06YlSdAv0O8B1g4tr+nWDbsIuBqgqr4APBJYNfuDqmpHVU1W1eTExMSxVSxJmlOfQL8Z2JBkfZJTGZz03DWrzzeAFwEk+WkGge4huCQtopGBXlWHgIuBa4F9DK5m2ZPk0iQXdt1eD7wqyZeBncArqqoWqmhJ0sP1OSlKVe0Gds9ad8nQ+73A88ZbmiTpaHinqCQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJakSvQE+yKcntSaaTbJ+nz68n2ZtkT5K/Hm+ZkqRRVo7qkGQFcBnwi8AB4OYku6pq71CfDcAbgedV1f1JfmyhCpYkza3PEfrZwHRV3VlVDwFXAZtn9XkVcFlV3Q9QVQfHW6YkaZQ+gb4a2D+0fKBbN+ypwFOT/FOSG5NsmuuDkmxLMpVkamZm5tgqliTNaVwnRVcCG4Bzga3AXyQ5fXanqtpRVZNVNTkxMTGmTUuSoF+g3wOsHVpe060bdgDYVVU/qKq7gDsYBLwkaZH0CfSbgQ1J1ic5FdgC7JrV5xMMjs5JsorBFMydY6xTkjTCyECvqkPAxcC1wD7g6qrak+TSJBd23a4F7kuyF7ge+IOqum+hipYkPdzIyxYBqmo3sHvWukuG3hfwuu4lSVoC3ikqSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIa0SvQk2xKcnuS6STbj9DvV5NUksnxlShJ6mNkoCdZAVwGnA9sBLYm2ThHv8cCrwVuGneRkqTR+hyhnw1MV9WdVfUQcBWweY5+bwXeDnx/jPVJknrqE+irgf1Dywe6df8ryVnA2qr61JE+KMm2JFNJpmZmZo66WEnS/I77pGiSRwDvBF4/qm9V7aiqyaqanJiYON5NS5KG9An0e4C1Q8trunWHPRZ4BvAPSe4GzgF2eWJUkhZXn0C/GdiQZH2SU4EtwK7DjVX1YFWtqqp1VbUOuBG4sKqmFqRiSdKcRgZ6VR0CLgauBfYBV1fVniSXJrlwoQuUJPWzsk+nqtoN7J617pJ5+p57/GVJko6Wd4pKUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RG9Ar0JJuS3J5kOsn2Odpfl2RvkluTXJfkKeMvVZJ0JCMDPckK4DLgfGAjsDXJxlndvghMVtUzgWuAPxl3oZKkI+tzhH42MF1Vd1bVQ8BVwObhDlV1fVV9r1u8EVgz3jIlSaP0CfTVwP6h5QPduvlcBHx6roYk25JMJZmamZnpX6UkaaSxnhRN8hvAJPCOudqrakdVTVbV5MTExDg3LUnL3soefe4B1g4tr+nW/T9JzgPeBLygqv5rPOVJkvrqc4R+M7AhyfokpwJbgF3DHZKcCbwfuLCqDo6/TEnSKCMDvaoOARcD1wL7gKurak+SS5Nc2HV7B/AY4KNJvpRk1zwfJ0laIH2mXKiq3cDuWesuGXp/3pjrkiQdJe8UlaRGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSI3r9J9H6P+u2f2pJtnv32168JNuVdPLwCF2SGtEr0JNsSnJ7kukk2+doPy3J33TtNyVZN+5CJUlHNjLQk6wALgPOBzYCW5NsnNXtIuD+qvop4F3A28ddqCTpyPrMoZ8NTFfVnQBJrgI2A3uH+mwG3tK9vwb48ySpqhpjrcvaUs3dLyXPG0hHp0+grwb2Dy0fAJ47X5+qOpTkQeAJwLeGOyXZBmzrFr+b5PajqHXV7M9bJpbjfq8CvpXl9z1v2Y71UhexBI5nv58yX8OiXuVSVTuAHcfys0mmqmpyzCWd8Jbjfi/HfYblud/LcZ9h4fa7z0nRe4C1Q8trunVz9kmyEngccN84CpQk9dMn0G8GNiRZn+RUYAuwa1afXcDLu/cvAT7n/LkkLa6RUy7dnPjFwLXACuCKqtqT5FJgqqp2AR8EPpJkGvg2g9Aft2OaqmnActzv5bjPsDz3eznuMyzQfscDaUlqg3eKSlIjDHRJasRJEeijHj3QgiRrk1yfZG+SPUle261/fJLPJvla9+cZS13rQkiyIskXk3yyW17fPUZiunusxKlLXeM4JTk9yTVJvppkX5KfXQ5jneT3u9/v25LsTPLI1sY6yRVJDia5bWjdnGObgfd0+35rkrOOZ9snfKD3fPRACw4Br6+qjcA5wKu7/dwOXFdVG4DruuUWvRbYN7T8duBd3eMk7mfweImW/Bnwmap6OvAsBvve9FgnWQ38LjBZVc9gcJHFFtob6yuBTbPWzTe25wMbutc24PLj2fAJH+gMPXqgqh4CDj96oClVdW9V/Uv3/j8Y/ANfzWBfP9R1+xDwy0tT4cJJsgZ4MfCBbjnACxk8RgIa2+8kjwOez+DqMKrqoap6gGUw1gyurPuR7n6VRwH30thYV9UNDK72Gzbf2G4GPlwDNwKnJ3nSsW77ZAj0uR49sHqJalkU3dMqzwRuAp5YVfd2Td8EnrhEZS2kdwN/CPywW34C8EBVHeqWWxvz9cAM8JfdNNMHkjyaxse6qu4B/hT4BoMgfxC4hbbH+rD5xnas+XYyBPqykuQxwMeA36uq7wy3dTdrNXWdaZJfAg5W1S1LXcsiWgmcBVxeVWcC/8ms6ZVGx/oMBkek64EfBx7Nw6cmmreQY3syBHqfRw80IckpDML8r6rq493qfz/8Faz78+BS1bdAngdcmORuBtNpL2Qwv3x697Uc2hvzA8CBqrqpW76GQcC3PtbnAXdV1UxV/QD4OIPxb3msD5tvbMeabydDoPd59MBJr5s3/iCwr6reOdQ0/FiFlwN/u9i1LaSqemNVramqdQzG9nNV9VLgegaPkYDG9ruqvgnsT/K0btWLGDyOuumxZjDVck6SR3W/74f3u9mxHjLf2O4CXtZd7XIO8ODQ1MzRq6oT/gVcANwBfB1401LXs0D7+PMMvobdCnype13AYD75OuBrwN8Dj1/qWhfw7+Bc4JPd+58A/hmYBj4KnLbU9Y15X58NTHXj/QngjOUw1sAfA18FbgM+ApzW2lgDOxmcI/gBg29jF803tkAYXMX3deArDK4AOuZte+u/JDXiZJhykST1YKBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRvwPKor4g8cDTZ4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Well I'm not sure if this is the best way, but there are so few above 5 I think I can group them all together\n",
        "rec_df['rating'] = rec_df['count'].apply(lambda x: 5 if (x >= 5) else x)"
      ],
      "metadata": {
        "id": "LzJ-mDyVKe_D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.hist(rec_df['rating']);"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "id": "Im4_xGcFKe8L",
        "outputId": "1cf636e5-8153-4b29-a0cc-103703d4a1ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEDCAYAAAAcI05xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAO1klEQVR4nO3da6xlZX3H8e/PmfEGRBpn11KGdkyjNJZWxBOKYoiFalAMvChtxlRbjHbS1mtvBvuixr7yRWNsq62ZIC1ewAuCoagoiRhqo6NnEJVh1FAcdSjtbFFAbKOi/77Y68DxuGfOOrrX3g+c7yc5mb3PeuasXx5m/1jn2WuvlapCktSuRyw6gCTp6CxqSWqcRS1JjbOoJalxFrUkNc6ilqTGDVbUSS5NcjjJLT3H/16SW5PsT3L5ULkk6aEmQ51HneQs4D7gHVV1yjpjnwS8Dzi7qr6d5Oer6vAgwSTpIWawI+qquhH41urvJfmVJNcl2Zfk35P8arfpj4C3VtW3u79rSUtSZ95r1HuAV1bV04G/BP6p+/6TgScn+Y8kn05y7pxzSVKzts5rR0mOBZ4JvD/JyrcftSrHk4BnAzuAG5P8elXdPa98ktSquRU1k6P3u6vq1CnbDgF7q+oHwFeTfIVJcX92jvkkqUlzW/qoqnuZlPDvAmTiqd3mDzI5mibJdiZLIbfPK5sktWzI0/OuAD4FnJzkUJKXAr8PvDTJ54H9wAXd8I8CdyW5FbgB+KuqumuobJL0UDLY6XmSpNnwk4mS1LhB3kzcvn177dy5c4gfLUkPS/v27ftmVY2mbRukqHfu3Mny8vIQP1qSHpaSfO1I21z6kKTGWdSS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcRa1JDXOopakxs3zetS97Lz4QwvZ78E3nreQ/UrSejyilqTGWdSS1DiLWpIa16uok/xZkv1JbklyRZJHDx1MkjSxblEnORF4FbBUVacAW4BdQweTJE30XfrYCjwmyVbgscB/DRdJkrTaukVdVXcAfwd8HbgTuKeqPrZ2XJLdSZaTLI/H49knlaRNqs/Sx88xuVv4E4FfBI5J8qK146pqT1UtVdXSaDT1bjKSpJ9Cn6WP3wa+WlXjqvoBcBXwzGFjSZJW9CnqrwNnJHlskgDnAAeGjSVJWtFnjXovcCVwE/DF7u/sGTiXJKnT61ofVfV64PUDZ5EkTeEnEyWpcRa1JDXOopakxlnUktQ4i1qSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1zqKWpMZZ1JLUOItakhpnUUtS4yxqSWqcRS1Jjetzc9uTk9y86uveJK+ZRzhJUo87vFTVl4FTAZJsAe4Arh44lySps9Glj3OA/6yqrw0RRpL0kzZa1LuAK6ZtSLI7yXKS5fF4/LMnkyQBGyjqJI8EzgfeP217Ve2pqqWqWhqNRrPKJ0mb3kaOqJ8H3FRV/zNUGEnST9pIUb+QIyx7SJKG06uokxwDPAe4atg4kqS11j09D6Cqvgs8fuAskqQp/GSiJDXOopakxlnUktQ4i1qSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1zqKWpMZZ1JLUOItakhpnUUtS4yxqSWqcRS1JjbOoJalxfe/wcnySK5N8KcmBJM8YOpgkaaLXHV6Avweuq6oLu7uRP3bATJKkVdYt6iSPA84CLgKoqu8D3x82liRpRZ+ljycCY+BfknwuySXdzW5/TJLdSZaTLI/H45kHlaTNqk9RbwVOA/65qp4GfBe4eO2gqtpTVUtVtTQajWYcU5I2rz5FfQg4VFV7u+dXMiluSdIcrFvUVfXfwDeSnNx96xzg1kFTSZIe0Pesj1cC7+7O+LgdeMlwkSRJq/Uq6qq6GVgaOIskaQo/mShJjbOoJalxFrUkNc6ilqTGWdSS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcRa1JDXOopakxlnUktQ4i1qSGmdRS1LjLGpJalyvGwckOQh8B/ghcH9VeRMBSZqTvrfiAvitqvrmYEkkSVO59CFJjetb1AV8LMm+JLunDUiyO8lykuXxeDy7hJK0yfUt6mdV1WnA84CXJzlr7YCq2lNVS1W1NBqNZhpSkjazXkVdVXd0fx4GrgZOHzKUJOlB6xZ1kmOSHLfyGHgucMvQwSRJE33O+ngCcHWSlfGXV9V1g6aSJD1g3aKuqtuBp84hiyRpCk/Pk6TGWdSS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcRa1JDXOopakxlnUktQ4i1qSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1rndRJ9mS5HNJrh0ykCTpx23kiPrVwIGhgkiSputV1El2AOcBlwwbR5K0Vt8j6jcDrwV+dKQBSXYnWU6yPB6PZxJOktSjqJO8ADhcVfuONq6q9lTVUlUtjUajmQWUpM2uzxH1mcD5SQ4C7wHOTvKuQVNJkh6wblFX1euqakdV7QR2AR+vqhcNnkySBHgetSQ1b+tGBlfVJ4BPDJJEkjSVR9SS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcRa1JDXOopakxlnUktQ4i1qSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1zqKWpMb1ubnto5N8Jsnnk+xP8oZ5BJMkTfS5w8v3gLOr6r4k24BPJvlIVX164GySJHoUdVUVcF/3dFv3VUOGkiQ9qNcadZItSW4GDgPXV9XeKWN2J1lOsjwej2edU5I2rV5FXVU/rKpTgR3A6UlOmTJmT1UtVdXSaDSadU5J2rQ2dNZHVd0N3ACcO0wcSdJafc76GCU5vnv8GOA5wJeGDiZJmuhz1scJwGVJtjAp9vdV1bXDxpIkrehz1scXgKfNIYskaQo/mShJjbOoJalxFrUkNc6ilqTGWdSS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcRa1JDWuz0WZNLCdF39oIfs9+MbzFrJfSRvjEbUkNc6ilqTGWdSS1DiLWpIa1+dWXCcluSHJrUn2J3n1PIJJkib6nPVxP/AXVXVTkuOAfUmur6pbB84mSaLHEXVV3VlVN3WPvwMcAE4cOpgkaWJDa9RJdjK5f+LeKdt2J1lOsjwej2eTTpLUv6iTHAt8AHhNVd27dntV7amqpapaGo1Gs8woSZtar6JOso1JSb+7qq4aNpIkabU+Z30EeDtwoKreNHwkSdJqfY6ozwReDJyd5Obu6/kD55IkddY9Pa+qPglkDlkkSVP4yURJapxFLUmNs6glqXEWtSQ1zju8SHrYebjdNckjaklqnEUtSY2zqCWpcRa1JDXOopakxlnUktQ4i1qSGmdRS1LjLGpJapxFLUmNs6glqXF9bsV1aZLDSW6ZRyBJ0o/rc1GmfwXeArxj2CjS8B5uF+vR5rDuEXVV3Qh8aw5ZJElTzGyNOsnuJMtJlsfj8ax+rCRtejMr6qraU1VLVbU0Go1m9WMladPzrA9JapxFLUmN63N63hXAp4CTkxxK8tLhY0mSVqx7el5VvXAeQSRJ07n0IUmNs6glqXEWtSQ1zqKWpMb1udaHpIcwr2/y0OcRtSQ1zqKWpMZZ1JLUOItakhpnUUtS4yxqSWqcRS1JjbOoJalxFrUkNc6ilqTGWdSS1LheRZ3k3CRfTnJbkouHDiVJelCfW3FtAd4KPA94CvDCJE8ZOpgkaaLPEfXpwG1VdXtVfR94D3DBsLEkSStSVUcfkFwInFtVL+uevxj4zap6xZpxu4Hd3dOTgS//lJm2A9/8Kf/ukMy1MebaGHNtzMMx1y9X1Wjahpldj7qq9gB7ftafk2S5qpZmEGmmzLUx5toYc23MZsvVZ+njDuCkVc93dN+TJM1Bn6L+LPCkJE9M8khgF3DNsLEkSSvWXfqoqvuTvAL4KLAFuLSq9g+Y6WdePhmIuTbGXBtjro3ZVLnWfTNRkrRYfjJRkhpnUUtS4xZS1EkuTXI4yS1H2J4k/9B9ZP0LSU5rJNezk9yT5Obu62/mlOukJDckuTXJ/iSvnjJm7nPWM9fc5yzJo5N8Jsnnu1xvmDLmUUne283X3iQ7G8l1UZLxqvl62dC5Vu17S5LPJbl2yra5z1fPXAuZryQHk3yx2+fylO2zfT1W1dy/gLOA04BbjrD9+cBHgABnAHsbyfVs4NoFzNcJwGnd4+OArwBPWfSc9cw19znr5uDY7vE2YC9wxpoxfwq8rXu8C3hvI7kuAt4y739j3b7/HLh82n+vRcxXz1wLmS/gILD9KNtn+npcyBF1Vd0IfOsoQy4A3lETnwaOT3JCA7kWoqrurKqbusffAQ4AJ64ZNvc565lr7ro5uK97uq37Wvuu+QXAZd3jK4FzkqSBXAuRZAdwHnDJEYbMfb565mrVTF+Pra5Rnwh8Y9XzQzRQAJ1ndL+6fiTJr817592vnE9jcjS22kLn7Ci5YAFz1v26fDNwGLi+qo44X1V1P3AP8PgGcgH8Tvfr8pVJTpqyfQhvBl4L/OgI2xcyXz1ywWLmq4CPJdmXyeUz1prp67HVom7VTUw+j/9U4B+BD85z50mOBT4AvKaq7p3nvo9mnVwLmbOq+mFVncrkk7SnJzllHvtdT49c/wbsrKrfAK7nwaPYwSR5AXC4qvYNva+N6Jlr7vPVeVZVncbkqqIvT3LWkDtrtaib/Nh6Vd278qtrVX0Y2JZk+zz2nWQbkzJ8d1VdNWXIQuZsvVyLnLNun3cDNwDnrtn0wHwl2Qo8Drhr0bmq6q6q+l739BLg6XOIcyZwfpKDTK6OeXaSd60Zs4j5WjfXguaLqrqj+/MwcDWTq4yuNtPXY6tFfQ3wB907p2cA91TVnYsOleQXVtblkpzOZP4Gf3F3+3w7cKCq3nSEYXOfsz65FjFnSUZJju8ePwZ4DvClNcOuAf6we3wh8PHq3gVaZK4165jnM1n3H1RVva6qdlTVTiZvFH68ql60Ztjc56tPrkXMV5Jjkhy38hh4LrD2TLGZvh5ndvW8jUhyBZOzAbYnOQS8nskbK1TV24APM3nX9Dbgf4GXNJLrQuBPktwP/B+wa+h/rJ0zgRcDX+zWNwH+GvilVdkWMWd9ci1izk4ALsvkphePAN5XVdcm+VtguaquYfI/mHcmuY3JG8i7Bs7UN9erkpwP3N/lumgOuaZqYL765FrEfD0BuLo7/tgKXF5V1yX5Yxjm9ehHyCWpca0ufUiSOha1JDXOopakxlnUktQ4i1qSGmdRS1LjLGpJatz/A7kiCxPpJkm4AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_rec_df = rec_df.drop(columns='count')\n",
        "new_rec_df.tail()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "1b5I5PaOKe5F",
        "outputId": "0d216ecd-383b-485c-f214-91cb04bac7a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          user_id  product_id  rating\n",
              "13863741   206209       43961       3\n",
              "13863742   206209       44325       1\n",
              "13863743   206209       48370       1\n",
              "13863744   206209       48697       1\n",
              "13863745   206209       48742       2"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c869125f-5069-4365-909a-3ddb60f94358\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>product_id</th>\n",
              "      <th>rating</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>13863741</th>\n",
              "      <td>206209</td>\n",
              "      <td>43961</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863742</th>\n",
              "      <td>206209</td>\n",
              "      <td>44325</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863743</th>\n",
              "      <td>206209</td>\n",
              "      <td>48370</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863744</th>\n",
              "      <td>206209</td>\n",
              "      <td>48697</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863745</th>\n",
              "      <td>206209</td>\n",
              "      <td>48742</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c869125f-5069-4365-909a-3ddb60f94358')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c869125f-5069-4365-909a-3ddb60f94358 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c869125f-5069-4365-909a-3ddb60f94358');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pickle.dump(new_rec_df, open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/new_rec_df.p\", \"wb\"))\n"
      ],
      "metadata": {
        "id": "9JAZ1ypfKe1Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_rec_df = pickle.load(open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/new_rec_df.p\", \"rb\"))\n",
        "new_rec_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "MRyUKdPAV9vS",
        "outputId": "7314cc94-9628-4e52-af4a-07162b755e4e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          user_id  product_id  rating\n",
              "0               1         196       5\n",
              "1               1       10258       5\n",
              "2               1       10326       1\n",
              "3               1       12427       5\n",
              "4               1       13032       4\n",
              "...           ...         ...     ...\n",
              "13863741   206209       43961       3\n",
              "13863742   206209       44325       1\n",
              "13863743   206209       48370       1\n",
              "13863744   206209       48697       1\n",
              "13863745   206209       48742       2\n",
              "\n",
              "[13863746 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1459d14d-b605-4b34-84d9-c622e4ac3d57\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>product_id</th>\n",
              "      <th>rating</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>196</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>10258</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>10326</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>12427</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>13032</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863741</th>\n",
              "      <td>206209</td>\n",
              "      <td>43961</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863742</th>\n",
              "      <td>206209</td>\n",
              "      <td>44325</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863743</th>\n",
              "      <td>206209</td>\n",
              "      <td>48370</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863744</th>\n",
              "      <td>206209</td>\n",
              "      <td>48697</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863745</th>\n",
              "      <td>206209</td>\n",
              "      <td>48742</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13863746 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1459d14d-b605-4b34-84d9-c622e4ac3d57')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1459d14d-b605-4b34-84d9-c622e4ac3d57 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1459d14d-b605-4b34-84d9-c622e4ac3d57');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Passing pieces of dataframe in FLASK to be put back together there\n",
        "pickle.dump(new_rec_df.columns.values,open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_columns.p','wb'))\n",
        "pickle.dump(new_rec_df.index.values, open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_index.p', 'wb'))\n",
        "pickle.dump(new_rec_df.user_id.values, open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_user.p', 'wb'))\n",
        "pickle.dump(new_rec_df.rating.values, open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_rating.p', 'wb'))\n",
        "pickle.dump(new_rec_df.product_id.values, open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_prod_id.p', 'wb'))"
      ],
      "metadata": {
        "id": "iQcSKgHWKep1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import time"
      ],
      "metadata": {
        "id": "RVtwae_fV2Za"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rec_columns = pickle.load(open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_columns.p','rb'))\n",
        "rec_index = pickle.load(open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_index.p', 'rb'))\n",
        "rec_user = pickle.load(open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_user.p', 'rb'))\n",
        "rec_rating = pickle.load(open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_rating.p', 'rb'))\n",
        "rec_prod_id = pickle.load(open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/rec_prod_id.p', 'rb'))"
      ],
      "metadata": {
        "id": "rOr5_8G_K_Y5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_df = pd.DataFrame(np.column_stack([rec_user, rec_prod_id, rec_rating]), index=rec_index, columns=rec_columns)\n",
        "new_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "AvzrwdGsK_Vc",
        "outputId": "2ac22988-11da-405d-8e9b-f0fe8d509b5a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          user_id  product_id  rating\n",
              "0               1         196       5\n",
              "1               1       10258       5\n",
              "2               1       10326       1\n",
              "3               1       12427       5\n",
              "4               1       13032       4\n",
              "...           ...         ...     ...\n",
              "13863741   206209       43961       3\n",
              "13863742   206209       44325       1\n",
              "13863743   206209       48370       1\n",
              "13863744   206209       48697       1\n",
              "13863745   206209       48742       2\n",
              "\n",
              "[13863746 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c665bdff-3696-4aba-a5f3-cdd7161114c4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>product_id</th>\n",
              "      <th>rating</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>196</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>10258</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>10326</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>12427</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>13032</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863741</th>\n",
              "      <td>206209</td>\n",
              "      <td>43961</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863742</th>\n",
              "      <td>206209</td>\n",
              "      <td>44325</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863743</th>\n",
              "      <td>206209</td>\n",
              "      <td>48370</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863744</th>\n",
              "      <td>206209</td>\n",
              "      <td>48697</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863745</th>\n",
              "      <td>206209</td>\n",
              "      <td>48742</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13863746 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c665bdff-3696-4aba-a5f3-cdd7161114c4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c665bdff-3696-4aba-a5f3-cdd7161114c4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c665bdff-3696-4aba-a5f3-cdd7161114c4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Must start with a new reader\n",
        "\n",
        "tic = time.time()\n",
        "reader = Reader(rating_scale=(1, 5))\n",
        "readtime = time.time()-tic\n",
        "print(readtime)\n",
        "# The columns must correspond to user id, item id and ratings (in that order).\n",
        "toc = time.time()\n",
        "new_rec_data = Dataset.load_from_df(new_rec_df, reader)\n",
        "loadtime = time.time()-toc\n",
        "print(loadtime)\n",
        "# sample random trainset and testset\n",
        "# test set is made of 25% of the ratings.\n",
        "tac = time.time()\n",
        "new_trainset, new_testset = train_test_split(new_rec_data, test_size=.25, random_state=12)\n",
        "splittime = time.time()-tac\n",
        "print(splittime)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VI5KU36qK_Sg",
        "outputId": "85aadf3b-8bca-4b3c-8f32-73784b4a5495"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.0009174346923828125\n",
            "14.556567668914795\n",
            "73.20622706413269\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "kszKiPXCK_Po"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pickle.dump(reader, open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/reader.p\", \"wb\"))"
      ],
      "metadata": {
        "id": "pP_zQyXdK_M5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating the model from the best params after grid search\n",
        "\n",
        "tic = time.time()\n",
        "\n",
        "new_svd_best_gs = SVD(n_factors = 20, n_epochs = 10, lr_all = 0.005, reg_all = 0.4)\n",
        "\n",
        "new_svd_best_gs.fit(new_trainset)\n",
        "\n",
        "fit_time = time.time()-tic\n",
        "print('fit ', fit_time)\n",
        "toc = time.time()\n",
        "\n",
        "new_svd_best_gs_predictions = new_svd_best_gs.test(new_testset)\n",
        "\n",
        "pred_time = time.time()-toc\n",
        "print('pred ', pred_time)\n",
        "\n",
        "# Then compute RMSE\n",
        "accuracy.rmse(new_svd_best_gs_predictions)\n",
        "\n",
        "# RMSE went down on grid search model from 1.2649 to 1.2648.  lol, not much."
      ],
      "metadata": {
        "id": "RNyogbrwK_J7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "10192b05-8526-49a8-a2f8-2260982aa00c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fit  205.3625226020813\n",
            "pred  84.0440661907196\n",
            "RMSE: 1.2636\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.263636931312993"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pickle.dump(new_svd_best_gs, open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/new_svd_best_gs.p\", \"wb\"))\n",
        "pickle.dump(new_svd_best_gs_predictions, open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/new_svd_best_gs_predictions.p\", \"wb\"))"
      ],
      "metadata": {
        "id": "1CjpToY1K_G3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_svd_best_gs = pickle.load(open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/new_svd_best_gs.p\", \"rb\"))\n",
        "new_svd_best_gs_predictions = pickle.load(open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/new_svd_best_gs_predictions.p\", \"rb\"))"
      ],
      "metadata": {
        "id": "14xgrKJOkkTR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Get top n predictions for a user"
      ],
      "metadata": {
        "id": "UA1A8kYqWvTX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Get top predictions for a user\n",
        "\n",
        "from collections import defaultdict\n",
        "\n",
        "def get_top_n(predictions, n=10):\n",
        "    \"\"\"Return the top-N recommendation for each user from a set of predictions.\n",
        "\n",
        "    Args:\n",
        "        predictions(list of Prediction objects): The list of predictions, as\n",
        "            returned by the test method of an algorithm.\n",
        "        n(int): The number of recommendation to output for each user. Default\n",
        "            is 10.\n",
        "\n",
        "    Returns:\n",
        "    A dict where keys are user (raw) ids and values are lists of tuples:\n",
        "        [(raw item id, rating estimation), ...] of size n.\n",
        "    \"\"\"\n",
        "\n",
        "    # First map the predictions to each user.\n",
        "    top_n = defaultdict(list)\n",
        "    for uid, iid, true_r, est, _ in predictions:\n",
        "        top_n[uid].append((iid, est))\n",
        "\n",
        "    # Then sort the predictions for each user and retrieve the k highest ones.\n",
        "    for uid, user_ratings in top_n.items():\n",
        "        user_ratings.sort(key=lambda x: x[1], reverse=True)\n",
        "        top_n[uid] = user_ratings[:n]\n",
        "\n",
        "    return top_n\n",
        "\n",
        "\n",
        "# First train an SVD algorithm on the movielens dataset.\n",
        "# data = Dataset.load_builtin('ml-100k')\n",
        "# trainset = data.build_full_trainset()\n",
        "# algo = SVD()\n",
        "# algo.fit(trainset)\n",
        "\n",
        "# Than predict ratings for all pairs (u, i) that are NOT in the training set.\n",
        "# anti_testset = new_trainset.build_anti_testset()\n",
        "# predictions = anti_new_svd_best_gs.test(anti_testset)"
      ],
      "metadata": {
        "id": "SvTSZJOkWzrP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "top_n = get_top_n(new_svd_best_gs_predictions, n=10)"
      ],
      "metadata": {
        "id": "Riui15-NW0IL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Analyze popularity basis"
      ],
      "metadata": {
        "id": "REqe5lYJaEb1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "1 in top_n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xS5fYI2qW0Ez",
        "outputId": "a74b1393-eff9-4248-c862-a8a855cc31ae"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "top_n[1] # Recommendations for user number 1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nO9gK_E0W0B-",
        "outputId": "08ceae15-da06-4941-9d56-0bb20748a0d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(196, 2.873946326155466),\n",
              " (46149, 2.712573906027794),\n",
              " (39657, 2.509711929862778),\n",
              " (13176, 2.32925516020976)]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(top_n.values()) "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_0cC3SkEWz_O",
        "outputId": "e71a44e7-1fc7-4f45-97ea-283085d80091"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "203501"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# order_products_desc['product_name'].value_counts()  "
      ],
      "metadata": {
        "id": "JGTMTzjCWz8M"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_rec_items=[]  # Making a list of all recommended items in the top for each user, then count them\n",
        "for idx in range(len(top_n.values())-200000):\n",
        "    all_rec_items.extend([x[0] for x in [*top_n.values()][idx]]) # product_ids recommended for first user\n",
        "all_rec_items    "
      ],
      "metadata": {
        "id": "cxXN3AQ0aXrG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "\n",
        "ctr = Counter(all_rec_items)\n",
        "most_ctr = ctr.most_common(20)"
      ],
      "metadata": {
        "id": "j6m66Y8MaXnc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "products_desc = pd.read_csv('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/df_products_description.csv')"
      ],
      "metadata": {
        "id": "DcH-RWrgaXhO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "[products_desc[products_desc['product_id']==x]['product_name'].iloc[0] for x in [x[0] for x in most_ctr]]"
      ],
      "metadata": {
        "id": "luJLuJaDaXeL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4842d1b9-d6b4-48b8-ab20-094dad8c5ab6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Banana',\n",
              " 'Bag of Organic Bananas',\n",
              " 'Organic Baby Spinach',\n",
              " 'Strawberries',\n",
              " 'Organic Avocado',\n",
              " 'Organic Strawberries',\n",
              " 'Organic Hass Avocado',\n",
              " 'Organic Zucchini',\n",
              " 'Organic Fuji Apple',\n",
              " 'Organic Whole Milk',\n",
              " 'Organic Blueberries',\n",
              " 'Honeycrisp Apple',\n",
              " 'Half & Half',\n",
              " 'Organic Large Extra Fancy Fuji Apple',\n",
              " 'Michigan Organic Kale',\n",
              " 'Sparkling Water Grapefruit',\n",
              " 'Organic Raspberries',\n",
              " 'Organic Baby Arugula',\n",
              " 'Apple Honeycrisp Organic',\n",
              " 'Organic Half & Half']"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_rec_df = pickle.load(open('/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/new_rec_df.p','rb'))"
      ],
      "metadata": {
        "id": "iAcN-D8uaXbS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_rec_df.product_id.value_counts().sum() # 13,863,746 ratings, so the top 80% would be 11,091,000"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2W7fi4ws_6bx",
        "outputId": "4def890d-6303-43e3-ad1e-1dffbcba3a3f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "13863746"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_rec_df.product_id.value_counts()[:6207].cumsum().values # I admit this was trial and error"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LHSMv7VKAUV9",
        "outputId": "2e798a11-6d5d-4298-b7bf-559a82c81af6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([   76125,   141780,   202909, ..., 11090416, 11090809, 11091202])"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# so the top 6200 products account for 80% of the ratings (or purchases in this case)\n",
        "short_head = list(new_rec_df.product_id.value_counts()[:6207].cumsum().index)"
      ],
      "metadata": {
        "id": "35wAx2MDAXRn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pickle.dump(short_head, open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/short_head.p\", \"wb\"))"
      ],
      "metadata": {
        "id": "3TtWLmiDAZo2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "13176 not in short_head"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N7b_T9N3Ajem",
        "outputId": "44d4a56f-b52a-4993-9c8b-6adfe6cb3074"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# So now when I generate recommendations I will make sure 20% of the recommendations come from the long tail\n",
        "# Personalized re-ranking"
      ],
      "metadata": {
        "id": "R7L5GLd8Al_F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# products_desc = pickle.load(open(\"Pickle/products_desc.p\", \"rb\"))\n",
        "#already loaded"
      ],
      "metadata": {
        "id": "nEKOLghxAn3g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Get Recommendation for a new user\n"
      ],
      "metadata": {
        "id": "kfaNUfDGGsII"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def grocery_rater(df, num, aisle=None):\n",
        "    userID = 300000\n",
        "    rating_list = []\n",
        "    while num > 0:\n",
        "        if aisle:\n",
        "            product = df[df['aisle'].str.contains(aisle)].sample(1)\n",
        "        else:\n",
        "            product = df.sample(1)\n",
        "        print('\\n', product['product_name'].iloc[0])\n",
        "        rating = input('How do you rate this product on a scale of 1-5, choose 0 to rate a different product:\\n')\n",
        "        if rating == '0':\n",
        "            continue\n",
        "        else:\n",
        "            rating_one_product = {'user_id':userID,'product_id':product['product_id'].iloc[0],'rating':int(rating)}\n",
        "            rating_list.append(rating_one_product) \n",
        "            num -= 1\n",
        "    return rating_list  "
      ],
      "metadata": {
        "id": "g-XWlZukBFWM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# try out the new function here!\n",
        "user_rating = grocery_rater(products_desc, 4, aisle='candy')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TwTTUF9WBIIo",
        "outputId": "51d0f280-65e2-43eb-bd17-dcbd765513c3"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            " Caramel Lovers Variety Candy Bars\n",
            "How do you rate this product on a scale of 1-5, choose 0 to rate a different product:\n",
            "3\n",
            "\n",
            " Original Bite Size Candies\n",
            "How do you rate this product on a scale of 1-5, choose 0 to rate a different product:\n",
            "1\n",
            "\n",
            " Licorice, Strawberry\n",
            "How do you rate this product on a scale of 1-5, choose 0 to rate a different product:\n",
            "2\n",
            "\n",
            " Go Bar Chocolate Brownie\n",
            "How do you rate this product on a scale of 1-5, choose 0 to rate a different product:\n",
            "4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "user_rating"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RrI6CPIdCUn2",
        "outputId": "89d16fcc-6952-447d-dbc4-41f3bb7a633b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'product_id': 3189, 'rating': 3, 'user_id': 300000},\n",
              " {'product_id': 44022, 'rating': 1, 'user_id': 300000},\n",
              " {'product_id': 24545, 'rating': 2, 'user_id': 300000},\n",
              " {'product_id': 45602, 'rating': 4, 'user_id': 300000}]"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a clean copy of the rec dataframe\n",
        "new_rec_df = pickle.load(open(\"/content/drive/MyDrive/Grocery_Recommendation/Kaggle_data/models/new_rec_df.p\", \"rb\"))\n",
        "\n",
        "# add the new ratings to the original ratings DataFrame\n",
        "new_ratings_df = new_rec_df.append(user_rating, ignore_index=True)\n",
        "new_data = Dataset.load_from_df(new_ratings_df, reader)"
      ],
      "metadata": {
        "id": "FLhy8iwYCV2g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_ratings_df.tail() # You can see the new user's ratings"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "G9N3I6UTCqfJ",
        "outputId": "f6a3e304-ee97-40c6-e57f-7c4af85724c7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          user_id  product_id  rating\n",
              "13863745   206209       48742       2\n",
              "13863746   300000        3189       3\n",
              "13863747   300000       44022       1\n",
              "13863748   300000       24545       2\n",
              "13863749   300000       45602       4"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-543a4729-6919-43af-959c-0b001bc81548\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>product_id</th>\n",
              "      <th>rating</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>13863745</th>\n",
              "      <td>206209</td>\n",
              "      <td>48742</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863746</th>\n",
              "      <td>300000</td>\n",
              "      <td>3189</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863747</th>\n",
              "      <td>300000</td>\n",
              "      <td>44022</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863748</th>\n",
              "      <td>300000</td>\n",
              "      <td>24545</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13863749</th>\n",
              "      <td>300000</td>\n",
              "      <td>45602</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-543a4729-6919-43af-959c-0b001bc81548')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-543a4729-6919-43af-959c-0b001bc81548 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-543a4729-6919-43af-959c-0b001bc81548');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# train a model using the new combined DataFrame\n",
        "new_user_svd = SVD(n_factors = 20, n_epochs = 10, lr_all = 0.005, reg_all = 0.4)\n",
        "new_user_svd.fit(new_data.build_full_trainset())"
      ],
      "metadata": {
        "id": "P0rPywpIEubR"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "SVD_Recommender.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}