data_processing 3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting features of 2018 movies from Wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "link = \"https://en.wikipedia.org/wiki/List_of_American_films_of_2018\"\n",
    "df1 = pd.read_html(link, header=0)[2]\n",
    "df2 = pd.read_html(link, header=0)[3]\n",
    "df3 = pd.read_html(link, header=0)[4]\n",
    "df4 = pd.read_html(link, header=0)[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = df1.append(df2.append(df3.append(df4,ignore_index=True),ignore_index=True),ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Opening</th>\n",
       "      <th>Opening.1</th>\n",
       "      <th>Title</th>\n",
       "      <th>Production company</th>\n",
       "      <th>Cast and crew</th>\n",
       "      <th>Ref.</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>5</td>\n",
       "      <td>Insidious: The Last Key</td>\n",
       "      <td>Universal Pictures / Blumhouse Productions / S...</td>\n",
       "      <td>Adam Robitel (director); Leigh Whannell (scree...</td>\n",
       "      <td>[2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>5</td>\n",
       "      <td>The Strange Ones</td>\n",
       "      <td>Vertical Entertainment</td>\n",
       "      <td>Lauren Wolkstein (director); Christopher Radcl...</td>\n",
       "      <td>[3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>5</td>\n",
       "      <td>Stratton</td>\n",
       "      <td>Momentum Pictures</td>\n",
       "      <td>Simon West (director); Duncan Falconer, Warren...</td>\n",
       "      <td>[4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>10</td>\n",
       "      <td>Sweet Country</td>\n",
       "      <td>Samuel Goldwyn Films</td>\n",
       "      <td>Warwick Thornton (director); David Tranter, St...</td>\n",
       "      <td>[5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>12</td>\n",
       "      <td>The Commuter</td>\n",
       "      <td>Lionsgate / StudioCanal / The Picture Company</td>\n",
       "      <td>Jaume Collet-Serra (director); Byron Willinger...</td>\n",
       "      <td>[6]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Holmes &amp; Watson</td>\n",
       "      <td>Columbia Pictures / Gary Sanchez Productions</td>\n",
       "      <td>Etan Cohen (director/screenplay); Will Ferrell...</td>\n",
       "      <td>[163]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Vice</td>\n",
       "      <td>Annapurna Pictures / Plan B Entertainment</td>\n",
       "      <td>Adam McKay (director/screenplay); Christian Ba...</td>\n",
       "      <td>[137]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>On the Basis of Sex</td>\n",
       "      <td>Focus Features</td>\n",
       "      <td>Mimi Leder (director); Daniel Stiepleman (scre...</td>\n",
       "      <td>[226]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Destroyer</td>\n",
       "      <td>Annapurna Pictures</td>\n",
       "      <td>Karyn Kusama (director); Phil Hay, Matt Manfre...</td>\n",
       "      <td>[258]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>28</td>\n",
       "      <td>Black Mirror: Bandersnatch</td>\n",
       "      <td>Netflix</td>\n",
       "      <td>David Slade (director); Charlie Brooker (scree...</td>\n",
       "      <td>[259]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Opening  Opening.1                       Title  \\\n",
       "0     JANUARY          5     Insidious: The Last Key   \n",
       "1     JANUARY          5            The Strange Ones   \n",
       "2     JANUARY          5                    Stratton   \n",
       "3     JANUARY         10               Sweet Country   \n",
       "4     JANUARY         12                The Commuter   \n",
       "..        ...        ...                         ...   \n",
       "265  DECEMBER         25             Holmes & Watson   \n",
       "266  DECEMBER         25                        Vice   \n",
       "267  DECEMBER         25         On the Basis of Sex   \n",
       "268  DECEMBER         25                   Destroyer   \n",
       "269  DECEMBER         28  Black Mirror: Bandersnatch   \n",
       "\n",
       "                                    Production company  \\\n",
       "0    Universal Pictures / Blumhouse Productions / S...   \n",
       "1                               Vertical Entertainment   \n",
       "2                                    Momentum Pictures   \n",
       "3                                 Samuel Goldwyn Films   \n",
       "4        Lionsgate / StudioCanal / The Picture Company   \n",
       "..                                                 ...   \n",
       "265       Columbia Pictures / Gary Sanchez Productions   \n",
       "266          Annapurna Pictures / Plan B Entertainment   \n",
       "267                                     Focus Features   \n",
       "268                                 Annapurna Pictures   \n",
       "269                                            Netflix   \n",
       "\n",
       "                                         Cast and crew   Ref.  \n",
       "0    Adam Robitel (director); Leigh Whannell (scree...    [2]  \n",
       "1    Lauren Wolkstein (director); Christopher Radcl...    [3]  \n",
       "2    Simon West (director); Duncan Falconer, Warren...    [4]  \n",
       "3    Warwick Thornton (director); David Tranter, St...    [5]  \n",
       "4    Jaume Collet-Serra (director); Byron Willinger...    [6]  \n",
       "..                                                 ...    ...  \n",
       "265  Etan Cohen (director/screenplay); Will Ferrell...  [163]  \n",
       "266  Adam McKay (director/screenplay); Christian Ba...  [137]  \n",
       "267  Mimi Leder (director); Daniel Stiepleman (scre...  [226]  \n",
       "268  Karyn Kusama (director); Phil Hay, Matt Manfre...  [258]  \n",
       "269  David Slade (director); Charlie Brooker (scree...  [259]  \n",
       "\n",
       "[270 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tmdbv3api import TMDb\n",
    "import json\n",
    "import requests\n",
    "tmdb = TMDb()\n",
    "tmdb.api_key = '15d4971d36d91bfb4d6b2cfd774d753c'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tmdbv3api import Movie\n",
    "tmdb_movie = Movie()\n",
    "def get_genre(x):\n",
    "    genres = []\n",
    "    result = tmdb_movie.search(x)\n",
    "    movie_id = result[0].id\n",
    "    response = requests.get('https://api.themoviedb.org/3/movie/{}?api_key=15d4971d36d91bfb4d6b2cfd774d753c'.format(movie_id,tmdb.api_key))\n",
    "    data_json = response.json()\n",
    "    if data_json['genres']:\n",
    "        genre_str = \" \" \n",
    "        for i in range(0,len(data_json['genres'])):\n",
    "            genres.append(data_json['genres'][i]['name'])\n",
    "        return genre_str.join(genres)\n",
    "    else:\n",
    "        np.NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['genres'] = df['Title'].map(lambda x: get_genre(str(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Opening</th>\n",
       "      <th>Opening.1</th>\n",
       "      <th>Title</th>\n",
       "      <th>Production company</th>\n",
       "      <th>Cast and crew</th>\n",
       "      <th>Ref.</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>5</td>\n",
       "      <td>Insidious: The Last Key</td>\n",
       "      <td>Universal Pictures / Blumhouse Productions / S...</td>\n",
       "      <td>Adam Robitel (director); Leigh Whannell (scree...</td>\n",
       "      <td>[2]</td>\n",
       "      <td>Horror Mystery Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>5</td>\n",
       "      <td>The Strange Ones</td>\n",
       "      <td>Vertical Entertainment</td>\n",
       "      <td>Lauren Wolkstein (director); Christopher Radcl...</td>\n",
       "      <td>[3]</td>\n",
       "      <td>Thriller Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>5</td>\n",
       "      <td>Stratton</td>\n",
       "      <td>Momentum Pictures</td>\n",
       "      <td>Simon West (director); Duncan Falconer, Warren...</td>\n",
       "      <td>[4]</td>\n",
       "      <td>Action Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>10</td>\n",
       "      <td>Sweet Country</td>\n",
       "      <td>Samuel Goldwyn Films</td>\n",
       "      <td>Warwick Thornton (director); David Tranter, St...</td>\n",
       "      <td>[5]</td>\n",
       "      <td>Drama History Western</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>12</td>\n",
       "      <td>The Commuter</td>\n",
       "      <td>Lionsgate / StudioCanal / The Picture Company</td>\n",
       "      <td>Jaume Collet-Serra (director); Byron Willinger...</td>\n",
       "      <td>[6]</td>\n",
       "      <td>Action Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Holmes &amp; Watson</td>\n",
       "      <td>Columbia Pictures / Gary Sanchez Productions</td>\n",
       "      <td>Etan Cohen (director/screenplay); Will Ferrell...</td>\n",
       "      <td>[163]</td>\n",
       "      <td>Mystery Adventure Comedy Crime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Vice</td>\n",
       "      <td>Annapurna Pictures / Plan B Entertainment</td>\n",
       "      <td>Adam McKay (director/screenplay); Christian Ba...</td>\n",
       "      <td>[137]</td>\n",
       "      <td>Thriller Science Fiction Action Adventure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>On the Basis of Sex</td>\n",
       "      <td>Focus Features</td>\n",
       "      <td>Mimi Leder (director); Daniel Stiepleman (scre...</td>\n",
       "      <td>[226]</td>\n",
       "      <td>Drama History</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Destroyer</td>\n",
       "      <td>Annapurna Pictures</td>\n",
       "      <td>Karyn Kusama (director); Phil Hay, Matt Manfre...</td>\n",
       "      <td>[258]</td>\n",
       "      <td>Thriller Crime Drama Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>28</td>\n",
       "      <td>Black Mirror: Bandersnatch</td>\n",
       "      <td>Netflix</td>\n",
       "      <td>David Slade (director); Charlie Brooker (scree...</td>\n",
       "      <td>[259]</td>\n",
       "      <td>Science Fiction Mystery Drama Thriller TV Movie</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Opening  Opening.1                       Title  \\\n",
       "0     JANUARY          5     Insidious: The Last Key   \n",
       "1     JANUARY          5            The Strange Ones   \n",
       "2     JANUARY          5                    Stratton   \n",
       "3     JANUARY         10               Sweet Country   \n",
       "4     JANUARY         12                The Commuter   \n",
       "..        ...        ...                         ...   \n",
       "265  DECEMBER         25             Holmes & Watson   \n",
       "266  DECEMBER         25                        Vice   \n",
       "267  DECEMBER         25         On the Basis of Sex   \n",
       "268  DECEMBER         25                   Destroyer   \n",
       "269  DECEMBER         28  Black Mirror: Bandersnatch   \n",
       "\n",
       "                                    Production company  \\\n",
       "0    Universal Pictures / Blumhouse Productions / S...   \n",
       "1                               Vertical Entertainment   \n",
       "2                                    Momentum Pictures   \n",
       "3                                 Samuel Goldwyn Films   \n",
       "4        Lionsgate / StudioCanal / The Picture Company   \n",
       "..                                                 ...   \n",
       "265       Columbia Pictures / Gary Sanchez Productions   \n",
       "266          Annapurna Pictures / Plan B Entertainment   \n",
       "267                                     Focus Features   \n",
       "268                                 Annapurna Pictures   \n",
       "269                                            Netflix   \n",
       "\n",
       "                                         Cast and crew   Ref.  \\\n",
       "0    Adam Robitel (director); Leigh Whannell (scree...    [2]   \n",
       "1    Lauren Wolkstein (director); Christopher Radcl...    [3]   \n",
       "2    Simon West (director); Duncan Falconer, Warren...    [4]   \n",
       "3    Warwick Thornton (director); David Tranter, St...    [5]   \n",
       "4    Jaume Collet-Serra (director); Byron Willinger...    [6]   \n",
       "..                                                 ...    ...   \n",
       "265  Etan Cohen (director/screenplay); Will Ferrell...  [163]   \n",
       "266  Adam McKay (director/screenplay); Christian Ba...  [137]   \n",
       "267  Mimi Leder (director); Daniel Stiepleman (scre...  [226]   \n",
       "268  Karyn Kusama (director); Phil Hay, Matt Manfre...  [258]   \n",
       "269  David Slade (director); Charlie Brooker (scree...  [259]   \n",
       "\n",
       "                                              genres  \n",
       "0                            Horror Mystery Thriller  \n",
       "1                                     Thriller Drama  \n",
       "2                                    Action Thriller  \n",
       "3                              Drama History Western  \n",
       "4                                    Action Thriller  \n",
       "..                                               ...  \n",
       "265                   Mystery Adventure Comedy Crime  \n",
       "266        Thriller Science Fiction Action Adventure  \n",
       "267                                    Drama History  \n",
       "268                      Thriller Crime Drama Action  \n",
       "269  Science Fiction Mystery Drama Thriller TV Movie  \n",
       "\n",
       "[270 rows x 7 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2018 = df[['Title','Cast and crew','genres']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Cast and crew</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Insidious: The Last Key</td>\n",
       "      <td>Adam Robitel (director); Leigh Whannell (scree...</td>\n",
       "      <td>Horror Mystery Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Strange Ones</td>\n",
       "      <td>Lauren Wolkstein (director); Christopher Radcl...</td>\n",
       "      <td>Thriller Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Stratton</td>\n",
       "      <td>Simon West (director); Duncan Falconer, Warren...</td>\n",
       "      <td>Action Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sweet Country</td>\n",
       "      <td>Warwick Thornton (director); David Tranter, St...</td>\n",
       "      <td>Drama History Western</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Commuter</td>\n",
       "      <td>Jaume Collet-Serra (director); Byron Willinger...</td>\n",
       "      <td>Action Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>Holmes &amp; Watson</td>\n",
       "      <td>Etan Cohen (director/screenplay); Will Ferrell...</td>\n",
       "      <td>Mystery Adventure Comedy Crime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>Vice</td>\n",
       "      <td>Adam McKay (director/screenplay); Christian Ba...</td>\n",
       "      <td>Thriller Science Fiction Action Adventure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>On the Basis of Sex</td>\n",
       "      <td>Mimi Leder (director); Daniel Stiepleman (scre...</td>\n",
       "      <td>Drama History</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>Destroyer</td>\n",
       "      <td>Karyn Kusama (director); Phil Hay, Matt Manfre...</td>\n",
       "      <td>Thriller Crime Drama Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>Black Mirror: Bandersnatch</td>\n",
       "      <td>David Slade (director); Charlie Brooker (scree...</td>\n",
       "      <td>Science Fiction Mystery Drama Thriller TV Movie</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Title  \\\n",
       "0       Insidious: The Last Key   \n",
       "1              The Strange Ones   \n",
       "2                      Stratton   \n",
       "3                 Sweet Country   \n",
       "4                  The Commuter   \n",
       "..                          ...   \n",
       "265             Holmes & Watson   \n",
       "266                        Vice   \n",
       "267         On the Basis of Sex   \n",
       "268                   Destroyer   \n",
       "269  Black Mirror: Bandersnatch   \n",
       "\n",
       "                                         Cast and crew  \\\n",
       "0    Adam Robitel (director); Leigh Whannell (scree...   \n",
       "1    Lauren Wolkstein (director); Christopher Radcl...   \n",
       "2    Simon West (director); Duncan Falconer, Warren...   \n",
       "3    Warwick Thornton (director); David Tranter, St...   \n",
       "4    Jaume Collet-Serra (director); Byron Willinger...   \n",
       "..                                                 ...   \n",
       "265  Etan Cohen (director/screenplay); Will Ferrell...   \n",
       "266  Adam McKay (director/screenplay); Christian Ba...   \n",
       "267  Mimi Leder (director); Daniel Stiepleman (scre...   \n",
       "268  Karyn Kusama (director); Phil Hay, Matt Manfre...   \n",
       "269  David Slade (director); Charlie Brooker (scree...   \n",
       "\n",
       "                                              genres  \n",
       "0                            Horror Mystery Thriller  \n",
       "1                                     Thriller Drama  \n",
       "2                                    Action Thriller  \n",
       "3                              Drama History Western  \n",
       "4                                    Action Thriller  \n",
       "..                                               ...  \n",
       "265                   Mystery Adventure Comedy Crime  \n",
       "266        Thriller Science Fiction Action Adventure  \n",
       "267                                    Drama History  \n",
       "268                      Thriller Crime Drama Action  \n",
       "269  Science Fiction Mystery Drama Thriller TV Movie  \n",
       "\n",
       "[270 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_director(x):\n",
    "    if \" (director)\" in x:\n",
    "        return x.split(\" (director)\")[0]\n",
    "    elif \" (directors)\" in x:\n",
    "        return x.split(\" (directors)\")[0]\n",
    "    else:\n",
    "        return x.split(\" (director/screenplay)\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-12-4838efec492a>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_2018['director_name'] = df_2018['Cast and crew'].map(lambda x: get_director(x))\n"
     ]
    }
   ],
   "source": [
    "df_2018['director_name'] = df_2018['Cast and crew'].map(lambda x: get_director(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actor1(x):\n",
    "    return ((x.split(\"screenplay); \")[-1]).split(\", \")[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-14-94b1a1569c5d>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_2018['actor_1_name'] = df_2018['Cast and crew'].map(lambda x: get_actor1(x))\n"
     ]
    }
   ],
   "source": [
    "df_2018['actor_1_name'] = df_2018['Cast and crew'].map(lambda x: get_actor1(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actor2(x):\n",
    "    if len((x.split(\"screenplay); \")[-1]).split(\", \")) < 2:\n",
    "        return np.NaN\n",
    "    else:\n",
    "        return ((x.split(\"screenplay); \")[-1]).split(\", \")[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-16-75f2be6b779a>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_2018['actor_2_name'] = df_2018['Cast and crew'].map(lambda x: get_actor2(x))\n"
     ]
    }
   ],
   "source": [
    "df_2018['actor_2_name'] = df_2018['Cast and crew'].map(lambda x: get_actor2(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actor3(x):\n",
    "    if len((x.split(\"screenplay); \")[-1]).split(\", \")) < 3:\n",
    "        return np.NaN\n",
    "    else:\n",
    "        return ((x.split(\"screenplay); \")[-1]).split(\", \")[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2018['actor_3_name'] = df_2018['Cast and crew'].map(lambda x: get_actor3(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Cast and crew</th>\n",
       "      <th>genres</th>\n",
       "      <th>director_name</th>\n",
       "      <th>actor_1_name</th>\n",
       "      <th>actor_2_name</th>\n",
       "      <th>actor_3_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Insidious: The Last Key</td>\n",
       "      <td>Adam Robitel (director); Leigh Whannell (scree...</td>\n",
       "      <td>Horror Mystery Thriller</td>\n",
       "      <td>Adam Robitel</td>\n",
       "      <td>Lin Shaye</td>\n",
       "      <td>Angus Sampson</td>\n",
       "      <td>Leigh Whannell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Strange Ones</td>\n",
       "      <td>Lauren Wolkstein (director); Christopher Radcl...</td>\n",
       "      <td>Thriller Drama</td>\n",
       "      <td>Lauren Wolkstein</td>\n",
       "      <td>Alex Pettyfer</td>\n",
       "      <td>James Freedson-Jackson</td>\n",
       "      <td>Emily Althaus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Stratton</td>\n",
       "      <td>Simon West (director); Duncan Falconer, Warren...</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>Simon West</td>\n",
       "      <td>Dominic Cooper</td>\n",
       "      <td>Austin Stowell</td>\n",
       "      <td>Gemma Chan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sweet Country</td>\n",
       "      <td>Warwick Thornton (director); David Tranter, St...</td>\n",
       "      <td>Drama History Western</td>\n",
       "      <td>Warwick Thornton</td>\n",
       "      <td>Bryan Brown</td>\n",
       "      <td>Sam Neill</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Commuter</td>\n",
       "      <td>Jaume Collet-Serra (director); Byron Willinger...</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>Jaume Collet-Serra</td>\n",
       "      <td>Liam Neeson</td>\n",
       "      <td>Vera Farmiga</td>\n",
       "      <td>Patrick Wilson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>Holmes &amp; Watson</td>\n",
       "      <td>Etan Cohen (director/screenplay); Will Ferrell...</td>\n",
       "      <td>Mystery Adventure Comedy Crime</td>\n",
       "      <td>Etan Cohen</td>\n",
       "      <td>Will Ferrell</td>\n",
       "      <td>John C. Reilly</td>\n",
       "      <td>Rebecca Hall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>Vice</td>\n",
       "      <td>Adam McKay (director/screenplay); Christian Ba...</td>\n",
       "      <td>Thriller Science Fiction Action Adventure</td>\n",
       "      <td>Adam McKay</td>\n",
       "      <td>Christian Bale</td>\n",
       "      <td>Amy Adams</td>\n",
       "      <td>Steve Carell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>On the Basis of Sex</td>\n",
       "      <td>Mimi Leder (director); Daniel Stiepleman (scre...</td>\n",
       "      <td>Drama History</td>\n",
       "      <td>Mimi Leder</td>\n",
       "      <td>Felicity Jones</td>\n",
       "      <td>Armie Hammer</td>\n",
       "      <td>Justin Theroux</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>Destroyer</td>\n",
       "      <td>Karyn Kusama (director); Phil Hay, Matt Manfre...</td>\n",
       "      <td>Thriller Crime Drama Action</td>\n",
       "      <td>Karyn Kusama</td>\n",
       "      <td>Nicole Kidman</td>\n",
       "      <td>Sebastian Stan</td>\n",
       "      <td>Toby Kebbell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>Black Mirror: Bandersnatch</td>\n",
       "      <td>David Slade (director); Charlie Brooker (scree...</td>\n",
       "      <td>Science Fiction Mystery Drama Thriller TV Movie</td>\n",
       "      <td>David Slade</td>\n",
       "      <td>Fionn Whitehead</td>\n",
       "      <td>Will Poulter</td>\n",
       "      <td>Asim Chaudhry</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Title  \\\n",
       "0       Insidious: The Last Key   \n",
       "1              The Strange Ones   \n",
       "2                      Stratton   \n",
       "3                 Sweet Country   \n",
       "4                  The Commuter   \n",
       "..                          ...   \n",
       "265             Holmes & Watson   \n",
       "266                        Vice   \n",
       "267         On the Basis of Sex   \n",
       "268                   Destroyer   \n",
       "269  Black Mirror: Bandersnatch   \n",
       "\n",
       "                                         Cast and crew  \\\n",
       "0    Adam Robitel (director); Leigh Whannell (scree...   \n",
       "1    Lauren Wolkstein (director); Christopher Radcl...   \n",
       "2    Simon West (director); Duncan Falconer, Warren...   \n",
       "3    Warwick Thornton (director); David Tranter, St...   \n",
       "4    Jaume Collet-Serra (director); Byron Willinger...   \n",
       "..                                                 ...   \n",
       "265  Etan Cohen (director/screenplay); Will Ferrell...   \n",
       "266  Adam McKay (director/screenplay); Christian Ba...   \n",
       "267  Mimi Leder (director); Daniel Stiepleman (scre...   \n",
       "268  Karyn Kusama (director); Phil Hay, Matt Manfre...   \n",
       "269  David Slade (director); Charlie Brooker (scree...   \n",
       "\n",
       "                                              genres       director_name  \\\n",
       "0                            Horror Mystery Thriller        Adam Robitel   \n",
       "1                                     Thriller Drama    Lauren Wolkstein   \n",
       "2                                    Action Thriller          Simon West   \n",
       "3                              Drama History Western    Warwick Thornton   \n",
       "4                                    Action Thriller  Jaume Collet-Serra   \n",
       "..                                               ...                 ...   \n",
       "265                   Mystery Adventure Comedy Crime          Etan Cohen   \n",
       "266        Thriller Science Fiction Action Adventure          Adam McKay   \n",
       "267                                    Drama History          Mimi Leder   \n",
       "268                      Thriller Crime Drama Action        Karyn Kusama   \n",
       "269  Science Fiction Mystery Drama Thriller TV Movie         David Slade   \n",
       "\n",
       "        actor_1_name            actor_2_name    actor_3_name  \n",
       "0          Lin Shaye           Angus Sampson  Leigh Whannell  \n",
       "1      Alex Pettyfer  James Freedson-Jackson   Emily Althaus  \n",
       "2     Dominic Cooper          Austin Stowell      Gemma Chan  \n",
       "3        Bryan Brown               Sam Neill             NaN  \n",
       "4        Liam Neeson            Vera Farmiga  Patrick Wilson  \n",
       "..               ...                     ...             ...  \n",
       "265     Will Ferrell          John C. Reilly    Rebecca Hall  \n",
       "266   Christian Bale               Amy Adams    Steve Carell  \n",
       "267   Felicity Jones            Armie Hammer  Justin Theroux  \n",
       "268    Nicole Kidman          Sebastian Stan    Toby Kebbell  \n",
       "269  Fionn Whitehead            Will Poulter   Asim Chaudhry  \n",
       "\n",
       "[270 rows x 7 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2018 = df_2018.rename(columns={'Title':'movie_title'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df18 = df_2018.loc[:,['director_name','actor_1_name','actor_2_name','actor_3_name','genres','movie_title']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>director_name</th>\n",
       "      <th>actor_1_name</th>\n",
       "      <th>actor_2_name</th>\n",
       "      <th>actor_3_name</th>\n",
       "      <th>genres</th>\n",
       "      <th>movie_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adam Robitel</td>\n",
       "      <td>Lin Shaye</td>\n",
       "      <td>Angus Sampson</td>\n",
       "      <td>Leigh Whannell</td>\n",
       "      <td>Horror Mystery Thriller</td>\n",
       "      <td>Insidious: The Last Key</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Lauren Wolkstein</td>\n",
       "      <td>Alex Pettyfer</td>\n",
       "      <td>James Freedson-Jackson</td>\n",
       "      <td>Emily Althaus</td>\n",
       "      <td>Thriller Drama</td>\n",
       "      <td>The Strange Ones</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Simon West</td>\n",
       "      <td>Dominic Cooper</td>\n",
       "      <td>Austin Stowell</td>\n",
       "      <td>Gemma Chan</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>Stratton</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Warwick Thornton</td>\n",
       "      <td>Bryan Brown</td>\n",
       "      <td>Sam Neill</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Drama History Western</td>\n",
       "      <td>Sweet Country</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jaume Collet-Serra</td>\n",
       "      <td>Liam Neeson</td>\n",
       "      <td>Vera Farmiga</td>\n",
       "      <td>Patrick Wilson</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>The Commuter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>Etan Cohen</td>\n",
       "      <td>Will Ferrell</td>\n",
       "      <td>John C. Reilly</td>\n",
       "      <td>Rebecca Hall</td>\n",
       "      <td>Mystery Adventure Comedy Crime</td>\n",
       "      <td>Holmes &amp; Watson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>Adam McKay</td>\n",
       "      <td>Christian Bale</td>\n",
       "      <td>Amy Adams</td>\n",
       "      <td>Steve Carell</td>\n",
       "      <td>Thriller Science Fiction Action Adventure</td>\n",
       "      <td>Vice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>Mimi Leder</td>\n",
       "      <td>Felicity Jones</td>\n",
       "      <td>Armie Hammer</td>\n",
       "      <td>Justin Theroux</td>\n",
       "      <td>Drama History</td>\n",
       "      <td>On the Basis of Sex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>Karyn Kusama</td>\n",
       "      <td>Nicole Kidman</td>\n",
       "      <td>Sebastian Stan</td>\n",
       "      <td>Toby Kebbell</td>\n",
       "      <td>Thriller Crime Drama Action</td>\n",
       "      <td>Destroyer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>David Slade</td>\n",
       "      <td>Fionn Whitehead</td>\n",
       "      <td>Will Poulter</td>\n",
       "      <td>Asim Chaudhry</td>\n",
       "      <td>Science Fiction Mystery Drama Thriller TV Movie</td>\n",
       "      <td>Black Mirror: Bandersnatch</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          director_name     actor_1_name            actor_2_name  \\\n",
       "0          Adam Robitel        Lin Shaye           Angus Sampson   \n",
       "1      Lauren Wolkstein    Alex Pettyfer  James Freedson-Jackson   \n",
       "2            Simon West   Dominic Cooper          Austin Stowell   \n",
       "3      Warwick Thornton      Bryan Brown               Sam Neill   \n",
       "4    Jaume Collet-Serra      Liam Neeson            Vera Farmiga   \n",
       "..                  ...              ...                     ...   \n",
       "265          Etan Cohen     Will Ferrell          John C. Reilly   \n",
       "266          Adam McKay   Christian Bale               Amy Adams   \n",
       "267          Mimi Leder   Felicity Jones            Armie Hammer   \n",
       "268        Karyn Kusama    Nicole Kidman          Sebastian Stan   \n",
       "269         David Slade  Fionn Whitehead            Will Poulter   \n",
       "\n",
       "       actor_3_name                                           genres  \\\n",
       "0    Leigh Whannell                          Horror Mystery Thriller   \n",
       "1     Emily Althaus                                   Thriller Drama   \n",
       "2        Gemma Chan                                  Action Thriller   \n",
       "3               NaN                            Drama History Western   \n",
       "4    Patrick Wilson                                  Action Thriller   \n",
       "..              ...                                              ...   \n",
       "265    Rebecca Hall                   Mystery Adventure Comedy Crime   \n",
       "266    Steve Carell        Thriller Science Fiction Action Adventure   \n",
       "267  Justin Theroux                                    Drama History   \n",
       "268    Toby Kebbell                      Thriller Crime Drama Action   \n",
       "269   Asim Chaudhry  Science Fiction Mystery Drama Thriller TV Movie   \n",
       "\n",
       "                    movie_title  \n",
       "0       Insidious: The Last Key  \n",
       "1              The Strange Ones  \n",
       "2                      Stratton  \n",
       "3                 Sweet Country  \n",
       "4                  The Commuter  \n",
       "..                          ...  \n",
       "265             Holmes & Watson  \n",
       "266                        Vice  \n",
       "267         On the Basis of Sex  \n",
       "268                   Destroyer  \n",
       "269  Black Mirror: Bandersnatch  \n",
       "\n",
       "[270 rows x 6 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df18['actor_2_name'] = new_df18['actor_2_name'].replace(np.nan, 'unknown')\n",
    "new_df18['actor_3_name'] = new_df18['actor_3_name'].replace(np.nan, 'unknown')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df18['movie_title'] = new_df18['movie_title'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df18['comb'] = new_df18['actor_1_name'] + ' ' + new_df18['actor_2_name'] + ' '+ new_df18['actor_3_name'] + ' '+ new_df18['director_name'] +' ' + new_df18['genres']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>director_name</th>\n",
       "      <th>actor_1_name</th>\n",
       "      <th>actor_2_name</th>\n",
       "      <th>actor_3_name</th>\n",
       "      <th>genres</th>\n",
       "      <th>movie_title</th>\n",
       "      <th>comb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adam Robitel</td>\n",
       "      <td>Lin Shaye</td>\n",
       "      <td>Angus Sampson</td>\n",
       "      <td>Leigh Whannell</td>\n",
       "      <td>Horror Mystery Thriller</td>\n",
       "      <td>insidious: the last key</td>\n",
       "      <td>Lin Shaye Angus Sampson Leigh Whannell Adam Ro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Lauren Wolkstein</td>\n",
       "      <td>Alex Pettyfer</td>\n",
       "      <td>James Freedson-Jackson</td>\n",
       "      <td>Emily Althaus</td>\n",
       "      <td>Thriller Drama</td>\n",
       "      <td>the strange ones</td>\n",
       "      <td>Alex Pettyfer James Freedson-Jackson Emily Alt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Simon West</td>\n",
       "      <td>Dominic Cooper</td>\n",
       "      <td>Austin Stowell</td>\n",
       "      <td>Gemma Chan</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>stratton</td>\n",
       "      <td>Dominic Cooper Austin Stowell Gemma Chan Simon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Warwick Thornton</td>\n",
       "      <td>Bryan Brown</td>\n",
       "      <td>Sam Neill</td>\n",
       "      <td>unknown</td>\n",
       "      <td>Drama History Western</td>\n",
       "      <td>sweet country</td>\n",
       "      <td>Bryan Brown Sam Neill unknown Warwick Thornton...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jaume Collet-Serra</td>\n",
       "      <td>Liam Neeson</td>\n",
       "      <td>Vera Farmiga</td>\n",
       "      <td>Patrick Wilson</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>the commuter</td>\n",
       "      <td>Liam Neeson Vera Farmiga Patrick Wilson Jaume ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>Etan Cohen</td>\n",
       "      <td>Will Ferrell</td>\n",
       "      <td>John C. Reilly</td>\n",
       "      <td>Rebecca Hall</td>\n",
       "      <td>Mystery Adventure Comedy Crime</td>\n",
       "      <td>holmes &amp; watson</td>\n",
       "      <td>Will Ferrell John C. Reilly Rebecca Hall Etan ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>Adam McKay</td>\n",
       "      <td>Christian Bale</td>\n",
       "      <td>Amy Adams</td>\n",
       "      <td>Steve Carell</td>\n",
       "      <td>Thriller Science Fiction Action Adventure</td>\n",
       "      <td>vice</td>\n",
       "      <td>Christian Bale Amy Adams Steve Carell Adam McK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>Mimi Leder</td>\n",
       "      <td>Felicity Jones</td>\n",
       "      <td>Armie Hammer</td>\n",
       "      <td>Justin Theroux</td>\n",
       "      <td>Drama History</td>\n",
       "      <td>on the basis of sex</td>\n",
       "      <td>Felicity Jones Armie Hammer Justin Theroux Mim...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>Karyn Kusama</td>\n",
       "      <td>Nicole Kidman</td>\n",
       "      <td>Sebastian Stan</td>\n",
       "      <td>Toby Kebbell</td>\n",
       "      <td>Thriller Crime Drama Action</td>\n",
       "      <td>destroyer</td>\n",
       "      <td>Nicole Kidman Sebastian Stan Toby Kebbell Kary...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>David Slade</td>\n",
       "      <td>Fionn Whitehead</td>\n",
       "      <td>Will Poulter</td>\n",
       "      <td>Asim Chaudhry</td>\n",
       "      <td>Science Fiction Mystery Drama Thriller TV Movie</td>\n",
       "      <td>black mirror: bandersnatch</td>\n",
       "      <td>Fionn Whitehead Will Poulter Asim Chaudhry Dav...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          director_name     actor_1_name            actor_2_name  \\\n",
       "0          Adam Robitel        Lin Shaye           Angus Sampson   \n",
       "1      Lauren Wolkstein    Alex Pettyfer  James Freedson-Jackson   \n",
       "2            Simon West   Dominic Cooper          Austin Stowell   \n",
       "3      Warwick Thornton      Bryan Brown               Sam Neill   \n",
       "4    Jaume Collet-Serra      Liam Neeson            Vera Farmiga   \n",
       "..                  ...              ...                     ...   \n",
       "265          Etan Cohen     Will Ferrell          John C. Reilly   \n",
       "266          Adam McKay   Christian Bale               Amy Adams   \n",
       "267          Mimi Leder   Felicity Jones            Armie Hammer   \n",
       "268        Karyn Kusama    Nicole Kidman          Sebastian Stan   \n",
       "269         David Slade  Fionn Whitehead            Will Poulter   \n",
       "\n",
       "       actor_3_name                                           genres  \\\n",
       "0    Leigh Whannell                          Horror Mystery Thriller   \n",
       "1     Emily Althaus                                   Thriller Drama   \n",
       "2        Gemma Chan                                  Action Thriller   \n",
       "3           unknown                            Drama History Western   \n",
       "4    Patrick Wilson                                  Action Thriller   \n",
       "..              ...                                              ...   \n",
       "265    Rebecca Hall                   Mystery Adventure Comedy Crime   \n",
       "266    Steve Carell        Thriller Science Fiction Action Adventure   \n",
       "267  Justin Theroux                                    Drama History   \n",
       "268    Toby Kebbell                      Thriller Crime Drama Action   \n",
       "269   Asim Chaudhry  Science Fiction Mystery Drama Thriller TV Movie   \n",
       "\n",
       "                    movie_title  \\\n",
       "0       insidious: the last key   \n",
       "1              the strange ones   \n",
       "2                      stratton   \n",
       "3                 sweet country   \n",
       "4                  the commuter   \n",
       "..                          ...   \n",
       "265             holmes & watson   \n",
       "266                        vice   \n",
       "267         on the basis of sex   \n",
       "268                   destroyer   \n",
       "269  black mirror: bandersnatch   \n",
       "\n",
       "                                                  comb  \n",
       "0    Lin Shaye Angus Sampson Leigh Whannell Adam Ro...  \n",
       "1    Alex Pettyfer James Freedson-Jackson Emily Alt...  \n",
       "2    Dominic Cooper Austin Stowell Gemma Chan Simon...  \n",
       "3    Bryan Brown Sam Neill unknown Warwick Thornton...  \n",
       "4    Liam Neeson Vera Farmiga Patrick Wilson Jaume ...  \n",
       "..                                                 ...  \n",
       "265  Will Ferrell John C. Reilly Rebecca Hall Etan ...  \n",
       "266  Christian Bale Amy Adams Steve Carell Adam McK...  \n",
       "267  Felicity Jones Armie Hammer Justin Theroux Mim...  \n",
       "268  Nicole Kidman Sebastian Stan Toby Kebbell Kary...  \n",
       "269  Fionn Whitehead Will Poulter Asim Chaudhry Dav...  \n",
       "\n",
       "[270 rows x 7 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df18"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting features of 2019 movies from Wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "link = \"https://en.wikipedia.org/wiki/List_of_American_films_of_2019\"\n",
    "df1 = pd.read_html(link, header=0)[2]\n",
    "df2 = pd.read_html(link, header=0)[3]\n",
    "df3 = pd.read_html(link, header=0)[4]\n",
    "df4 = pd.read_html(link, header=0)[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df1.append(df2.append(df3.append(df4,ignore_index=True),ignore_index=True),ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Opening</th>\n",
       "      <th>Opening.1</th>\n",
       "      <th>Title</th>\n",
       "      <th>Production company</th>\n",
       "      <th>Cast and crew</th>\n",
       "      <th>Ref.</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>4</td>\n",
       "      <td>Escape Room</td>\n",
       "      <td>Columbia Pictures</td>\n",
       "      <td>Adam Robitel (director); Bragi F. Schut, Maria...</td>\n",
       "      <td>[2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>4</td>\n",
       "      <td>Rust Creek</td>\n",
       "      <td>IFC Films</td>\n",
       "      <td>Jen McGowan (director); Julie Lipson (screenpl...</td>\n",
       "      <td>[3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>4</td>\n",
       "      <td>American Hangman</td>\n",
       "      <td>Hangman Justice Productions</td>\n",
       "      <td>Wilson Coneybeare (director/screenplay); Donal...</td>\n",
       "      <td>[4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>11</td>\n",
       "      <td>A Dog's Way Home</td>\n",
       "      <td>Columbia Pictures</td>\n",
       "      <td>Charles Martin Smith (director); W. Bruce Came...</td>\n",
       "      <td>[5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>JANUARY</td>\n",
       "      <td>11</td>\n",
       "      <td>The Upside</td>\n",
       "      <td>STX Entertainment</td>\n",
       "      <td>Neil Burger (director); Jon Hartmere (screenpl...</td>\n",
       "      <td>[6]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Spies in Disguise</td>\n",
       "      <td>20th Century Fox / Blue Sky Studios / Chernin ...</td>\n",
       "      <td>Nick Bruno, Troy Quane (directors); Brad Copel...</td>\n",
       "      <td>[132]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Little Women</td>\n",
       "      <td>Columbia Pictures / Regency Enterprises</td>\n",
       "      <td>Greta Gerwig (director/screenplay); Saoirse Ro...</td>\n",
       "      <td>[221]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>1917</td>\n",
       "      <td>Universal Pictures / DreamWorks Pictures / Ent...</td>\n",
       "      <td>Sam Mendes (director/screenplay); Krysty Wilso...</td>\n",
       "      <td>[222]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>25</td>\n",
       "      <td>Just Mercy</td>\n",
       "      <td>Warner Bros. Pictures / Participant Media</td>\n",
       "      <td>Destin Daniel Cretton (director/screenplay), A...</td>\n",
       "      <td>[223]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>DECEMBER</td>\n",
       "      <td>27</td>\n",
       "      <td>Clemency</td>\n",
       "      <td>Neon</td>\n",
       "      <td>Chinonye Chukwu (director/screenplay); Alfre W...</td>\n",
       "      <td>[224]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>242 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Opening  Opening.1              Title  \\\n",
       "0     JANUARY          4        Escape Room   \n",
       "1     JANUARY          4         Rust Creek   \n",
       "2     JANUARY          4   American Hangman   \n",
       "3     JANUARY         11   A Dog's Way Home   \n",
       "4     JANUARY         11         The Upside   \n",
       "..        ...        ...                ...   \n",
       "237  DECEMBER         25  Spies in Disguise   \n",
       "238  DECEMBER         25       Little Women   \n",
       "239  DECEMBER         25               1917   \n",
       "240  DECEMBER         25         Just Mercy   \n",
       "241  DECEMBER         27           Clemency   \n",
       "\n",
       "                                    Production company  \\\n",
       "0                                    Columbia Pictures   \n",
       "1                                            IFC Films   \n",
       "2                          Hangman Justice Productions   \n",
       "3                                    Columbia Pictures   \n",
       "4                                    STX Entertainment   \n",
       "..                                                 ...   \n",
       "237  20th Century Fox / Blue Sky Studios / Chernin ...   \n",
       "238            Columbia Pictures / Regency Enterprises   \n",
       "239  Universal Pictures / DreamWorks Pictures / Ent...   \n",
       "240          Warner Bros. Pictures / Participant Media   \n",
       "241                                               Neon   \n",
       "\n",
       "                                         Cast and crew   Ref.  \n",
       "0    Adam Robitel (director); Bragi F. Schut, Maria...    [2]  \n",
       "1    Jen McGowan (director); Julie Lipson (screenpl...    [3]  \n",
       "2    Wilson Coneybeare (director/screenplay); Donal...    [4]  \n",
       "3    Charles Martin Smith (director); W. Bruce Came...    [5]  \n",
       "4    Neil Burger (director); Jon Hartmere (screenpl...    [6]  \n",
       "..                                                 ...    ...  \n",
       "237  Nick Bruno, Troy Quane (directors); Brad Copel...  [132]  \n",
       "238  Greta Gerwig (director/screenplay); Saoirse Ro...  [221]  \n",
       "239  Sam Mendes (director/screenplay); Krysty Wilso...  [222]  \n",
       "240  Destin Daniel Cretton (director/screenplay), A...  [223]  \n",
       "241  Chinonye Chukwu (director/screenplay); Alfre W...  [224]  \n",
       "\n",
       "[242 rows x 6 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['genres'] = df['Title'].map(lambda x: get_genre(str(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2019 = df[['Title','Cast and crew','genres']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Cast and crew</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Escape Room</td>\n",
       "      <td>Adam Robitel (director); Bragi F. Schut, Maria...</td>\n",
       "      <td>Thriller Action Mystery Adventure Horror</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Rust Creek</td>\n",
       "      <td>Jen McGowan (director); Julie Lipson (screenpl...</td>\n",
       "      <td>Thriller Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>American Hangman</td>\n",
       "      <td>Wilson Coneybeare (director/screenplay); Donal...</td>\n",
       "      <td>Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A Dog's Way Home</td>\n",
       "      <td>Charles Martin Smith (director); W. Bruce Came...</td>\n",
       "      <td>Drama Adventure Family</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Upside</td>\n",
       "      <td>Neil Burger (director); Jon Hartmere (screenpl...</td>\n",
       "      <td>Comedy Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>Spies in Disguise</td>\n",
       "      <td>Nick Bruno, Troy Quane (directors); Brad Copel...</td>\n",
       "      <td>Animation Action Adventure Comedy Family</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>Little Women</td>\n",
       "      <td>Greta Gerwig (director/screenplay); Saoirse Ro...</td>\n",
       "      <td>Drama Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>1917</td>\n",
       "      <td>Sam Mendes (director/screenplay); Krysty Wilso...</td>\n",
       "      <td>War Drama Action Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>Just Mercy</td>\n",
       "      <td>Destin Daniel Cretton (director/screenplay), A...</td>\n",
       "      <td>Drama Crime History</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>Clemency</td>\n",
       "      <td>Chinonye Chukwu (director/screenplay); Alfre W...</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>242 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Title                                      Cast and crew  \\\n",
       "0          Escape Room  Adam Robitel (director); Bragi F. Schut, Maria...   \n",
       "1           Rust Creek  Jen McGowan (director); Julie Lipson (screenpl...   \n",
       "2     American Hangman  Wilson Coneybeare (director/screenplay); Donal...   \n",
       "3     A Dog's Way Home  Charles Martin Smith (director); W. Bruce Came...   \n",
       "4           The Upside  Neil Burger (director); Jon Hartmere (screenpl...   \n",
       "..                 ...                                                ...   \n",
       "237  Spies in Disguise  Nick Bruno, Troy Quane (directors); Brad Copel...   \n",
       "238       Little Women  Greta Gerwig (director/screenplay); Saoirse Ro...   \n",
       "239               1917  Sam Mendes (director/screenplay); Krysty Wilso...   \n",
       "240         Just Mercy  Destin Daniel Cretton (director/screenplay), A...   \n",
       "241           Clemency  Chinonye Chukwu (director/screenplay); Alfre W...   \n",
       "\n",
       "                                       genres  \n",
       "0    Thriller Action Mystery Adventure Horror  \n",
       "1                              Thriller Drama  \n",
       "2                                    Thriller  \n",
       "3                      Drama Adventure Family  \n",
       "4                                Comedy Drama  \n",
       "..                                        ...  \n",
       "237  Animation Action Adventure Comedy Family  \n",
       "238                             Drama Romance  \n",
       "239                 War Drama Action Thriller  \n",
       "240                       Drama Crime History  \n",
       "241                                     Drama  \n",
       "\n",
       "[242 rows x 3 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_director(x):\n",
    "    if \" (director)\" in x:\n",
    "        return x.split(\" (director)\")[0]\n",
    "    elif \" (directors)\" in x:\n",
    "        return x.split(\" (directors)\")[0]\n",
    "    else:\n",
    "        return x.split(\" (director/screenplay)\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-34-a433a4faa834>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_2019['director_name'] = df_2019['Cast and crew'].map(lambda x: get_director(str(x)))\n"
     ]
    }
   ],
   "source": [
    "df_2019['director_name'] = df_2019['Cast and crew'].map(lambda x: get_director(str(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actor1(x):\n",
    "    return ((x.split(\"screenplay); \")[-1]).split(\", \")[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-36-eee71c5586b8>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_2019['actor_1_name'] = df_2019['Cast and crew'].map(lambda x: get_actor1(x))\n"
     ]
    }
   ],
   "source": [
    "df_2019['actor_1_name'] = df_2019['Cast and crew'].map(lambda x: get_actor1(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actor2(x):\n",
    "    if len((x.split(\"screenplay); \")[-1]).split(\", \")) < 2:\n",
    "        return np.NaN\n",
    "    else:\n",
    "        return ((x.split(\"screenplay); \")[-1]).split(\", \")[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-38-48d0aadc78c0>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_2019['actor_2_name'] = df_2019['Cast and crew'].map(lambda x: get_actor2(x))\n"
     ]
    }
   ],
   "source": [
    "df_2019['actor_2_name'] = df_2019['Cast and crew'].map(lambda x: get_actor2(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_actor3(x):\n",
    "    if len((x.split(\"screenplay); \")[-1]).split(\", \")) < 3:\n",
    "        return np.NaN\n",
    "    else:\n",
    "        return ((x.split(\"screenplay); \")[-1]).split(\", \")[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2019['actor_3_name'] = df_2019['Cast and crew'].map(lambda x: get_actor3(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2019 = df_2019.rename(columns={'Title':'movie_title'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df19 = df_2019.loc[:,['director_name','actor_1_name','actor_2_name','actor_3_name','genres','movie_title']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df19['actor_2_name'] = new_df19['actor_2_name'].replace(np.nan, 'unknown')\n",
    "new_df19['actor_3_name'] = new_df19['actor_3_name'].replace(np.nan, 'unknown')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df19['movie_title'] = new_df19['movie_title'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df19['comb'] = new_df19['actor_1_name'] + ' ' + new_df19['actor_2_name'] + ' '+ new_df19['actor_3_name'] + ' '+ new_df19['director_name'] +' ' + new_df19['genres']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>director_name</th>\n",
       "      <th>actor_1_name</th>\n",
       "      <th>actor_2_name</th>\n",
       "      <th>actor_3_name</th>\n",
       "      <th>genres</th>\n",
       "      <th>movie_title</th>\n",
       "      <th>comb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adam Robitel</td>\n",
       "      <td>Taylor Russell</td>\n",
       "      <td>Logan Miller</td>\n",
       "      <td>Deborah Ann Woll</td>\n",
       "      <td>Thriller Action Mystery Adventure Horror</td>\n",
       "      <td>escape room</td>\n",
       "      <td>Taylor Russell Logan Miller Deborah Ann Woll A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Jen McGowan</td>\n",
       "      <td>Hermione Corfield</td>\n",
       "      <td>Jay Paulson</td>\n",
       "      <td>Sean O'Bryan</td>\n",
       "      <td>Thriller Drama</td>\n",
       "      <td>rust creek</td>\n",
       "      <td>Hermione Corfield Jay Paulson Sean O'Bryan Jen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Wilson Coneybeare</td>\n",
       "      <td>Donald Sutherland</td>\n",
       "      <td>Vincent Kartheiser</td>\n",
       "      <td>Oliver Dennis</td>\n",
       "      <td>Thriller</td>\n",
       "      <td>american hangman</td>\n",
       "      <td>Donald Sutherland Vincent Kartheiser Oliver De...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Charles Martin Smith</td>\n",
       "      <td>Bryce Dallas Howard</td>\n",
       "      <td>Edward James Olmos</td>\n",
       "      <td>Alexandra Shipp</td>\n",
       "      <td>Drama Adventure Family</td>\n",
       "      <td>a dog's way home</td>\n",
       "      <td>Bryce Dallas Howard Edward James Olmos Alexand...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Neil Burger</td>\n",
       "      <td>Bryan Cranston</td>\n",
       "      <td>Kevin Hart</td>\n",
       "      <td>Nicole Kidman</td>\n",
       "      <td>Comedy Drama</td>\n",
       "      <td>the upside</td>\n",
       "      <td>Bryan Cranston Kevin Hart Nicole Kidman Neil B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>Nick Bruno, Troy Quane</td>\n",
       "      <td>Will Smith</td>\n",
       "      <td>Tom Holland</td>\n",
       "      <td>Rashida Jones</td>\n",
       "      <td>Animation Action Adventure Comedy Family</td>\n",
       "      <td>spies in disguise</td>\n",
       "      <td>Will Smith Tom Holland Rashida Jones Nick Brun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>Greta Gerwig</td>\n",
       "      <td>Saoirse Ronan</td>\n",
       "      <td>Emma Watson</td>\n",
       "      <td>Florence Pugh</td>\n",
       "      <td>Drama Romance</td>\n",
       "      <td>little women</td>\n",
       "      <td>Saoirse Ronan Emma Watson Florence Pugh Greta ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>Sam Mendes</td>\n",
       "      <td>George MacKay</td>\n",
       "      <td>Dean-Charles Chapman</td>\n",
       "      <td>Mark Strong</td>\n",
       "      <td>War Drama Action Thriller</td>\n",
       "      <td>1917</td>\n",
       "      <td>George MacKay Dean-Charles Chapman Mark Strong...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>Destin Daniel Cretton</td>\n",
       "      <td>Michael B. Jordan</td>\n",
       "      <td>Jamie Foxx</td>\n",
       "      <td>Brie Larson</td>\n",
       "      <td>Drama Crime History</td>\n",
       "      <td>just mercy</td>\n",
       "      <td>Michael B. Jordan Jamie Foxx Brie Larson Desti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>Chinonye Chukwu</td>\n",
       "      <td>Alfre Woodard</td>\n",
       "      <td>Wendell Pierce</td>\n",
       "      <td>Aldis Hodge</td>\n",
       "      <td>Drama</td>\n",
       "      <td>clemency</td>\n",
       "      <td>Alfre Woodard Wendell Pierce Aldis Hodge Chino...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>242 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              director_name         actor_1_name          actor_2_name  \\\n",
       "0              Adam Robitel       Taylor Russell          Logan Miller   \n",
       "1               Jen McGowan    Hermione Corfield           Jay Paulson   \n",
       "2         Wilson Coneybeare    Donald Sutherland    Vincent Kartheiser   \n",
       "3      Charles Martin Smith  Bryce Dallas Howard    Edward James Olmos   \n",
       "4               Neil Burger       Bryan Cranston            Kevin Hart   \n",
       "..                      ...                  ...                   ...   \n",
       "237  Nick Bruno, Troy Quane           Will Smith           Tom Holland   \n",
       "238            Greta Gerwig        Saoirse Ronan           Emma Watson   \n",
       "239              Sam Mendes        George MacKay  Dean-Charles Chapman   \n",
       "240   Destin Daniel Cretton    Michael B. Jordan            Jamie Foxx   \n",
       "241         Chinonye Chukwu        Alfre Woodard        Wendell Pierce   \n",
       "\n",
       "         actor_3_name                                    genres  \\\n",
       "0    Deborah Ann Woll  Thriller Action Mystery Adventure Horror   \n",
       "1        Sean O'Bryan                            Thriller Drama   \n",
       "2       Oliver Dennis                                  Thriller   \n",
       "3     Alexandra Shipp                    Drama Adventure Family   \n",
       "4       Nicole Kidman                              Comedy Drama   \n",
       "..                ...                                       ...   \n",
       "237     Rashida Jones  Animation Action Adventure Comedy Family   \n",
       "238     Florence Pugh                             Drama Romance   \n",
       "239       Mark Strong                 War Drama Action Thriller   \n",
       "240       Brie Larson                       Drama Crime History   \n",
       "241       Aldis Hodge                                     Drama   \n",
       "\n",
       "           movie_title                                               comb  \n",
       "0          escape room  Taylor Russell Logan Miller Deborah Ann Woll A...  \n",
       "1           rust creek  Hermione Corfield Jay Paulson Sean O'Bryan Jen...  \n",
       "2     american hangman  Donald Sutherland Vincent Kartheiser Oliver De...  \n",
       "3     a dog's way home  Bryce Dallas Howard Edward James Olmos Alexand...  \n",
       "4           the upside  Bryan Cranston Kevin Hart Nicole Kidman Neil B...  \n",
       "..                 ...                                                ...  \n",
       "237  spies in disguise  Will Smith Tom Holland Rashida Jones Nick Brun...  \n",
       "238       little women  Saoirse Ronan Emma Watson Florence Pugh Greta ...  \n",
       "239               1917  George MacKay Dean-Charles Chapman Mark Strong...  \n",
       "240         just mercy  Michael B. Jordan Jamie Foxx Brie Larson Desti...  \n",
       "241           clemency  Alfre Woodard Wendell Pierce Aldis Hodge Chino...  \n",
       "\n",
       "[242 rows x 7 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_df = new_df18.append(new_df19,ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>director_name</th>\n",
       "      <th>actor_1_name</th>\n",
       "      <th>actor_2_name</th>\n",
       "      <th>actor_3_name</th>\n",
       "      <th>genres</th>\n",
       "      <th>movie_title</th>\n",
       "      <th>comb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adam Robitel</td>\n",
       "      <td>Lin Shaye</td>\n",
       "      <td>Angus Sampson</td>\n",
       "      <td>Leigh Whannell</td>\n",
       "      <td>Horror Mystery Thriller</td>\n",
       "      <td>insidious: the last key</td>\n",
       "      <td>Lin Shaye Angus Sampson Leigh Whannell Adam Ro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Lauren Wolkstein</td>\n",
       "      <td>Alex Pettyfer</td>\n",
       "      <td>James Freedson-Jackson</td>\n",
       "      <td>Emily Althaus</td>\n",
       "      <td>Thriller Drama</td>\n",
       "      <td>the strange ones</td>\n",
       "      <td>Alex Pettyfer James Freedson-Jackson Emily Alt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Simon West</td>\n",
       "      <td>Dominic Cooper</td>\n",
       "      <td>Austin Stowell</td>\n",
       "      <td>Gemma Chan</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>stratton</td>\n",
       "      <td>Dominic Cooper Austin Stowell Gemma Chan Simon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Warwick Thornton</td>\n",
       "      <td>Bryan Brown</td>\n",
       "      <td>Sam Neill</td>\n",
       "      <td>unknown</td>\n",
       "      <td>Drama History Western</td>\n",
       "      <td>sweet country</td>\n",
       "      <td>Bryan Brown Sam Neill unknown Warwick Thornton...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jaume Collet-Serra</td>\n",
       "      <td>Liam Neeson</td>\n",
       "      <td>Vera Farmiga</td>\n",
       "      <td>Patrick Wilson</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>the commuter</td>\n",
       "      <td>Liam Neeson Vera Farmiga Patrick Wilson Jaume ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>Nick Bruno, Troy Quane</td>\n",
       "      <td>Will Smith</td>\n",
       "      <td>Tom Holland</td>\n",
       "      <td>Rashida Jones</td>\n",
       "      <td>Animation Action Adventure Comedy Family</td>\n",
       "      <td>spies in disguise</td>\n",
       "      <td>Will Smith Tom Holland Rashida Jones Nick Brun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>508</th>\n",
       "      <td>Greta Gerwig</td>\n",
       "      <td>Saoirse Ronan</td>\n",
       "      <td>Emma Watson</td>\n",
       "      <td>Florence Pugh</td>\n",
       "      <td>Drama Romance</td>\n",
       "      <td>little women</td>\n",
       "      <td>Saoirse Ronan Emma Watson Florence Pugh Greta ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509</th>\n",
       "      <td>Sam Mendes</td>\n",
       "      <td>George MacKay</td>\n",
       "      <td>Dean-Charles Chapman</td>\n",
       "      <td>Mark Strong</td>\n",
       "      <td>War Drama Action Thriller</td>\n",
       "      <td>1917</td>\n",
       "      <td>George MacKay Dean-Charles Chapman Mark Strong...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>510</th>\n",
       "      <td>Destin Daniel Cretton</td>\n",
       "      <td>Michael B. Jordan</td>\n",
       "      <td>Jamie Foxx</td>\n",
       "      <td>Brie Larson</td>\n",
       "      <td>Drama Crime History</td>\n",
       "      <td>just mercy</td>\n",
       "      <td>Michael B. Jordan Jamie Foxx Brie Larson Desti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511</th>\n",
       "      <td>Chinonye Chukwu</td>\n",
       "      <td>Alfre Woodard</td>\n",
       "      <td>Wendell Pierce</td>\n",
       "      <td>Aldis Hodge</td>\n",
       "      <td>Drama</td>\n",
       "      <td>clemency</td>\n",
       "      <td>Alfre Woodard Wendell Pierce Aldis Hodge Chino...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>512 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              director_name       actor_1_name            actor_2_name  \\\n",
       "0              Adam Robitel          Lin Shaye           Angus Sampson   \n",
       "1          Lauren Wolkstein      Alex Pettyfer  James Freedson-Jackson   \n",
       "2                Simon West     Dominic Cooper          Austin Stowell   \n",
       "3          Warwick Thornton        Bryan Brown               Sam Neill   \n",
       "4        Jaume Collet-Serra        Liam Neeson            Vera Farmiga   \n",
       "..                      ...                ...                     ...   \n",
       "507  Nick Bruno, Troy Quane         Will Smith             Tom Holland   \n",
       "508            Greta Gerwig      Saoirse Ronan             Emma Watson   \n",
       "509              Sam Mendes      George MacKay    Dean-Charles Chapman   \n",
       "510   Destin Daniel Cretton  Michael B. Jordan              Jamie Foxx   \n",
       "511         Chinonye Chukwu      Alfre Woodard          Wendell Pierce   \n",
       "\n",
       "       actor_3_name                                    genres  \\\n",
       "0    Leigh Whannell                   Horror Mystery Thriller   \n",
       "1     Emily Althaus                            Thriller Drama   \n",
       "2        Gemma Chan                           Action Thriller   \n",
       "3           unknown                     Drama History Western   \n",
       "4    Patrick Wilson                           Action Thriller   \n",
       "..              ...                                       ...   \n",
       "507   Rashida Jones  Animation Action Adventure Comedy Family   \n",
       "508   Florence Pugh                             Drama Romance   \n",
       "509     Mark Strong                 War Drama Action Thriller   \n",
       "510     Brie Larson                       Drama Crime History   \n",
       "511     Aldis Hodge                                     Drama   \n",
       "\n",
       "                 movie_title  \\\n",
       "0    insidious: the last key   \n",
       "1           the strange ones   \n",
       "2                   stratton   \n",
       "3              sweet country   \n",
       "4               the commuter   \n",
       "..                       ...   \n",
       "507        spies in disguise   \n",
       "508             little women   \n",
       "509                     1917   \n",
       "510               just mercy   \n",
       "511                 clemency   \n",
       "\n",
       "                                                  comb  \n",
       "0    Lin Shaye Angus Sampson Leigh Whannell Adam Ro...  \n",
       "1    Alex Pettyfer James Freedson-Jackson Emily Alt...  \n",
       "2    Dominic Cooper Austin Stowell Gemma Chan Simon...  \n",
       "3    Bryan Brown Sam Neill unknown Warwick Thornton...  \n",
       "4    Liam Neeson Vera Farmiga Patrick Wilson Jaume ...  \n",
       "..                                                 ...  \n",
       "507  Will Smith Tom Holland Rashida Jones Nick Brun...  \n",
       "508  Saoirse Ronan Emma Watson Florence Pugh Greta ...  \n",
       "509  George MacKay Dean-Charles Chapman Mark Strong...  \n",
       "510  Michael B. Jordan Jamie Foxx Brie Larson Desti...  \n",
       "511  Alfre Woodard Wendell Pierce Aldis Hodge Chino...  \n",
       "\n",
       "[512 rows x 7 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_df = pd.read_csv('new_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>director_name</th>\n",
       "      <th>actor_1_name</th>\n",
       "      <th>actor_2_name</th>\n",
       "      <th>actor_3_name</th>\n",
       "      <th>genres</th>\n",
       "      <th>movie_title</th>\n",
       "      <th>comb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>James Cameron</td>\n",
       "      <td>CCH Pounder</td>\n",
       "      <td>Joel David Moore</td>\n",
       "      <td>Wes Studi</td>\n",
       "      <td>Action Adventure Fantasy Sci-Fi</td>\n",
       "      <td>avatar</td>\n",
       "      <td>CCH Pounder Joel David Moore Wes Studi James C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gore Verbinski</td>\n",
       "      <td>Johnny Depp</td>\n",
       "      <td>Orlando Bloom</td>\n",
       "      <td>Jack Davenport</td>\n",
       "      <td>Action Adventure Fantasy</td>\n",
       "      <td>pirates of the caribbean: at world's end</td>\n",
       "      <td>Johnny Depp Orlando Bloom Jack Davenport Gore ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sam Mendes</td>\n",
       "      <td>Christoph Waltz</td>\n",
       "      <td>Rory Kinnear</td>\n",
       "      <td>Stephanie Sigman</td>\n",
       "      <td>Action Adventure Thriller</td>\n",
       "      <td>spectre</td>\n",
       "      <td>Christoph Waltz Rory Kinnear Stephanie Sigman ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Christopher Nolan</td>\n",
       "      <td>Tom Hardy</td>\n",
       "      <td>Christian Bale</td>\n",
       "      <td>Joseph Gordon-Levitt</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>the dark knight rises</td>\n",
       "      <td>Tom Hardy Christian Bale Joseph Gordon-Levitt ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Doug Walker</td>\n",
       "      <td>Doug Walker</td>\n",
       "      <td>Rob Walker</td>\n",
       "      <td>unknown</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>star wars: episode vii - the force awakens    ...</td>\n",
       "      <td>Doug Walker Rob Walker unknown Doug Walker Doc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5359</th>\n",
       "      <td>Jim Strouse</td>\n",
       "      <td>Jessica Williams</td>\n",
       "      <td>Chris O'Dowd</td>\n",
       "      <td>Keith Stanfield</td>\n",
       "      <td>Romance Comedy</td>\n",
       "      <td>the incredible jessica james</td>\n",
       "      <td>Jessica Williams Chris O'Dowd Keith Stanfield ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5360</th>\n",
       "      <td>Farhad Mann</td>\n",
       "      <td>Adelaide Kane</td>\n",
       "      <td>Benjamin Hollingsworth</td>\n",
       "      <td>Jean Louisa Kelly</td>\n",
       "      <td>Romance</td>\n",
       "      <td>can't buy my love</td>\n",
       "      <td>Adelaide Kane Benjamin Hollingsworth Jean Loui...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5361</th>\n",
       "      <td>Hannaleena Hauru</td>\n",
       "      <td>Inka Haapamäki</td>\n",
       "      <td>Rosa Honkonen</td>\n",
       "      <td>Tiitus Rantala</td>\n",
       "      <td>Romance Comedy</td>\n",
       "      <td>thick lashes of lauri mäntyvaara</td>\n",
       "      <td>Inka Haapamäki Rosa Honkonen Tiitus Rantala Ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5362</th>\n",
       "      <td>Jonathan A. Rosenbaum</td>\n",
       "      <td>Lou Diamond Phillips</td>\n",
       "      <td>Wallace Shawn</td>\n",
       "      <td>Gina Holden</td>\n",
       "      <td>Crime Comedy Action Family</td>\n",
       "      <td>cop and a half: new recruit</td>\n",
       "      <td>Lou Diamond Phillips Wallace Shawn Gina Holden...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5363</th>\n",
       "      <td>Ravi Udyawar</td>\n",
       "      <td>Sridevi Kapoor</td>\n",
       "      <td>Sajal Ali</td>\n",
       "      <td>Akshaye Khanna</td>\n",
       "      <td>Crime Drama Thriller</td>\n",
       "      <td>mom</td>\n",
       "      <td>Sridevi Kapoor Sajal Ali Akshaye Khanna Ravi U...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5364 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              director_name          actor_1_name            actor_2_name  \\\n",
       "0             James Cameron           CCH Pounder        Joel David Moore   \n",
       "1            Gore Verbinski           Johnny Depp           Orlando Bloom   \n",
       "2                Sam Mendes       Christoph Waltz            Rory Kinnear   \n",
       "3         Christopher Nolan             Tom Hardy          Christian Bale   \n",
       "4               Doug Walker           Doug Walker              Rob Walker   \n",
       "...                     ...                   ...                     ...   \n",
       "5359            Jim Strouse      Jessica Williams            Chris O'Dowd   \n",
       "5360            Farhad Mann         Adelaide Kane  Benjamin Hollingsworth   \n",
       "5361       Hannaleena Hauru        Inka Haapamäki           Rosa Honkonen   \n",
       "5362  Jonathan A. Rosenbaum  Lou Diamond Phillips           Wallace Shawn   \n",
       "5363           Ravi Udyawar        Sridevi Kapoor               Sajal Ali   \n",
       "\n",
       "              actor_3_name                           genres  \\\n",
       "0                Wes Studi  Action Adventure Fantasy Sci-Fi   \n",
       "1           Jack Davenport         Action Adventure Fantasy   \n",
       "2         Stephanie Sigman        Action Adventure Thriller   \n",
       "3     Joseph Gordon-Levitt                  Action Thriller   \n",
       "4                  unknown                      Documentary   \n",
       "...                    ...                              ...   \n",
       "5359       Keith Stanfield                   Romance Comedy   \n",
       "5360     Jean Louisa Kelly                          Romance   \n",
       "5361        Tiitus Rantala                   Romance Comedy   \n",
       "5362           Gina Holden       Crime Comedy Action Family   \n",
       "5363        Akshaye Khanna             Crime Drama Thriller   \n",
       "\n",
       "                                            movie_title  \\\n",
       "0                                                avatar   \n",
       "1              pirates of the caribbean: at world's end   \n",
       "2                                               spectre   \n",
       "3                                 the dark knight rises   \n",
       "4     star wars: episode vii - the force awakens    ...   \n",
       "...                                                 ...   \n",
       "5359                       the incredible jessica james   \n",
       "5360                                  can't buy my love   \n",
       "5361                   thick lashes of lauri mäntyvaara   \n",
       "5362                        cop and a half: new recruit   \n",
       "5363                                                mom   \n",
       "\n",
       "                                                   comb  \n",
       "0     CCH Pounder Joel David Moore Wes Studi James C...  \n",
       "1     Johnny Depp Orlando Bloom Jack Davenport Gore ...  \n",
       "2     Christoph Waltz Rory Kinnear Stephanie Sigman ...  \n",
       "3     Tom Hardy Christian Bale Joseph Gordon-Levitt ...  \n",
       "4     Doug Walker Rob Walker unknown Doug Walker Doc...  \n",
       "...                                                 ...  \n",
       "5359  Jessica Williams Chris O'Dowd Keith Stanfield ...  \n",
       "5360  Adelaide Kane Benjamin Hollingsworth Jean Loui...  \n",
       "5361  Inka Haapamäki Rosa Honkonen Tiitus Rantala Ha...  \n",
       "5362  Lou Diamond Phillips Wallace Shawn Gina Holden...  \n",
       "5363  Sridevi Kapoor Sajal Ali Akshaye Khanna Ravi U...  \n",
       "\n",
       "[5364 rows x 7 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "old_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = old_df.append(my_df,ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>director_name</th>\n",
       "      <th>actor_1_name</th>\n",
       "      <th>actor_2_name</th>\n",
       "      <th>actor_3_name</th>\n",
       "      <th>genres</th>\n",
       "      <th>movie_title</th>\n",
       "      <th>comb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>James Cameron</td>\n",
       "      <td>CCH Pounder</td>\n",
       "      <td>Joel David Moore</td>\n",
       "      <td>Wes Studi</td>\n",
       "      <td>Action Adventure Fantasy Sci-Fi</td>\n",
       "      <td>avatar</td>\n",
       "      <td>CCH Pounder Joel David Moore Wes Studi James C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gore Verbinski</td>\n",
       "      <td>Johnny Depp</td>\n",
       "      <td>Orlando Bloom</td>\n",
       "      <td>Jack Davenport</td>\n",
       "      <td>Action Adventure Fantasy</td>\n",
       "      <td>pirates of the caribbean: at world's end</td>\n",
       "      <td>Johnny Depp Orlando Bloom Jack Davenport Gore ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sam Mendes</td>\n",
       "      <td>Christoph Waltz</td>\n",
       "      <td>Rory Kinnear</td>\n",
       "      <td>Stephanie Sigman</td>\n",
       "      <td>Action Adventure Thriller</td>\n",
       "      <td>spectre</td>\n",
       "      <td>Christoph Waltz Rory Kinnear Stephanie Sigman ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Christopher Nolan</td>\n",
       "      <td>Tom Hardy</td>\n",
       "      <td>Christian Bale</td>\n",
       "      <td>Joseph Gordon-Levitt</td>\n",
       "      <td>Action Thriller</td>\n",
       "      <td>the dark knight rises</td>\n",
       "      <td>Tom Hardy Christian Bale Joseph Gordon-Levitt ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Doug Walker</td>\n",
       "      <td>Doug Walker</td>\n",
       "      <td>Rob Walker</td>\n",
       "      <td>unknown</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>star wars: episode vii - the force awakens    ...</td>\n",
       "      <td>Doug Walker Rob Walker unknown Doug Walker Doc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5871</th>\n",
       "      <td>Nick Bruno, Troy Quane</td>\n",
       "      <td>Will Smith</td>\n",
       "      <td>Tom Holland</td>\n",
       "      <td>Rashida Jones</td>\n",
       "      <td>Animation Action Adventure Comedy Family</td>\n",
       "      <td>spies in disguise</td>\n",
       "      <td>Will Smith Tom Holland Rashida Jones Nick Brun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5872</th>\n",
       "      <td>Greta Gerwig</td>\n",
       "      <td>Saoirse Ronan</td>\n",
       "      <td>Emma Watson</td>\n",
       "      <td>Florence Pugh</td>\n",
       "      <td>Drama Romance</td>\n",
       "      <td>little women</td>\n",
       "      <td>Saoirse Ronan Emma Watson Florence Pugh Greta ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5873</th>\n",
       "      <td>Sam Mendes</td>\n",
       "      <td>George MacKay</td>\n",
       "      <td>Dean-Charles Chapman</td>\n",
       "      <td>Mark Strong</td>\n",
       "      <td>War Drama Action Thriller</td>\n",
       "      <td>1917</td>\n",
       "      <td>George MacKay Dean-Charles Chapman Mark Strong...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5874</th>\n",
       "      <td>Destin Daniel Cretton</td>\n",
       "      <td>Michael B. Jordan</td>\n",
       "      <td>Jamie Foxx</td>\n",
       "      <td>Brie Larson</td>\n",
       "      <td>Drama Crime History</td>\n",
       "      <td>just mercy</td>\n",
       "      <td>Michael B. Jordan Jamie Foxx Brie Larson Desti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5875</th>\n",
       "      <td>Chinonye Chukwu</td>\n",
       "      <td>Alfre Woodard</td>\n",
       "      <td>Wendell Pierce</td>\n",
       "      <td>Aldis Hodge</td>\n",
       "      <td>Drama</td>\n",
       "      <td>clemency</td>\n",
       "      <td>Alfre Woodard Wendell Pierce Aldis Hodge Chino...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5876 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               director_name       actor_1_name          actor_2_name  \\\n",
       "0              James Cameron        CCH Pounder      Joel David Moore   \n",
       "1             Gore Verbinski        Johnny Depp         Orlando Bloom   \n",
       "2                 Sam Mendes    Christoph Waltz          Rory Kinnear   \n",
       "3          Christopher Nolan          Tom Hardy        Christian Bale   \n",
       "4                Doug Walker        Doug Walker            Rob Walker   \n",
       "...                      ...                ...                   ...   \n",
       "5871  Nick Bruno, Troy Quane         Will Smith           Tom Holland   \n",
       "5872            Greta Gerwig      Saoirse Ronan           Emma Watson   \n",
       "5873              Sam Mendes      George MacKay  Dean-Charles Chapman   \n",
       "5874   Destin Daniel Cretton  Michael B. Jordan            Jamie Foxx   \n",
       "5875         Chinonye Chukwu      Alfre Woodard        Wendell Pierce   \n",
       "\n",
       "              actor_3_name                                    genres  \\\n",
       "0                Wes Studi           Action Adventure Fantasy Sci-Fi   \n",
       "1           Jack Davenport                  Action Adventure Fantasy   \n",
       "2         Stephanie Sigman                 Action Adventure Thriller   \n",
       "3     Joseph Gordon-Levitt                           Action Thriller   \n",
       "4                  unknown                               Documentary   \n",
       "...                    ...                                       ...   \n",
       "5871         Rashida Jones  Animation Action Adventure Comedy Family   \n",
       "5872         Florence Pugh                             Drama Romance   \n",
       "5873           Mark Strong                 War Drama Action Thriller   \n",
       "5874           Brie Larson                       Drama Crime History   \n",
       "5875           Aldis Hodge                                     Drama   \n",
       "\n",
       "                                            movie_title  \\\n",
       "0                                                avatar   \n",
       "1              pirates of the caribbean: at world's end   \n",
       "2                                               spectre   \n",
       "3                                 the dark knight rises   \n",
       "4     star wars: episode vii - the force awakens    ...   \n",
       "...                                                 ...   \n",
       "5871                                  spies in disguise   \n",
       "5872                                       little women   \n",
       "5873                                               1917   \n",
       "5874                                         just mercy   \n",
       "5875                                           clemency   \n",
       "\n",
       "                                                   comb  \n",
       "0     CCH Pounder Joel David Moore Wes Studi James C...  \n",
       "1     Johnny Depp Orlando Bloom Jack Davenport Gore ...  \n",
       "2     Christoph Waltz Rory Kinnear Stephanie Sigman ...  \n",
       "3     Tom Hardy Christian Bale Joseph Gordon-Levitt ...  \n",
       "4     Doug Walker Rob Walker unknown Doug Walker Doc...  \n",
       "...                                                 ...  \n",
       "5871  Will Smith Tom Holland Rashida Jones Nick Brun...  \n",
       "5872  Saoirse Ronan Emma Watson Florence Pugh Greta ...  \n",
       "5873  George MacKay Dean-Charles Chapman Mark Strong...  \n",
       "5874  Michael B. Jordan Jamie Foxx Brie Larson Desti...  \n",
       "5875  Alfre Woodard Wendell Pierce Aldis Hodge Chino...  \n",
       "\n",
       "[5876 rows x 7 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "director_name    0\n",
       "actor_1_name     0\n",
       "actor_2_name     0\n",
       "actor_3_name     0\n",
       "genres           3\n",
       "movie_title      0\n",
       "comb             3\n",
       "dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = final_df.dropna(how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "director_name    0\n",
       "actor_1_name     0\n",
       "actor_2_name     0\n",
       "actor_3_name     0\n",
       "genres           0\n",
       "movie_title      0\n",
       "comb             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv('final_data.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}