# Import our modules
# Pandas and numpy
import pandas as pd
import numpy as np
# Modules for webscraping from webpages and apis
import urllib
import requests
import json
from bs4 import BeautifulSoup
# Using re for string matching
import re
# matplotlib 
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('ggplot')

# import Imdb from imdbpie
from imdbpie import Imdb

# grab top 250 movies using imdbpie
imdb = Imdb()
imdb = Imdb(anonymize=True)

top_250 = imdb.top_250()

df_250 = pd.DataFrame(top_250)

# Check for the oldest movie in the list
df_250['year'].min()

# Pull ids out into list
ids_250 = df_250['tconst']

import pickle

# Save list of ids out with pickle

with open('ids_250_pickle', 'wb') as fp:
    pickle.dump(ids_250, fp)

with open ('ids_250_pickle', 'rb') as fp:
    ids_250 = pickle.load(fp)

# Function to request and soupify our results
def soupy(url):
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) \
    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
    response = requests.get(url, headers=headers)
    content = response.content
    soup = BeautifulSoup(content, "lxml")
    return soup

# url formaters, search imdb by year (from 1910 -) and grab first 3 pages (150 most popular movies) for each year
year_url = 'http://www.imdb.com/search/title?year={},{}&title_type=feature&sort=moviemeter,asc&page={}&ref_=adv_nxt'
years = range(1910,2018)
pages = range(1,4)

# Function to grab title ids from imdb search pages
def get_titles(url):
    soup = soupy(url)
    try:
        titles = soup.findAll('h3', class_="lister-item-header")
        for i in titles:
            tt_link = i.find('a')['href']
            tt_id = tt_link.split('/')[2]
            title_ids.append(tt_id)      
    except AttributeError:
        tt_id = None     

# Instantiate an empty list to hold our ids
title_ids = []

# Loop over each year and the first three pages to grab title ids
for year in years:
    for page in pages:
        get_titles(year_url.format(year,year,page))

len(title_ids)

# Append top 250 ids to most popular per year
for i in ids_250:
    title_ids.append(i)

# Delete duplicate ids
title_ids_set = list(set(title_ids))
len(title_ids_set)

# Pickle ids
with open('ids_pickle', 'wb') as fp:
    pickle.dump(title_ids_set, fp)

%%timeit -o
# Time first 100 from OMDB
title_request = 'http://www.omdbapi.com/?i={}&plot=full'
info_omdb = []
for i in title_ids_set[:50]:
    url = title_request.format(i)
    response = requests.get(url)
    info_omdb.append(response.json()) 

%%timeit -o
# Time first 100 from OMDB
title_request = 'http://www.omdbapi.com/?i={}&plot=full'
info_omdb = []
for i in title_ids_set[:50]:
    url = title_request.format(i)
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) \
    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
    response = requests.get(url, headers=headers)
    info_omdb.append(response.json())

%%timeit -o
# Time first 100 from imdbpie
info_imdbpie = []
for i in title_ids_set[:50]:
    title = imdb.get_title_by_id(i)
    info_imdbpie.append(title)

# Take a look at the OMDB info
info_omdb_df = pd.DataFrame(info_omdb)
info_omdb_df.head(2)

# Take a look at the imdbpie info
# info_imdbpie

feats = ['rating','title', 'year', 'cast_summary', 'genres', 'imdb_id', 'directors_summary', 'certification','type']
df_imdbpie_test = pd.DataFrame([{fn: getattr(f, fn) for fn in feats} for f in info_imdbpie])

df_imdbpie_test.head(2)

# List to capture title info
movie_info = []

# Grab info for all titles using omdbapi
# The list of ids was subsetted and rerun for each group to avoid overloading the api
for t in title_ids_set[16000:]:
    args = {'i':t,'plot':'short','r':'json', 'tomatoes':'True'}
    url = 'http://www.omdbapi.com/' 
    r = requests.get(URL, params=args)
    movie_info.append(r.json())

# Convert info to dataframe
movie_df = pd.DataFrame(movie_info)

# Drop any duplicates
movie_df.drop_duplicates(subset='imdbID', inplace=True)

# Drop any null ids that resulted from our connection being kicked
movie_df = movie_df[movie_df.imdbID.notnull()]

movie_df.shape

# Export dataframe with pickle
movie_df.to_pickle('movie_df.pkl')

# Import dataframe with pickle
movie_df = pd.read_pickle('movie_df.pkl')

movie_df.head(2)

movie_df.columns

# Drop unneeded columns
drop_columns = [u'DVD', u'Error', u'Poster', u'Response', u'Type', u'Website',
       u'tomatoConsensus', u'tomatoFresh', u'tomatoImage',
       u'tomatoMeter', u'tomatoRating', u'tomatoReviews', u'tomatoRotten',
       u'tomatoURL', u'tomatoUserMeter', u'tomatoUserRating',
       u'tomatoUserReviews']

movie_df.drop(drop_columns, axis=1, inplace=True)

movie_df.head(2)

# Replace N/A with nan
movie_df = movie_df.applymap(lambda x: np.nan if x == 'N/A' else x)

movie_df.info()

# Drop titles without an imdb rating
rated_df = movie_df[movie_df['imdbRating'].notnull()]

rated_df.info()

rated_df.reset_index(drop=True, inplace=True)

rated_df.to_pickle('rated_films_first.pkl')

import warnings
warnings.filterwarnings('ignore')

# Strip imdbvotes string of commas
rated_df['imdbVotes'] = rated_df.loc[:,'imdbVotes'].str.replace(',','')

# List of columns to convert to numeric
to_convert = ['imdbRating', 'Metascore', 'Year', 'imdbVotes']

# Loop through list of columns and convert
for i in to_convert:
    rated_df[i] = rated_df.loc[:,i].apply(pd.to_numeric)

rated_df.describe()

# Lets convert release date to datetime
from datetime import datetime
rated_df['Release_Date'] = pd.to_datetime(rated_df.loc[:,'Released'],format='%d %b %Y')

rated_df.Production.nunique()

rated_df.Production.value_counts()[:25]

from fuzzywuzzy import fuzz
from fuzzywuzzy import process

choices = set(rated_df['Production'])
process.extract('Fox', choices, limit=15)

# Create a dictionary of production company replacements 
replacements = {
    'Production': {
        r'^Howco.*': 'Howco International Pictures',
        r'^Sony.*': 'Sony Pictures',
        r'^Twentieth Century.*': '20th Century Fox',
        r'^20th.*': '20th Century Fox',
        r'^Century Fox': '20th Century Fox',
        r'^Fox Film.*': '20th Century Fox',
        r'^Fox International': '20th Century Fox',
        r'^Fox Home.*': '20th Century Fox',
        r'^Fox': '20th Century Fox',
        r'^Fox Searchlight.*': 'Fox Searchlight',
        r'^21st Century.*': '21st Century Film',
        r'^Tri[\sSs][St].*': 'TriStar Pictures',
        r'^Columbia Tr.*': 'TriStar Pictures',
        r'^Columbia.*': 'Columbia Pictures',
        r'^Cosm.*': 'Cosmopolitan Productions',
        r'^Toho.*': 'Toho Company',
        r'^MGM.*': 'MGM',
        r'^Metro[ -]G.*': 'MGM',
        r'^Metro.*':'Metro Pictures',
        r'^Vestron.*':'Vestron Pictures',
        r'^Milestone Fil.*':'Milestone Film',
        r'^W[Aa][Rr]?[Nn][Ee][Rr]s?\s?[BH].*':'Warner Brothers',
        r'^WARNER': 'Warner Brothers',
        r'^Warner I.*': 'Warner Independent',
        r'^Walter W.*':'Walter Wanger',
        r'^Water Bearer.*':'Water Bearer Films',
        r'^Arthur Mayer.*':'Arthur Mayer Edward Kingsley I',
        r'^Path[^f].*': 'Pathe',
        r"^Lions?[Gg].*.*": 'Lionsgate Films',
        r"^Lions?'?s?\sG.*": "Lion's Gate Films",
        r"^MCA.*": 'MCA Universal',
        r"^Universum F.*":'Universum Film',
        r'^New Li.*': "New Line Cinema",
        r'^New World.*':'New World Pictures',
        r'^RKO.*':'RKO Pictures',
        r'^Dream\s?W.*': 'DreamWorks',
        r'^Paramou.*':'Paramount Pictures',
        r'^Mir[ai]max.*': 'Miramax Films',
        r'^Allied Artists.*':'Allied Artists Pictures',
        r'^Amazon.*': 'Amazon Studios',
        r'^Cinerama.*':'Cinerama',
        r'^Artisan.*':'Artisan Pictures',
        r'^Pacific Arts.*':'Pacific Arts',
        r'^IFC.*':'IFC Films',
        r'^Magnolia.*':'Magnolia Pictures',
        r'^Magnet.*':'Magnet Releasing',
        r'^Weinstein.*':'The Weinstein Co.',
        r'^The Weinstein Co.*': 'The Weinstein Co.',
        r'^Winstar.*':'Winstar Cinema',
        r'^BFI.*':'British Film Institute',
        r'^VCI.*':'VCI Entertainment',
        r'^Home Box.*':'HBO',
        r'^HBO.*':'HBO',
        r'^Criterion.*':'The Criterion Collection',
        r'^American Int.*':'American International Pictures',
        r'^Independent Int.*':'Independent International Pictures',
        r'^Independ[ae]nt Pi.*':'Independent Pictures',
        r'^Relativity.*':'Relativity Media',
        r'^STARZ MEDIA.*': 'Starz',
        r'^Associated First National P.*': 'First National Pictures',
        r'^First National P.*': 'First National Pictures',
        r'^Tartan.*':'Tartan Films',
        r'^Burroughs-Tarzan Ente.*':'Burroughs-Tarzan Enterprises',
        r'^Focus .*':'Focus Features',
        r'.*?Rogue.*':'Rogue Pictures',
        r'^Anchor Bay.*':'Anchor Bay Films',
        r'^Touchstone.*':'Touchstone Pictures',
        r'^CBS .*':'CBS Films',
        r'^De Laurentiis Ent.*':'De Laurentiis Entertainment Group',
        r'.*Concorde.*':'New Horizons Pictures',
        r'New Horizons.*':'New Horizons Pictures',
        r'American Pop.*':'American Pop Classics',
        r'Republic.*':'Republic Pictures',
        r'.*Embassy.*':'Embassy Pictures',
        r'^Screen Gems.*':'Screen Gems',
        r'^Rialto.*':'Rialto Pictures',
        r'^Summit.*':'Summit Entertainment',
        r'^Dimension.*':'Dimension Films',
        r'^Monogram.*':'Monogram Pictures',
        r'^Open Road.*':'Open Road Films',
        r'.*Goldwyn.*':'Samuel Goldwyn Films',
        r'^Roadside At.*':'Roadside Attractions',
        r'^PolyGram.*':'PolyGram Films',
        r'^First Nation.*':'First National Pictures',
        r'^Hollywood Pic.*':'Hollywood Pictures',
        r'^A24.*':'A24 Films',
        r'^Kino.*':'Kino',
        r"^Loew's.*":"Loew's Inc.",
        r'^Artkino Pict.*':'Artkino Pictures',
        r'^Gramercy.*':'Gramercy Pictures',
        r'Buena Vista.*': 'Buena Vista Pictures',
        r'.*Disney.*': 'Disney',
        r'^Dream[Ww]orks.*': 'DreamWorks',
        r'^Twentieth Century Fox.*': '20th Century Fox',
        r'Warner Bros\..*': 'Warner Bros.',
        r'Walt Disney': 'Disney',
        r'Hollywood/Buena Vista Pictures': 'Buena Vista Pictures',
        r'UTV.*': 'UTV Motion Pictures',
        r'United Artists.*': 'United Artists',
        r'Universal.*': 'Universal Pictures',
        r'Paramount.*': 'Paramount Pictures',
        r'Orion.*': 'Orion Pictures',
        r'Newmarket Film.*': 'Newmarket Film Group',
        r'^United Film Distribution.*':'United Film Distribution Company',
        r'^Cannon.*':'Cannon Films',
        r'^Group 1.*':'Group 1',
        r'^UN': 'Universal Pictures',
        r'^Millenn?ium En.*': 'Millennium Entertainment',
    }
}

rated_df.replace(replacements, regex=True, inplace=True)

rated_df.Production.nunique()

rated_df.Production.value_counts()[:25]

rated_df.Production.isnull().sum()

rated_df['Production'][rated_df['Title'] == 'Mad Max'] = 'Kennedy Miller Productions'

rated_df['Production'][rated_df['Title'] == 'The Night Porter'] = 'Lotar Film Productions'

rated_df['Month'] = rated_df.loc[:,"Release_Date"].apply(lambda x: x.month)

def awards(string):
    wins = re.compile(r"\d\d?")
    #awards = []
    ow = 0
    on = 0
    w = 0
    n = 0
    try:
        listed = string.split('.')
        for l in listed:
            if 'Oscar' in l and l[0] == 'W':
                ow = wins.findall(l)[0]
            elif 'Oscar' in l and l[0] == 'N':
                on = wins.findall(l)[0]
            elif 'win' in l and 'nomination' in l:
                w = wins.findall(l)[0]
                n = wins.findall(l)[1]
            elif 'nomination' in l and 'win' not in l:
                n = wins.findall(l)[0]
            else:
                None
        
    except:
        None
    return [ow,on,w,n]

rated_df['Awards_'] = rated_df['Awards'].apply(awards)

rated_df['Oscar_wins'] = rated_df['Awards_'].apply(lambda x: x[0])

rated_df['Oscar_noms'] = rated_df['Awards_'].apply(lambda x: x[1])

rated_df['Award_wins'] = rated_df['Awards_'].apply(lambda x: x[2])

rated_df['Award_noms'] = rated_df['Awards_'].apply(lambda x: x[3])

rated_df.drop(['Awards','Awards_'], axis=1, inplace=True)

def runtime(string):
    ints = re.compile(r"\d\d?\d?")
    try:
        if 'h' in string and 'min' in string:
            run = int(ints.findall(string)[0])*60 + int(ints.findall(string)[1])
        elif 'h' in string:
            run = int(ints.findall(string)[0])*60
        elif 'min' in string:
            run = int(ints.findall(string)[0])
        else:
            run = np.nan
    except:
        run = np.nan
    return run       

rated_df['RunTime'] = rated_df['Runtime'].apply(runtime)

rated_df.drop(['Runtime'], axis=1, inplace=True)

rated_df['RunTime'].isnull().sum()

# Impute missing Runtimes with median for year
rated_df['RunTime'].fillna(rated_df.groupby(['Year'])['RunTime'].transform("median"), inplace=True)

rated_df['RunTime'].isnull().sum()

def genre_1(string):
    clean_list = []
    try:
        listy = string.split(',')
        for l in listy:
            clean = l.strip()
            clean_list.append(clean)
        return clean_list[0] 
    except:
        return 'None'

rated_df['genre_1'] = rated_df["Genre"].apply(genre_1)

def genre_2(string):
    clean_list = []
    try:
        listy = string.split(',')
        for l in listy:
            clean = l.strip()
            clean_list.append(clean)
        return clean_list[1] 
    except:
        return 'None'

rated_df['genre_2'] = rated_df["Genre"].apply(genre_2)

def genre_3(string):
    clean_list = []
    try:
        listy = string.split(',')
        for l in listy:
            clean = l.strip()
            clean_list.append(clean)
        return clean_list[2] 
    except:
        return 'None'

rated_df['genre_3'] = rated_df["Genre"].apply(genre_3)

rated_df['genre_1'].unique()

rated_df['genre_1'].nunique()

genre_1 = rated_df['genre_1'].unique()

# Dummy genre
for genre in genre_1:
    if genre in rated_df['genre_1']:
        rated_df[genre] = 1
    else:
        rated_df[genre] = 0

rated_df.Director.nunique()

rated_df.Director.value_counts()[:20]

choices_d = set(rated_df['Director'])
process.extract('Lana Wachowski', choices_d, limit=15)

rated_df['Director'] = rated_df['Director'].str.replace('Ethan Coen, Joel Coen', 'Joel Coen, Ethan Coen')

rated_df['Director'] = rated_df['Director'].str.replace('Joel Coen, Ethan Coen', 'Joel & Ethan Coen')

rated_df['Director'] = rated_df['Director'].str.replace('Bobby Farrelly, Peter Farrelly', 'Peter Farrelly, Bobby Farrelly')

rated_df['Director'] = rated_df['Director'].str.replace('Peter Farrelly, Bobby Farrelly', 'Bobby & Peter Farrelly')

rated_df['Director'] = rated_df['Director'].str.replace('Michael Powell, Emeric Pressburger', 'Michael Powell & Emeric Pressburger')

rated_df['Director'] = rated_df['Director'].str.replace('Lana Wachowski, Lilly Wachowski', "The Wachowski's")

# Define a function to grab to first Director for each feature
def director(string):
    clean_list = []
    try:
        listy = string.split(',')
        for l in listy:
            clean = l.strip()
            clean_list.append(clean)
        return clean_list[0] 
    except:
        return 'Unknown'

rated_df['Director_1'] = rated_df['Director'].apply(director)

rated_df['Director_1'].nunique()

rated_df['Director_1'].isnull().sum()

rated_df['Director_1'] = rated_df['Director_1'].fillna('Unknown')

rated_df['Director_1'].isnull().sum()

director_40 = rated_df['Director_1'].value_counts().head(40)
director_40 = pd.DataFrame(director_40)
director_40.reset_index(inplace=True)
director_40.columns = ['Director_1', 'Value']

directors = rated_df[["Director_1", "imdbRating"]].groupby(['Director_1'], \
                                            as_index=False).mean()

top_directors = rated_df[["Director_1", "imdbRating"]].groupby(['Director_1'], \
                                            as_index=False).mean().sort_values('imdbRating', ascending=False)

director_info = pd.merge(director_40, directors, how='left')
director_info = director_info.sort_values('imdbRating', ascending=False)

director_info.plot('Director_1', 'imdbRating', kind='bar', figsize=(10,7), legend=False, xticks=None,\
                title = 'Average Rating for 40 most prolific Directors', ylim=(0,10), colormap='Paired')

director_count = rated_df['Director_1'].value_counts()
director_count = pd.DataFrame(director_count)
director_count.reset_index(inplace=True)
director_count.columns = ['Director_1', 'Value']

director_5 = director_count[director_count.Value > 5]
director_5.head()

director_5_rating = pd.merge(director_5, top_directors, how='left')
director_5_rating = director_5_rating.sort_values('imdbRating', ascending=False)
director_5_rating = director_5_rating.head(40)
director_5_rating

director_5_rating.plot('Director_1', 'imdbRating', kind='bar', figsize=(10,7), legend=False, xticks=None,\
                title = 'Top Rated Directors (>5 films represented)', ylim=(0,10), colormap='Paired')

directors_5up = director_5['Director_1']

len(directors_5up)

# Dummy important directors
# Since director seams so important we will use the 658 for dummies
for dirc in directors_5up:
    if dirc in rated_df['Director_1']:
        rated_df[dirc] = 1
    else:
        rated_df[dirc] = 0

rated_df['Language'].nunique()

rated_df['Language_main'] = rated_df['Language'].apply(director)

rated_df['Language_main'].nunique()

rated_df['Language_main'].value_counts()[:20]

rated_df['Language_main'] = rated_df['Language_main'].str.replace('Unknown','English')

rated_df['Language_main'].value_counts()[:20]

rated_df['Language_main'].isnull().sum()

rated_df['Rated'] = rated_df['Rated'].str.lower()

rated_df['Rated'].unique()

rated_df['Rated'] = rated_df['Rated'].fillna('None')

rated_df['Rated'].isnull().sum()

rating_dum = rated_df['Rated'].unique()

# Dummy rating
for rate in rating_dum:
    if rate in rated_df['Rated']:
        rated_df[rate+' Rated'] = 1
    else:
        rated_df[rate+' Rated'] = 0

rated_df.columns

rated_df['Country'].nunique()

rated_df['Country_Main'] = rated_df['Country'].apply(director)

rated_df['Country_Main'].nunique()

rated_df['Country_Main'].value_counts()[:20]

rated_df['Country_Main'] = rated_df['Country_Main'].str.replace('Unknown','USA')

rated_df['Country_Main'].isnull().sum()

country_values = rated_df['Country_Main'].value_counts()
country_values = pd.DataFrame(country_values)
country_values.reset_index(inplace=True)
country_values.columns = ['Country_Main', 'Values']
country_values.head()

Top_countries = rated_df[["Country_Main", "imdbRating"]].groupby(['Country_Main'], as_index=False).mean()

country_info = pd.merge(Top_countries, country_values, how='left')

country_20 = country_info.sort_values('Values', ascending=False).head(20)

country_20.plot('Country_Main', 'imdbRating', kind='bar', figsize=(10,7), legend=False, xticks=None,\
                title = 'Average Rating for 20 most prolific Countries', ylim=(0,10), colormap='Paired')

country_dummy = rated_df['Country_Main'].value_counts()
country_dummy = pd.DataFrame(country_dummy)
country_dummy = country_dummy[country_dummy.Country_Main > 5]
country_index = country_dummy.index
country_index

# Dummy country
for country in country_index:
    if country in rated_df['Country_Main']:
        rated_df[country] = 1
    else:
        rated_df[country] = 0

rated_df['Production'].nunique()

rated_df['Production'].isnull().sum()

rated_df['Production'] = rated_df['Production'].fillna('None')

rated_df['Production'].isnull().sum()

production = pd.DataFrame(rated_df['Production'].value_counts()[:100]).T
production.drop('None', axis=1, inplace=True)
production.head()

top_production = production.columns

for pro in top_production:
    if pro in rated_df['Production']:
        rated_df['P_'+pro] = 1
    else:
        rated_df['P_'+pro] = 0

production_20 = pd.DataFrame(top_production[:20])
production_20.columns = ['Production']

production_ratings = rated_df[["Production", "imdbRating"]].groupby(['Production'], as_index=False).mean()

production_ratings_20 = pd.merge(production_20, production_ratings, how='left')

production_sort = production_ratings_20.sort_values('imdbRating', ascending=False)

production_sort.plot('Production', 'imdbRating', kind='bar', figsize=(10,7), legend=False, xticks=None,\
                title = 'Average Rating for 20 most prolific Production Companies', ylim=(0,10), colormap='Paired')

rated_df['Actors'].describe()

rated_df['Actors'].isnull().sum()

rated_df['Actors'].fillna(value='None', inplace=True)

rated_df['Actors'].isnull().sum()

# Countvectorize actors

def grams_per_comma(string):
    for ln in string.split(','):
        terms = re.findall(r'\w{2,}', ln)
        for gram in zip(terms, terms[1:]):
            yield '%s %s' % gram

from sklearn.feature_extraction.text import CountVectorizer

cvec = CountVectorizer(analyzer=grams_per_comma, ngram_range=(3, 3), max_features=100)
cvec.fit(rated_df['Actors'])

actors_df  = pd.DataFrame(cvec.transform(rated_df['Actors']).todense(),
             columns=cvec.get_feature_names())

actors = actors_df.transpose().sort_values(0, ascending=False).transpose()
print actors.shape
actors.head()

actors_columns = []
for i in actors.columns:
    name = i+'_Actor'
    actors_columns.append(name)

actors.columns = actors_columns
actors.head(1)

merge_df = rated_df.merge(actors, how='left', left_index=True, right_index=True)
print merge_df.shape
merge_df.head(1)

rated_df['Plot'].fillna('None',inplace=True)

cvec = CountVectorizer(stop_words='english', max_features=40)
cvec.fit(rated_df['Plot'])

plot_tdf  = pd.DataFrame(cvec.transform(rated_df['Plot']).todense(),
             columns=cvec.get_feature_names())

plot_feats = plot_tdf.transpose().sort_values(0, ascending=False).transpose().head()
plot_feats.head()

plot_columns = []
for i in plot_feats.columns:
    name = 'Plot_'+i
    plot_columns.append(name)

plot_feats.columns = plot_columns
plot_feats.head()

merge_df = merge_df.merge(plot_feats, how='left', left_index=True, right_index=True)
print merge_df.shape
merge_df.head(1)

cvec_t = CountVectorizer(stop_words='english', max_features=40)
cvec_t.fit(rated_df['Title'])

title_df  = pd.DataFrame(cvec_t.transform(rated_df['Title']).todense(),
             columns=cvec_t.get_feature_names())

title = title_df.transpose().sort_values(0, ascending=False).transpose()
print title.shape
title.head()

title_columns = []
for i in title.columns:
    name = 'Title_'+i
    title_columns.append(name)

title.columns = title_columns
title.head()

merge_df = merge_df.merge(title, how='left', left_index=True, right_index=True)
print merge_df.shape
merge_df.head(1)

producers = pd.read_pickle('producer_df.pkl')

# Define a function to grab to first Director for each feature
def producer(listy):
    try:
        return listy[0] 
    except:
        return 'None'

producers['Producer_1'] = producers['Producers'].apply(producer)

producer_clean = producers.drop('Producers', axis=1)
producer_clean.head()

top_producers = producer_clean['Producer_1'].value_counts()[:100]

top_producers = pd.DataFrame(top_producers)
top_producers.reset_index(inplace=True)
top_producers = top_producers.iloc[1:,:]
top_producers_list = top_producers['index']
top_producers_list

for duce in top_producers_list:
    if duce in producer_clean['Producer_1']:
        producer_clean['Producer_'+duce] = 1
    else:
        producer_clean['Producer_'+duce] = 0

producer_clean.head()

# Merge producer_1
merge_df = pd.merge(merge_df, producer_clean, how='left')
print merge_df.shape
merge_df.head(1)

gross = pd.read_pickle('gross_perf.pkl')

gross.reset_index(drop=True, inplace=True)
gross.head()

gross.info()

# Only 4800 films with info for both gross and budget
# Performance is a measure of how well a film did in the box office in relation to its budget
# Will subset the data later and run another model using performance

merge_df.to_pickle('merged_cleaned.pkl')

merge_df.columns

drop = [u'Actors', u'BoxOffice', u'Country', u'Director',
        u'Genre', u'Language', u'Metascore', u'Plot',
        u'Production', 'Released', 'Title', 'Writer',
        'Release_Date']

df = merge_df.drop(drop, axis=1)

# Fill votes with the mdeian for the rating
df.imdbVotes.fillna(df.groupby(["imdbRating"])["imdbVotes"].transform("median"), inplace=True)

# Fill month with the mode
df.Month.fillna(df.Month.mode()[0], inplace=True)

import seaborn as sns

# Create a list of quantitative data columns
quant = ['imdbRating',
         'imdbVotes',
         'Year',
         'Month',
         'RunTime']
quant

# Plot Histograms of numerical data columns
sns.set(rc={"figure.figsize": (8, 4)})
for i in quant:
    sns.distplot(df[i])
    plt.xlabel(i)
    plt.ylabel('Count')
    plt.show()

df['imdbRating'].describe(percentiles=[.25,.33,.66, .75])

# Bin rating into high, mid, low (3,2,1)

rating_labels = []
for i in df['imdbRating']:
    if i >= 7.2:
        rate = 2
    elif i >= 6.2:
        rate = 1
    else:
        rate = 0
    rating_labels.append(rate)

df['Rating_Labels'] = rating_labels

df.iloc[:,:15].head(1)

drop_for_sklearn = ['Rated', 'imdbID', 'imdbRating', 'genre_1', 'genre_2', 'genre_3',\
                   u'Director_1',u'Language_main',u'Country_Main', 'Producer_1',]

df.imdbVotes = df.imdbVotes.apply(int)

df.to_pickle('df.pkl')

df_1 = df.drop(drop_for_sklearn, axis=1)

df_1.columns

df_1.to_pickle('df_1.pkl')

dropped = ['Rating_Labels']
X = df_1.drop(dropped, axis=1)
y = df_1['Rating_Labels']

X.Oscar_wins.isnull().sum()

process.extract('Fox', choices, limit=15)

X = X.fillna(method='ffill')

from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier

cv = StratifiedKFold(n_splits=3, random_state=21, shuffle=True)

dt = DecisionTreeClassifier()
bc = BaggingClassifier(dt)
rf = RandomForestClassifier(n_estimators=1000, class_weight='balanced')
xt = ExtraTreesClassifier(n_estimators=1000, class_weight='balanced')

def score(model, string):
    s = cross_val_score(model, X, y, cv=cv)
    print("{} Score:\t{:0.3} ± {:0.3}".format(string, s.mean().round(3), s.std().round(3)))

score(bc, 'Bagging')

score(rf, 'Random')

score(xt, 'Extra Trees')

rf = RandomForestClassifier(n_estimators=1000, max_depth = 5, class_weight='balanced')
model = rf.fit(X, y)

# importances = rf.feature_importances_
# importances =pd.DataFrame(importances)
# importances.sort_values(0,ascending=False).head(20)

feature_importances = pd.DataFrame(rf.feature_importances_,
                                   index = X.columns,
                                    columns=['importance'])
importances = feature_importances.sort_values(by='importance', ascending=False).head(20).T
importances

fig, ax = plt.subplots(figsize=(16,8))
importances.plot(kind='bar', ax=ax)

pred = model.predict(X)

from sklearn.metrics import confusion_matrix, classification_report


# conf matrix
cm = confusion_matrix(y, pred)
cm = pd.DataFrame(cm, columns=["Predict Low Rating", "Predict Average Rating", "Predict High Rating"], \
                  index=["Low Rating", "Average Rating", "High Rating"] )
cm

print(classification_report(y, pred))