import pandas as pd
print("Pandas version: {}".format(pd.__version__))

import numpy as np
print("Numpy version: {}".format(np.__version__))

import matplotlib
import matplotlib.pyplot as plt
fs = (16,6) # Figure size
print("Matplotlib version: {}".format(matplotlib.__version__))

import scipy as sp
print("Scipy version: {}".format(sp.__version__))

import sklearn
from sklearn.linear_model import LogisticRegression, PassiveAggressiveClassifier, RidgeClassifier, SGDClassifier, Perceptron
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.model_selection import cross_val_score,GridSearchCV, StratifiedKFold, train_test_split
from sklearn.naive_bayes import GaussianNB, BernoulliNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import VotingClassifier, RandomForestClassifier, GradientBoostingClassifier, BaggingClassifier, AdaBoostClassifier, ExtraTreesClassifier
from sklearn.svm import SVC, NuSVC, LinearSVC
from sklearn.preprocessing import StandardScaler
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.neural_network import MLPClassifier
print("Sklearn version: {}".format(sklearn.__version__))

import xgboost
from xgboost import XGBClassifier
print("Xgboost version: {}".format(xgboost.__version__))

import seaborn as sns
print("Seaborn version: {}".format(sns.__version__))

import IPython
from IPython.display import display
print("IPython version: {}".format(IPython.__version__))

%matplotlib inline

# Do not show warnings (used when the notebook is completed)
import warnings
# warnings.filterwarnings('ignore') !!!

train0 = pd.read_csv('train.csv')
test0 = pd.read_csv('test.csv')

train = train0.copy() # train will be modified throughout this notebook
test = test0.copy() # test will be modified throughout this notebook

print('The train dataset contains {} entries. Preview:'.format(str(train.shape[0])))
display(train.head())
print('The test dataset contains {} entries. Preview:'.format(str(test.shape[0])))
display(test.head())

# Delete PassengerId field
train.drop('PassengerId', axis=1, inplace=True)
test.drop('PassengerId', axis=1, inplace=True)

# Combine train and test sets into ds ('dataset'), dropping the 'Survived' column for the train set.
ds = pd.concat([train.drop('Survived', axis=1), test], axis=0)
print("Preview of combined dataset 'ds':")
display(ds.head())
print('...')
ds.tail()

# Inspect and look for missing data in the training set
temp = train.isna().sum(axis=0)
print('Training set missing data:')
print(temp[temp>0])

# Inspect and look for missing data in the training set
temp = test.isna().sum(axis=0)
print('Test set missing data:')
print(temp[temp>0])

# Correlation between cabin and class
cabin_temp = ds.dropna(axis=0, subset=['Cabin'])
cabin_temp.head(5)

# Extract the letter from the cabin
cabin_temp['Deck']=cabin_temp['Cabin'].apply(lambda x: x[0])
cabin_temp.head(5)

# Show Cabin Letter Distribution with Pclass
cabin_temp[['Pclass', 'Deck', 'Fare']].groupby(['Pclass', 'Deck']).mean()

temp = train.dropna(axis=0, subset=['Cabin'])
temp['Deck']=temp['Cabin'].apply(lambda x: x[0])
temp[['Pclass', 'Survived', 'Deck']].groupby(['Pclass', 'Deck']).mean()

temp[['Pclass', 'Survived', 'Deck']].groupby(['Pclass', 'Deck']).count()

sns.set(rc={'figure.figsize':(12,8)})
sns.boxplot(x=temp['Deck'], y=temp['Fare'], hue=temp['Pclass'])
plt.ylim(0,300)

train[train['Embarked'].isna()]

sns.boxplot(x=ds['Embarked'], y=ds['Fare'], hue=ds['Pclass'])
plt.ylim(0,200);

# Fill the missing information
train.loc[train['Embarked'].isna(),'Embarked']=['C']

print(train[['Pclass', 'Survived']].groupby(['Pclass']).mean().round(3)*100)
sns.barplot(x=train['Pclass'], y=train['Survived']*100)
plt.title('Survival rate (%) per class')

print(train[['Fare', 'Pclass']].groupby(['Pclass']).mean().round(1))
sns.barplot(x=train['Pclass'], y=train['Fare'])
plt.title('Survival rate (%) per class')

sns.boxplot(train['Pclass'], train['Fare'], hue=train['Survived'])
plt.ylim(0,200); # Extreme fares removed for clarity

sns.barplot(train['Sex'], train['Survived'], hue=train['Pclass'])
plt.title("Impact of gender (and class) on survival rate")

sns.kdeplot(train.loc[train['Survived']==1,'Age'], label="Survived", shade=True, color='green')
sns.kdeplot(train.loc[train['Survived']==0,'Age'], label="Did not survive", shade=True, color='gray')
sns.kdeplot(train['Age'], label="All passengers", shade=False, color='black')
plt.xticks(np.arange(0, 80, 2.0))
plt.title('Survival rate as a function of age')
plt.xlabel('Age'); plt.ylabel('Frequency');
plt.gcf().set_size_inches(20,12)

sns.kdeplot(train.loc[(train['Survived']==1) & (train['Sex']=='female'),'Age'], label="Survived female", shade=True, color='green')
# sns.kdeplot(train.loc[(train['Survived']==1) & (train['Sex']=='male'),'Age'], label="Survived male", shade=True, color='gray')
sns.kdeplot(train.loc[train['Sex']=='female','Age'], label="All female passengers", shade=False, color='black')
plt.xticks(np.arange(0, 80, 2.0))
plt.title('Survival rate for women as a function of age')
plt.xlabel('Age'); plt.ylabel('Frequency');
plt.gcf().set_size_inches(20,12)

sns.kdeplot(train.loc[(train['Survived']==1) & (train['Sex']=='male'),'Age'], label="Survived male", shade=True, color='green')
# sns.kdeplot(train.loc[(train['Survived']==1) & (train['Sex']=='male'),'Age'], label="Survived male", shade=True, color='gray')
sns.kdeplot(train.loc[train['Sex']=='male','Age'], label="All male passengers", shade=False, color='black')
plt.xticks(np.arange(0, 80, 2.0))
plt.title('Survival rate for men as a function of age')
plt.xlabel('Age'); plt.ylabel('Frequency');
plt.gcf().set_size_inches(20,12)

sns.barplot(train['SibSp'], train['Survived'])

sns.barplot(train['Parch'], train['Survived'])

temp = (train.loc[:,['Survived','SibSp', 'Parch']].groupby(['SibSp', 'Parch']).mean())
temp = pd.pivot_table(temp,index='SibSp',columns='Parch')
sns.heatmap(temp,xticklabels=range(7))
plt.xlabel('Parch')
plt.title('Survival heat map as a function of SibSp and Parch')

sns.barplot(train['Embarked'], train['Survived'])

sns.barplot(train['Embarked'], train['Survived'], hue=train['Pclass'])

# Inspect and look for missing data in the training set
temp = train.isna().sum(axis=0)
print('Training set missing data:')
temp[temp>0] # good practice ????? !!!

ds.drop(['Cabin'], axis=1, inplace=True)
train.drop(['Cabin'], axis=1, inplace=True)
test.drop(['Cabin'], axis=1, inplace=True);

train['Sex']=train['Sex'].apply(lambda s: 0 if s=='female' else 1)
test['Sex']=test['Sex'].apply(lambda s: 0 if s=='female' else 1)
ds['Sex']=ds['Sex'].apply(lambda s: 0 if s=='female' else 1)

train.describe()

train.corr()

n_colors = 256 # Number of colors in the legend bar
color_min, color_max = [-1, 1] # Range of values that will be mapped to the palette, i.e. min and max possible correlation

heatmap_columns = ['Survived', 'Age', 'Pclass', 'Sex', 'Fare', 'SibSp', 'Parch'] 

def heatmap(x, y, size, color, palette):
#     fig, ax = plt.subplots()
    plot_grid = plt.GridSpec(1, 15, hspace=0.2, wspace=0.1) # Setup a 1x15 grid
    ax = plt.subplot(plot_grid[:,:-1]) # Use the leftmost 14 columns of the grid for the main plot
    # Mapping from column names to integer coordinates
    x_labels = heatmap_columns[:-1:]
    y_labels = heatmap_columns[::-1][:-1]
    x_to_num = {p[1]:p[0] for p in enumerate(x_labels)} 
    y_to_num = {p[1]:p[0] for p in enumerate(y_labels)} 
    size_scale = 3000
    
    ax.scatter(x=x.map(x_to_num), y=y.map(y_to_num),s=size*size_scale, c=color, cmap=palette, marker='s')
    
    plt.title('Correlation between the main features.')
    ax.set_xticks([x_to_num[v] for v in x_labels])
    ax.set_xticklabels(x_labels, rotation=45, horizontalalignment='right')
    ax.set_yticks([y_to_num[v] for v in y_labels])
    ax.set_yticklabels(y_labels)
    ax.grid(False, 'major')
    ax.grid(True, 'minor')
    ax.set_xticks([t + 0.5 for t in ax.get_xticks()], minor=True)
    ax.set_yticks([t + 0.5 for t in ax.get_yticks()], minor=True)
    ax.set_xlim([-0.5, max([v for v in x_to_num.values()]) + 0.5]) 
    ax.set_ylim([-0.5, max([v for v in y_to_num.values()]) + 0.5])
    
#     # Add color legend on the right side of the plot
    ax = plt.subplot(plot_grid[:,-1]) # Use the rightmost column of the plot

    col_x = [0]*50 # Fixed x coordinate for the bars
    bar_y=np.linspace(color_min, color_max, n_colors) # y coordinates for each of the n_colors bars
    bar_height = bar_y[1] - bar_y[0]

    ax.barh(y=bar_y,
        width=5, # Make bars 5 units wide
        left=0, # Make bars start at 0
        height=bar_height,
        color=palette(bar_y+0.5),
        linewidth=0)
    
    ax.set_xticklabels([])
    ax.yaxis.tick_right()
    ax.set_ylim(-1,1)

corr = train[heatmap_columns].corr()
corr = corr.mask(np.tril(np.ones(corr.shape)).astype(np.bool)) # Removes the upper triangle diagonal
corr = pd.melt(corr.reset_index(), id_vars='index') # Unpivot the dataframe, so we can get pair of arrays for x and y
corr.columns = ['x', 'y', 'value']
# print(corr)
heatmap(
    x=corr['x'],
    y=corr['y'],
    size=corr['value'].abs(),
    color = corr['value'],
    palette=plt.cm.bwr
)

plt.gcf().set_size_inches(10,10)

# Extract the title from the names
ds['Title']=ds['Name'].str.extract(r'([A-Za-z]+)\.',expand=False)
train['Title']=train['Name'].str.extract(r'([A-Za-z]+)\.',expand=False)
test['Title']=test['Name'].str.extract(r'([A-Za-z]+)\.',expand=False)
ds['Title'].value_counts()

ds.groupby(['Title'])['Age'].agg(['mean','std']).sort_values(['mean'])

def fill_age(df):
    titles_map = ds.loc[:,['Title','Age']].groupby(['Title']).mean().sort_values(['Age'])
    age_from_titles = df.loc[:,'Title'].apply(lambda x: titles_map.loc[x].values[0])
    df['Age']=df['Age'].fillna(age_from_titles)
    return df

fill_age(train)
fill_age(test)
fill_age(ds)

test.loc[test['Fare'].isna(),:]

temp_fare = ds.loc[:,['Pclass','Title','Fare']].groupby(['Pclass','Title']).mean().loc[3,:].loc['Mr'][0]
print('Average fare from Mister in the 3rd class: {} years.'.format(round(temp_fare,2)))
test.loc[test['Fare'].isna(),['Fare']] = temp_fare

# def assign_age_group(age):
#     if age <= 12:
#         return "Child"
#     elif age <= 34:
#         return "Young adult"
#     elif age <= 50:
#         return "Adult"
#     else:
#         return "Senior"
    
def assign_age_group(age):
    if age <= 2:
        return "Toddler"
    elif age <= 12:
        return "Child"
    elif age <= 34:
        return "Young adult"
    elif age <= 50:
        return "Adult"
    else:
        return "Senior"

train['AgeGroup']=train['Age'].apply(lambda x: assign_age_group(x))
test['AgeGroup']=test['Age'].apply(lambda x: assign_age_group(x))
ds['AgeGroup']=ds['Age'].apply(lambda x: assign_age_group(x))

train = pd.concat((train, pd.get_dummies(train['AgeGroup'], prefix = 'AgeGroup', drop_first=True)), axis = 1)
test = pd.concat((test, pd.get_dummies(test['AgeGroup'], prefix = 'AgeGroup', drop_first=True)), axis = 1)
ds = pd.concat((ds, pd.get_dummies(ds['AgeGroup'], prefix = 'AgeGroup', drop_first=True)), axis = 1)

ds.head(3)

age_groups = ['AgeGroup_Child', 'AgeGroup_Senior', 'AgeGroup_Young adult']

test['Pclass'].unique()

target = 'Pclass'
prefix = 'Pclass'

train = pd.concat((train, pd.get_dummies(train[target], prefix = prefix, drop_first=True)), axis = 1)
test = pd.concat((test, pd.get_dummies(test[target], prefix = prefix, drop_first=True)), axis = 1)
ds = pd.concat((ds, pd.get_dummies(ds[target], prefix = prefix, drop_first=True)), axis = 1)

class_groups = train.columns[-2:].to_list()

class_groups

test.head(1)

test['Pclass'].unique()

ds.head(3)

selected_fields = ['Survived', 'Sex', 'SibSp', 'Parch', 'Fare']+age_groups+class_groups
print(selected_fields)
selected_fields_y = selected_fields.copy()
selected_fields_y.remove('Survived')
print(selected_fields_y)

X_train = train[selected_fields].drop(['Survived'],axis=1)
y_train = train['Survived']

X_test = test[selected_fields_y]

X_test.head(3)

X_train.head(3)

scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Cross validate model with Kfold stratified cross val
kfold = StratifiedKFold(n_splits=10)
random_state = 1986
n_jobs=-1 # The number of jobs to run in parallel for fit.

# classifiers
classifiers_list = [
    #Ensemble Methods
    AdaBoostClassifier(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state)),
    BaggingClassifier(random_state=random_state),
#     ExtraTreesClassifier(random_state=random_state),
    GradientBoostingClassifier(random_state=random_state),
    RandomForestClassifier(random_state=random_state),

    #Gaussian Processes
#     GaussianProcessClassifier(random_state=random_state),
    
    #GLM
    LogisticRegression(random_state=random_state),
#     PassiveAggressiveClassifier(random_state=random_state),
    RidgeClassifier(),
#     SGDClassifier(random_state=random_state),
#     Perceptron(random_state=random_state),
    MLPClassifier(random_state=random_state),
    
    #Navies Bayes
    BernoulliNB(),
#     GaussianNB(),
    
    #Nearest Neighbor
    KNeighborsClassifier(),
    
    #SVM
    SVC(probability=True, random_state=random_state),
#     NuSVC(probability=True, random_state=random_state),
    LinearSVC(random_state=random_state),
    
    #Trees    
    DecisionTreeClassifier(random_state=random_state),
    ExtraTreesClassifier(random_state=random_state),
    
    #Discriminant Analysis
    LinearDiscriminantAnalysis(),
#     QuadraticDiscriminantAnalysis(),

    
    #xgboost: http://xgboost.readthedocs.io/en/latest/model.html
    XGBClassifier()    
    ]

# store cv results in list
cv_results_list = []
cv_means_list = []
cv_std_list = []

# perform cross-validation
for clf in classifiers_list:
    cv_results_list.append(cross_val_score(clf,
                                           X_train,
                                           y_train,
                                           scoring = "accuracy",
                                           cv = kfold,
                                           n_jobs=n_jobs))

# store mean and std accuracy
for cv_result in cv_results_list:
    cv_means_list.append(cv_result.mean())
    cv_std_list.append(cv_result.std())
                      
cv_res_df = pd.DataFrame({"CrossValMeans":cv_means_list,
                          "CrossValerrors": cv_std_list,
                          "Algorithm":[clf.__class__.__name__ for clf in classifiers_list]})                    

cv_res_df = cv_res_df.sort_values(by='CrossValMeans',ascending=False)             
cv_res_df.set_index('Algorithm')

# Display the results as a bar plot
sns.barplot(cv_res_df['CrossValMeans'],
                cv_res_df['Algorithm'],
                **{'xerr':cv_std_list}
               )
plt.xlabel("Mean Accuracy")
plt.title("Cross validation scores with errors")

best_estimators = []

def gridsearch(clf, X_train, y_train, param_grid, random_state=1986, suffix='_best', scoring='accuracy', 
               n_jobs=4, kfold=StratifiedKFold(n_splits=10), verbose=1, other_args={}, print_best=1):

    estimator = clf(random_state=random_state, **other_args)

    gs = GridSearchCV(estimator,
                            param_grid=param_grid,
                            cv=kfold,
                            scoring="accuracy",
                            n_jobs=n_jobs,
                            verbose=verbose)
    
    gs.fit(X_train,y_train)

    name_of_best_estimator=clf.__name__+suffix
    best_estimators.append((name_of_best_estimator, gs.best_estimator_))
    if print_best==1:
        print(gs.best_estimator_)
        
    print('Best {} score: {}%.'.format(clf.__name__, round(100*gs.best_score_,2)))

gridsearch(XGBClassifier, X_train, y_train, 
           param_grid= {
#                'nthread':[4], #when use hyperthread, xgboost may become slower
#               'objective':['binary:logistic'],
#               "learning_rate": [0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2],
              'max_depth': [3,5,6,7],
#               'min_child_weight': [11],
#               'silent': [1],
#               'subsample': [0.8],
#               'colsample_bytree': [0.7],
              'n_estimators': [100,120], #number of trees, change it to 1000 for better results
#               'missing':[-999],
#               'seed': [1337]
           }
          )

gridsearch(GradientBoostingClassifier, X_train, y_train, 
           param_grid= {
#     "loss":["deviance"],
    "learning_rate": [0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2],
#     "min_samples_split": np.linspace(0.1, 0.5, 4),
#     "min_samples_leaf": np.linspace(0.1, 0.5, 4),
#     "max_depth":[3,5,8],
#     "max_features":["log2","sqrt"],
#     "criterion": ["friedman_mse",  "mae"],
    "subsample":[0.5, 0.618, 0.8, 0.83, 0.85, 0.87, 0.9, 0.95, 1.0],
#     "n_estimators":[10]
    })

gridsearch(ExtraTreesClassifier, X_train, y_train, 
           param_grid={"max_depth": [None],
              "max_features": [1, 3, 10],
              "min_samples_split": [2, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [False],
              "n_estimators" :[100,300],
              "criterion": ["gini"]})

gridsearch(SVC, X_train, y_train, 
           param_grid={'kernel': ['rbf'], 
                  'gamma': [0.0005, 0.0008, 0.001, 0.005, 0.01],
                  'C': [1, 10, 50, 100, 150, 200, 250]},
          other_args={"probability": True})

gridsearch(RandomForestClassifier, X_train, y_train, 
           param_grid={"max_depth": [None],
                  "max_features": [1, 3, 10],
                  "min_samples_split": [2, 3, 10],
                  "min_samples_leaf": [1, 3, 10],
                  "bootstrap": [False],
                  "n_estimators" :[100,300],
                  "criterion": ["gini"]}
          )

gridsearch(LogisticRegression, X_train, y_train, 
           param_grid={"C":np.logspace(-3,3,10), "penalty":["l1","l2"]},
           other_args={'max_iter':500}
          )

LDA = LinearDiscriminantAnalysis()

LDA.fit(X_train,y_train)

LDA_best = LDA

best_estimators.append(("LinearDiscriminantAnalysis_best", LDA_best))

best_estimators

votingC = VotingClassifier(estimators=best_estimators,
                           voting='soft', n_jobs=n_jobs)

votingC = votingC.fit(X_train, y_train)

Y_test_final_pred = votingC.predict(X_test).astype(int)

submit_df = pd.DataFrame({ 'PassengerId': test0['PassengerId'],'Survived': Y_test_final_pred})
submit_df.to_csv("voting_submission_df.csv", index=False)