credit_risk_ensemble (2).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Learning\n",
    "\n",
    "## Initial Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from imblearn.metrics import classification_report_imbalanced"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the CSV and Perform Basic Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>issue_d</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>dti</th>\n",
       "      <th>...</th>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10500.0</td>\n",
       "      <td>0.1719</td>\n",
       "      <td>375.35</td>\n",
       "      <td>RENT</td>\n",
       "      <td>66000.0</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>low_risk</td>\n",
       "      <td>n</td>\n",
       "      <td>27.24</td>\n",
       "      <td>...</td>\n",
       "      <td>85.7</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>65687.0</td>\n",
       "      <td>38199.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>61987.0</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25000.0</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>929.09</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>105000.0</td>\n",
       "      <td>Verified</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>low_risk</td>\n",
       "      <td>n</td>\n",
       "      <td>20.23</td>\n",
       "      <td>...</td>\n",
       "      <td>91.2</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>271427.0</td>\n",
       "      <td>60641.0</td>\n",
       "      <td>41200.0</td>\n",
       "      <td>49197.0</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20000.0</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>529.88</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>56000.0</td>\n",
       "      <td>Verified</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>low_risk</td>\n",
       "      <td>n</td>\n",
       "      <td>24.26</td>\n",
       "      <td>...</td>\n",
       "      <td>66.7</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60644.0</td>\n",
       "      <td>45684.0</td>\n",
       "      <td>7500.0</td>\n",
       "      <td>43144.0</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10000.0</td>\n",
       "      <td>0.1640</td>\n",
       "      <td>353.55</td>\n",
       "      <td>RENT</td>\n",
       "      <td>92000.0</td>\n",
       "      <td>Verified</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>low_risk</td>\n",
       "      <td>n</td>\n",
       "      <td>31.44</td>\n",
       "      <td>...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99506.0</td>\n",
       "      <td>68784.0</td>\n",
       "      <td>19700.0</td>\n",
       "      <td>76506.0</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>22000.0</td>\n",
       "      <td>0.1474</td>\n",
       "      <td>520.39</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>52000.0</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>low_risk</td>\n",
       "      <td>n</td>\n",
       "      <td>18.76</td>\n",
       "      <td>...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>219750.0</td>\n",
       "      <td>25919.0</td>\n",
       "      <td>27600.0</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 86 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  int_rate  installment home_ownership  annual_inc  \\\n",
       "0    10500.0    0.1719       375.35           RENT     66000.0   \n",
       "1    25000.0    0.2000       929.09       MORTGAGE    105000.0   \n",
       "2    20000.0    0.2000       529.88       MORTGAGE     56000.0   \n",
       "3    10000.0    0.1640       353.55           RENT     92000.0   \n",
       "4    22000.0    0.1474       520.39       MORTGAGE     52000.0   \n",
       "\n",
       "  verification_status   issue_d loan_status pymnt_plan    dti  ...  \\\n",
       "0     Source Verified  Mar-2019    low_risk          n  27.24  ...   \n",
       "1            Verified  Mar-2019    low_risk          n  20.23  ...   \n",
       "2            Verified  Mar-2019    low_risk          n  24.26  ...   \n",
       "3            Verified  Mar-2019    low_risk          n  31.44  ...   \n",
       "4        Not Verified  Mar-2019    low_risk          n  18.76  ...   \n",
       "\n",
       "   pct_tl_nvr_dlq  percent_bc_gt_75  pub_rec_bankruptcies  tax_liens  \\\n",
       "0            85.7             100.0                   0.0        0.0   \n",
       "1            91.2              50.0                   1.0        0.0   \n",
       "2            66.7              50.0                   0.0        0.0   \n",
       "3           100.0              50.0                   1.0        0.0   \n",
       "4           100.0               0.0                   0.0        0.0   \n",
       "\n",
       "   tot_hi_cred_lim  total_bal_ex_mort total_bc_limit  \\\n",
       "0          65687.0            38199.0         2000.0   \n",
       "1         271427.0            60641.0        41200.0   \n",
       "2          60644.0            45684.0         7500.0   \n",
       "3          99506.0            68784.0        19700.0   \n",
       "4         219750.0            25919.0        27600.0   \n",
       "\n",
       "   total_il_high_credit_limit  hardship_flag  debt_settlement_flag  \n",
       "0                     61987.0              N                     N  \n",
       "1                     49197.0              N                     N  \n",
       "2                     43144.0              N                     N  \n",
       "3                     76506.0              N                     N  \n",
       "4                     20000.0              N                     N  \n",
       "\n",
       "[5 rows x 86 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the data\n",
    "file_path = Path('Resources/LoanStats_2019Q1.csv')\n",
    "df = pd.read_csv(file_path)\n",
    "\n",
    "# Preview the data\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split the Data into Training and Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create our features\n",
    "X = df.drop(\"loan_status\", axis=1)\n",
    "X = pd.get_dummies(X)\n",
    "\n",
    "# Create our target\n",
    "y = df[\"loan_status\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>...</th>\n",
       "      <th>issue_d_Mar-2019</th>\n",
       "      <th>pymnt_plan_n</th>\n",
       "      <th>initial_list_status_f</th>\n",
       "      <th>initial_list_status_w</th>\n",
       "      <th>next_pymnt_d_Apr-2019</th>\n",
       "      <th>next_pymnt_d_May-2019</th>\n",
       "      <th>application_type_Individual</th>\n",
       "      <th>application_type_Joint App</th>\n",
       "      <th>hardship_flag_N</th>\n",
       "      <th>debt_settlement_flag_N</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>6.881700e+04</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.0</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.000000</td>\n",
       "      <td>68817.0</td>\n",
       "      <td>68817.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>16677.594562</td>\n",
       "      <td>0.127718</td>\n",
       "      <td>480.652863</td>\n",
       "      <td>8.821371e+04</td>\n",
       "      <td>21.778153</td>\n",
       "      <td>0.217766</td>\n",
       "      <td>0.497697</td>\n",
       "      <td>12.587340</td>\n",
       "      <td>0.126030</td>\n",
       "      <td>17604.142828</td>\n",
       "      <td>...</td>\n",
       "      <td>0.177238</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.123879</td>\n",
       "      <td>0.876121</td>\n",
       "      <td>0.383161</td>\n",
       "      <td>0.616839</td>\n",
       "      <td>0.860340</td>\n",
       "      <td>0.139660</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>10277.348590</td>\n",
       "      <td>0.048130</td>\n",
       "      <td>288.062432</td>\n",
       "      <td>1.155800e+05</td>\n",
       "      <td>20.199244</td>\n",
       "      <td>0.718367</td>\n",
       "      <td>0.758122</td>\n",
       "      <td>6.022869</td>\n",
       "      <td>0.336797</td>\n",
       "      <td>21835.880400</td>\n",
       "      <td>...</td>\n",
       "      <td>0.381873</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.329446</td>\n",
       "      <td>0.329446</td>\n",
       "      <td>0.486161</td>\n",
       "      <td>0.486161</td>\n",
       "      <td>0.346637</td>\n",
       "      <td>0.346637</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>0.060000</td>\n",
       "      <td>30.890000</td>\n",
       "      <td>4.000000e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>9000.000000</td>\n",
       "      <td>0.088100</td>\n",
       "      <td>265.730000</td>\n",
       "      <td>5.000000e+04</td>\n",
       "      <td>13.890000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6293.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>15000.000000</td>\n",
       "      <td>0.118000</td>\n",
       "      <td>404.560000</td>\n",
       "      <td>7.300000e+04</td>\n",
       "      <td>19.760000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12068.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>24000.000000</td>\n",
       "      <td>0.155700</td>\n",
       "      <td>648.100000</td>\n",
       "      <td>1.040000e+05</td>\n",
       "      <td>26.660000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>21735.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>40000.000000</td>\n",
       "      <td>0.308400</td>\n",
       "      <td>1676.230000</td>\n",
       "      <td>8.797500e+06</td>\n",
       "      <td>999.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>587191.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          loan_amnt      int_rate   installment    annual_inc           dti  \\\n",
       "count  68817.000000  68817.000000  68817.000000  6.881700e+04  68817.000000   \n",
       "mean   16677.594562      0.127718    480.652863  8.821371e+04     21.778153   \n",
       "std    10277.348590      0.048130    288.062432  1.155800e+05     20.199244   \n",
       "min     1000.000000      0.060000     30.890000  4.000000e+01      0.000000   \n",
       "25%     9000.000000      0.088100    265.730000  5.000000e+04     13.890000   \n",
       "50%    15000.000000      0.118000    404.560000  7.300000e+04     19.760000   \n",
       "75%    24000.000000      0.155700    648.100000  1.040000e+05     26.660000   \n",
       "max    40000.000000      0.308400   1676.230000  8.797500e+06    999.000000   \n",
       "\n",
       "        delinq_2yrs  inq_last_6mths      open_acc       pub_rec  \\\n",
       "count  68817.000000    68817.000000  68817.000000  68817.000000   \n",
       "mean       0.217766        0.497697     12.587340      0.126030   \n",
       "std        0.718367        0.758122      6.022869      0.336797   \n",
       "min        0.000000        0.000000      2.000000      0.000000   \n",
       "25%        0.000000        0.000000      8.000000      0.000000   \n",
       "50%        0.000000        0.000000     11.000000      0.000000   \n",
       "75%        0.000000        1.000000     16.000000      0.000000   \n",
       "max       18.000000        5.000000     72.000000      4.000000   \n",
       "\n",
       "           revol_bal  ...  issue_d_Mar-2019  pymnt_plan_n  \\\n",
       "count   68817.000000  ...      68817.000000       68817.0   \n",
       "mean    17604.142828  ...          0.177238           1.0   \n",
       "std     21835.880400  ...          0.381873           0.0   \n",
       "min         0.000000  ...          0.000000           1.0   \n",
       "25%      6293.000000  ...          0.000000           1.0   \n",
       "50%     12068.000000  ...          0.000000           1.0   \n",
       "75%     21735.000000  ...          0.000000           1.0   \n",
       "max    587191.000000  ...          1.000000           1.0   \n",
       "\n",
       "       initial_list_status_f  initial_list_status_w  next_pymnt_d_Apr-2019  \\\n",
       "count           68817.000000           68817.000000           68817.000000   \n",
       "mean                0.123879               0.876121               0.383161   \n",
       "std                 0.329446               0.329446               0.486161   \n",
       "min                 0.000000               0.000000               0.000000   \n",
       "25%                 0.000000               1.000000               0.000000   \n",
       "50%                 0.000000               1.000000               0.000000   \n",
       "75%                 0.000000               1.000000               1.000000   \n",
       "max                 1.000000               1.000000               1.000000   \n",
       "\n",
       "       next_pymnt_d_May-2019  application_type_Individual  \\\n",
       "count           68817.000000                 68817.000000   \n",
       "mean                0.616839                     0.860340   \n",
       "std                 0.486161                     0.346637   \n",
       "min                 0.000000                     0.000000   \n",
       "25%                 0.000000                     1.000000   \n",
       "50%                 1.000000                     1.000000   \n",
       "75%                 1.000000                     1.000000   \n",
       "max                 1.000000                     1.000000   \n",
       "\n",
       "       application_type_Joint App  hardship_flag_N  debt_settlement_flag_N  \n",
       "count                68817.000000          68817.0                 68817.0  \n",
       "mean                     0.139660              1.0                     1.0  \n",
       "std                      0.346637              0.0                     0.0  \n",
       "min                      0.000000              1.0                     1.0  \n",
       "25%                      0.000000              1.0                     1.0  \n",
       "50%                      0.000000              1.0                     1.0  \n",
       "75%                      0.000000              1.0                     1.0  \n",
       "max                      1.000000              1.0                     1.0  \n",
       "\n",
       "[8 rows x 95 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "low_risk     68470\n",
       "high_risk      347\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the balance of our target values\n",
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the X and y into X_train, X_test, y_train, y_test\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Pre-Processing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the StandardScaler instance\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the Standard Scaler with the training data\n",
    "X_scaler = scaler.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale the training and testing data\n",
    "X_scaled = X_scaler.transform(X_train)\n",
    "X_tested = X_scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensemble Learners\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Balanced Random Forest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BalancedRandomForestClassifier(min_samples_leaf=2, n_jobs=1, random_state=1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Resample the training data with the BalancedRandomForestClassifier\n",
    "from imblearn.ensemble import BalancedRandomForestClassifier\n",
    "brf = BalancedRandomForestClassifier(bootstrap=True, class_weight=None,\n",
    "                criterion='gini', max_depth=None, max_features='auto',\n",
    "                max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
    "                min_samples_leaf=2, min_samples_split=2,\n",
    "                min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=1,\n",
    "                oob_score=False, random_state=1, replacement=False,\n",
    "                sampling_strategy='auto', verbose=0, warm_start=False)\n",
    "brf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7855052723466922"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculated the balanced accuracy score\n",
    "# Define model 1st\n",
    "brf_predictions = brf.predict(X_test)\n",
    "balanced_accuracy_score(y_test, brf_predictions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   68,    33],\n",
       "       [ 1749, 15355]], dtype=int64)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display the confusion matrix\n",
    "confusion_matrix(y_test, brf_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   pre       rec       spe        f1       geo       iba       sup\n",
      "\n",
      "  high_risk       0.04      0.67      0.90      0.07      0.78      0.59       101\n",
      "   low_risk       1.00      0.90      0.67      0.95      0.78      0.62     17104\n",
      "\n",
      "avg / total       0.99      0.90      0.67      0.94      0.78      0.62     17205\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print the imbalanced classification report\n",
    "print(classification_report_imbalanced(y_test, brf_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <td>0.091758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <td>0.064100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_pymnt</th>\n",
       "      <td>0.057649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_rec_int</th>\n",
       "      <td>0.057297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <td>0.051748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>int_rate</th>\n",
       "      <td>0.031956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>issue_d_Jan-2019</th>\n",
       "      <td>0.023537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installment</th>\n",
       "      <td>0.017079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <td>0.017015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <td>0.016538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max_bal_bc</th>\n",
       "      <td>0.016170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dti</th>\n",
       "      <td>0.016070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>revol_bal</th>\n",
       "      <td>0.015999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>annual_inc</th>\n",
       "      <td>0.015776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <td>0.015356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <td>0.015029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>out_prncp</th>\n",
       "      <td>0.014828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_bc_limit</th>\n",
       "      <td>0.014649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_bal_il</th>\n",
       "      <td>0.014402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_rcnt_il</th>\n",
       "      <td>0.014319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>issue_d_Mar-2019</th>\n",
       "      <td>0.013520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>il_util</th>\n",
       "      <td>0.013152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <td>0.013102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_util</th>\n",
       "      <td>0.012785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <td>0.012637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <td>0.012633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <td>0.012406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <td>0.011687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>all_util</th>\n",
       "      <td>0.011565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_rev_accts</th>\n",
       "      <td>0.011456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <td>0.011409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <td>0.010736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <td>0.010380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <td>0.010098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>loan_amnt</th>\n",
       "      <td>0.009954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <td>0.009914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_il_tl</th>\n",
       "      <td>0.009822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inq_last_12m</th>\n",
       "      <td>0.009604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <td>0.009537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_acc</th>\n",
       "      <td>0.008977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_bc_tl</th>\n",
       "      <td>0.008871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <td>0.008745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <td>0.008046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>next_pymnt_d_Apr-2019</th>\n",
       "      <td>0.007906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_acc</th>\n",
       "      <td>0.007821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_sats</th>\n",
       "      <td>0.007799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inq_fi</th>\n",
       "      <td>0.007608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_bc_sats</th>\n",
       "      <td>0.007586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_op_past_12m</th>\n",
       "      <td>0.007555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_acc_6m</th>\n",
       "      <td>0.007472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_rev_tl_bal_gt_0</th>\n",
       "      <td>0.007274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mort_acc</th>\n",
       "      <td>0.006875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_cu_tl</th>\n",
       "      <td>0.006862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <td>0.006839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_il_24m</th>\n",
       "      <td>0.006414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <td>0.006319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <td>0.006160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_rv_12m</th>\n",
       "      <td>0.006066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <td>0.005981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_act_il</th>\n",
       "      <td>0.005530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>issue_d_Feb-2019</th>\n",
       "      <td>0.004962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>next_pymnt_d_May-2019</th>\n",
       "      <td>0.004685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_rv_24m</th>\n",
       "      <td>0.004587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <td>0.004165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>open_il_12m</th>\n",
       "      <td>0.004016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <td>0.003275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>verification_status_Not Verified</th>\n",
       "      <td>0.002757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <td>0.002617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>home_ownership_MORTGAGE</th>\n",
       "      <td>0.002280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <td>0.002190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>home_ownership_RENT</th>\n",
       "      <td>0.002085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>home_ownership_OWN</th>\n",
       "      <td>0.001840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>verification_status_Verified</th>\n",
       "      <td>0.001736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>verification_status_Source Verified</th>\n",
       "      <td>0.001547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>application_type_Joint App</th>\n",
       "      <td>0.001226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>application_type_Individual</th>\n",
       "      <td>0.001221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <td>0.001215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pub_rec</th>\n",
       "      <td>0.000898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>initial_list_status_w</th>\n",
       "      <td>0.000813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <td>0.000573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <td>0.000517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>initial_list_status_f</th>\n",
       "      <td>0.000419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pymnt_plan_n</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hardship_flag_N</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_30dpd</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>home_ownership_ANY</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tax_liens</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recoveries</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>policy_code</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delinq_amnt</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>debt_settlement_flag_N</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Importance\n",
       "total_rec_prncp                        0.091758\n",
       "total_pymnt_inv                        0.064100\n",
       "total_pymnt                            0.057649\n",
       "total_rec_int                          0.057297\n",
       "last_pymnt_amnt                        0.051748\n",
       "int_rate                               0.031956\n",
       "issue_d_Jan-2019                       0.023537\n",
       "installment                            0.017079\n",
       "mths_since_recent_inq                  0.017015\n",
       "out_prncp_inv                          0.016538\n",
       "max_bal_bc                             0.016170\n",
       "dti                                    0.016070\n",
       "revol_bal                              0.015999\n",
       "annual_inc                             0.015776\n",
       "tot_hi_cred_lim                        0.015356\n",
       "mo_sin_old_rev_tl_op                   0.015029\n",
       "out_prncp                              0.014828\n",
       "total_bc_limit                         0.014649\n",
       "total_bal_il                           0.014402\n",
       "mths_since_rcnt_il                     0.014319\n",
       "issue_d_Mar-2019                       0.013520\n",
       "il_util                                0.013152\n",
       "total_il_high_credit_limit             0.013102\n",
       "bc_util                                0.012785\n",
       "total_bal_ex_mort                      0.012637\n",
       "avg_cur_bal                            0.012633\n",
       "total_rev_hi_lim                       0.012406\n",
       "mo_sin_old_il_acct                     0.011687\n",
       "all_util                               0.011565\n",
       "num_rev_accts                          0.011456\n",
       "bc_open_to_buy                         0.011409\n",
       "tot_cur_bal                            0.010736\n",
       "acc_open_past_24mths                   0.010380\n",
       "mths_since_recent_bc                   0.010098\n",
       "loan_amnt                              0.009954\n",
       "pct_tl_nvr_dlq                         0.009914\n",
       "num_il_tl                              0.009822\n",
       "inq_last_12m                           0.009604\n",
       "num_actv_rev_tl                        0.009537\n",
       "total_acc                              0.008977\n",
       "num_bc_tl                              0.008871\n",
       "num_op_rev_tl                          0.008745\n",
       "mo_sin_rcnt_tl                         0.008046\n",
       "next_pymnt_d_Apr-2019                  0.007906\n",
       "open_acc                               0.007821\n",
       "num_sats                               0.007799\n",
       "inq_fi                                 0.007608\n",
       "num_bc_sats                            0.007586\n",
       "num_tl_op_past_12m                     0.007555\n",
       "open_acc_6m                            0.007472\n",
       "num_rev_tl_bal_gt_0                    0.007274\n",
       "mort_acc                               0.006875\n",
       "total_cu_tl                            0.006862\n",
       "percent_bc_gt_75                       0.006839\n",
       "open_il_24m                            0.006414\n",
       "mo_sin_rcnt_rev_tl_op                  0.006319\n",
       "num_actv_bc_tl                         0.006160\n",
       "open_rv_12m                            0.006066\n",
       "total_rec_late_fee                     0.005981\n",
       "open_act_il                            0.005530\n",
       "issue_d_Feb-2019                       0.004962\n",
       "next_pymnt_d_May-2019                  0.004685\n",
       "open_rv_24m                            0.004587\n",
       "inq_last_6mths                         0.004165\n",
       "open_il_12m                            0.004016\n",
       "delinq_2yrs                            0.003275\n",
       "verification_status_Not Verified       0.002757\n",
       "num_accts_ever_120_pd                  0.002617\n",
       "home_ownership_MORTGAGE                0.002280\n",
       "tot_coll_amt                           0.002190\n",
       "home_ownership_RENT                    0.002085\n",
       "home_ownership_OWN                     0.001840\n",
       "verification_status_Verified           0.001736\n",
       "verification_status_Source Verified    0.001547\n",
       "application_type_Joint App             0.001226\n",
       "application_type_Individual            0.001221\n",
       "pub_rec_bankruptcies                   0.001215\n",
       "pub_rec                                0.000898\n",
       "initial_list_status_w                  0.000813\n",
       "num_tl_90g_dpd_24m                     0.000573\n",
       "collections_12_mths_ex_med             0.000517\n",
       "initial_list_status_f                  0.000419\n",
       "pymnt_plan_n                           0.000000\n",
       "hardship_flag_N                        0.000000\n",
       "num_tl_30dpd                           0.000000\n",
       "home_ownership_ANY                     0.000000\n",
       "tax_liens                              0.000000\n",
       "num_tl_120dpd_2m                       0.000000\n",
       "recoveries                             0.000000\n",
       "collection_recovery_fee                0.000000\n",
       "policy_code                            0.000000\n",
       "acc_now_delinq                         0.000000\n",
       "delinq_amnt                            0.000000\n",
       "chargeoff_within_12_mths               0.000000\n",
       "debt_settlement_flag_N                 0.000000"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List the features sorted in descending order by feature importance\n",
    "pd.set_option(\"display.max_rows\", None, \"display.max_columns\", None)\n",
    "\n",
    "importances = pd.DataFrame(brf.feature_importances_, index = X_train.columns, columns=[\"Importance\"]).sort_values(\"Importance\", ascending=False)\n",
    "importances.loc[:,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Easy Ensemble Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EasyEnsembleClassifier(n_estimators=1000, n_jobs=1, random_state=1)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the Classifier\n",
    "from imblearn.ensemble import EasyEnsembleClassifier\n",
    "ee_model = EasyEnsembleClassifier(base_estimator=None, n_estimators=1000, n_jobs=1, random_state=1, replacement=False, sampling_strategy=\"auto\", verbose=0, warm_start=False)\n",
    "ee_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9333555812316499"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculated the balanced accuracy score\n",
    "# Define model 1st\n",
    "ee_predicitions = ee_model.predict(X_test)\n",
    "balanced_accuracy_score(y_test, ee_predicitions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   93,     8],\n",
       "       [  925, 16179]], dtype=int64)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display the confusion matrix\n",
    "confusion_matrix(y_test, ee_predicitions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   pre       rec       spe        f1       geo       iba       sup\n",
      "\n",
      "  high_risk       0.09      0.92      0.95      0.17      0.93      0.87       101\n",
      "   low_risk       1.00      0.95      0.92      0.97      0.93      0.87     17104\n",
      "\n",
      "avg / total       0.99      0.95      0.92      0.97      0.93      0.87     17205\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print the imbalanced classification report\n",
    "print(classification_report_imbalanced(y_test, ee_predicitions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Questions\n",
    "\n",
    "1. Which model had the best balanced accuracy score?\n",
    "\n",
    "    Easy Ensemble Classifier \n",
    "\n",
    "2. Which model had the best recall score?\n",
    "\n",
    "    Easy Ensemble Classifier\n",
    "    \n",
    "3. Which model had the best geometric mean score?\n",
    "\n",
    "    Easy Ensemble Classifier\n",
    "\n",
    "4. What are the top three features?\n",
    "\n",
    "     1.Regression\n",
    "     2.Classification\n",
    "     3.Predict \n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}