Capstone 3 Gradient Boost Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import subprocess\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, cross_val_predict\n",
    "from itertools import cycle\n",
    "from sklearn import ensemble\n",
    "import time\n",
    "from sklearn.metrics import recall_score\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from scipy import interp\n",
    "import statsmodels.api as sm\n",
    "from sklearn.metrics import recall_score\n",
    "\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.utils import resample\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LB</th>\n",
       "      <th>AC</th>\n",
       "      <th>FM</th>\n",
       "      <th>UC</th>\n",
       "      <th>DL</th>\n",
       "      <th>DS</th>\n",
       "      <th>DP</th>\n",
       "      <th>ASTV</th>\n",
       "      <th>MSTV</th>\n",
       "      <th>ALTV</th>\n",
       "      <th>...</th>\n",
       "      <th>Min</th>\n",
       "      <th>Max</th>\n",
       "      <th>Nmax</th>\n",
       "      <th>Nzeros</th>\n",
       "      <th>Mode</th>\n",
       "      <th>Mean</th>\n",
       "      <th>Median</th>\n",
       "      <th>Variance</th>\n",
       "      <th>Tendency</th>\n",
       "      <th>NSP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>120.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>43.0</td>\n",
       "      <td>...</td>\n",
       "      <td>62.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>137.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>132.0</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>2.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>68.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>133.0</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>68.0</td>\n",
       "      <td>198.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>134.0</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>53.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>137.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>137.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>132.0</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>53.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>137.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      LB     AC   FM     UC     DL   DS   DP  ASTV  MSTV  ALTV ...    Min  \\\n",
       "0  120.0  0.000  0.0  0.000  0.000  0.0  0.0  73.0   0.5  43.0 ...   62.0   \n",
       "1  132.0  0.006  0.0  0.006  0.003  0.0  0.0  17.0   2.1   0.0 ...   68.0   \n",
       "2  133.0  0.003  0.0  0.008  0.003  0.0  0.0  16.0   2.1   0.0 ...   68.0   \n",
       "3  134.0  0.003  0.0  0.008  0.003  0.0  0.0  16.0   2.4   0.0 ...   53.0   \n",
       "4  132.0  0.007  0.0  0.008  0.000  0.0  0.0  16.0   2.4   0.0 ...   53.0   \n",
       "\n",
       "     Max  Nmax  Nzeros   Mode   Mean  Median  Variance  Tendency  NSP  \n",
       "0  126.0   2.0     0.0  120.0  137.0   121.0      73.0       1.0  2.0  \n",
       "1  198.0   6.0     1.0  141.0  136.0   140.0      12.0       0.0  1.0  \n",
       "2  198.0   5.0     1.0  141.0  135.0   138.0      13.0       0.0  1.0  \n",
       "3  170.0  11.0     0.0  137.0  134.0   137.0      13.0       1.0  1.0  \n",
       "4  170.0   9.0     0.0  137.0  136.0   138.0      11.0       1.0  1.0  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_cardio = pd.read_csv('Cardiotocography.csv', sep=',')\n",
    "\n",
    "df_cardio.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LB</th>\n",
       "      <th>AC</th>\n",
       "      <th>FM</th>\n",
       "      <th>UC</th>\n",
       "      <th>DL</th>\n",
       "      <th>DS</th>\n",
       "      <th>DP</th>\n",
       "      <th>ASTV</th>\n",
       "      <th>MSTV</th>\n",
       "      <th>ALTV</th>\n",
       "      <th>...</th>\n",
       "      <th>Min</th>\n",
       "      <th>Max</th>\n",
       "      <th>Nmax</th>\n",
       "      <th>Nzeros</th>\n",
       "      <th>Mode</th>\n",
       "      <th>Mean</th>\n",
       "      <th>Median</th>\n",
       "      <th>Variance</th>\n",
       "      <th>Tendency</th>\n",
       "      <th>NSP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2126</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2127</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2128</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.481</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.005</td>\n",
       "      <td>87.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      LB  AC     FM     UC     DL     DS     DP  ASTV  MSTV  ALTV ...   Min  \\\n",
       "2126 NaN NaN    NaN    NaN    NaN    NaN    NaN   NaN   NaN   NaN ...   NaN   \n",
       "2127 NaN NaN    NaN    NaN  0.000  0.000  0.000   NaN   NaN   NaN ...   NaN   \n",
       "2128 NaN NaN  0.481  0.015  0.015  0.001  0.005  87.0   7.0  91.0 ...   NaN   \n",
       "\n",
       "      Max  Nmax  Nzeros  Mode  Mean  Median  Variance  Tendency  NSP  \n",
       "2126  NaN   NaN     NaN   NaN   NaN     NaN       NaN       NaN  NaN  \n",
       "2127  NaN   NaN     NaN   NaN   NaN     NaN       NaN       NaN  NaN  \n",
       "2128  NaN   NaN     NaN   NaN   NaN     NaN       NaN       NaN  NaN  \n",
       "\n",
       "[3 rows x 22 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null_data = df_cardio[df_cardio.isnull().any(axis=1)]\n",
    "null_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cardio = df_cardio.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cardio = df_cardio.drop(['Min','Mean','Mode'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEgCAYAAACq+TSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGmRJREFUeJzt3Xm4XFWZ7/Hva8KkCFGCikwBZ9RuoBGIaBtHEL0gSiMODXhRnFttaRvFR0FtL6gt6m0UUBH14oCAiqgIKoGLKBhQJoEmCkiUQYggEGbf/mOtI5VKZZ1T4dSQ8P08Tz1Ve+9Ve7+1kqrf2XNkJpIkLc9DRl2AJGm8GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFtBwRkfVxdUSsuZw2V9U2M7vGz4iIN0TEGRGxOCLuiYgbIuLCiPhCROzS1X5ex/ImHndHxKKI+FZEzB3kZ5VaZk7eRHrQ2wR4J3DIVBpHxAzgZGAn4Gbg+8Ai4JHA44BXA08GTurx9quBY+rrhwHbA7sDL4+I3TPz2yv8KaQVFJ6ZLfUWEQn8GUjKH1WPy8wbu9pcBWwKrJaZ99ZxrwW+ClwAPCczb+l6z0OB7TLz9I5x84DTgTMyc15X+4OBDwBXZubm0/gRpSlx05PUtgT4MLAO8MEpvueZ9fmY7pAAyMwlnSExBYfX580iYv0+3idNC4NCmtzhwG+BN0bEE6fQ/qb6PJW2UxEdr90EoKEzKKRJZOY9wAHAakxtP8WJwD3AmyLiqxHx8ojY9AGU8Nb6/LvuTV/SMLiPQlqOuo/iD5m5UR0+G5gLPDszz6rjrqJrH0UdvwfwaeAxHbNcDJwJHJ2Z3+ta1jzKPorundnbAc8G/gq4M1sjYVBIy9EjKOYCZwPnAttnZi4vKGr71YDnAs8CtqrPs+rkrwD7ZP0CdgRFp3uBPwE/B/4zM8+e7s8oTYVBIS1Hd1DUcd+iHK66Z2Z+sxUUPeY3A3gFcDRlbWG3zPxOnTaP5Rz1JI2a+yik/hxA2f/wfyJi9X7emJn3ZeZxwGF11POmuzhpEAwKqQ+Z+Vvgs8BmwNtXcDa31udotpLGhEEh9e9DlDOuDwTW7p4YEa+KiBdGxDLfr4h4DPCGOnjmQKuUpomX8JD6lJmLI+KjwMeW02Q74B3AdRFxFnBlHb8Z8BJgLeC7wPGDrlWaDgaFtGI+A7wFmNNj2n8CVwAvAP4O2BFYk3Ii3nzga8DX0iNJtJLwqCdJUpP7KCRJTQaFJKnJoJAkNRkUkqSmVeKop9mzZ+ecOXNGXYYkrVTOO++8GzNz0nucrBJBMWfOHBYsWDDqMiRppRIRV0+lnZueJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlqWiVuXHTJTZfw9C8/fdRlcNHeF426BEmadq5RSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1DTUoImLNiDg3Ii6IiEsi4uAebdaIiG9GxMKIOCci5gyzRknS0oa9RnEX8LzM/HtgS2CniNi+q82+wJ8z8/HAYcChQ65RktRhqEGRxW11cLX6yK5muwJfrq+PB54fETGkEiVJXYa+jyIiZkTEr4EbgNMy85yuJhsC1wBk5r3ALcB6w61SkjRh6EGRmfdl5pbARsC2EfG0ria91h661zqIiP0iYkFELLjv1vsGUaokiREe9ZSZNwPzgZ26Ji0CNgaIiJnAusDiHu8/KjO3ycxtZjx8xoCrlaQHr2Ef9bR+RMyqr9cCXgBc1tXsJGDv+np34KeZucwahSRpOGYOeXkbAF+OiBmUkDouM0+OiA8BCzLzJOCLwFcjYiFlTWLPIdcoSeow1KDIzAuBrXqM/0DH6zuBfxpmXZKk5fPMbElSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUN+zLjA/HU9Z7Kgr0XjLoMSVoluUYhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpaeaoC5gWf/wVHLTuqKsYrINuGXUFkh6kXKOQJDVNOSgi4hURsW/H8GYRcXZE3BwRJ0TErMGUKEkapX7WKN4PrNMx/H+B2cAhwNbAf0xjXZKkMdHPPorNgYsAImJd4EXAbpn5/Yj4PSUw3jr9JUqSRqnffRRZn58D3Af8uA4vAtafrqIkSeOjn6C4AHhNRDwMeD1wembeVadtAtww3cVJkkavn01P7wO+B+wN3EbZ9DThZcA501iXJGlMTDkoMvOsiNgEeCLw28y8uWPy0cDC6S5OkjR6fZ1wl5m3Aud1jouIWZn5g2mtSpI0Nvo5j+LNEfGejuEtI2IRcFNEnBcRGw2kQknSSPWzM/vtwF86hj8D/BF4TZ3PIdNYlyRpTPSz6WkT4HKAiFgf2AF4fmbOj4i7gf8aQH2SpBHrZ43iLmD1+vq5wBLg/9fhxYCX8JCkVVA/axTnAm+t+yX+BTglM++r0zanbIaSJK1i+lmjeDewBeUyHhsDB3ZMeyXws8lmEBEbR8TpEXFpRFwSEe/o0SYi4jMRsTAiLoyIrfuoUZI0zfo5j+I3wOMjYj1gcWZmx+T9geumMJt7gXdn5vkR8XDgvIg4rc57wouBJ9THdsDn6rMkaQT6vh9FZt7UFRJk5kWZ+acpvPfazDy/vr4VuBTYsKvZrsBXsvgFMCsiNui3TknS9OjrhLuImAO8lnJ29prd0zNzjz7ntRXLXvpjQ+CajuFFddy1Xe/fD9gPYJN1Y6qLlST1acpBERH/AJxB+RF/InAhsC4wh/JjPuVLeETE2sAJwDsz8y/dk3u8JZcZkXkUcBTANo+dscx0SdL06GfT08cpP+5Po/yY75uZmwPPovyQf2wqM4mI1ep8js3ME3s0WUTZWT5hIzyiSpJGpp+g2BL4GvDXOrwmQGaeDRzMFM7MjogAvghcmpmfXE6zk4C96tFP2wO3ZOa1y2krSRqwfvZRJHB3ZmZE3ABsCpxdp11DOUppMjsA/wxcFBG/ruPeRznrm8w8AvgBsDNlU9YS4HV91ChJmmb9BMVvgMcBpwM/B94VEQuAu4H3AL+dbAaZeRa990F0tkm8paokjY1+guIoyloElLWAU4HL6vDtwO7TWJckaUz0c8LdVzteXxoRTwHmAmsBv8hMb4UqSaugvs6j6JSZtwGnTWMtkqQx1AyKiNi5n5l5pztJWvVMtkZxch/zSmDGA6hFkjSGJguKzYZShSRpbE12wt1twCeBJ2fm1b0ewJNrmzsGXawkafgmC4p3Um5KdGqjzamUNY93T1dRkqTxMVlQ7AEc0X1Z8U512pGUy4NLklYxkwXFppQzsidzKeUqspKkVcxkQXEHsM4U5rM27qOQpFXSZEFxPrDLFOaza20rSVrFTHZ47OHAcRFxdmZ+uVeDiNiLcoXXV053cVP22K3goAUjW7wkrcqaQZGZJ0bEp4EvRcTbgFOA31NOrtsE2BHYBjgsM7896GIlScM36bWeMvPdETGfcqjs/sAaddJdwM+AXTOznzO4JUkrkSldFDAzvwd8LyJmAuvV0Tdl5r0Dq0ySNBb6unpsDYbrB1SLJGkM9XPPbEnSg5BBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktQ0c9QFTIeL/nALcw74/qjLkKShuuqQlwxlOa5RSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1DTUoIuLoiLghIi5ezvSIiM9ExMKIuDAith5mfZKkZQ17jeIYYKfG9BcDT6iP/YDPDaEmSVLDUIMiM88EFjea7Ap8JYtfALMiYoPhVCdJ6mXc9lFsCFzTMbyojpMkjci4BUX0GJc9G0bsFxELImLBfUtuGXBZkvTgNW5BsQjYuGN4I+CPvRpm5lGZuU1mbjPjoesOpThJejAat6A4CdirHv20PXBLZl476qIk6cFs5jAXFhFfB+YBsyNiEfBBYDWAzDwC+AGwM7AQWAK8bpj1SZKWNdSgyMxXTTI9gbcOqRxJ0hSM26YnSdKYMSgkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1GRSSpKahXj12UJ6+4bosOOQloy5DklZJrlFIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmiIzR13DAxYRtwKXj7qO5ZgN3DjqInoY17pgfGsb17pgfGsb17pgfGsbZl2bZub6kzVaJW6FClyemduMuoheImLBONY2rnXB+NY2rnXB+NY2rnXB+NY2jnW56UmS1GRQSJKaVpWgOGrUBTSMa23jWheMb23jWheMb23jWheMb21jV9cqsTNbkjQ4q8oahSRpQAwKSVLTSh8UEbFTRFweEQsj4oAhL3vjiDg9Ii6NiEsi4h11/CMj4rSIuKI+P6KOj4j4TK31wojYesD1zYiIX0XEyXV4s4g4p9b1zYhYvY5fow4vrNPnDLiuWRFxfERcVvtu7jj0WUS8q/47XhwRX4+INUfVZxFxdETcEBEXd4zru48iYu/a/oqI2HuAtX28/nteGBHfjohZHdPeW2u7PCJ27Bg/rd/dXnV1TNs/IjIiZtfhkfdZHf/22geXRMTHOsYPpc+mLDNX2gcwA/gtsDmwOnABsMUQl78BsHV9/XDgv4EtgI8BB9TxBwCH1tc7Az8EAtgeOGfA9f0r8DXg5Dp8HLBnfX0E8Ob6+i3AEfX1nsA3B1zXl4HX19erA7NG3WfAhsCVwFodfbXPqPoM+Edga+DijnF99RHwSOB39fkR9fUjBlTbi4CZ9fWhHbVtUb+XawCb1e/rjEF8d3vVVcdvDPwIuBqYPUZ99lzgx8AadfhRw+6zKdc/jIUMrHiYC/yoY/i9wHtHWM93gRdSzhLfoI7bgHJCIMCRwKs62v+t3QBq2Qj4CfA84OT6hbix48v8t76rX6K59fXM2i4GVNc6lB/k6Bo/0j6jBMU19QdiZu2zHUfZZ8Ccrh+WvvoIeBVwZMf4pdpNZ21d03YDjq2vl/pOTvTboL67veoCjgf+HriK+4Ni5H1G+SPkBT3aDbXPpvJY2Tc9TXy5Jyyq44aubnrYCjgHeHRmXgtQnx9Vmw2z3k8B7wH+WofXA27OzHt7LPtvddXpt9T2g7A58CfgS3Wz2Bci4mGMuM8y8w/AJ4DfA9dS+uA8xqPPJvTbR6P6fvxvyl/rI68tInYB/pCZF3RNGoc+eyLw7Lrp8oyIeMYY1baUlT0oose4oR/vGxFrAycA78zMv7Sa9hg37fVGxEuBGzLzvCkue5j9OJOyCv65zNwKuJ2yGWV5htVnjwB2pazqPxZ4GPDixrLH4v9etbxahl5jRBwI3AscOzFqOTUMvLaIeChwIPCBXpNHVVeHmZTNW9sD/wYcFxExJrUtZWUPikWU7Y8TNgL+OMwCImI1Skgcm5kn1tHXR8QGdfoGwA11/LDq3QHYJSKuAr5B2fz0KWBWRExc36tz2X+rq05fF1g8gLomlrUoM8+pw8dTgmPUffYC4MrM/FNm3gOcCDyT8eizCf320VC/H3XH70uB12TdNjLi2h5HCf4L6ndhI+D8iHjMiOuasAg4MYtzKWv/s8ektqWs7EHxS+AJ9ciU1Sk7FU8a1sJr+n8RuDQzP9kx6SRg4miJvSn7LibG71WPuNgeuGViU8J0ysz3ZuZGmTmH0ic/zczXAKcDuy+nrol6d6/tB/KXSmZeB1wTEU+qo54P/IYR9xllk9P2EfHQ+u86UdfI+6xDv330I+BFEfGIusb0ojpu2kXETsC/A7tk5pKumveMcpTYZsATgHMZwnc3My/KzEdl5pz6XVhEOfjkOsagz4DvUP6IIyKeSNlBfSMj7LPlGsaOkEE+KEcv/DflaIADh7zsZ1FW/S4Efl0fO1O2Vf8EuKI+P7K2D+DwWutFwDZDqHEe9x/1tDnlP9xC4Fvcf7TFmnV4YZ2++YBr2hJYUPvtO5TV75H3GXAwcBlwMfBVylEnI+kz4OuUfSX3UH7g9l2RPqLsL1hYH68bYG0LKdvPJ74HR3S0P7DWdjnw4o7x0/rd7VVX1/SruH9n9jj02erA/6v/384HnjfsPpvqw0t4SJKaVvZNT5KkATMoJElNBoUkqcmgkCQ1GRSSpCaDQtMuIvaplyW4PSL+Ui9PsEuPdvMj4vhR1DjdImKPiNinx/ihf8Z6/P3+9RIpt0fEkoj4ZUTsV4+/H7qIWD0iDoqILUexfD0wMydvIk1dRHwOeAPwWeD9lP9jewLfjYgDMvPQUdY3QHtQzqo9pmv8WyjHzg9FRKwFnAo8nXI2/ll10lzgI8BawKeHVU+H1YEPUs5l+PUIlq8HwKDQtImIlwFvolyK+4iOST+MiOuAj0bEaZl5/mgqvF9ErJWZdwx6OZn5m0Evo8tHKJdE2S4zO+998OOIOBx48pDr0SrATU+aTu+gnM36+R7TPgrcCryte0LdJHJVRNwREd+PiA27pk/cxOXOiLg+Ik6p1+uZmP7IiDiyTrszIs6OiO265pER8a8R8amI+BNwUUQcHBHXRcRDutq+tLZ/fB3eKyLOiojFEfHnKDer2qaj/THAK4Dn1PdlRBxUpy2z6Skinlc3zU18ns9GubDkxPR5dR7zIuJbEXFbRPwuIt7S6vwoF8F7I+Ws6GVu3pOZizPz7I72W0bET+qmqT9HxLER8egedTytazlLfaaIOCYiFkTEC6PcBOj22l9P7XjbrfX5Sx19NKf1eTQ+DApNiygXxpsLfC8z7+uenpm3UK6b9I9dk+YCb6fcZGlf4O8ol/WYmO9ewPuAT1LuD/FmShg9rE5fg3LzlxdSrsD5MsplzH/cGSbVv1HuOfDPwL9QLpj4aOA5Xe32AM7LzIV1eA7wFeCfgFdTLsFwZkRsXqd/uH62X3H/fQO+sJx+2gI4hXJNn1dQNse8mnJxxG6fp9ycZjdgPnB4RGzba77VP1D65ZRGm4k61q/zfGhd/tsp/XDaCu7H2AT4OPAflHs6PIr7r4YK9ZpGlDWeiT4axDW7NAjDulaIj1X7ATyGct2rdzTafAq4o2N4PmX7/aYd43ao89mpDv8XcEJjnvsCdwNP6Bg3k3ItnI93jEvgVz3efwFLX5doDcq9JfZfzvIeUud/GfCBjvHHA/N7tJ8PHN8x/A3KtZpmdIzbo9Y3cSOkeXX4Qx1tVqME4CGNvtizvu9JU/j3OgS4GVinY9y29f2v6qrjaZN8pmMolxbv/Dd4WX3vk+vw2nV4n1H/X/XR/8M1Co3a+Zl59cRAZv6Mcvnsib+cfw3sXDcTbRsRM7re/wLKDYaujIiZcf8lwc8Atulq+/0ey/8m8IqO972Yclvb4yYaRMRTotwH+nrgPkq4PYly45l+bQt8O5de6zqB8kP7rK62p068yHLp8ysol5aezFQu4LYtcGp23D8ly6Wur+pRx1RclZlXdAxP7JuZSr0acwaFpsuNwF3Apo02mwJ/6Bp3Q492N1A2EQEcTdn0tAfl7oHXR8SHOwJjNuXGL/d0PV7H0tfuB7i+x7K+UecxsWnklcDPM/P3ABHxcMoP9saUzWPPBp5BWRNZs/FZl2eD7jpqaNxEuQ1rp5u7hu+eZJkTfbvJitRRXd+jjqnoVSusWB9pzBgUmhZZbgf6c+Al3TuHASJiHcqmjDO7Jj2qu20dN3HLz79m5mGZ+RTKD+AnKMHxhtp2MeWS5c/o8ditu8wedf+uvv+VdWfw/6KsZUyYS/mr+LWZeWxmnpWZCyg3KloRnbcwBaCG3no88BsfLaDcMXDHFamjenRHHXfW5+59FisSJFqJGRSaTp+mbI55fY9pBwDrUPY5dNo6Iv72F3BE7ED5ATu3ewaZeU1mHkLZmb1FHf0T4PHA7zNzQdfjoinW/Q1KqOxGOc/gWx3T1qrPd3XU+EzKDu5Ok/21P+EcYLeuTWgvp+z3OKv3W6Ymy+G+RwJvrjvNlxIRsyJibkcdO9Y1ponpz6B8rok6FtXnp3S02Ziy2a1frmGsxDyPQtMmM78TEUdQjs7ZAjiZ8n/slcA+wHtz2XMobgBOroeTrgkcStlvcQpARBxJ+Qv3F5SdzM+l3PHr3+v7v0I5d2N+RHwC+B3lr/Ntgesy87AplH4c5YidjwNn5tJ30PsFcBvw+Yj4GGXt4iCW3YR2GbBrPZdkEfDHzOx1m8qPUI6O+k49OXGj+pl/lJk/n0Ktk3k/5bP/LCIOA35Wx29HObLpEMqa3ycpR5D9KCIOpexsPoRyE58TADJzUUT8EvhwRCyh/GH5PlZgzScz746IK4E9IuJiytrKhZl59yRv1TgY9d50H6vWg3LnsH0of7HeTjl+/gzKLTK7286nHC30JsqtSO8Afghs3NFmH8qP3WJgCeWueN13LluXsjZzDeUv10WUe17v0NEmgbc16j6rtnljj2k7Ue5Cdkdd/s4se+TPbODbtc4EDur8jF3ze37tnzspQflZYO2O6fOYwtFGjc+yBrA/5UCAJfXxS+BdwJod7bYCflqn3wx8DXh017weX5d7O+Vua7v2+OzHAAu63jenfoaXdox7Ue2/O+u0OaP+/+pjag/vcCdJanIfhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElN/wOfXXlNgQKQcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_cardio['NSP'].value_counts()[:].plot(kind='barh')\n",
    "plt.title('NSP',fontsize=20)\n",
    "plt.xlabel('Observation Count',fontsize=15)\n",
    "plt.ylabel('Class',fontsize=15)\n",
    "plt.savefig('target__distribution.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train-Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop('NSP',axis=1)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train-Validation Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline GBRF Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boost classifier on test set: 0.999\n",
      "\n",
      "Cross validation score: 0.9115304303539598\n",
      "\n",
      "Recall score: 0.9647887323943662\n",
      "\n",
      "--- 14.724570035934448 seconds ---\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test)\n",
    "print('Accuracy of gradient boost classifier on test set: {:.3f}\\n'.format(clf.score(X_train_sub, y_train_sub)))\n",
    "\n",
    "print('Cross validation score: {}\\n'.format(cross_val_score(clf, X_test_sub, y_test_sub, cv=10).mean()))\n",
    "\n",
    "print('Recall score: {}\\n'.format(recall_score(y_test, y_pred_clf, average = 'micro')))\n",
    "\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 2. 3.]\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(y_pred_clf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average AUC score: 0.9845810000634594\n"
     ]
    }
   ],
   "source": [
    "# Binarize the output\n",
    "y = label_binarize(y, classes=[1, 2, 3])\n",
    "n_classes = 3\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=0)\n",
    "\n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "\n",
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "classifier = OneVsRestClassifier(ensemble.GradientBoostingClassifier(**params))\n",
    "y_score = classifier.fit(X_train_sub, y_train_sub).decision_function(X_test_sub)\n",
    "\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test_sub[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "print('Average AUC score: {}'.format((roc_auc[0]+roc_auc[1]+roc_auc[2])/3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f_classif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop('NSP',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_selectKbest_scores(feature_amount):\n",
    "    score = []\n",
    "    y = df_cardio['NSP']\n",
    "    X = df_cardio.drop('NSP',axis=1)\n",
    "    \n",
    "    for i in range(1,feature_amount+1):\n",
    "        \n",
    "        selector = SelectKBest(f_classif, k = i)\n",
    "\n",
    "        selector.fit(X, y)\n",
    "\n",
    "        X_new = selector.transform(X)\n",
    "\n",
    "        y_new = df_cardio['NSP']\n",
    "\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X_new, y_new, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "        X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, \n",
    "                                                                            random_state = 465) \n",
    "                   \n",
    "        params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "        # Initialize and fit the model.\n",
    "        clf = ensemble.GradientBoostingClassifier(**params)\n",
    "        clf.fit(X_train_sub, y_train_sub)\n",
    "        \n",
    "        score.append(cross_val_score(clf, X_test, y_test, cv=20).mean())\n",
    "        \n",
    "    #return score\n",
    "    plt.figure(figsize=(12, 6)) #visualize mean prediction errors in relation to k number\n",
    "    plt.plot(range(1,feature_amount+1), score, color='blue', linestyle='dashed', marker='o',  \n",
    "    markerfacecolor='blue', markersize=10)    \n",
    "    plt.title('Score vs Best Feature Count')  \n",
    "    plt.xlabel('Feature Count')  \n",
    "    plt.ylabel('Score')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAGDCAYAAAAVh7eRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XeYXGXd//H3N8lCDCmURFroTUBQMRQrCKIElRqlSBMlah5QEBQQlOLDIxbEQpMmAkJElKICASkCiv4IVSEbINQQSjAQEgKk3b8/7lmz2WxNZuZMeb+ua6/ZOefMzHfPzs589p67REoJSZIkSeXVr+gCJEmSpEZk0JYkSZIqwKAtSZIkVYBBW5IkSaoAg7YkSZJUAQZtSZIkqQIM2pIkSVIFGLQl1ZWI+HBE/D0iZkbEjIj4W0RsXXRd5RARKSLeiIjZEfFKRFwZESuW6X437Gb/IRGxoPS4bV9nleFx74iILy3r/fTxMZeLiJMj4vHSuXw6Ii6OiHUr/Lg7RMTUSj6GpPpj0JZUNyJiKPAn4BfAysCawCnA22V+nP7lvL8+ek9KaTCwPrAScHKVHveelNLgdl+HV+lxuxQRA5biZlcDuwH7A8OA9wD3ATuVsTRJ6hWDtqR6sjFASunKlNKClNKbKaWbU0oPtx0QEYdFxKSImBURj0bEVqXtm5ZaWF+LiEciYrd2t7kkIs6NiBsi4g3gYxGxfET8OCKejYiXIuK8iHhHx4JKx70WEe9ut21ERLwZEe+MiOER8afSMTMi4q6I6PG1N6X0OnA9sFm7+x0WERdFxAsR8XxE/G/bPwURsWFE/LXU0v9KRPy2tP3O0s0fKrVU79OXE97deYiIlUo/2/SIeLX0/cjSvtOAjwBntbWQR8S6pdb1Ae3u/7+t3qWW9b9FxJkRMYPSPxkRcWjpd/pqREyIiHW6qPXjwM7A7imle1NK81NKM1NKZ6eULiods0ZEXF/6XTwREYe1u/0lEfG/7a4v1kpdah0/JiIeLp3n30bEwIhYAbgRWKPdJwJr9OU8S2pMBm1J9eQxYEFE/DoiRkfESu13RsRnyeHsIGAouWXzPxHRAvwRuBl4J3AE8JuI2KTdzfcHTgOGAHcDPyAH+/cCG5Jbz7/bsaCU0tvAH4D92m3+HPDXlNLLwNHAVGAEsCrwbSD19IOWfrY9gH+02/xrYH6pnvcBnwDaumZ8r/TzrQSMJLf6k1L6aGn/e0ot1b/t6bE76O489AN+BawDrA28CZxVetwTgLuAw/vYQr4t8CT593RaROxBPmd7kc/hXcCVXdz248D/Syk91839X0n+fawBjAH+LyL60tr9OWAXYD1gS+CQlNIbwGhgWrtPBKb14T4lNSiDtqS6UWrl/TA5qF4ATC+1Tq5aOuRLwA9LrZkppfRESukZYDtgMHB6SmluSuk2cheU9uH4upTS31JKC8ldUQ4DjkopzUgpzQL+D9i3i9Ku6HBf+5e2AcwDVgfWSSnNSyndlVLqLmjfHxGvAa+Qw+svAUo/42jgyJTSG6UQf2a7muaRA+8aKaW3Ukp3d/MYndmu1Ore9rVdRATdnIeU0n9SSr9PKc0p7TsN2L6Pj9vRtJTSL0qt0W8CXwa+n1KalFKaX3r893bRqr0K8EJXdxwRa5GfP8eWztGDwIXAgX2o7+cppWkppRnkf97e24fbSmoyBm1JdaUUuA5JKY0E3k1umfxpafdawJRObrYG8FwpRLd5htw626Z9K+gIYBBwX1vwBG4qbe/MbcA7ImLbUgB8L3BNad+PgCeAmyPiyYg4rocfcauU0orAQOBc4K6IGEgO0S3AC+1q+iW55RfgW0AA/6/UNebQHh6no3+klFZs9/WPns5DRAyKiF9GxDMR8TpwJ7BiLFsf946t0esAP2v3+DNKP+eaS9wS/kP+p6YrawBt/zC06fg86MmL7b6fQ/4HTpI6ZdCWVLdSSq3AJeTADTmkbdDJodOAtTr0jV4beL793bX7/hVyN4jN2wXPYaVBip3VsRC4ityqvT/wp7Ywl1KalVI6OqW0PvAZ4Bu96aqQUppHbm1dr/TzPUduaR/erqahKaXNS8e/mFI6LKW0BrkV+JzoZqaRXurpPBwNbAJsm1IaCrR1U4m2H6PD/b1RuhzUbttqHY7peJvngC93+CfgHSmlv3dS71+Abdr6iXdiGrByRAxpt6398+CNHmrrTo/dgSQ1H4O2pLoREe+KiKPbDbhbixxu2/oxXwgcExHvj2zDUgvzP8kh6lsR0RIRO5BD7/jOHqcUnC8AzoyId5Yea82I+GQ35V0B7AN8nkXdRoiIT5fqCOB1YEHpq6eftT/wBXLQfTKl9AK5D/YZETE0IvpFxAYRsX3p+M+2C5ivkoNf2+O8RJ7FpE96cR6GlOp7LSJWBk7qcBeLPW5KaTo51B4QEf1Lre6d/WPU3nnA8RGxeenxh5X64ndW71+AW4BrSs+BARExJCK+EhGHlvpu/x34fmkQ45bAF4HflO7iQWDXiFg5IlYDjuzpHHX4WVeJiGF9uI2kBmfQllRPZpEHy/0z8uwg/wD+TW5ZJaX0O3I/4StKx14LrJxSmkseGDma3Ep7DnBQqUW8K8eSu3z8o9Qt4i/k1ttOpZTawvwa5Bko2mxUuu1s4B7gnJTSHd087kMRMZsclg8G9iz1B4Y8yHM54NHS/qtZ1FVi69J5mU2ereTrKaWnSvtOBn5d6n7xuW4euzPdnYefAu8gn9N/kLuVtPczYExptpCfl7YdBnyT3M1jc3Lw7VJK6RrygMzxpcf/N/n32JUxwA3Ab4GZpeNHleqG/I/ZuuTW7WuAk1JKt5T2XQY8BDxN/qem1wNHS8+lK4EnS+fZWUckEd2PyZEkSZK0NGzRliRJkirAoC1JkiRVgEFbkiRJqgCDtiRJklQBBm1JkiSpAgYUXUC5DB8+PK277rpFlyFJkqQGd999972SUupqteD/apigve666zJx4sSiy5AkSVKDi4hnenOcXUckSZKkCjBoS5IkSRVg0JYkSZIqwKAtSZIkVYBBW5IkSaoAg7YkSZJUAQZtSZIkqQIM2pLUjSlTYNw4GDoU+vXLl+PG5e2SJHXHoC1JXbjxRthyS7jwQpg1C1LKlxdemLffeGPRFUqSaplBW5I6MWUKjBkDc+bAvHmL75s3L28fM8aWbUlS1wzaktSJM85YMmB3NG8enHlmdeqRJNUfg7YkdeLyy3sXtC+7rDr1SJLqj0Fbkjoxe3Z5j5MkNR+DtiR1MGcOLLdc744dPLiytUiS6ldFg3ZE7BIRkyPiiYg4rpP960TErRHxcETcEREjO+wfGhHPR8RZlaxTkto79VR4++2ej2tpgQMPrHw9UqNx2kw1i4oF7YjoD5wNjAY2A/aLiM06HPZj4NKU0pbAqcD3O+z/HvDXStUoSQCvvAInngh33pmvf+1rMH48DBrU/e1aWuCooypfn9RInDZTzaSSLdrbAE+klJ5MKc0FxgO7dzhmM+DW0ve3t98fEe8HVgVurmCNkprY88/noLzOOvB//wd33ZW3r7EG7LMPXH11DtstLZ3f/vTTYYMNqlevVO+cNlPNppJBe03guXbXp5a2tfcQsHfp+z2BIRGxSkT0A84AvtndA0TE2IiYGBETp0+fXqayJTWDE0+E9deHX/wiv7E/8giccMLix4weDQ8/DGPHLv4R98EHw+qrw2mnwVNPFVO/VI+cNlPNppJBOzrZljpcPwbYPiIeALYHngfmA+OAG1JKz9GNlNL5KaVRKaVRI0aMKEfNkhrYpEkwf37+fvhwOPRQePxx+PWvYdNNO7/NBhvAWWfBzJmwYEG+vOQSuPVW2G47WHHFqpUv1T2nzVSzGVDB+54KrNXu+khgWvsDUkrTgL0AImIwsHdKaWZEfAD4SESMAwYDy0XE7JTSEgMqJakn992XW5+vuSa/0X/+83Dkkct2n5tuCtdem79/6y1YuLDnPt1Ss+vtdJivvw733APbbps/SZLqVSWfvvcCG0XEehGxHLAvcH37AyJieKmbCMDxwMUAKaXPp5TWTimtS271vtSQLamv7roLdtkFRo2C22+H73wHPvnJ8j7GwoWw666w776LWsslda4v02F+8IN5/MRRR8HcuZWrqRY5K0vjqFjQTinNBw4HJgCTgKtSSo9ExKkRsVvpsB2AyRHxGHng42mVqkdSc1m4EL78Zbj/fvj+9+GZZ/K0fcOHl/dx+vXLfbz/+Mf8Rpg6dpCT9F8HHAADevgsvaUFvvQluPRSeN/78mxAbfPaX3YZ/P3v+e+7UTkrS2OJ1CDvCqNGjUoTJ04sugxJBVm4MHflOOec3EVkyBBobYW1165Ol44TTsgzl5x6am45l7TI66/nsQ277grveU+eXaQrgwblQchtM/osWAD9++dPjFZfPU/HueaasPfe8NnP5pbvRuleMmVKDtN9OT8qRkTcl1Ia1dNxDfLUlNSs5s/PrVzvfnd+433mmUUzgbzrXdXrN/2//wsHHQTf/W4eXCkpe+yxPHD4G9/ILbNdTZvZ0pK3X3314iGyf/98OWBADqKXX567g/3yl/CRj8App+T9CxfWf0u3s7I0HoO2pLo1YwZsvHEOuP37wxVX5JlFttyy+rVE5I92Dz0Uttmm+o8v1aI//zn/PUyfDrfckruCdDVt5tixefvo0V3f39CheTDztdfCyy/Db34D++2X9912G4wcCUcckcdn1EvoTgleeimPI/nVr5yVpdEYtNXwam1QSa3VU2t6Oj9vvJHfsAFWXhl22w2uuw4eeii/4fbU/7OSWlrgoovyjCQp5QVxpGb105/CZz6T56ufOBE+9rFF+zqbNvOss/rWHWLoUNh///zJVdv1D3wg/8P70Y8uCt0zZ/Z8X9V4XU4Jnn0WJkzILdKXXrpo+/rrw4475hmMemPWLHjhhfLVpsqxj7Ya2o035oFq8+Yt3krQ0pK/rr66+9aTRq+n1vR0fvbeG264Iff3nDoV3vnO4mrtyamn5sVw7rkHNtyw6Gqk6pswIXfz+OUvqzv15ezZ8Kc/we9+lwdDP/FE/sTrD3/I/5x/5COLuqNA+V+XFyzI3ddeeSV3mYHcl/ymmxaf3nCXXRYNbLziChgxAvbaq/dTIP7zn/nTgtZWePVVp0Kstt720TZoq2HV2qCSWqun1vTm/ADssEMedPiBD1SlrKX2+ON5kNbQoXmWhFVXLboiqfKmTMndNg45pOhKsraBlJBbvidPzn+Le+2Vw++aa+buLEvzujx//qJP0C6/PHeTefTR/Bhvv50HYj/zTN5/0knw2mv5067NNsuXna2zN25cbpHvrvtIS0v+9O5Xv8rBetw4OPfc/HN95jOwxx6w004wcGDvz5P6zqCtptfbF6yxY/NAtlmzlty/VmnJpRkzcpeF9iLyR5MA//nPki/U/frlF3HI/ROPPhquvLL7uZbb6jnrrO5/tkbUm9/XgAF5yr56OT///Gf+uHyzzeCOO/o2h7BUb26+Oc8n379/bkUeNqzoihY3e3b+ROx3v8uh+M0389/m44/3/D6xzz65BXrSpBymH300f6r22mv5denww/N9twXptq9tt+1bjUvTIPPaa7ll/Lrrcg2zZuV9jz+e36fmzl00PaLKx6Ctpjd0aOfhubPjvvQl+MlPltzX9ucxdixccMHi+wYPXnT/++0H48cvvn/11WFaaS3UT386v7D3tu7e9ClsNH35fdXT+fnTn2D33fNHz3/8Y37jkxpJSnm2jGOPhc03zwMV11+/6Kq698YbOZwecsiSjSidWX753Erdvz9stNGiQH3ccfm9IKXy/W0vS1eWt9/O/9S/8koeNJpSDt3rrJNfh3bfHdZbrzx1NjuDtppev369WzykX7/80f6//73kvi9+MV/efXf+OLC9lpY82wXk0eJPPrn4/kGDFo2Gv/nm3q9I2K9f/riz2fTl91Vv5+fii3N/8k9/uuhKpPJKCQ4+OM+CMWZM7s5QT5/c9PZ1JyK/R2y4YXVah6dMyQMmL7sst8QPHgwHHphXyexL18K33sqf2F57LTzySN625ZZ5DMnuu1em9mZh0FbTq7UW0t7W09IC996bF3VoFpdemt+se6PeWrQ7mjwZNtmk6Cqk8jnzzBzojjuu/j6xqbX3iUqaMiV3L7nuOvjmN/M//o88kvt377EHbL/9knObt7/tGWfkvuhtwf+AA3KXyCLGFNVCPS5Yo6Z3wAFdv2i0aWnJrQS1Uk9b68p73wsPPFCduoqwcGF+sb/ttnz905/Oo/Nr6fdVCXfemT9uPvfcoiuRls3tt+dZNCC3sh5/fP2FbKi994lK2mCDvGjQX/+66NO1f/0rf+K28855cOb++8NVV+UuKG1qbUn4WqunJ7Zoq2FNmZL7C7Z/weioFmcdueuuPOfsYYflN65zzoF118198urxjay9efNyX/Yf/CC3pOyxR14uHZpjVpb582HPPfOApd//Pv/8Uj1JCX7+89xyuO22uVtdPb8uNcPrTk/mzIG//CV3L/njH3ML8SuvwAor5AaR/fbLA0e7Uovvo9WoxxZtNb3llsujwfv16/1Sv5W0wQa9W3p4q63y4MuI3Bf5rLPgU5/Krdw9zVpSy8aPz4OIDjoo/05+85s8+r9Nb89PPb/ZDRiQz8PWW+c3r7//veiKpN578808ePDII/M0cjfdVN8hG5rjdacngwblhb8uvhhefDE39KywQt532GHdh2yo7pLw9bhEvUFbDWnhwkUDFW++eemW+q2Evi493L9/XvHwkkvyFE3775/797Z1uah1r7226EX69dfzdId//GP+mfbff8lVHJdlaeZ6scIK+RystVYOK67upnowa1ZebfHSS+GUU/InMkOGFF1VeTTD605v9e+fPwlu09O6BpCD7dln59sOGADHHLPotoMH53O54op5saDhw+H00/P+6dPze8Jaa+VZUdZbL/9Dc/75ef8zz+Rattgij1l63/vy4kf1tkS9XUfUkH70I/jWt/Jy2IceWnQ15dHWr/n003PLw+ab5+kDhwypvTe8F1/Myy+fe24e8X7EEbn+iPpvASuXJ5/Mv88jj6zfc1ILA5JquZ5GkhJ87Wu5L+9uuxVdjaqlL7OynHBCfp3/4Afzp7Bvvw3f/na+/cKFi7522SX3EZ85M/9tdty/77759i+8kJ9zCxcuOua663pfd6Vnp3LWETW1H/wgL707fnz9hpje2Gef3GJ/+OH5Bamzlcaq6amn8j85F1+cWxU++9n8QrvllsXWVetaW2GNNXLLT70o97LVjVZPI0gptyBuv32eN1rNp9ZmZamleuyjraZ27LGNH7Ihf0S3445w2mn5o7evfQ2efba4eg4+OH+KcNBBeRq78eMN2T2ZPTsHmb32yt2D6sGUKTnUzpmz5Me48+bl7WPG5OOasZ5G8PbbuX/uV7/qLDnNrNZmZam1enrDoK2GcvLJi6b2afSQDXlQ3e9/n2fw2Gef/IZYzeXJ//532HtveOmlfP3ss3Or9vnn54Ud1LPBg+HHP4Zbb4UvfCF/PFrram1AUq3VU++mTYMddsj/NJ94Yu4GpuZ09NG9C7ZHHdWc9fSGXUfUMG68EXbdNfd5bdY31GefzSPIhw/PMwKcf36e33brrcv3GCnBhAnw/e/neaFXWSXPu7rjjuV7jGZ0+un5d/XNb8IPf1h0Nd3ry8e3v/pVng6uo/HjYbXVcn/qCy9ccv9118GwYfk5fMUVS+6/5Zb8hvqzn+U33970x2yERUcqbfLkHLJnzYJf/zr/I63mVmvdsmqlHruOqKm8/HJuDdxiixwAm9Xaa+eQDXlA4u23wzbbwMc/nudJXdb/q99+O4f20aPzYL6f/jSPDDdkL7tjj4X/+Z/cx/2qq4qupmtz5+buLr3R2+OWVW8HPVWrnnq27rrwsY/BP/5hyFZWa7Oy1Fo9PbFFW3Uvpbzwx0035fk/t9ii6Ipqx+uv58FMP/lJDt7tF4hp09NMDW+/nVcS+8Qn8vHHHJNnPPn85/Nc5SqfBQvgF7+Ar3wFBg4suppF3nwzD7r9/e/h+ut73yrcjAOkallXf+tHHAG//S18/euw0kpFVynVB2cdUdNo6zLyk5/UVr+sWvLWW3n+23e8Iw8SmTs3LxYzdGieSqmrj+D22y+f3xdegMces991Nc2YkYNRObv99EVKeZzD/Pl5rtuXX84hbPfdc5/8v/yl+37RLS25hakaYwbGjcvdT7qrp3///A9MNccw1JKuPm4fMGDRtGqXXJIHNEvqmUFbTSOlvADIpz+dP0JSz373O/jc53KQ6uklYLvt4NRTc/eTZhhgWiv23jsPkLz7bnj3u6vzmK+9Bn/6U+7j+Mor+bEBzjsP1l8/dyloaamtZZChd/VA7prz85833+tEb87P8svnQdXONy71jn201fDmzYOnn87hb7fdmu/Nc1mMGZM/BegpZA8YAO9/f16kwpBdXWeemVeR3GUXeO65yj7WzTfn58M735k/8Zg4Mf/e58/P+7/yldx1qG20f60tW92benbeOc+Kc9FF1amplvRmVpaFC5t3ELlUSUYT1a1TTsn9sSsdQhpRBNx1V8/HzZ9fW0vZNpO1184f98+alQf3vPpq+e77hRfgnHNyv33If0OtrbmP7j335Nlrfvaz/I9WV2ptQFJP9UyYkMP4F76Qj6/0qnG15PLL62/ZaqlR2HVEdenuu/MiHwcfnFchVN/1dmndaixlq67ddltu1f7c53JgWlrPPAN/+EMe0Pj3v+fffVuf3Pnzcx/mZvnU4qWXYKed8gxFn/lM0dVUnn/rUvnZdUQNa+bMPFJ+vfVyq5uWzuDB5T1OlbHjjnmmmB/9KPe1HTdu8RbbceO6XvHwzTfz5fTp+e/lG9/ILeQnnwz//ndewRNyy3WzhGzIrbcDB+YuZ9/7Xn0sErQsVlihd8f5ty6Vn0FbdeeII2Dq1Ny6N2RI0dXUr3pcyrZZfepT8OCDeUDb+efnsJxSvrzwwry9bUXURx/Ng1ff857cFx9gxIi8cMzjj8NDD8F3v5unaGymcN3eyJG569QBB+RzMWZM76YHrDcp5dmGeuo2Av6tS5Vi1xHVlQUL8swBq62WW+W09Gpt5gh1rTe/q+WWywHyySdzgP7gB/P0jP/zP9Wrs96klD8VO+YYOOwwOPfcoisqr7feyuNYhgyBSZPy9a74ty71TW+7jnQz1EWqPf3756nGGuT/w0K1zdTQ01K2vvEWrzezRsyfnxcXOvts2HNPWH316tRWzyLgyCPhfe9btNDV/PndDwKtdXPn5ufA2LG5y8htt+V50CdM8G9dKoJdR1QXFizILU733ZevN+tH3uVWazNHqHO9mTVi4cLc/WHcOEN2X22/Pay8cg6pO+4Ip59en//M3303vPe9uS9+2wqwa62V/679W5eKYdcR1YUf/Qi+9a08B+6hhxZdjVRdzhpRHXPmwBe/COPHw2c/m/u193YgYZFefTW/Pl54IayzTp66cdddi65KamzOOqKG8cADcMIJsNdei+bAlZqJM8RUx6BBcMUV8MMf5mkQP/CB3Oe91n3pS/mfgm9+M6/uaMiWaodBWzVtzhz4/Odh+PA824JdRtSMnCGmeiJyYL3hhryQz/7712Y3kiefXLTg0Pe/n1fz/OEP66MFXmomBm3VtLPPzqPlf/1rWGWVoquRinH00b0L2kcdVZ16msEnPwn33ptbiiNyH/laCNzz5uU+5JtvDscem7dtvHHumy2p9hi0VdOOPDK3LO28c9GVSMVpmyFm0KAlA3dLS97urBHlt+GGsOmmOWB/5Sv5k4XuplistHvuga22guOPz4MXTzutuFok9Y5BWzXplVfyV0uLo+ElcNaIoq2/Plx5JXz4w3k5+2q79FL40Ifgtdfg2mvhD3/I86ZLqm3OOqKakxLssQf861+528jyyxddkSTBn/+c+2wvtxz87newww6VfbyU4PXXYdgwmD49z6d+wgmuiCvVAmcdUd264AK4/vq81LohW1Kt+NSn4P/9vzw4e++9K7ts+zPPwGc+A7vskudIHzEi9802ZEv1xaCtmvLYY3lA18c/Dl//etHVSNLiNtkE/vlP+OMfc+hNKS90Uy7z5+eW6802gzvugM99rjYGYUpaOhUN2hGxS0RMjognIuK4TvavExG3RsTDEXFHRIwsbX9vRNwTEY+U9u1TyTpVG+bNy1P5DRyYZxnp57+BkmrQ0KHwwQ/m73/xi9xve+rUZb/fZ56BrbeGY46BnXaCRx/NDQ/9+y/7fUsqRsWiTET0B84GRgObAftFxGYdDvsxcGlKaUvgVOD7pe1zgINSSpsDuwA/jYgVK1WrasMbb8Cqq+auI2usUXQ1ktSztdfOY0ne//68BPqyWHXV3B/76qvhuuvyfUuqb5VsM9wGeCKl9GRKaS4wHti9wzGbAbeWvr+9bX9K6bGU0uOl76cBLwMjKlirasCKK+aPY/faq+hKJKl39tgjdyUZOhQ+9jE477y+dfW49lr4yEdyQ8PAgbm7yN57uziX1CgqGbTXBJ5rd31qaVt7DwF7l77fExgSEYstSxIR2wDLAVMqVKcKNnNm7jLy9NO+uUiqP5ttlhe32XlnGDcuz5gEMGVKvt5+OsZx4/L2557LIX3PPfPMIi+9VOzPIKkyKhm0O4tMHf/PPwbYPiIeALYHngfm//cOIlYHLgO+kFJauMQDRIyNiIkRMXH69Onlq1xVdcQR8NvfwgsvFF2JJC2dtk/kbr0VttwSbrwxX154YZ6dJKV8eeGFOZhvvDHcfHNeNn3ixDxPt6TGM6CC9z0VWKvd9ZHAtPYHlLqF7AUQEYOBvVNKM0vXhwJ/Bk5MKf2jswdIKZ0PnA95Hu1y/wCqvN/+Fi67DE46CT7wgaKrkaSl179/7j4yZUpuqX777SWPmTdv0bF/+Uvl5+KWVKxKtmjfC2wUEetFxHLAvsD17Q+IiOER0VbD8cDFpe3LAdeQB0r+roI1qkDPPZeXNd5uOzjxxKKrkaTyOOMMWLCg+2P69cuDHiU1tooF7ZTSfOBwYAIwCbgqpfRIRJwaEbuVDtsBmBwRjwGrAqeVtn8O+ChwSEQ8WPp6b6XX8QkMAAAgAElEQVRqVTG+8508Z+zll8OASn62IklVdPnl+bWtO/Pm5U/zJDU2l2BXYWbNggcfzCPuJalR9OvXu5lH+vXrueVbUm1yCXbVrKeegjffzKuqGbIlNZrBg8t7nKT6ZdBWVc2ZA5/6FOzecUZ1SWoQBxwALS3dH9PSAgceWJ16JBXHoK2q+ta38ipq3/xm0ZVIUmUcfXTvgvZRR1WnHknFMWiram64Ac4+G448Mi/sIEmNaIMN8owigwYtGbhbWvL2q6/Ox0lqbAZtVUTHFdGGDIExY2CjjeD73y+6OkmqrNGj4eGHYezYxVeGHDs2bx89uugKJVWDs46o7G68MYfqefMWLc7QZuBA+MMffJORJEn1y1lHVIgpU3LInjNnyZAN8NZbef+UKdWvTZIkqZoM2iqrM87oPGC3N28enHlmdeqRJEkqikFbZXX55b0L2q6IJkmSGp1BW2U1e3Z5j5MkSapXBm2VlSuiSZIkZQZtlZUrokmSJGUGbZWVK6JJkiRlBm2VlSuiSZIkZQZtld3o0XD//fDFL7oimiRJal4Dii5AjWnGDLjgApgwAXbaqehqJEmSqs8WbVVEayssWADrrFN0JZIkScUwaKsiWlthueVg3XWLrkSSJKkYBm1VRGsrbLQRDLBzkiRJalIGbVVEayu8611FVyFJklQc2xtVEV/5it1GJElSczNoqyJckEaSJDU7u46o7F5+GZ57DlIquhJJkqTiGLRVdhddBGuvDW+8UXQlkiRJxTFoq+wmTYKRI2Hw4KIrkSRJKo5BW2XnjCOSJEkGbZVZSgZtSZIkMGirzF54AWbNMmhLkiQ5vZ/KasgQuOIK2HrroiuRJEkqlkFbZTVkCOy3X9FVSJIkFc+uIyqrf/4T7r236CokSZKKZ4u2yuo734FXXzVsS5Ik2aKtsnLGEUmSpMygrbKZPTsvvW7QliRJMmirjCZPzpcGbUmSJIO2yqi1NV8atCVJkgzaKqNPfxruvBM22qjoSiRJkornrCMqm2HD4CMfKboKSZKk2mCLtsrmnHPg7ruLrkKSJKk2GLRVFgsWwFFHwfXXF12JJElSbaho0I6IXSJickQ8ERHHdbJ/nYi4NSIejog7ImJku30HR8Tjpa+DK1mnlt3TT8PcuQ6ElCRJalOxoB0R/YGzgdHAZsB+EbFZh8N+DFyaUtoSOBX4fum2KwMnAdsC2wAnRcRKlapVy84ZRyRJkhZXyRbtbYAnUkpPppTmAuOB3Tscsxlwa+n729vt/yRwS0ppRkrpVeAWYJcK1qplNGlSvtxkk2LrkCRJqhWVDNprAs+1uz61tK29h4C9S9/vCQyJiFV6eVvVkMcegxEjYJVViq5EkiSpNlQyaEcn21KH68cA20fEA8D2wPPA/F7elogYGxETI2Li9OnTl7VeLYPzzoOHHy66CkmSpNpRyaA9FVir3fWRwLT2B6SUpqWU9kopvQ84obRtZm9uWzr2/JTSqJTSqBEjRpS7fvVBv36w2mpFVyFJklQ7Khm07wU2ioj1ImI5YF9gscnfImJ4RLTVcDxwcen7CcAnImKl0iDIT5S2qQbNmAFjx8J99xVdiSRJUu2oWNBOKc0HDicH5EnAVSmlRyLi1IjYrXTYDsDkiHgMWBU4rXTbGcD3yGH9XuDU0jbVoEcfhQsugJdfLroSSZKk2lHRJdhTSjcAN3TY9t12318NXN3FbS9mUQu3aphT+0mSJC3JlSG1zFpbYeBAWHvtoiuRJEmqHQZtLbPWVth4Y+jfv+hKJEmSaodBW8tszhzYrOOan5IkSU2uon201Rxuuw3SErOcS5IkNTdbtFUW0dkSQ5IkSU3MoK1lctNNsOuuMG2J5YQkSZKam0Fby+Tee+HGG2HYsKIrkSRJqi0GbS2T1lZYZx1YYYWiK5EkSaotBm0tk9ZWF6qRJEnqjEFbS23hQoO2JElSVwzaWmqzZsHWW8OoUUVXIkmSVHucR1tLbdgwuOOOoquQJEmqTbZoS5IkSRVg0NZSO/po+OhHi65CkiSpNhm0tdQefBDmzi26CkmSpNpk0NZSc8YRSZKkrhm0tVRefz0vu77ppkVXIkmSVJsM2loqkyfnS1u0JUmSOmfQ1lIZOBA+/3nYcsuiK5EkSapNzqOtpbLFFnD55UVXIUmSVLts0dZSmTULUiq6CkmSpNpl0NZS2Xbb3HVEkiRJnTNoq8/mzYMnnoB11im6EkmSpNpl0FafPfVUDtvOOCJJktQ1g7b6rLU1Xxq0JUmSumbQVp8ZtCVJknpm0FafffCDcMopMGxY0ZVIkiTVLufRVp99+MP5S5IkSV2zRVt9khI88AC88UbRlUiSJNW2XgftiPhwRHyh9P2IiFivcmWpVk2fDlttBRddVHQlkiRJta1XQTsiTgKOBY4vbWoBXIC7CTkQUpIkqXd626K9J7Ab8AZASmkaMKRSRal2TZqULw3akiRJ3ett0J6bUkpAAoiIFSpXkmpZaysMGgQjRxZdiSRJUm3rbdC+KiJ+CawYEYcBfwEuqFxZqlWtrbDJJtDPYbSSJEnd6tX0fimlH0fEzsDrwCbAd1NKt1S0MtWkE0+E2bOLrkKSJKn29Ri0I6I/MCGl9HHAcN3kPvShoiuQJEmqDz12AEgpLQDmRITrADa5F1+E666DmTOLrkSSJKn29XZlyLeAf0XELZRmHgFIKX2tIlWpJv31r7DvvvDww7DFFkVXI0mSVNt6G7T/XPpSE2tthQjYaKOiK5EkSap9vR0M+euIWA7YuLRpckppXuXKUi1qbYX11oOBA4uuRJIkqfb1dmXIHYDHgbOBc4DHIuKjvbjdLhExOSKeiIjjOtm/dkTcHhEPRMTDEbFraXtLRPw6Iv4VEZMi4vgl713VNmmSC9VIkiT1Vm+7jpwBfCKlNBkgIjYGrgTe39UNSrOVnA3sDEwF7o2I61NKj7Y77ETgqpTSuRGxGXADsC7wWWD5lNIWETEIeDQirkwpPd2nn05ls3AhTJ4MO+1UdCWSJEn1obdBu6UtZAOklB6LiJYebrMN8ERK6UmAiBgP7A60D9oJGFr6fhgwrd32FSJiAPAOYC55Dm8VJAIeeADe8Y6iK5EkSaoPvQ3aEyPiIuCy0vXPA/f1cJs1gefaXZ8KbNvhmJOBmyPiCGAF4OOl7VeTQ/kLwCDgqJTSjF7WqgqIsNuIJElSX/R2Ie2vAo8AXwO+Tm6V/koPt4lOtqUO1/cDLkkpjQR2BS6LiH7k1vAFwBrAesDREbH+Eg8QMTYiJkbExOnTp/fyR9HSuOMOOPtsWLCg6EokSZLqQ2+D9gDgZymlvVJKewI/B/r3cJupwFrtro9kUdeQNl8ErgJIKd0DDASGA/sDN6WU5qWUXgb+Bozq+AAppfNTSqNSSqNGjBjRyx9FS2P8eDjpJOjf029dkiRJQO+D9q3kvtJt3gH8pYfb3AtsFBHrlaYG3Be4vsMxzwI7AUTEpuSgPb20fcfIVgC2A1p7WasqoLXVriOSJEl90dugPTClNLvtSun7Qd3dIKU0HzgcmABMIs8u8khEnBoRu5UOOxo4LCIeIs9ickhKKZFnKxkM/Jsc2H+VUnq4Dz+XysygLUmS1De9HQz5RkRslVK6HyAiRgFv9nSjlNIN5Cn72m/7brvvHwU+1MntZpOn+FMNePVVeOklg7YkSVJf9DZoHwn8LiKmkQc0rgHsU7GqVFOeeCJfGrQlSZJ6r9ugHRFbA8+llO6NiHcBXwb2Am4CnqpCfaoBW28Nr78OLT3NnC5JkqT/6qmP9i/Ji8UAfAD4Nrn/9KvA+RWsSzVmyBAYOLDoKiRJkupHT11H+rdbKGYf4PyU0u+B30fEg5UtTbXi9NNhhRXgiCOKrkSSJKl+9NSi3b+0DDrkafhua7evt/27Ved+9Sv461+LrkKSJKm+9BS0rwT+GhHXkWcZuQsgIjYEZla4NtWAuXNhyhTYdNOiK5EkSaov3bZKp5ROi4hbgdWBm0tzXEMO6HYkaAJTpuRl151xRJIkqW967P6RUvpHJ9seq0w5qjWtpfU4DdqSJEl909uVIdWkZs6EFVeETTYpuhJJkqT6YtBWtw45BGbMgMGDi65EkiSpvhi01aOIoiuQJEmqPwZtdSkl2H57uPTSoiuRJEmqPwZtdemFF+DOO2HWrKIrkSRJqj8GbXXJGUckSZKWnkFbXTJoS5IkLT2DtrrU2gpDhsAaaxRdiSRJUv0xaKtLI0bApz7lrCOSJElLo8eVIdW8vvOdoiuQJEmqX7ZoS5IkSRVg0Fan7r8fVl89T+8nSZKkvjNoq1OTJsGLL8Lw4UVXIkmSVJ8M2upUayv07w8bbFB0JZIkSfXJoK1OtbbC+uvD8ssXXYkkSVJ9MmirU62tsOmmRVchSZJUv5zeT5361Kdg442LrkKSJKl+GbTVqdNPL7oCSZKk+mbXES3hjTfg7beLrkKSJKm+GbS1hPPOg0GDYObMoiuRJEmqXwZtLaG1Nc+fPWxY0ZVIkiTVL4O2ljBpErzrXUVXIUmSVN8M2lpCa6tBW5IkaVkZtLWYV16B//zHoC1JkrSsDNpaTP/+8OMfw047FV2JJElSfXMebS1mpZXg6KOLrkKSJKn+2aKtxTz2GDz7bNFVSJIk1T9btLWYb3wDpk6FBx8suhJJkqT6Zou2FuOMI5IkSeVh0NZ/vfUWPPUUbLJJ0ZVIkiTVP4O2/uvxx2HhQlu0JUmSysGgrf9qbc2XBm1JkqRlV9GgHRG7RMTkiHgiIo7rZP/aEXF7RDwQEQ9HxK7t9m0ZEfdExCMR8a+IGFjJWgUf/jBceaVBW5IkqRwqNutIRPQHzgZ2BqYC90bE9SmlR9sddiJwVUrp3IjYDLgBWDciBgCXAwemlB6KiFWAeZWqVdnqq8O++xZdhSRJUmOoZIv2NsATKaUnU0pzgfHA7h2OScDQ0vfDgGml7z8BPJxSegggpfSflNKCCtYq4Jpr4JFHiq5CkiSpMVQyaK8JPNfu+tTStvZOBg6IiKnk1uwjSts3BlJETIiI+yPiW509QESMjYiJETFx+vTp5a2+ySxcCAccABdcUHQlkiRJjaGSQTs62ZY6XN8PuCSlNBLYFbgsIvqRu7R8GPh86XLPiNhpiTtL6fyU0qiU0qgRI0aUt/om8/zzMGeO/bMlSZLKpZJBeyqwVrvrI1nUNaTNF4GrAFJK9wADgeGl2/41pfRKSmkOubV7qwrW2vSccUSSJKm8Khm07wU2ioj1ImI5YF/g+g7HPAvsBBARm5KD9nRgArBlRAwqDYzcHngUVYxBW5IkqbwqNutISml+RBxODs39gYtTSo9ExKnAxJTS9cDRwAURcRS5W8khKaUEvBoRPyGH9QTckFL6c6VqVQ7aw4bBqqsWXYkkSVJjiJxr69+oUaPSxIkTiy6jbs2YAc89B+95T9GVSJIk1baIuC+lNKqn4yrWoq36svLK+UuSJEnl4RLsYtYsOOUUeNRe8JIkSWVj0BaTJsHJJ8NjjxVdiSRJUuMwaOu/M45summxdUiSJDUSg7ZobYUBA2D99YuuRJIkqXEYtEVrK2y4IbS0FF2JJElS4zBoi6eecqEaSZKkcnN6P3HffTB7dtFVSJIkNRZbtEW/fjB0aNFVSJIkNRaDdpO7+2447DB48cWiK5EkSWosBu0m97e/wYUXwsCBRVciSZLUWAzaTa61FVZbDVZcsehKJEmSGotBu8m1tjrjiCRJUiUYtJtYSjlouyKkJElS+Rm0m9isWTB8OGy+edGVSJIkNR7n0W5iQ4fC448XXYUkSVJjskVbkiRJqgCDdhP7v/+DMWOKrkKSJKkx2XWkid15J7z8ctFVSJIkNSZbtJuYU/tJkiRVjkG7Sc2ZA888Y9CWJEmqFIN2k3rssXxp0JYkSaoMg3aTmj8fdtgBttii6EokSZIak4Mhm9SoUXD77UVXIUmS1Lhs0W5SKRVdgSRJUmMzaDep7baDr3616CokSZIal0G7CS1cCA8/DIMGFV2JJElS4zJoN6Fnn4W33nLGEUmSpEoyaDehSZPypUFbkiSpcgzaTai1NV8atCVJkirHoN2ENtkEDjsMhg8vuhJJkqTG5TzaTWjXXfOXJEmSKscW7Sb08svOoy1JklRpBu0mM2MGrLoq/PSnRVciSZLU2AzaTWby5Hy50UbF1iFJktToDNpNxhlHJEmSqsOg3WRaW2G55WDddYuuRJIkqbEZtJtMa2vuNjLA+WYkSZIqyrjVZA49FGbNKroKSZKkxmfQbjK77150BZIkSc2hol1HImKXiJgcEU9ExHGd7F87Im6PiAci4uGI2LWT/bMj4phK1tksXn0VJk6Et94quhJJkqTGV7GgHRH9gbOB0cBmwH4RsVmHw04ErkopvQ/YFzinw/4zgRsrVWOzuf122Hpr+Pe/i65EkiSp8VWyRXsb4ImU0pMppbnAeKBjx4UEDC19PwyY1rYjIvYAngQeqWCNTcWp/SRJkqqnkkF7TeC5dtenlra1dzJwQERMBW4AjgCIiBWAY4FTunuAiBgbERMjYuL06dPLVXfDam2FkSNh8OCiK5EkSWp8lQza0cm21OH6fsAlKaWRwK7AZRHRjxywz0wpze7uAVJK56eURqWURo0YMaIsRTey1lZbsyVJkqqlkrOOTAXWand9JO26hpR8EdgFIKV0T0QMBIYD2wJjIuKHwIrAwoh4K6V0VgXrbWgp5aB98MFFVyJJktQcKhm07wU2ioj1gOfJgx3373DMs8BOwCURsSkwEJieUvpI2wERcTIw25C9bFKCa68FG/4lSZKqo2JBO6U0PyIOByYA/YGLU0qPRMSpwMSU0vXA0cAFEXEUuVvJISmljt1LVAb9+sGOOxZdhSRJUvOIRsm1o0aNShMnTiy6jJp1//3w7LPwmc9A//5FVyNJklS/IuK+lNKono6r6II1qh2XXAIHHZRbtiVJklR5xq4m0TbjSHQ2F4wkSZLKzqDdJJzaT5IkqboM2k1g9mx47jmDtiRJUjUZtJvAY4/lS4O2JElS9VRyHm3ViC23zF1HVlut6EokSZKah0G7CQwYAJtsUnQVkiRJzcWuI03goovgiiuKrkKSJKm5GLSbwE9/CuPHF12FJElSczFoN7gFC/JgSAdCSpIkVZdBu8E9/TTMnWvQliRJqjaDdoNrbc2Xm25abB2SJEnNxqDd4J59Nl8664gkSVJ1GbQb3Fe/CrNmwcorF12JJElSczFoN4HBg4uuQJIkqfkYtJfSlCkwbhwMHQr9+uXLcePy9lpywAFwzTVFVyFJktR8DNpL4cYb87LmF16Yu2WklC8vvDBvv/HGoivMXnkFfvObPPOIJEmSqsug3UdTpsCYMTBnDsybt/i+efPy9jFjaqNlu23GEaf2kyRJqj6Ddh+dccaSAbujefPgzDOrU093DNqSJEnFMWj30eWX9y5oX3ZZderpTmsrDBwIa69ddCWSJEnNx6DdR7Nnl/e4Stt2W+jfv+gqJEmSms+AoguoN4MH54GPvTmuaD/+cdEVSJIkNS9btPvogAOgpaX7YyLgwAOrU48kSZJqk0G7j44+uueg/Y53wFFHwQMPwPHH55lIqm3SJBg1Cu65p/qPLUmSJIN2n22wAVx9NQwatGTgbmnJ26++Oh83YQKcfjq8+91w003VrfORR+C++3LolyRJUvUZtJfC6NHw8MMwduziK0OOHZu3jx6djzvuOLj9dlhuubxtv/3gxRerU2Pb1H4bb1ydx5MkSdLiDNpLaYMN4KyzYOZMWLAgX551Vt7e3g47wEMPwSmnwB/+ABddVJ36Jk2CddbJLeySJEmqPmcdqYLll4fvfje3aLfNaX3nnbDSSrDFFpV5zNZWF6qRJEkqki3aVbTRRjl0p5QHS261VeUGS265Jey0U/nvV5IkSb1j0C5ARB4oeeCBlRss+atfwTe/Wd77lCRJUu8ZtAsyfDhcfPHigyXvuKM8971wYXnuR5IkSUvPoF2wtsGSF18M22+ft/3738sWls8+G1ZbDV59tSwlSpIkaSkYtGvA8svDF76Qu5S89BJ88IPwoQ/Bv/61dPfX2gpvvw0rrljeOiVJktR7Bu0a88535mkCH388D5Y87ri+D5Zsm3EkojI1SpIkqWcG7RoTAQcdlMPygQfCD36QB0u+/nrv78Op/SRJkopn0K5RbYMl77gDDj44rzwJPbduz5wJ06YZtCVJkopm0K5x228PJ52Uv3/ggbzgzXnndT1Yct68PEf3Rz9avRolSZK0JIN2HRkyBN7zHvjqV+HDH158sOSUKTBuHKy/Pvz0p/DJT+brU6YUV68kSVIzi5RS0TWUxahRo9LEiROLLqPiUoLLL4dvfANeew2+/W3YbjsYMya3Zs+bt+jYlpb8dfXVeZ5uSZIkLbuIuC+lNKqn42zRrjMReZBka2seNDlzZg7Zc+YsHrIhX58zJ++3ZVuSJKm6Khq0I2KXiJgcEU9ExHGd7F87Im6PiAci4uGI2LW0feeIuC8i/lW63LGSddajVVaBiy6CuXOXDNgdzZsHZ55ZnbokSZKUVazrSET0Bx4DdgamAvcC+6WUHm13zPnAAymlcyNiM+CGlNK6EfE+4KWU0rSIeDcwIaW0ZneP1yxdRzoaOhRmzerdcTNnVr4eSZKkRlcLXUe2AZ5IKT2ZUpoLjAd273BMAkoT1zEMmAaQUnogpTSttP0RYGBELF/BWuvW7NnlPU6SJEnlUcmgvSbwXLvrU0vb2jsZOCAipgI3AEd0cj97k1u93+64IyLGRsTEiJg4ffr08lRdZwYPLu9xkiRJKo9KBu3OFgDv2E9lP+CSlNJIYFfgsoj4b00RsTnwA+DLnT1ASun8lNKolNKoESNGlKns+nLAAXlmke60tOQBlJIkSaqeSgbtqcBa7a6PpNQ1pJ0vAlcBpJTuAQYCwwEiYiRwDXBQSsk5M7pw9NG9C9pHHVWdeiRJkpRVMmjfC2wUEetFxHLAvsD1HY55FtgJICI2JQft6RGxIvBn4PiU0t8qWGPd22CDPE/2oEFLBu6Wlrz96qvzcZIkSaqeigXtlNJ84HBgAjAJuCql9EhEnBoRu5UOOxo4LCIeAq4EDkl5GpTDgQ2B70TEg6Wvd1aq1no3ejQ8/DCMHZtnF+nXL1+OHZu3u1iNJElS9bkypCRJktQHtTC9nyRJktS0DNqSJElSBRi0JUmSpAowaEuSJEkVYNCWJEmSKsCgLUmSJFWAQVuSJEmqAIO2JEmSVAENs2BNREwHnim6jgYzHHil6CIanOe4OjzPlec5rg7Pc+V5jquj3s/zOimlET0d1DBBW+UXERN7s+qRlp7nuDo8z5XnOa4Oz3PleY6ro1nOs11HJEmSpAowaEuSJEkVYNBWd84vuoAm4DmuDs9z5XmOq8PzXHme4+poivNsH21JkiSpAmzRliRJkirAoN3EImKtiLg9IiZFxCMR8fVOjtkhImZGxIOlr+8WUWu9i4inI+JfpXM4sZP9ERE/j4gnIuLhiNiqiDrrVURs0u45+mBEvB4RR3Y4xufyUoiIiyPi5Yj4d7ttK0fELRHxeOlypS5ue3DpmMcj4uDqVV1/ujjPP4qI1tJrwjURsWIXt+329UVZF+f45Ih4vt3rwq5d3HaXiJhceo0+rnpV158uzvNv253jpyPiwS5u23DPZbuONLGIWB1YPaV0f0QMAe4D9kgpPdrumB2AY1JKny6ozIYQEU8Do1JKnc4ZWnpxPwLYFdgW+FlKadvqVdg4IqI/8DywbUrpmXbbd8Dncp9FxEeB2cClKaV3l7b9EJiRUjq9FDpWSikd2+F2KwMTgVFAIr++vD+l9GpVf4A60cV5/gRwW0ppfkT8AKDjeS4d9zTdvL4o6+IcnwzMTin9uJvb9QceA3YGpgL3Avu1f6/UIp2d5w77zwBmppRO7WTf0zTYc9kW7SaWUnohpXR/6ftZwCRgzWKralq7k1+UUkrpH8CKpX+E1Hc7AVPah2wtvZTSncCMDpt3B35d+v7XwB6d3PSTwC0ppRmlcH0LsEvFCq1znZ3nlNLNKaX5pav/AEZWvbAG0sVzuTe2AZ5IKT2ZUpoLjCf/DagT3Z3niAjgc8CVVS2qQAZtARAR6wLvA/7Zye4PRMRDEXFjRGxe1cIaRwJujoj7ImJsJ/vXBJ5rd30q/tOztPal6xdxn8vlsWpK6QXI/7AD7+zkGJ/T5XUocGMX+3p6fVH3Di91z7m4i25QPpfL5yPASymlx7vY33DPZYO2iIjBwO+BI1NKr3fYfT95mdH3AL8Arq12fQ3iQymlrYDRwP+UPlprLzq5jf26+igilgN2A37XyW6fy9Xlc7pMIuIEYD7wmy4O6en1RV07F9gAeC/wAnBGJ8f4XC6f/ei+NbvhnssG7SYXES3kkP2blNIfOu5PKb2eUppd+v4GoCUihle5zLqXUppWunwZuIb8UWR7U4G12l0fCUyrTnUNZTRwf0rppY47fC6X1UttXZtKly93cozP6TIoDSL9NPD51MWgql68vqgLKaWXUkoLUkoLgQvo/Nz5XC6DiBgA7AX8tqtjGvG5bNBuYqW+UhcBk1JKP+nimNVKxxER25CfM/+pXpX1LyJWKA02JSJWAD4B/LvDYdcDB+XJR2I78kCRF6pcaiPosrXE53JZXQ+0zSJyMHBdJ8dMAD4RESuVPo7/RGmbeikidgGOBXZLKc3p4pjevL6oCx3GwuxJ5+fuXmCjiFiv9KnZvuS/AfXNx4HWlNLUznY26nN5QNEFqFAfAg4E/tVuqp1vA2sDpJTOA8YAX42I+cCbwL5dtaqoS6sC15Qy3gDgipTSTRHxFfjveb6BPOPIE8Ac4AsF1Vq3ImIQeVaAL7fb1v4c+1xeChFxJbADMDwipgInAacDV0XEF4Fngc+Wjh0FfCWl9KWU0oyI+B45pACcmlJamoFoTaGL83w8sN+3nJwAAAPmSURBVDxwS+n14x8ppa9ExBrAhSmlXeni9aWAH6HmdXGOd4iI95K7gjxN6fWj/TkuzfpyOPkfxf7AxSmlRwr4EepCZ+c5pXQRnYyfaYbnstP7SZIkSRVg1xFJkiSpAgzakiRJUgUYtCVJkqQKMGhLkiRJFWDQliRJkirAoC1JBYiIBRHxYLuvdZfiPlaM/9/e/YRYVYZxHP/+FLPIcFOLClKQJGroDxrSH0oLgmgxGQmBBJUQFGTSIqJFuyKa2ogI/Vn0B4n+rEIoAzFII8wsJ0uhhYbSIgrKjIq0p8V5J+8Mk86Ul5mG72dzDue+73neezf34eE950kePP2rGxXj1iS7kuxLsj/Js32Isa69nlGSZhRf7ydJUyDJ0aqa9x/vsRDYXFUDk5w3u6qOT2DcAF0zmtuqan/r7HZ/VW38N+s9SZyDwNKq+v503leSppoVbUmaJpLMTjKU5JMkw0lGmmfMS7I1ye4kXyQZbFOeBha1ivhQkuVJNvfcb0OSe9r5wSRPJNkOrEqyKMl7ST5N8mGSS8ZZ0qPAk1W1H6Cqjo0k2UkWtDUNt+NF7frLSe7sWcPRdlye5IMkb7fK+KbWCXUtcAGwLcm20/uLStLUsjOkJE2Ns3o6sh6oqpXAGuCnqro6yVxgR5L3gUPAyqo6kuRc4OMk7wCPAQNVdSV0yewpYv5WVde3sVvpujh+nWQZsBG4acz4AeC5f7jXBuDVqnolyX3AeuD2U8S/CrgM+BbYAVxXVeuTPAKssKItaaYx0ZakqfHrSILc4xbg8p6K8HzgYuAw8FSSG4A/gQvp2hVP1hvQVciBa4G3Wrtj6Fp9T8Y1wB3t/DXgmQnM2VlVh9saPgcWAtsnGVeS/jdMtCVp+gjwUFVtGXWx2/5xHrCkqv5oe5rPHGf+MUZvCRw75pd2nAX8OE6iP9aXwBJgzwTWPvLAz99rSJfFn9Ez5vee8+P4HyRphnOPtiRNH1uAB5LMAUiyOMnZdJXt71qSvQJY0Mb/DJzTM/8b4NIkc5PMB24eL0hVHQEOJFnV4iTJFeMMHQIeT7K4jZvVtnkAfATc1c5Xc6IyfZAuOQcYBOZM4HuP/R6SNCOYaEvS9PES8BWwO8le4Hm6qu8mYGmSXXRJ7cjDiT/Q7ePem2Soqg4BbwLDbc5nJ4m1GliTZA9d5Xpw7ICqGgbWAa8n2QfsBc5vH68F7k0yDNwNPNyuvwjcmGQnsIwTVfSTeQF414chJc00vt5PkiRJ6gMr2pIkSVIfmGhLkiRJfWCiLUmSJPWBibYkSZLUBybakiRJUh+YaEuSJEl9YKItSZIk9YGJtiRJktQHfwGnobOrd1Y4RQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "get_selectKbest_scores(18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature indices: [ 0  1  6  7  8  9 15 16]\n",
      "\n",
      "Accuracy of gradient boost classifier on test set: 0.951\n",
      "\n",
      "AUC score: 0.8298950014788524\n",
      "\n",
      "Recall score: 0.95\n",
      "\n",
      "0.9418831168831169\n"
     ]
    }
   ],
   "source": [
    "selector = SelectKBest(f_classif, k=8)\n",
    "\n",
    "selector.fit(X, y)\n",
    "\n",
    "X_new = selector.transform(X)\n",
    "\n",
    "X_new.shape\n",
    "print(\"feature indices: {}\\n\".format(selector.get_support(indices=True)))\n",
    "\n",
    "y = df_cardio['NSP']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_new, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test_sub)\n",
    "print('Accuracy of gradient boost classifier on test set: {:.3f}\\n'.format(clf.score(X_test, y_test)))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_test_sub, y_pred_clf, pos_label=2)\n",
    "print('AUC score: {}\\n'.format(auc(fpr, tpr)))\n",
    "\n",
    "print('Recall score: {}\\n'.format(recall_score(y_test_sub, y_pred_clf, average = 'micro')))\n",
    "\n",
    "\n",
    "print(cross_val_score(clf, X_test, y_test, cv=20).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio[['LB','AC','DP','ASTV','MSTV','ALTV','Median','Variance']]\n",
    "                   \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boost classifier on test set: 0.999\n",
      "\n",
      "Recall score: 0.95\n",
      "\n",
      "0.9169278895768833\n"
     ]
    }
   ],
   "source": [
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test_sub)\n",
    "print('Accuracy of gradient boost classifier on test set: {:.3f}\\n'.format(clf.score(X_train_sub, y_train_sub)))\n",
    "\n",
    "print('Recall score: {}\\n'.format(recall_score(y_test_sub, y_pred_clf, average = 'micro')))\n",
    "\n",
    "print(cross_val_score(clf, X_test_sub, y_test_sub, cv=20).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average AUC score: 0.9802663858382007\n"
     ]
    }
   ],
   "source": [
    "# Binarize the output\n",
    "y = label_binarize(y, classes=[1, 2, 3])\n",
    "n_classes = 3\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=0)\n",
    "\n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "\n",
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "classifier = OneVsRestClassifier(ensemble.GradientBoostingClassifier(**params))\n",
    "y_score = classifier.fit(X_train_sub, y_train_sub).decision_function(X_test_sub)\n",
    "\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test_sub[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "print('Average AUC score: {}'.format((roc_auc[0]+roc_auc[1]+roc_auc[2])/3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection with Recursive Feature Elimination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Features: 17\n",
      "Selected Features: [ True  True  True  True  True False  True  True  True  True  True  True\n",
      "  True  True False  True  True  True]\n",
      "Feature Ranking: [1 1 1 1 1 3 1 1 1 1 1 1 1 1 2 1 1 1]\n"
     ]
    }
   ],
   "source": [
    "# Feature Extraction with RFE\n",
    "from sklearn.feature_selection import RFE\n",
    "\n",
    "array = df_cardio.values\n",
    "X = array[:,0:18]\n",
    "Y = array[:,18]\n",
    "# feature extraction\n",
    "\n",
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "rfe = RFE(clf, 16)\n",
    "fit = rfe.fit(X, Y)\n",
    "print(\"Num Features: {}\".format(fit.n_features_+1))\n",
    "print(\"Selected Features: {}\".format(fit.support_))\n",
    "print(\"Feature Ranking: {}\".format(fit.ranking_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop(['NSP','DS'],axis=1)\n",
    "                   \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boost classifier on test set: 0.999\n",
      "\n",
      "0.9085001273236566\n"
     ]
    }
   ],
   "source": [
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test_sub)\n",
    "print('Accuracy of gradient boost classifier on test set: {:.3f}\\n'.format(clf.score(X_train_sub, y_train_sub)))\n",
    "\n",
    "print(cross_val_score(clf, X_test_sub, y_test_sub, cv=10).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop(['NSP','DS','Nzeros'],axis=1)\n",
    "                   \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boost classifier on test set: 0.999\n",
      "\n",
      "AUC score: 0.8356994971901804\n",
      "\n",
      "Recall score: 0.9529411764705882\n",
      "\n",
      "0.9055589508530684\n"
     ]
    }
   ],
   "source": [
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test_sub)\n",
    "print('Accuracy of gradient boost classifier on test set: {:.3f}\\n'.format(clf.score(X_train_sub, y_train_sub)))\n",
    "\n",
    "print('Recall score: {}\\n'.format(recall_score(y_test_sub, y_pred_clf, average = 'micro')))\n",
    "\n",
    "print(cross_val_score(clf, X_test_sub, y_test_sub, cv=10).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average AUC score: 0.9862838440356273\n"
     ]
    }
   ],
   "source": [
    "# Binarize the output\n",
    "y = label_binarize(y, classes=[1, 2, 3])\n",
    "n_classes = 3\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=0)\n",
    "\n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "\n",
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "classifier = OneVsRestClassifier(ensemble.GradientBoostingClassifier(**params))\n",
    "y_score = classifier.fit(X_train_sub, y_train_sub).decision_function(X_test_sub)\n",
    "\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test_sub[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "print('Average AUC score: {}'.format((roc_auc[0]+roc_auc[1]+roc_auc[2])/3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection based on Importance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Measure: Importance Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature ranking:\n",
      "1. feature 7 (0.213688)\n",
      "2. feature 6 (0.146642)\n",
      "3. feature 8 (0.140586)\n",
      "4. feature 13 (0.115770)\n",
      "5. feature 5 (0.110974)\n",
      "6. feature 1 (0.047933)\n",
      "7. feature 0 (0.039282)\n",
      "8. feature 3 (0.039090)\n",
      "9. feature 12 (0.032703)\n",
      "10. feature 9 (0.032216)\n",
      "11. feature 10 (0.029334)\n",
      "12. feature 14 (0.019953)\n",
      "13. feature 11 (0.018414)\n",
      "14. feature 2 (0.009069)\n",
      "15. feature 4 (0.002501)\n",
      "16. feature 15 (0.001844)\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "index 16 is out of bounds for axis 0 with size 16",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-159-887b205bfd96>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train_sub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%d. feature %d (%f)\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimportances\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m: index 16 is out of bounds for axis 0 with size 16"
     ]
    }
   ],
   "source": [
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "forest = ensemble.GradientBoostingClassifier(**params)\n",
    "forest.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "importances = clf.feature_importances_\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X_train_sub.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop(['NSP','Tendency'],axis=1)\n",
    "                   \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boost classifier on test set: 0.999\n",
      "\n",
      "AUC score: 0.830930198166223\n",
      "\n",
      "Recall score: 0.9470588235294117\n",
      "\n",
      "0.9055589508530684\n"
     ]
    }
   ],
   "source": [
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test_sub)\n",
    "print('Accuracy of gradient boost classifier on test set: {:.3f}\\n'.format(clf.score(X_train_sub, y_train_sub)))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_test_sub, y_pred_clf, pos_label=2)\n",
    "print('AUC score: {}\\n'.format(auc(fpr, tpr)))\n",
    "\n",
    "print('Recall score: {}\\n'.format(recall_score(y_test_sub, y_pred_clf, average = 'micro')))\n",
    "\n",
    "print(cross_val_score(clf, X_test_sub, y_test_sub, cv=10).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Measure: F-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaYAAAEWCAYAAAAtuzN2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xl4VPX1+PH3AUQ2AQOK7IhsISREcMGq/EJpUBFRhAqUKgi2oqVVv0BFEYu4AAoKInWpbIoGFMrihiIw4gIqaNi0EZUgO7Jpwg6e3x/3Jgxhkkwgk7kzc17PMw/3fu4y52Qgh7vMuaKqGGOMMV5RKtwBGGOMMf6sMBljjPEUK0zGGGM8xQqTMcYYT7HCZIwxxlOsMBljjPEUK0zGRBAReUFEhoU7DmNCSex7TCYWiEgmUAM47jfcRFW3nsE+U4DpqlrnzKKLTCIyFdisqg+FOxYTXeyIycSSG1S1kt/rtItScRCRMuF8/zMhIqXDHYOJXlaYTMwTkTYi8pmI7BORVe6RUM6y20XkWxHJEpEfReROd7wi8B5QS0Sy3VctEZkqIo/5bZ8iIpv95jNF5H4RWQ3sF5Ey7nazReRnEdkgIv8oINbc/efsW0T+KSI7RWSbiNwkIh1F5DsR2SMiD/ptO1xEZonITDefr0Skpd/yeBHxuT+HdSLSOc/7Pi8i74rIfqAf0Av4p5v7W+56Q0TkB3f/34hIF7999BGRT0RkjIjsdXO9zm95nIhMEZGt7vK5fss6iUi6G9tnIpIU9AdsIo4VJhPTRKQ28A7wGBAHDAJmi8h57io7gU5AZeB24BkRaaWq+4HrgK2ncQTWE7geqAr8BrwFrAJqA+2Be0XkmiD3dQFQzt32YeA/wJ+B1sDVwMMi0tBv/RuBN91cXwfmishZInKWG8cHwPnA34HXRKSp37Z/Ah4HzgFeAV4DnnRzv8Fd5wf3fasAjwDTRaSm3z4uBzKA6sCTwCQREXfZq0AFIMGN4RkAEWkFTAbuBKoBLwLzReTsIH9GJsJYYTKxZK77P+59fv8b/zPwrqq+q6q/qepCYAXQEUBV31HVH9TxEc4v7qvPMI5nVXWTqh4ELgXOU9URqnpEVX/EKS49gtzXUeBxVT0KzMD5hT9eVbNUdR2wDvA/ulipqrPc9Z/GKWpt3FclYJQbx2LgbZwimmOeqn7q/pwOBQpGVd9U1a3uOjOB9cBlfqtsVNX/qOpxYBpQE6jhFq/rgP6quldVj7o/b4C/AC+q6ueqelxVpwGH3ZhNFIrYc9zGnIabVPXDPGP1gT+KyA1+Y2cBSwDcU03/Aprg/EeuArDmDOPYlOf9a4nIPr+x0sDHQe5rt/tLHuCg++cOv+UHcQrOKe+tqr+5pxlr5SxT1d/81t2IcyQWKO6AROQ24P+ABu5QJZximWO73/sfcA+WKuEcwe1R1b0Bdlsf6C0if/cbK+sXt4kyVphMrNsEvKqqf8m7wD1VNBu4Dedo4ah7pJVz6inQLa37cYpXjgsCrOO/3SZgg6o2Pp3gT0PdnAkRKQXUAXJOQdYVkVJ+xake8J3ftnnzPWleROrjHO21B5ap6nERSefEz6sgm4A4EamqqvsCLHtcVR8PYj8mCtipPBPrpgM3iMg1IlJaRMq5NxXUwflf+dnAz8Ax9+ipg9+2O4BqIlLFbywd6OheyL8AuLeQ9/8C+NW9IaK8G0MLEbm02DI8WWsRudm9I/BenFNiy4HPcYrqP91rTinADTinB/OzA/C/flURp1j9DM6NI0CLYIJS1W04N5P8W0TOdWNo6y7+D9BfRC4XR0URuV5EzgkyZxNhrDCZmKaqm3BuCHgQ5xfqJmAwUEpVs4B/AG8Ae3Eu/s/32/Z/QBrwo3vdqhbOBfxVQCbO9aiZhbz/cZwCkAxsAHYBL+PcPBAK84DuOPncCtzsXs85AnTGuc6zC/g3cJubY34mAc1zrtmp6jfAWGAZTtFKBD4tQmy34lwz+x/OTSf3AqjqCpzrTM+5cX8P9CnCfk2EsS/YGhMjRGQ40EhV/xzuWIwpiB0xGWOM8RQrTMYYYzzFTuUZY4zxFDtiMsYY4yn2PabTULVqVW3UqFG4wzgj+/fvp2LFiuEO44xEQw4QHXlYDt7g9RxWrly5S1XPK2w9K0ynoUaNGqxYsSLcYZwRn89HSkpKuMM4I9GQA0RHHpaDN3g9BxHZGMx6dirPGGOMp1hhMsYY4ylWmIwxxniKFSZjjDGeYoXJGGOMp1hhMsYY4ylWmIwxJgYdOnSIyy67jJYtW5KQkMC//vUvADZs2MDll19O48aN6d69O0eOHAFg48aNtG/fnqSkJFJSUti8eXPIYovYwiQiXURERaSZO19KRJ4VkbUiskZEvhSRC0XkcxFJF5GfRORndzpdRKaKyJ159nmTiLwbnoyMMabknH322SxevJhVq1aRnp7OggULWL58Offffz/33Xcf69ev59xzz2XSpEkADBo0iNtuu43Vq1fz8MMP88ADD4QstogtTEBP4BOghzvfHedRy0mqmgh0Afap6uWqmgw8DMxU1WR3Ps1v2xw93HFjjIlqIkKlSpUAOHr0KEePHkVEWLx4Md26dQOgd+/ezJ07F4BvvvmG9u3bA9CuXTvmzZsXstgisvODiFQCrgTa4Ty4bThQE9iW81hoVS3sOPNDYKqI1FTVbSJSAfgDzgPJCnTw6HEaDHnnDDIIv4GJx+hjOXhCNORhOXhDMDlkjro+d/r48eO0bt2a77//nr/97W9cdNFFVK1alTJlnNJQp04dtmzZAkDLli2ZPXs299xzD3PmzCErK4vdu3dTrVq1Ys8jIgsTcBOwQFW/E5E9ItIK5ymjn4jI1cAiYLqqfp3fDlT1uIj8F7gFGI/z9M4l7lNLTyEifwX+ClC9+nk8nHiseDMqYTXKO3+JI1k05ADRkYfl4A3B5ODz+U6aHzduHNnZ2QwbNozatWtz8ODB3HV27tzJgQMH8Pl83HzzzTz77LM899xzJCUlUb16dZYtW5Z71FWcIrUw9QTGudMzgJ6qOlhEmgK/d1+LROSPqrqogP2kAU/hFKYewCv5raiqLwEvAdRr2EjHronUH51jYOIxLAdviIY8LAdvCCaHzF4pAcdXrlzJ4cOHOXz4MFdddRVlypRh2bJlNG7cOLf/Xs4pvuzsbJo1a0anTp2KM/wTVDWiXkA14CCwEcgENgE/4T5bym+9QcAEv/k+wHN51hF3Hy2BnUC5YGJo0qSJRrolS5aEO4QzFg05qEZHHpaDNxQlh507d+revXtVVfXAgQN61VVX6VtvvaXdunXTtLQ0VVW98847deLEiaqq+vPPP+vx48dVVfXBBx/UYcOGFTk+YIUG8Ts2Em9+6Aa8oqr1VbWBqtYFNgBtRaQWOHfoAUk4xStf7g/qDWAa8K6qHgpt6MYY4w3btm2jXbt2JCUlcemll5KamkqnTp0YPXo0Tz/9NI0aNWL37t3069cPcE4BNm3alCZNmrBjxw6GDh0astgi8bi1JzAqz9hsYCqwR0TOdse+AJ4LYn9pwGBgSHEFaIwxXpeUlMTXX596Gb5hw4Z88cUXp4x369Yt91ReqEVcYVLVlABjzwLPFrLdVJzilXf8a5xTesYYYzwgEk/lGWOMiWJWmIwxxniKFSZjTInZtGkT7dq1Iz4+noSEBMaPHw/Anj17SE1NpXHjxqSmprJ3714AfvnlF2644Ybcfm5TpkwJZ/imhHimMLl97171my/j9rZ7u4j78YnIJe70uyJStbhjNcacnjJlyjB27Fi+/fZbli9fzsSJE/nmm28YNWoU7du3Z/369bRv355Ro5z7myZOnEjz5s1ZtWoVPp+PgQMH5jYVNdHLM4UJ2A+0EJHy7nwqsOVMdqiqHVV13xlHZowpFjVr1qRVq1YAnHPOOcTHx7NlyxbmzZtH7969gZP7s4kIWVlZqCrZ2dnExcXltssx0ctrn/B7wPXALJzbwtOAqwFEpCIwAUjEiXu4qs5zC9kUoDnwLZBT2BCRTOASVd0lInOBukA5YLw6nRwQkWyczg+dcL64e6Oq7igoSOuV5w3RkANERx6F5eDfny13LDOTr7/+mssvv5wdO3ZQs2ZNwCleO3fuBGDAgAF07tyZWrVqkZWVxcyZMylVykv/nzah4LVPeAbQQ0TK4XxB9nO/ZUOBxap6KU7z1qfcYnUXcEBVk4DHgdb57LuvqrYGLgH+ISI5nQcrAstVtSWwlCCauBpjzkx2djZdu3Zl3LhxVK5cOd/13n//fZKTk9m6dSvp6ekMGDCAX3/9tQQjNeHgqSMmVV0tIg1wjpbyPhepA9BZRAa58+WAekBb3O8wuduvzmf3/xCRLu50XaAxsBs4AuRcx1qJcwrxFNbE1XuiIQeIjjwKy8G/ceixY8d44IEHuPzyy4mLi8Pn81G5cmVmz55NtWrV2L17N+eccw4+n48xY8bwpz/9iY8++giAc889l9dee434+PhizyE7O/uUBqeRJhpyAI8VJtd8YAyQgtMXL4cAXVU1w39lEQHQgnYoIik4j7S4QlUPiIgPp7ABHHVbEwEcJ5+fifo1cW3atKn+vdeNQSfkRT6fj1vcxoyRKhpygOjII9gcVJXevXtz5ZVXMm7cuNzx7t27s379erp27cqoUaPo0aMHKSkpXHzxxezZs4eUlBR27NjBjh07+OMf/0j16tVDkkNKFHwOkZ4DeO9UHsBkYISqrskz/j7wd3ErkYhc7I4vBXq5Yy1wTgHmVQXY6xalZkCbkERujCnQp59+yquvvsrixYtJTk4mOTmZd999lyFDhrBw4UIaN27MwoULGTLE6RA2bNgwPvvsMxITE2nfvj2jR48OSVEy3uK5IyZ1HvA3PsCiR3EedbHaLU6ZODcsPA9McU/hpeP0yMtrAdDfXScDWB6C0I0xhbjqqqs4cYLiZIsWnfqEmlq1avHBBx+EOizjMZ4pTKp6ytOmVNUH+Nzpg8CdAdY5yKmPSM9Z1sBv9rrC3ldVZ+HcEWiMMSZMvHgqzxhjTAyzwmSMMcZTrDAZEyP69u3L+eefT4sWLXLHunfvnnsTQoMGDUhOTgacL7+WL18+d1n//v3DFbaJQZ65xnQ63O8l/ReIV9X/ud+BeltVW/it8wLOXXhlgQtxbn4AeA1IUtVb/datAawBaqvq0RJJwpgS0qdPHwYMGMBtt92WOzZz5szc6YEDB1KlSpXc+Ysuuoj09PQSjdEYiPDChPNF3E9wbn4YHmgFVe0PICKNgFmqmuzOnwusF5Fyfo9U/yMwx4qSiUZt27YlMzMz4DJV5Y033mDx4sUlG5QxAURsYRKRSsCVOO2J5pNPYcqPqu4Vkc9wevPNdod7AA8Vtq31yvOGaMgBQp9HoD51eX388cfUqFGDxo0b545t2LCBiy++mMqVK/PYY49x9dVXhyxGY/xFbGECbgIWqOp3IrJHRFoBe4q4jzScYjRbROoCDXC+sGtMTElLS6Nnz5658zVr1uSnn36iWrVqrFy5kptuuol169YV2NfOmOISyYWpJ84XbsFp/toTmFjEfcwHnnWPvroDb6jqb4FWtF553hMNOUDo8/DvnbZ9+3b2799/0tjx48eZOXMmL774Yr591qpVq0ZaWhpNmzYNuDwaerRZDh6iqhH3wumhdxDYiNMBYhPwE87NDWvz2aYRkB5gPA2npdEK4NJg3r9JkyYa6ZYsWRLuEM5YNOSgWrJ5bNiwQRMSEk4ae++997Rt27Ynje3cuVOPHTumqqo//PCD1qpVS3fv3p3vfqPhs7AcQg9YoUH8jo3U28W7Aa+oan1VbaCqdYENQJ3T2FcaMBioqqpfFmeQxnhJz549ueKKK8jIyKBOnTpMmjQJgBkzZpx0Gg9g6dKlJCUl0bJlS7p168YLL7xAXFxcOMI2MShST+X1BEblGZsNPAg0FZHNfuP3qeqbBexrATAV+HexRmiMx6SlpQUcnzp16iljXbt2pWvXriGOyJjAIrIwqWpKgLFncZ/LlM823wPJAcaPAPZfQWOM8YhIPZVnjDEmSllhMsYY4ylWmIwxxniKFSZjPCZQs9Xhw4dTu3btk576CrBw4UJat25NYmIirVu3tpZCJirEVGESkewAY8NFZIuIpIvI/0TkeRGJqZ+L8ZY+ffqwYMGCU8bvu+8+0tPTSU9Pp2PHjgBUr16dt956izVr1jBt2jRuvfXWU7YzJtJE5F15IfCMqo5xC9JS4P8BS/Jb2XrleUM05AAn8sjpaVdQs9W8Lr744tzphIQEDh06xOHDhzn77LNDEaoxJcKODE5WFigH7A13IMbk9dxzz5GUlETfvn3Zu/fUv6KzZ8/m4osvtqJkIp44XSJig4hkq2qlPGPDgb8APwP1gfdU9U8BtvXvldf64XH/CX3AIVSjPOw4GO4ozkw05AAn8kisfeJZSNu3b+eBBx5gypQpAOzZs4cqVaogIkyePJndu3dz//33566/YcMGHnroIZ588klq165d4jlkZ2dTqVKlwlf0MMsh9Nq1a7dSVS8pdMVg+hZFywvIDjA2HBjkTp8FzAN6FLQf65XnDdGQg2rgPAL1tMtv2aZNm7Rx48b6ySefhCrEQkXDZ2E5hB5R3isvJNR5QOACoG24YzHG37Zt23Kn58yZk3vH3r59+7j++usZOXIkV155ZbjCM6ZY2c0PfkREgN8B9jxpEzY9e/bE5/Oxa9cu6tSpwyOPPILP5yM9PR0RoUGDBrz44ouAc93p+++/59FHH+XRRx8F4IMPPuD8888PZwrGnJFYK0wV8jR4fdr98z4R+TPOqbzVWENXE0aBmq3269cv4LoPPfQQDz1U6EOXjYkoMVWYVDW/U5fDSzIOY4wx+bNrTMYYYzzFCpMxxhhPscJkTJiNHj36lN54OcaMGYOIsGvXLgB++eUXbrjhBlq2bElCQkLu95yMiSYxUZhEpIGIrM0zNlxEBrnTg9w+eWtFZJWI3BaeSE0suvbaawP2xtu0aRMLFy6kXr16uWMTJ06kefPmrFq1Cp/Px8CBAzly5EhJhmtMyMVEYSqIiPQHUoHLVLUFzneYJLxRmVjSsmVL4uJOfYjyfffdx5NPPonzLQaHiJCVlYWqkp2dTVxcHGXKxNQ9TCYG2N9oeBBop6q/AqjqL8C0gjawJq7eEOk55DRtDWT+/PnUrl2bli1bnjQ+YMAAOnfuTK1atcjKymLmzJmUKhXz/780USbWC1N54BxV/aGwFfP0yuPhxGOhji2kapR3frFHskjPwefzAU5/s+XLl7N//358Ph+HDh3i/vvv56mnnsqd//TTT6lSpQofffQR1atX5/XXX2fr1q3ccccdvPzyy1SsWDGsuWRnZ+fmE6ksBw8Jpm9RpL9wmrOuzTM2HPgXsKeo+7Need4QDTmoOnn4979bvXq1nnfeeVq/fn2tX7++li5dWuvWravbtm3Tjh076tKlS3O3bdeunX7++efhCj1XNHwWlkPoYb3yTrIbODfPWByQCewXkYYlHpEx+UhMTGTnzp1kZmaSmZlJnTp1+Oqrr7jggguoV68eixYtAmDHjh1kZGTQsKH99TXRJSYKk6pmA9tEpD2AiMQB1wKfACOBiSJS2V1W2T1tZ0yJePTRR7niiivIyMigTp06TJo0Kd91hw0bxmeffUZiYiLt27dn9OjRVK9evQSjNSb0Yuka0204BWisO/+Iqv4gIs8DlYAvReQocBQYm99OjCluw4YNIyUlJd/l/k+zrVWrFh988EHogzImjGKmMKnqN0C7AOMKPOm+jDHGhFlMnMozxhgTOawwGWOM8RQrTMbk0bdv31N61w0ePJhmzZqRlJREly5d2LdvX+6ykSNH0qhRI5o2bcr7778fjpCNiSoxU5hEpIuIqIg08xtrIiLvisj3IvKtiLwhIjXCGacJvz59+pzSuy41NZW1a9eyevVqmjRpwsiRIwH45ptvmDFjBuvWrWPBggXcfffdHD9+PBxhGxM1YqYwAT1xbg/vASAi5YB3gOdVtZGqxgPPA+eFL0TjBW3btj2ld12HDh1ye9K1adOGzZudByHPmzePHj16cPbZZ3PhhRfSqFEjvvjiixKP2ZhoEhN35YlIJeBKnLvy5uN0ffgTsExV38pZT1WXBLM/65XnDcWdQ0G96/xNnjyZ7t27A7BlyxbatGmTu6xOnTps2bKl2GIyJhbFRGECbgIWqOp3IrJHRFoBLYCVwe7AeuV5T3Hn4N9jbPv27bm96/xNnz6dffv2Ubt2bXw+H5s3b+bbb7/NXW/btm2sW7euSF96jYb+ZpaDN0RDDkDM9Mp7B0h1p/8BPAU8DdxzOvuzXnneEMoc/HvX5Zg6daq2adNG9+/fnzv2xBNP6BNPPJE736FDB/3ss8+K9F72WXiD5RB6WK88h4hUA34PvCwimcBgoDuwDmgdxtBMBFmwYAGjR49m/vz5VKhQIXe8c+fOzJgxg8OHD7NhwwbWr1/PZZddFsZIjYl8UV+YgG7AK6paX1UbqGpdYAPwHfA7Ecm9sCAi14pIYrgCNd7Qs2fPU3rXDRgwgKysLFJTU0lOTqZ///4AJCQkcMstt9C8eXOuvfZaJk6cSOnSpcOcgTGRLRauMfUERuUZm41z80MnYJyIjMPpkbcauKdkwzNek5aWdspYv3798l1/6NChDB06NJQhGRNTor4wqWpKgLFn/WavLblojDHGFCYWTuUZY4yJIFaYjDHGeIoVJhOTAvXDe/PNN0lISKBUqVKsWLEid/yLL74gOTmZ5ORkWrZsyZw5c8IRsjExIyIKk4g8IyL3+s2/LyIv+82PFZEHRWRWPtv7ROQSd/pBv/EGIrI2lLEbbwrUD69Fixb897//pW3btqeMr1ixgvT0dBYsWMCdd97JsWOR/eVkY7wsIgoT8BnwOwARKQVUBxL8lv8OWKSq3YLY14OFr2KiXaB+ePHx8TRt2vSUdStUqJDbJ+/QoUOISInEaEysipS78j4FnnGnE4C1QE0RORc4AMQDe0Vkraq2EJHywBSgOfAtUB5AREYB5UUkHecLtkOB0iLyH5zitgW4UVUPFhSM9crzhtPJIdh+eHl9/vnn9O3bl40bN/Lqq6/mFipjTPGLiH9dqrpVRI6JSD2cArIMqA1cAfyC8/2jI36b3AUcUNUkEUkCvnL3M0REBqhqMjin8oDGQE9V/YuIvAF0BabnjcF65XnP6eQQTD+8ffv2sXLlSrKzs08anzhxIhs3buTBBx+kYsWKlC1b9nRDP0k09DezHLwhGnKACClMrk9xitLvcPrc1Xanf8E51eevLfAsgKquFpHVBex3g6qmu9MrgQaBVlLVl4CXAJo2bap/73Xj6WXhET6fj1tSUsIdxhk50xwyMzOpWLEiKXn2UbVqVVq3bs0ll1wScLupU6cSFxeX7/Ki8vl8p8QQaSwHb4iGHCByrjHBietMiTin8pbjHDH9Dqdo5aVB7vew3/RxIqtYmxKwYcOG3JsdNm7cSEZGBg0aNAhvUMZEsUgqTJ/itBDao6rHVXUPUBWnOC3Ls+5SoBeAiLQAkvyWHRWRs0ogXuNhgfrhzZkzhzp16rBs2TKuv/56rrnmGgA++eQTWrZsSXJyMl26dOHf//53kR5rYYwpmkg6OliDczfe63nGKqnqLvdhgDmeB6a4p/DSAf9Hir4ErBaRr3BufjAxKFA/PIAuXbqcMnbrrbdy6623hjokY4yryIXJvROurqoWdN2m2KnqcaBynrE+ftOZOA//w72rrkc++7kfuN9vqIXfsjHFFrAxxpjTEtSpPPcLqpVFJA5YhXM08nRoQzPGGBOLgr3GVEVVfwVuBqaoamvgD6ELyxhjTKwKtjCVEZGawC3A2yGMx5hiF6gv3p49e0hNTaVx48akpqayd+9eAH755RduuOEGWrZsSUJCAlOmTAlX2MbErGAL0wjgfeAHVf1SRBoC64vyRu7pwGvyjN0rIv8uwj7eFZGqRXlfYwL1xRs1ahTt27dn/fr1tG/fnlGjnGdJTpw4kebNm7Nq1Sp8Ph8DBw7kyJEjgXZrjAmRoAqTqr6pqkmqepc7/6Oqdi3ie6Vx6g0JPdzxAomjlKp2VNV9RXxfE+MC9cWbN28evXv3BqB3797MnTsXABEhKysLVSU7O5u4uDhrP2RMCQvqX5yINMG5BbuG24suCeisqo8V4b1mAY+JyNmqethtB1QLSBeRRcC5wFnAQ6o6z13+HrAE57tKN4nIR8Al7u3hc4G6QDlgvNuZARHJBsbjfOfpIE7vux0iUgN4AWjoxnOXqn4mIn8G/gGUBT4H7nbvAMyX9crzhsJyKKgv3o4dO6hZsyYANWvWZOfOnQAMGDCAzp07U6tWLbKyspg5cyalSkXS1/2MiXzB/lfwP8Bg4EXIbfPzOhB0YVLV3SLyBc6jzOfhHC3NxCkeXVT1VxGpDiwXkfnuZk2B21X1biBvV+e+qrrHbdj6pYjMVtXdQEVguaoOFZEngb+4cT4LfKSqXUSkNFBJROKB7sCVqnrUPa3YC3glb/zWK897CsuhoL54x44dO2l5zvxHH31E9erVef3119m6dSt33HEHL7/8MhUrVgxRFtHR38xy8IZoyAEAVS30BXzp/vm131h6MNvm2c+fgbSc7YFWOEdJz+E0Yk3HKVQX4PSs25Bn+0ygujs9HOfW9VU4/fLauOOHAXGnuwMvu9M/A2fn2d8AYKv7vulABjC8sDyaNGmikW7JkiXhDuGMFSWHDRs2aEJCQu58kyZNdOvWraqqunXrVs35TDt27KhLly7NXa9du3b6+eefF0/A+Yi1z8KrLIfQA1ZoELUi2HMUu0TkItz+cyLSDdgWbPHzMxdoLyKtgPKq+hXOEcp5QGt1un7vwDk9B7A/0E5EJAXndvUrVLUl8LXfNkfdHwAU3vtOgGmqmuy+mqrq8NPIy0SYzp07M23aNACmTZvGjTc6TXnr1avHokWLAOd0X0ZGBg0bNsx3P8aY4hdsYfobzmm8ZiKyBbgX6F/UN1PVbMAHTObETQ9VgJ3qnEprB9QPYldVgL2qekBEmgFtgthmEc7jMBCR0iJS2R3rJiLnu+NxIhLM+5sIEqgv3pAhQ1i4cCGNGzdm4cLqEwtVAAAgAElEQVSFDBkyBIBhw4bx2WefkZiYSPv27Rk9erT1xTOmhBV6jcl9YuwlqvoHEakIlFLVrDN4zzTgv5y4Q+814C0RWYFzOu1/QexjAdDf7YWXgdNpvDD3AC+JSD+cI6m7VHWZiDwEfODmeRSnCG8sSkLG2/Lri5dzZOSvVq1afPDBB6EOyRhTgEILk6r+JiIDgDdUNeCptaJQ1Tk4p9By5nfh3HUXSAv/GVVt4Dd7XT77r+Q3PQvnbkBUdQdwykOUVHUmzk0YxhhjPCDYU3kLRWSQiNR1T3fFuX3zjDHGmGIV7O3ifd0//+Y3ppz4TpAxxhhTLILt/HBhgJcVJeNZRemP5/P5qFKlCsnJySQnJzNixIhwhW2MIfjHXtwW6BXq4IqLiBwXkXQRWSciq0Tk/9ybHRCRFBH5RUS+FpFvReRf4Y7XnLmi9McDuPrqq0lPTyc9PZ2HH364pMM1xvgJ9hrTpX6vq3G+3No5RDGFwkH3e0oJQCrQEfAvQB+r6sXAJcCfRaR1OII0xaco/fGMMd4S1DUmVf27/7yIVAFeDUlEIaaqO932Ql+KyPA8y/aLyErgImBlfvuwXnneECiH0+mPB7Bs2TJatmxJrVq1GDNmDAkJCaEJ2hhTqNNtm3wAaFycgZQkVf3RPZV3vv+4iFTD+bLuo3m3sV553hMoh9Ppj7d//36mT59O+fLlWb58Oddccw3Tp08vgQwc0dDfzHLwhmjIAQi6V95bwHz39TbwIzA6mG298AKyA4ztA2oAKTi99r7GOUrqX9j+rFeeNxSWQ7D98fKqX7++/vzzz8UWZ2Fi4bOIBJZD6BFkr7xgj5jG+E0fAzaq6uYzLYrh4j7o8DiwE4jHucbUKbxRmVDL6Y83ZMiQk/rjbd++nRo1aiAifPHFF/z2229Uq1YtzNEaE7uCLUwdVfV+/wERGZ13LBKIyHk4z2V6TlU1z6M0TJTo2bMnPp+PXbt2UadOHR555BGGDBnCLbfcwqRJk6hXrx5vvvkmALNmzeL555+nTJkylC9fnhkzZuR9xIoxpgQFW5hSgbxF6LoAY15VXkTScR6xcQznxo2nwxuSCaWi9McbMGAAAwYMCHVIxpggFViYROQu4G6godswNcc5wKehDKw4qWrpApb5cDqeG2OM8YDCjphex3m8+UhgiN94lqruCVlUxhhjYlaBhUlVf8G5Y60ngPvconI4jyWvpKo/hT5EY4wxsSTYlkQ3iMh6YAPwEc4jzt8LYVwmhj3zzDMkJCTQokULevbsyaFDh7j66qtze9nVqlWLm266KdxhGmNCJNiWRI/hfPH0O1W9EGhPmK4xiYiKyKt+82VE5GcRedud7yMiz+XZZoXbK+8nd91097XIfXCg/7rdRGR+yWRj8tqyZQvPPvssK1asYO3atRw/fpwZM2bw8ccf5/ayu+KKK7j55pvDHaoxJkSCLUxHVXU3UEpESqnqEiA5hHEVZD/QQkTKu/OpwJaCNlDVS1Q1GRgBvKZO37xknDvzeuRZvQcnHvtuwuDYsWMcPHiQY8eOceDAAWrVqpW7LCsri8WLF9sRkzFRLNjbxfeJSCXgY+A1EdmJc9t1uLwHXI/zdNqeOIXk6tPYzwfAZBE5X50eepVwOkH0Lmgj65UXGpmjrqd27doMGjSIevXqUb58eTp06ECHDh1y15kzZw7t27encuXKYYzUGBNKwRamG4GDwL1AL6AKztFHuMwAHnZP3yUBkzmNwqSqR0VkLvBHYCJwE7BQAzxC3nrlhZ7P5yMrK4tp06Yxffp0KlWqxPDhwxk6dCipqakATJw4kY4dO+Lz+aKmL1g05GE5eEM05AAE1yvPaXFEfeAP7nQF4Jxgty3OF27fO2AFcDvwBM5RztvueB+crg6Btr0DGJdnLAWnJRE4fQA7FxaD9coLnTfeeEP79u2bOz9t2jS96667VFV1165dGhcXpwcPHlRV7+ZQVNGQh+XgDV7PgSB75QV7V95fcE6bvegO1QbC/TCb+Tg9/M70etBSoIGIJOE8b2pBIeubEKpXrx7Lly/nwIEDqCqLFi0iPj4egDfffJNOnTpRrly5MEdpjAmlYG9++BtwJfArgKquJ88jI8JgMjBCVdecyU5U9TfgTeAV4C1VPVIcwZnTc/nll9OtWzdatWpFYmIiv/32G3/9618BmDFjBj179gxzhMaYUAv2GtNhVT2S09hSRMoAGrKogqBOd/Px+SzuIyL+t2210YK7oacB9wGDiis+c/oeeeQRHnnkkVPGo+LcuTGmUMEWpo9E5EGcZqipOP3z3gpdWPlT1UoBxny4/e5UdSowNZ9tX85n/EvA2kkbY4wHBHsqbwjwM7AGuBN4F3goVEEZY4yJXYV1F6+nqj+512H+476MMcaYkCnsVN5coBWAiMxW1a6hD8lEi4yMDLp37547/+OPPzJixAh2797NvHnzKFWqFOeffz5Tp049qbuDMSa2FXYqz/+6S8NQBnKmCuuhZ0pe06ZNc/vbrVy5kgoVKtClSxcGDx7M6tWrSU9Pp1OnTowYEc7vahtjvKawIybNZ9qLcnvoqepBguihZ0rOokWLuOiii6hfv/5J4/v377fHmBtjTlJYYWopIr/iHDmVd6dx51VVvdawLN8eeiJyGTAOKI/TXul2Vc0Qkf8DWqhqXxFJdLe5TFUP5Pcm1iuvYJmjrj9lLO93kIYOHcorr7xClSpVWLJkSUjiMMZEJnG6REQ+EckGfgc8DPwZWI7T22+QqnYSkcrAAVU9JiJ/AO5S1a4iUgrnVvNngKHAPap6yiM98vTKa/3wuMi+D6RGedhxMDT7Tqxd5aT5o0eP0q1bN6ZMmUJcXNxJy1577TWOHDnC7bffXuT3yc7OplKlU749EHGiIQ/LwRu8nkO7du1Wquolha4YTN+iSHhReA+9usAcYC3Obe//89u2IZANjA3mvaxXXtHMnTtXU1NTAy7LzMzUhISE09qv1/uCBSsa8rAcvMHrOVCcvfIiTH499B4FlqhqC+AGnEfE52iMU5js1rAQSEtLO+k03vr163On58+fT7NmzcIRljHGo4Lt/BBJJgO/qOoaEUnxG6/CiZsh+uQMikgVnNZGbYHnRKSbqs4qoVij3oEDB1i4cCEvvvhi7tiQIUPIyMigVKlS1K9fnxdeeCGMERpjvCbqCpPm30PvSWCae7PDYr/xZ4B/q+p37mPWl4jIUlXdWQLhRr0KFSqwe/fuk8Zmz54dpmiMMZEgagqTFt5DbxnQxG/xMHe8r9/6m4BGoYzTGGNMwaLxGpMxxpgIZoXJGGOMp0TNqTwTXvv27eOOO+5g7dq1iAiTJ09m3LhxZGRk5C6vWrUq6enpYY7UGON1MVGYROQ4zneXctwENACWAHeo6iR3vYuBr4DBqjqmpOOMZPfccw/XXnsts2bN4siRIxw4cICZM2fmLh84cCBVqlQpYA/GGOOIicIEHFTVZP8BEWmAU6y6A5Pc4R7AqhKNLAr8+uuvLF26lKlTpwJQtmxZypYtm7tcVXnjjTdYvHhxPnswxpgTYqUw5ecnoLKI1AB2AtfiPASxQNYrz5HTE+/HH3/kvPPO4/bbb2fVqlW0bt2a8ePHU7FiRQA+/vhjatSoQePGjc84bmNM9IuaXnkFyXMqb4OqdnG/fDsI+AD4DfgauAPYiNPeaEyefVivvDxyeuJlZGRw9913M2HCBJo3b86ECROoWLEiffs6d+I/88wz1K5dm1tuueVMwz6J1/uCBSsa8rAcvMHrOQTbKy9WClN23u85+RWmO4CZOIVrPk4j2FMKk796DRtpqVsCfYc3cgxMPMbYNWd2wJxzxLR9+3batGlDZmYm4BwhjRo1infeeYdjx45Ru3ZtVq5cSZ06dc407JP4fD5SUlKKdZ/hEA15WA7e4PUcRCSowhTrp/JQ1e0ichTn+U334BSmApU/qzQZAR7tEEl8Ph+ZvVKKZV8XXHABdevWJSMjg6ZNm7Jo0SKaN28OwIcffkizZs2KvSgZY6JXzBcm18PA+ap63B5ad3omTJhAr169OHLkCA0bNmTKlCnAqc9hMsaYwlhhAlT1s3DHEOmSk5NZsWLFKeM5d+oZY0ywYqIwFdZHL8/48NBHZIwxJj/WksgYY4ynWGEyxhjjKVaYTJHt27ePbt260axZM+Lj41m2bBmDBw+mWbNmJCUl0aVLF/bt2xfuMI0xESqiCpOIqIi86jdfRkR+FpG33fkaIvK2iKwSkW9E5F0RSRSRdPe1R0Q2uNMfutNN87zHOBH5Z0nnFkly+uL973//Y9WqVcTHx5OamsratWtZvXo1TZo0YeTIkeEO0xgToSKqMAH7gRYiUt6dT+XE49IBRgALVbWlqjYHhqjqGlVNdnvlzcdp0Jqsqn8AZuD0xwNAREoB3XC+cGsCyOmL169fP8Dpi1e1alU6dOhAmTLOvTRt2rRh8+bN4QzTGBPBIvGuvPeA64FZQE8gDbjaXVYTp8UQAKq6upB9peEUoUfc+bZApqpuLGijWOyVF2xfPIDJkyfTvXv3Yo/ZGBMbIqolkYhk43RmeBj4M7AcuBcYpKqdROQanELzNfAhMEVVt/ptPxV4W1Vn+Y2tA/6kqqtE5AVgjapODPDeMd0rL9i+eNOnTycjI4MRI0YQ6i8re70vWLCiIQ/LwRu8nkOwvfIi7ohJVVe7j6zoSZ5O4Kr6vog0xOkSfh3wtYi0UNWfC9hlGtDDLVA34hS9QO/7EvASOL3yzrTPXLgVtVdeTvuiZs2aMXLkSO6++24ASpcuzahRo0hJSWHatGmsW7eORYsWUaFChVCEfRKv9wULVjTkYTl4QzTkABFYmFzzgTFAClDNf4Gq7gFeB153b4poC8wuYF9pOKf/PgJWq+rOwt48lnvl5dcXb8GCBYwePZqPPvqoRIqSMSZ6RWphmgz8oqpr3C7hAIjI74HlqnpARM4BLsJ55lK+VPUHEdkNjALGhTDmqBGoL96ll17K4cOHSU1NBZwbIF544YUwR2qMiUQRWZhUdTMQ6LkTrYHnROQYzh2HL6vql0HsMg0YCcwpviijV6C+eN9//32YojHGRJuIKkyF9bxT1aeApwrYvk8+488AzxRHjMYYY85MpH2PyRhjTJSzwmSMMcZTrDDFoOPHj/OXv/yFTp06AdCnTx8uvPBCkpOTSU5OJj09PcwRGmNiWcRcYxIRBZ5W1YHu/CCgkj0/qejGjx9PvXr1Thp76qmn6NatW5giMsaYEyLpiOkwcLOIVA93IJFs8+bNvPPOO1x/fWR/D8sYE70i5ogJOIbTeeE+YKj/ArfV0EGgGVAfuB3oDVwBfJ5zN56IPA9cCpQHZqnqv0SkCvAF0FlVM0QkDVisqvn2HIq0XnmZfl8Gvvfee3nyySf5+OOPT1pn6NChjBgxgvbt2zNq1CjOPvvskg7TGGOAyCpMABOB1SLyZIBl5wK/BzoDbwFXAncAX4pIsqqmA0NVdY+IlAYWiUiS2+JoADBVRMYD5wYqSnl65fFw4rGQJBgKPp8PgGXLlnH06FGysrI4ePAgu3fvxufzccMNN9C7d2+OHj3K2LFj6d+/P7179w5v0EHIzs7OzS2SRUMeloM3REMOEGGFSVV/FZFXgH/gHCH5e0tVVUTWADtUdQ3kNmltAKQDt7gFpgxOJ/LmOG2IForIH3EKX8t83jtie+XltB56//33WblyJX369OHXX3/l0KFDvPzyy0yfPj133bJlyzJmzJiI6LcVLX3BoiEPy8EboiEHiLDC5BoHfAVMyTN+2P3zN7/pnPkyInIhMAi4VFX3uqf/ykHuc5jicYpdHFDgw4QitVfeyJEjcx/gN27cOD788EOmT5/Otm3bqFmzJqrK3LlzadGiRZgjNcbEski6+QHIbdL6BtCviJtWxnnQ4C8iUgOn+3iO+4BvcTqWTxaRs4oj1kjRq1cvEhMTSUxMZNeuXTz00EPhDskYE8Mi8YgJYCwwoCgbuM9b+hpYB/wIfAogIk1wrkVdpqpZIrIUeAj4V/GG7C3Jycnce++9ACxevDjM0RhjzAkRU5j8++Sp6g6ggt98H7/pTKBFPstyp/OI91vn/4ohXGOMMacp4k7lGWOMiW5WmIwxxniKFSYP27RpE+3atSM+Pp6EhATGj3ceQTV8+HBq166d29vu3XffLWRPxhgTOSLmGtOZEpHjwBrgLJwuEtOAcar6m/sU3EGq2imMIZ6iTJkyjB07llatWpGVlUXr1q1znxB73333MWjQoDBHaIwxxS9mChNwUFWTAUTkfOB1oAoevvuuZs2a1KxZE4BzzjmH+Ph4tmzZEuaojDEmtGKpMOVS1Z1uB4gvRWR4UbcviV55mXm+wJuZmcnXX3/N5Zdfzqeffspzzz3HK6+8wiWXXMLYsWM599xzQxqPMcaUlJi9xqSqP+Lkf364YylMdnY2Xbt2Zdy4cVSuXJm77rqLH374gfT0dGrWrMnAgQPDHaIxxhQbUdVwx1AiRCTb/7tQ7tg+oCnO95gKvMaUp4lr64fH5dt8vFgk1q4CwLFjx3jggQe49NJLueWWW05Zb/v27TzwwANMmZK3Q1PBsrOzqVSpUuErelg05ADRkYfl4A1ez6Fdu3YrVfWSQldU1Zh4Adl55hsCuwEBUoC3g91XkyZNtCT89ttveuutt+o999xz0vjWrVtzp59++mnt3r17kfe9ZMmSMw0v7KIhB9XoyMNy8Aav5wCs0CB+x8bkNSYROQ94AXhOVVVEwh1SQJ9++imvvvoqiYmJJCcnA/DEE0+QlpZGeno6IkKDBg148cUXwxypMcYUn1gqTOVFJJ0Tt4u/Cjztt7y9iPh3Ff+jqi4ryQDzuuqqq3KO7k7SsWPHMERjjDElI2YKk6qWLmCZD+eptsYYY8IsZu/KM8YY401WmIwxxniKFaYS1LdvX84///yTnhA7ePBgmjVrRlJSEl26dGHfvn1hjNAYY8LPk4VJRFRExvrNDzqdDg1e06dPHxYsWHDSWGpqKmvXrmX16tU0adIk99HnxhgTqzxZmIDDwM0iUr24digiYb/Ro23btsTFxZ001qFDB8qUcUJr06YNmzdvDrSpMcbEjLD/ss7HMeAl4D5gqP8C95bvHE2Ba4EVwAQgESen4ao6T0T6ANcD5YCKItIeeBK4DlDgMVWdKSI1gZlAZXf7u1T14/yCK2qvvLx97/IzefJkunfvHvR+jTEmGnm1MAFMBFaLyJP+g3qiQ/gNwD+Bz4BHgMWq2ldEqgJfiMiH7iZXAEmqukdEugLJQEugOk4T16XAn4D3VfVxESmN32PbS8rjjz9OmTJl6NWrV0m/tTHGeIone+Xl9LUTkRHAUeAgUElVh7vLGwNvAb9X1a0isgLnqOiYu4s44BrgcuD/qert7nbPAGtUdbI7/yrwJrAPmAxMB+aqqv9RWU5Mp90rL6fvHQTubbdgwQLeeustxo4dS7ly5YLe75nwek+tYERDDhAdeVgO3uD1HCK6Vx5uXzucApOJ88yk4e5YReBr4Gq/9VcCTQPspw9O26Gc+XFAX7/5V4HO7nQt4C84DxO8raD4zqRX3oYNGzQhISF3/r333tP4+HjduXPnae/zdHi9p1YwoiEH1ejIw3LwBq/nQJC98rx68wMAqroHeAPo5zc8BZiiJ18Deh/4u7hN70Tk4nx2uRToLiKl3X55bXFO+9UHdqrqf4BJQKtiTgWAnj17csUVV5CRkUGdOnWYNGkSAwYMICsri9TUVJKTk+nfv38o3toYYyKGl68x5RgLDABwC0g3oImI9HWX3wE8inM0tNotTplAoEdYzMG55rQK5+aHf6rqdhHpDQwWkaNANnBbKBJJS0s7Zaxfv34B1jTGmNjlycKkfs9NUtUdnHwzQn5HeXcG2M9UYKrfvAKD3Zf/etOAaacdsDHGmGLj6VN5xhhjYo8VJmOMMZ5ihckYY4ynWGEKkX379tGtWzeaNWtGfHw8y5aF9ZmDxhgTMUJy84OIVAMWubMXAMeBn935y1T1yGnutwywS1WrnnmUoXXPPfdw7bXXMmvWLI4cOcKBAwfCHZIxxkSEkBQmVd2N0/oHtyt4tqqOCcV7edGvv/7K0qVLmTp1KgBly5albNmy4Q3KGGMiRInfLu5+Z+hvQFmcPncDcE4p7gJewGmwegC4UVV3ishFwOvuOu/n2dcQ4GacdkSzVHWEiDQC5gKfA22An4AuqnpIRJq471EN5yjuZmAkMF1V33H3OROYpqrv5pdDfk1cc5q1/vjjj5x33nncfvvtrFq1itatWzN+/HgqVqxY9B+YMcbEmJD3yvM/YhKRFsBjQDdVPSYiLwE+nO4OR4GOqvqeiDyN04lhlIi8i1M4XheRe4BHVLWqiHTE+RLt3wAB3gVGADuBb4FWqrpGRP4LvKGqM0RkJU5ro7dEpBxOsbsCp5t4NxE5F6e9UWNVPZ4nj0J75eX0xMvIyODuu+9mwoQJNG/enAkTJlCxYkX69u17yjbh4vWeWsGIhhwgOvKwHLzB6zl4plceMBwY5E7fC2wB0t1XBjAM58jtgN82vYAX3Om9QGk90Ttvn57oe7fBb1/f4/TGawR867evocAQ4FxgY4D4BPgG5yhqADCqsJwK65W3bds2rV+/fu780qVLtWPHjgVuU9K83lMrGNGQg2p05GE5eIPXcyDIXnklfSpPgMmqOuykQeemBv8bIo5z4jSjuq9A+3pMVSfl2VcjnAcN5revk6iqishrOI++6OP+eUYuuOAC6tatS0ZGBk2bNmXRokU0b978THdrjDExoaRvF/8QuCXnybQiUk1E6hWyzXLgFnfa/2FF7wP9RKSiu686BT3xVlX3Arvc5zghIuVEJKfV0RScNkWHVDWjqEkFMmHCBHr16kVSUhLp6ek8+OCDxbFbY4yJeiV6xKTONZ9HgA9FpBTOdaX+wNYCNvsH8JqI/B9OE9acfb0rIs2A5W5T8SwKP9rpBbwoIo/jHKF1xTm9t1VEvgNmnGZqp0hOTmbFihXFtTtjjIkZIS9M6j7cz2/+dZy77PKq6rfODNwioarf4zzwL8dIv/WeBp4OsK9kv3VG+U1nACl5V3aPuhriPF7dGGNMGMV85wcRuQbnLr5nVDUr3PEYY0ys8+RjL0qSqr4PFHadyxhjTAmJ+SMmY4wx3mKFyRhjjKdYYTLGGOMpVpiMMcZ4Ssh75UUjEcnCaacUyarjNM6NZNGQA0RHHpaDN3g9h/qqel5hK8X8XXmnKUODaUToYSKywnLwhmjIw3LwhmjIAexUnjHGGI+xwmSMMcZTrDCdnpfCHUAxsBy8IxrysBy8IRpysJsfjDHGeIsdMRljjPEUK0zGGGM8xQpTEYjItSKSISLfi8iQcMdTFCKSKSJrRCRdRFa4Y3EislBE1rt/nhvuOP2JyGQR2Skia/3GAsYsjmfdz2a1iLQKX+Qn5JPDcBHZ4n4W6SLS0W/ZA24OGW7n+7ATkboiskREvhWRdSJyjzseMZ9FATlE2mdRTkS+EJFVbh6PuOMXisjn7mcxU0TKuuNnu/Pfu8sbhDP+oAXz/HV7KUBp4Aec5zaVBVYBzcMdVxHizwSq5xl7EhjiTg8BRoc7zjzxtQVaAWsLixnoCLwHCNAG+Dzc8ReQw3BgUIB1m7t/r84GLnT/vpX2QA41gVbu9DnAd26sEfNZFJBDpH0WAlRyp88CPnd/xm8APdzxF4C73Om7gRfc6R7AzHDnEMzLjpiCdxnwvar+qKpHcB5keGOYYzpTNwLT3OlpwE1hjOUUqroU2JNnOL+YbwReUcdyoKqI1CyZSPOXTw75uRGYoaqHVXUD8D3O37uwUtVtqvqVO52F8/yy2kTQZ1FADvnx6mehqprtzp7lvhT4PTDLHc/7WeR8RrOA9uI+8tvLrDAFrzawyW9+MwX/xfYaBT4QkZUi8ld3rIaqbgPnHy5wftiiC15+MUfa5zPAPc012e8UqudzcE8FXYzzP/WI/Czy5AAR9lmISGkRSQd2Agtxjub2qeoxdxX/WHPzcJf/AlQr2YiLzgpT8AL9LyOS7rW/UlVbAdcBfxORtuEOqJhF0ufzPHARkAxsA8a6457OQUQqAbOBe1X114JWDTDmiTwC5BBxn4WqHlfVZKAOzlFcfKDV3D89m0dBrDAFbzNQ12++DrA1TLEUmapudf/cCczB+Qu9I+cUi/vnzvBFGLT8Yo6Yz0dVd7i/XH4D/sOJU0SezUFEzsL5hf6aqv7XHY6ozyJQDpH4WeRQ1X2AD+caU1URyel96h9rbh7u8ioEf2o5bKwwBe9LoLF790tZnAuJ88McU1BEpKKInJMzDXQA1uLE39tdrTcwLzwRFkl+Mc8HbnPvCGsD/JJzmslr8lxv6YLzWYCTQw/3TqoLgcbAFyUdX17uNYlJwLeq+rTfooj5LPLLIQI/i/NEpKo7XR74A871siVAN3e1vJ9FzmfUDVis7p0Qnhbuuy8i6YVzt9F3OOd0h4Y7niLE3RDnDqNVwLqc2HHONS8C1rt/xoU71jxxp+GcXjmK8z+/fvnFjHPKYqL72awBLgl3/AXk8Kob42qcXxw1/dYf6uaQAVwX7vjdmK7COf2zGkh3Xx0j6bMoIIdI+yySgK/deNcCD7vjDXEK5/fAm8DZ7ng5d/57d3nDcOcQzMtaEhljjPEUO5VnjDHGU6wwGWOM8RQrTMYYYzzFCpMxxhhPscJkjDHGU8oUvooxpiSIyHGcW5dz3KSqmWEKx5iwsdvFjfEIEclW1Uol+H5l9ER/NWM8w07lGRMhRKSmiCx1nxu0VkSudsevFZGv3Gf0LHLH4kRkrtucdLmIJLnjw0XkJRH5AHjFbQj6lIh86a57ZxhTNAawU3nGeEl5t2s0wAZV7ZJn+Z+A91X1cREpDVQQkfNwery1VdUNIhLnrvsI/P/27lg1ijCMwvB70limyAXEytIiYhfIIqsB7sEAAAFMSURBVGiTRkgZSG+R0j69l2CIoL2I3VpYLAQLIQm5DGMRQbOFxWcx/5IQEpsE9l94n2qY2WK6w8ws53BcVS+TPAPeMxSVAjwB1qtq2prmf1XV0yQPgMMkX2qYepDmwmCS+jGtoTX6Nt+Bg1ZG+qmqTpKMgMksSKpqVtC5Dmy1c1+TrCRZbtc+V9W0Hb8AHieZ9awtM/TCGUyaG4NJWhBVNWlzJZvAhyRvgHNunjH439zBn2u/262q8b3erHQHfmOSFkSSVeBHVb1laMpeA74BG60Bmyuv8ibAdjs3An7WzRtKY+BVewojyaPWQC/NjU9M0uIYAa+T/AV+AztVdda+E31MssSwifQc2APeJTkFLricPrhuH3gIHLVpiDMuZ7mlufDv4pKkrvgqT5LUFYNJktQVg0mS1BWDSZLUFYNJktQVg0mS1BWDSZLUlX+FqY8KvbrCkAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from matplotlib import pyplot\n",
    "from xgboost import plot_importance\n",
    "\n",
    "# fit model no training data\n",
    "model = XGBClassifier()\n",
    "model.fit(X_train_sub, y_train_sub)\n",
    "# plot feature importance\n",
    "plot_importance(model)\n",
    "plt.savefig('Feature_importance.png')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop(['NSP','Tendency','Nzeros','DL','Nmax','DS'],axis=1)\n",
    "                   \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boost classifier on test set: 0.999\n",
      "\n",
      "Recall score: 0.9529411764705882\n",
      "\n",
      "0.9117035905271198\n"
     ]
    }
   ],
   "source": [
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test_sub)\n",
    "print('Accuracy of gradient boost classifier on test set: {:.3f}\\n'.format(clf.score(X_train_sub, y_train_sub)))\n",
    "\n",
    "print('Recall score: {}\\n'.format(recall_score(y_test_sub, y_pred_clf, average = 'micro')))\n",
    "\n",
    "print(cross_val_score(clf, X_test_sub, y_test_sub, cv=10).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average AUC score: 0.9800611313451472\n"
     ]
    }
   ],
   "source": [
    "# Binarize the output\n",
    "y = label_binarize(y, classes=[1, 2, 3])\n",
    "n_classes = 3\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=0)\n",
    "\n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "\n",
    "params = {'n_estimators': 500,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .1}\n",
    "\n",
    "classifier = OneVsRestClassifier(ensemble.GradientBoostingClassifier(**params))\n",
    "y_score = classifier.fit(X_train_sub, y_train_sub).decision_function(X_test_sub)\n",
    "\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test_sub[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "print('Average AUC score: {}'.format((roc_auc[0]+roc_auc[1]+roc_auc[2])/3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Changing the Tree Depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gradtree(num_estimators,num_depth,num_rate):\n",
    "    params = {'n_estimators': num_estimators,\n",
    "          'max_depth': num_depth,\n",
    "          'learning_rate': num_rate,    \n",
    "          'loss': 'deviance'}\n",
    "\n",
    "    # Initialize and fit the model.\n",
    "    clf = ensemble.GradientBoostingClassifier(**params)\n",
    "    clf.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "    y_pred_clf = clf.predict(X_test_sub)\n",
    "    print('Accuracy of GBRF classifier on test set: {:.2f}\\n'.format(clf.score(X_test_sub, y_test_sub)))\n",
    "\n",
    "    print('Cross_validation score of GBRF: {}\\n'.format(cross_val_score(clf, X_test_sub, y_test_sub, cv=10).mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop(['NSP','Tendency','Nzeros','DL','Nmax','DS'],axis=1)\n",
    "                   \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.97\n",
      "\n",
      "Cross_validation score of GBRF: 0.9120550038197097\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(500,2,.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.95\n",
      "\n",
      "Cross_validation score of GBRF: 0.9151642475171886\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(500,4,.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.96\n",
      "\n",
      "Cross_validation score of GBRF: 0.9234733893557422\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(500,6,.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.95\n",
      "\n",
      "Cross_validation score of GBRF: 0.8997606315253373\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(500,8,.1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Change Number of Estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop(['NSP','Tendency','Nzeros','DL','Nmax','DS'],axis=1)\n",
    "                   \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.95\n",
      "\n",
      "Cross_validation score of GBRF: 0.9175910364145657\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(250,6,.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.95\n",
      "\n",
      "Cross_validation score of GBRF: 0.9205322128851542\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(1000,6,.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.95\n",
      "\n",
      "Cross_validation score of GBRF: 0.9175910364145657\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(1500,6,.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.96\n",
      "\n",
      "Cross_validation score of GBRF: 0.9204481792717086\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(2000,6,.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.95\n",
      "\n",
      "Cross_validation score of GBRF: 0.9145607333842627\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(2500,6,.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Change Learning Rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop(['NSP','Tendency','Nzeros','DL','Nmax','DS'],axis=1)\n",
    "                   \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.94\n",
      "\n",
      "Cross_validation score of GBRF: 0.9027960275019098\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(1000,6,.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.96\n",
      "\n",
      "Cross_validation score of GBRF: 0.914733893557423\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(1000,6,.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.96\n",
      "\n",
      "Cross_validation score of GBRF: 0.9146498599439775\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(1000,6,.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.95\n",
      "\n",
      "Cross_validation score of GBRF: 0.9235625159154571\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(1000,6,.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of GBRF classifier on test set: 0.95\n",
      "\n",
      "Cross_validation score of GBRF: 0.914644766997708\n",
      "\n"
     ]
    }
   ],
   "source": [
    "get_gradtree(1000,6,.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Gradient Tree Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boosted random forest on test set: 0.9648\n",
      "\n",
      "Cross validation score: 0.9301\n",
      "\n",
      "Recall score: 0.9647887323943662\n",
      "\n",
      "--- 35.93630409240723 seconds ---\n"
     ]
    }
   ],
   "source": [
    "y = df_cardio['NSP']\n",
    "\n",
    "X = df_cardio.drop(['NSP','Tendency','Nzeros','Nmax','DS','DL'],axis=1) #drop least important F-score features\n",
    "\n",
    "#X = df_cardio[['LB','AC','DP','ASTV','MSTV','ALTV','Median','Variance']] #selectKbest\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    " \n",
    "start_time = time.time()\n",
    "    \n",
    "params = {'n_estimators': 1000,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .2}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "predict_train_clf = clf.predict(X_train)\n",
    "predict_test_clf = clf.predict(X_test)\n",
    "\n",
    "# Accuracy tables.\n",
    "table_train = pd.crosstab(y_train, predict_train_clf, margins=True)\n",
    "table_test = pd.crosstab(y_test, predict_test_clf, margins=True)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test)\n",
    "print('Accuracy of gradient boosted random forest on test set: {:.4f}\\n'.format(clf.score(X_test, y_test)))\n",
    "\n",
    "print('Cross validation score: {:.4f}\\n'.format(cross_val_score(clf, X_test, y_test, cv=20).mean()))\n",
    "\n",
    "print('Recall score: {}\\n'.format(recall_score(y_test, y_pred_clf, average = 'micro')))\n",
    "\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 2. 3.]\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(y_pred_clf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         1.0       0.97      1.00      0.98       339\n",
      "         2.0       0.93      0.83      0.88        48\n",
      "         3.0       0.97      0.85      0.90        39\n",
      "\n",
      "    accuracy                           0.96       426\n",
      "   macro avg       0.96      0.89      0.92       426\n",
      "weighted avg       0.96      0.96      0.96       426\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, y_pred_clf))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The precision is the ratio tp / (tp + fp) where tp is the number of true positives and fp the number of false positives. The precision is intuitively the ability of the classifier to not label a sample as positive if it is negative.\n",
    "- The recall is the ratio tp / (tp + fn) where tp is the number of true positives and fn the number of false negatives. The recall is intuitively the ability of the classifier to find all the positive samples.\n",
    "- The F-beta score can be interpreted as a weighted harmonic mean of the precision and recall, where an F-beta score reaches its best value at 1 and worst score at 0.\n",
    "- The F-beta score weights the recall more than the precision by a factor of beta. beta = 1.0 means recall and precision are equally important.\n",
    "- The support is the number of occurrences of each class in y_test."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set accuracy:\n",
      "AVG Percent Type I errors: 0.0016028204718014545\n",
      "AVG Percent Type II errors: 0.0016028204718014545\n",
      "\n",
      "Test set accuracy:\n",
      "AVG Percent Type I errors: 0.04356594373606045\n",
      "AVG Percent Type II errors: 0.10782089100673171\n"
     ]
    }
   ],
   "source": [
    "# Accuracy tables.\n",
    "table_train = pd.crosstab(y_train, predict_train_clf, margins=True)\n",
    "table_test = pd.crosstab(y_test, predict_test_clf, margins=True)\n",
    "\n",
    "train_tI_errors = ((1-(table_train.loc[1.0,1.0]/table_train.loc['All',1.0]))\n",
    "                   +(1-(table_train.loc[2.0,2.0]/table_train.loc['All',2.0]))\n",
    "                   +(1-(table_train.loc[3.0,3.0]/table_train.loc['All',3.0])))/3 \n",
    "\n",
    "train_tII_errors = ((1-(table_train.loc[1.0,1.0]/table_train.loc[1.0,'All']))\n",
    "                   +(1-(table_train.loc[2.0,2.0]/table_train.loc[2.0,'All']))\n",
    "                   +(1-(table_train.loc[3.0,3.0]/table_train.loc[3.0,'All'])))/3 \n",
    "\n",
    "test_tI_errors = ((1-(table_test.loc[1.0,1.0]/table_test.loc['All',1.0]))\n",
    "                   +(1-(table_test.loc[2.0,2.0]/table_test.loc['All',2.0]))\n",
    "                   +(1-(table_test.loc[3.0,3.0]/table_test.loc['All',3.0])))/3 \n",
    "\n",
    "test_tII_errors = ((1-(table_test.loc[1.0,1.0]/table_test.loc[1.0,'All']))\n",
    "                   +(1-(table_test.loc[2.0,2.0]/table_test.loc[2.0,'All']))\n",
    "                   +(1-(table_test.loc[3.0,3.0]/table_test.loc[3.0,'All'])))/3 \n",
    "print((\n",
    "    'Training set accuracy:\\n'\n",
    "    'AVG Percent Type I errors: {}\\n'\n",
    "    'AVG Percent Type II errors: {}\\n\\n'\n",
    "    'Test set accuracy:\\n'\n",
    "    'AVG Percent Type I errors: {}\\n'\n",
    "    'AVG Percent Type II errors: {}'\n",
    ").format(train_tI_errors, train_tII_errors, test_tI_errors, test_tII_errors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Differentials:\n",
      "Type I error difference: 0.041963123264258995\n",
      "\n",
      "Type II error difference: 0.10621807053493026\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print((\n",
    "    'Accuracy Differentials:\\n'\n",
    "    'Type I error difference: {}\\n\\n'\n",
    "    'Type II error difference: {}\\n'\n",
    ").format(test_tI_errors - train_tI_errors, test_tII_errors - train_tII_errors))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_cardio['NSP']\n",
    "\n",
    "X = df_cardio.drop(['NSP','Tendency','Nzeros','Nmax','DS','DL'],axis=1) #drop least important F-score features\n",
    "\n",
    "#X = df_cardio[['LB','AC','DP','ASTV','MSTV','ALTV','Median','Variance']] #selectKbest\n",
    "\n",
    "#X = df_cardio.drop(['NSP','DS','Nzeros'],axis=1) #Recursive feature elimination\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xd4FOX2wPHvS0I1NAFRQQREQnooQUAgFIEovTdFqoUmKt6LiugPVCxcRAVRBOSiFClG0KugSJMmvXcQ6aFJCRDSzu+P2aybvoFsNuV8nmefZHba2dnZOTPvzJwxIoJSSimVmnzuDkAppVT2polCKaVUmjRRKKWUSpMmCqWUUmnSRKGUUipNmiiUUkqlSRNFJjDG9DTG/OLuONzNGFPBGBNpjPHIwnlWNMaIMcYzq+bpSsaYPcaYRrcx3m2tg8aYwsaYH4wxV4wx8zM6vqsZY44ZYx5Lo//Pxpinb2O6bxljvrmz6G6fMWalMaa/u+afUbkuUdhWrJu2DdZZY8wMY4yXK+cpIrNEpLkr55EdJf0Ri8hxEfESkTh3xuUutoRV5U6mISJ+IrIynfkkS453sA52AsoCpUSk822Mn2VS2riLyOMi8l93xZQV0kuWWSHXJQqb1iLiBQQD1YFX3RzPbXHnXnJu2UPPiDy6vB8EDopIbEZHzIvrSJ4lIrnqBRwDHnPo/gD4n0N3QWAccByIAD4HCjv0bwtsB64CR4Aw2/vFgWnAGeAU8DbgYevXG1hj+/9zYFySmBYBL9n+vx9YCJwH/gSGOgz3FrAA+MY2//4pfL7iwEzb+H8BI4F8DnGsBT4FrgD7gaZJxk3rM6wFPgIu2fo9BCwHLgIXgFlACdvwXwPxwE0gEvgXUBEQwNM2zEpgjG2614BfgNIO8fSyfYaLwBtJv7skn7sw8B/b8FeANbb3Eub5tO07vQC87jBebWA9cNn2uScCBRz6CzAIOAT8aXvvY+CE7TvYAjRwGN4DeM22blyz9X8AWG2b1nXb8uhqG74V1vp0GVgHBCZZV/8N7ARuAZ6Oy8AW+2ZbHBHAeNv7x23zirS96uKwDtqG8QN+tX2XEcBrKSzT/wOigRjbdPph7TyOtC3nc1jrWnHb8AnLup8thtUpTLMRcNK2PpyzLfN2wBPAQVs8rzkMPwN4O+n4SX/PQFiSWHc4rGPJfifpLQes39o3DsPNB85irVurAT+Hfk8Ae23f9ylguO390sCPtu/2EvA7tt9iCnE0w/o9XsFaB1clxE0Gf2fpxeuS7aorJ+6OF4l/aOWBXcDHDv0nAIuBu4GiwA/AWIcf5hXbl5oPKAdUs/X7HvgCuAu4B9gIPGvr15t/EkVDrI2MsXWXtH3J99umuQUYBRQAKgNHgRYOK28M1g8rHw4JzCH+mViJpyjWD/cg0M8hjljgRSA/0NX2ee528jPEAkOwNliFgSq2ZVEQKGNbISektKyTbEgcE8URoKpteiuB92z9fG0rfn3bshhn++ypJYpJtvHLYW2s69niSpjnl7Z5BGFtdH1s49UE6tg+U0VgHzDMYbqCtSG5O2F5A08CpWzjvIz1gyxk6/cK1jrlDRjb/Eo5TKuKw7RrYG0sH7HF/LRtmRV0WH7bsRJN4aTLFCvBPWX73wuok9JyTmEdLIq1gX4ZKGTrfiSV5foWiTeYfYHDWOumF/Ad8HWS+c7EWodSWj8bYa1Ho7DWwQFYOzWzbXH4AVFAZdvwM3AiUaQUq8M6lmKiSGs5pPK5i2KtUxOA7Q79zmDbWcD6Pdew/T8Wa8cwv+3VANvvPkkcpbGSfSfbcC/allFCosjQ7yy9eF2yXXXlxN3xsi3USKzsL8Bv/JOdDdYe30MOw9flnz3JL4CPUphmWayNj+ORR3dghe3/3vzzIzVYe1sNbd0DgOW2/x8BjieZ9qvAVw4rb7K9NIdhPWxx+Dq89yyw0iGO044rK1YyeMrJz3A8tXnbhmkHbEttBSblRDHSof9AYInt/1HAHId+RbD2GJMlCqykeRMISqFfwjzLJ/nM3VL5DMOAcIduAZqk87n/Tpg3cABom8pwSRPFZGBMkmEOAKEOy69vCutvwoZxNdZef+kkwyRazimsg90dv6d0PttbJN5g/gYMdOj2xkrgng7zrZzG9BrZvquEI9WitnEecRhmC9DO9v8MXJcoUl0OKU3LoV8JW8wJR1LHsX5nxZIMNxprp61KStNxGK4XsMGh22AddaUWd5q/s/TidcUrt56jaCciRbFWumpYGR2sbF0E2GKMuWyMuQwssb0P1p7dkRSm9yDWnsAZh/G+wNorT0Ssb24u1koK0APrUDJhOvcnTMM2ndewNuIJTqTxuUpj7X3/5fDeX1h72QlO2WJw7H+/k58h0byNMfcYY+YaY04ZY65iNYmVJmPOOvx/A2svFVtM9vmJyA2sQ++UlMbaI0zpu0lzPsaYqsaYH20XNlwF3iX5Z0j6uV82xuyzXQl0GavJLmGc1NaRlDwIvJzk+34A67OnOO8k+mEdje03xmwyxrRycr4ZiTGp+0m+fnni/DoKcFH+uaDhpu1vhEP/m/yzHmQa2xVjkbZXA5xcDsYYD2PMe8aYI7Z15JitV8J33hGr+ekvY8wqY0xd2/sfYh19/WKMOWqMGZHKLJKu6+LYndHfmRPxZrrcmigAEJFVWHss42xvXcBaSf1EpITtVVysE99gfXkPpTCpE1h746UdxismIn6pzHoO0MkY8yDWUcRCh+n86TCNEiJSVESecAw7jY90AWvv7kGH9ypgtZsmKGeMMUn6n3byMySd91jbe4EiUgyrScakMXxGnMFqGgSsyzSxmntScgGruSKl7yY9k7Hahh+2fYbXSPwZwOFz2DYw/wa6ACVFpARW813COKmtIyk5AbyT5PsuIiJzUpp3UiJySES6YyXz94EFxpi70hrnNmJM6jTJ169YEm/o7+R7T+o61s5bgnvTGDbN+Yp1xZiX7fU7zi+HHljnJh/D2imoaHvf2Ka7SUTaYn0P3wPzbO9fE5GXRaQy0Bp4yRjTNIXpn8FKWtZErd/nAw79M/o7SzNeV8jVicJmAtDMGBMsIvFYbdkfGWPuATDGlDPGtLANOw3oY4xpaozJZ+tXTUTOYJ2I/Y8xppit30PGmNCUZigi27DaZacCS0Xksq3XRuCqMebftuvXPYwx/saYEGc+iG0vbR7wjjGmqC0RvYS1B5LgHmCoMSa/MaYz4AP8lNHPYFMUqxnvsjGmHFb7vKMIrLbs27EAaG2MqWeMKYDVxJLiim773qYD440x99uWW11jTEEn5lMUq3040hhTDXjeieFjsb4/T2PMKKCYQ/+pwBhjzMPGEmiMSUhwSZfHl8BzxphHbMPeZYxpaYwp6kTcGGOeNMaUsX3+hHUozhZbPKkv+x+Be40xw4wxBW3ryiPOzBNrJ+dFY0wlY11W/i7wrdzGVVFO2g48YYy52xhzL1bTYGoigIrGGGe3W84uh6JYO1EXsZLWuwk9jDEFjHWPSnERicFal+Js/VoZY6rYNvwJ76d0afj/AD9jTAdjXSk2lMQJMaO/s1TjdZVcnyhE5DzWybc3bG/9G+twcYPtsG0ZVjssIrIR6IN15c8VrCsTEvauemE1++zFarNeANyXxqznYGX82Q6xxGHteQRjXfF0AWvDUzwDH2kI1l7YUawrf2ZjbUQT/AE8bJv2O0AnEUlo0snoZ/g/rBOyV7BW9u+S9B8LjLQ1qwzPwGdARPbYPstcrD2ua1gnfm+lMspwrJPIm7CuMHkf59bf4Vh7YNewNtzfpjP8UuBnrIsE/sI6knFsahmPlax/wdo4TMM6iQ5Wu/d/bcuji4hsxjpHNRFreR/GOpfgrDBgjzEmEutKrG4iEmVrpnsHWGubVx3HkUTkGtbJ0dZYTXKHgMZOznM61pU2q7HW0Sis78lVvgZ2YDWf/ELa30/CDYEXjTFb05twBpbDTKzv+hTWb2NDkv5PAcds24vnsPb4wfqdLcPayK8HPpMU7oERkQtAZ+A9rI37w1hXAibI6O8svXgzXcKVOSoXMMb0xjpBVt/dsWSUbe/1MlYT0Z/ujkcp9Y9cf0Shsi9jTGtjTBFbu/s4rCOGY+6NSimVlCYK5U5tsU6ensY6HO8meoirVLajTU9KKaXSpEcUSiml0pTjinqVLl1aKlas6O4wlFIqR9myZcsFESmT/pDJ5bhEUbFiRTZv3uzuMJRSKkcxxvyV/lAp06YnpZRSadJEoZRSKk2aKJRSSqVJE4VSSqk0aaJQSimVJk0USiml0uSyRGGMmW6MOWeM2Z1Kf2OM+cQYc9gYs9MYU8NVsSillLp9rryPYgZWeeWZqfR/HKu+z8NYD/eZbPurVLYmIkTFRrk7DKWyjMsShYisNsZUTGOQtsBMWxG4DcaYEsaY+2wP2FHZ3JQpMHt2+sNlpVanp/DYucwN6qZnHEdKXedgmescKn2dQ6UjOVTmOlcLueo5Pkplsgvc8UNS3XlndjkSPxDmpO29ZInCGPMM8AxAhQoVsiS4nMQdG+1Vq6y/oWk9Hy+LPXZuNlUit3PYKzjD48YjnC4exaHS1zloSwaHSl/nRImbiO25e0WiPahy4S6aHSzDfVcLkU/raaps7GZ0HKsPXeTg2et0rX0/czl929NyZ6JI6bGXKf70RGQKMAWgVq1aue7neacbendstENDoUcPeOaZrJtnuhoBBBO8cmWag12JusKuc7vYGbHT/tp1bheR0ZEAGAxV7q5CSNlQ+pUNJND2qliiIvmcfgqnUu4hIsybN48hQ4Zw+XI0o958k9dee425BZ15cnDK3JkoTpL4AePl4Q5SXg42ezZs3w7BGd8RBrLpRjs9rjgMSrIQ4+LjOHzp8D8J4Zz199jlY/ZhShQqQWDZQHoH9Sbo3iACywbiV8aPuwrclbmxKZUFRIRu3boxb948QkJCmDZtGgEBAXc8XXcmisXAYGPMXKyT2Ffy0vkJx+1kwvYtnR3h3OVOs2MSFz1j2NnoQXaGlmXnon7sPLeT3ed22086exgPvEt7U6d8HZ6p8Yz9KKF8sfIYk9LBrVI5h4hgjMEYQ0hICLVr1+aFF17A0zNzNvEue3CRMWYOVmNAaSACeBPIDyAinxvr1zkR6wHyN4A+tofRp6lWrVqSG6rHNmqUeDuZ444I7lSjRtbfDGbH6LhoDlw4kOwo4fS1fw5GyxQpYx0d3PNPs5FPGR8KeRbKvPiVyiaOHDnCgAEDeOGFF2jbtm2qwxljtohIrduZhyuveuqeTn8BBrlq/jlBnjuKyAARIeJ6RKLzCDsjdrL3/F5i4mMAKOBRAN8yvjxW+bFESaGsV1k3R6+U68XFxfHxxx8zcuRI8ufPz40bN1w2rxz3PIqcJrWm+ExsdXGPOz3H4LAAomKj2Ht+b7KkcP7Gefvg5YqWI7BsIGFVwuwJwbuUN/k98t/pJ1Eqx9m9ezf9+vVj48aNtG7dmsmTJ1OuXDmXzU8ThYul1hQfHGw1N+VYt3GOQRBOFLzFTq/r7Gxbkp2+19k5yZeDFw8SJ3EAFPYsjP89/rTxbmNPCAH3BFCqSClXfRKlcpxt27Zx9OhR5syZQ9euXV1+ns1l5yhcJbuco0hoK09Pn77W36+muzigrNa3j/V3+lepDhIZHcnuc7vZGbGTHRE72Bmxkyu3rtj7VypRyZ4MEl4PlXwIj3wero5eqRxn48aNHD16lG7duiEiXL58mZIlSzo9/p2co9BEcZuGLRnGx3987O4wcgSvAl5WInA4jxBQNoBiBYu5OzSlsr0bN24watQoPvroIx5++GF27959W1czZcuT2bndpZuXKFOkDJNbTk5zuFFvWn9H/18WBJXUm6Pgz2NQqaJrpl+/ATRrlmrvgp4F8Svjx4MlHtQb1ZS6DStWrKB///4cPXqUZ599lvfffz/TLnnNCE0Ud8CrgBcdfTsme9/xPO8pWzN+R98sDg7g/KfgVQbmr3TDzJVSd+LgwYM0bdqUypUrs2LFCholXFLuBrqb5wIJ53khF5y0VkplqX379gFQtWpV5s+fz86dO92aJECPKNK07/y+RJdoOtqyP4IzV/65b8xRltxp7czlqTn+Glyl8o7z588zdOhQ5s2bx+bNm6levTodOyZvsXAHTRSp+Pvm3/hP9ide4lMdJl9kYIrvZ8lRhDOXp+rhjFLZnogwZ84chg4dytWrV3nrrbfw8/Nzd1iJaKJIxY2YG8RLPC/VeYmWVVsC8OOP8NtvVv8jRyDg/qruvbNab+1WKkcTETp37szChQupU6cOU6dOzXZJAjRRpKta6Wo0qdQEgNEL4S/bTnytUtAjexwVKqVyGMcifvXq1aN+/foMGTIED4/seQ+RJooMcnon3tVPE9LzD0rlSIcOHWLAgAEMGzaMdu3a8dJLL7k7pHTpVU+p2H1u951NwPHSJ1fQ8w9K5SixsbGMGzeOwMBAtm/fTlRUznnuuh5RpKLLgi4AlCzs/C3yyeg5BKUUsHPnTvr168fmzZtp27Ytn332Gffff7+7w3KaJopUxMbH0qxyMy7+3pFGA6330m3tSelpREqpPG/Hjh0cP36cefPm0alTpxz3sCxtekpF/nz58S3jy5w5xvmb5/ROO6WUzfr165lt23F88sknOXDgAJ07d85xSQL0iMIpGWpB0uYmpfK069ev8/rrr/PJJ59QrVo1unTpgqenJyVKlHB3aLctTyaKpYeXcuraKXv3qlWwYUPiYa5VvcWCBRCpLUhKKSctW7aMAQMGcOzYMQYNGsTYsWPdUsQvs+X8T5BB125dI2xWWPIePsnfKhBd1vkWpClTrIwTGnrHMSqlcp6DBw/SvHlzqlSpwurVq2nQoIG7Q8o0eS5RJDxv+c3QN+lb3XqqUBfrAifmzftnOIOhfLHyON2cmHASW89LKJWn7N27F19fX6pWrcrChQsJCwujcOHC7g4rU+W5RJHg7sJ3U6F4BQAK3bLeq1D8DicaGgrPPHOHE1FK5QQREREMGTKEhQsX2ov4tW/f3t1huUSeTRR3LOmd13o5rFJ5gojwzTffMGzYMCIjIxkzZgz+/v7uDsul9PLY25X0zmu9HFapXE9E6NixI7169cLb25vt27fz2muvkT9/fneH5lJ6RHEn9FJYpfIExyJ+DRs2pFGjRgwaNCjbFvHLbHpEoZRSaThw4AANGzYkPDwcgGHDhjF06NA8kyRAE0XGTZliPdbOlQX/lFJuFxsby3vvvUdQUBB79uwhNjbW3SG5jTY9ZZTjk+X0nIRSudL27dvp168fW7dupWPHjkycOJF7773X3WG5TZ5LFOH7wu98InpuQqlcbc+ePZw6dYoFCxZkm+dWu1Oea3oa8dsIALxLebs5EqVUdrJu3TpmzZoFQI8ePTh48KAmCZs8lyg883nyTI1naFGlhbtDUUplA5GRkQwdOpT69evz7rvvEhsbizGGYsWKuTu0bCPPJQqllErwyy+/4O/vz8SJExk8eDAbNmzIFUX8MpsuEaVUnnTgwAHCwsKoWrUqv//+O48++qi7Q8q2XJoojDFhwMeABzBVRN5L0r8C8F+ghG2YESLykytjSpDqw+iSluZISkt1KJWj7dq1i4CAALy9vQkPD6dFixYUKlTI3WFlay5rejLGeACTgMcBX6C7McY3yWAjgXkiUh3oBnzmqniSSvVhdElLcySll8UqlSOdOXOGjh07EhQUxNatWwFo27atJgknuPKIojZwWESOAhhj5gJtgb0OwwiQcMaoOHDahfEkk+pVrnr5q1K5hojw3//+lxdffJGbN28yduxYAgMD3R1WjuLKRFEOOOHQfRJ4JMkwbwG/GGOGAHcBj6U0IWPMM8AzABUqVMj0QJVSuZOI0K5dOxYvXkz9+vWZOnUq3t56aXxGufKqp5Qe+SNJursDM0SkPPAE8LUxJllMIjJFRGqJSK0yZcq4IFS0NIdSuUh8fDwAxhiaNGnCpEmTWLVqlSaJ2+TKRHESeMChuzzJm5b6AfMARGQ9UAgo7cKYUqelOZTKFfbt20eDBg1YuHAhAC+88AIDBw4kXz69G+B2uXLJbQIeNsZUMsYUwDpZvTjJMMeBpgDGGB+sRHHehTGlLeHchD6lTqkcJyYmhnfffZfg4GD279+PSNIGDHW7XJYoRCQWGAwsBfZhXd20xxgz2hjTxjbYy8AAY8wOYA7QW9zx7U6ZAqtWZflslVKZY9u2bdSuXZvXX3+ddu3asXfvXjp16uTusHINl95HYbsn4qck741y+H8v4P67XBLum9AmJ6VypH379nH27FnCw8Np166du8PJdbTRLkFoqDY5KZWD/P7773z99dcAdO/enYMHD2qScBFNFEqpHOXatWsMGjSIhg0b8sEHHxAXF4cxhqJFi7o7tFwrzyaKVqenMGF7I70kVqkc5Oeff8bPz4/JkyczbNgwNmzYkKceSeouebYo4GPnZlMlcjsQrJfEKpUDHDhwgJYtW+Lj48O6deuoU6eOu0PKM/JsogA47BVMsJbqUCrbEhF27txJUFAQ3t7eLF68mGbNmlGwYEF3h5an5M2mp/37CL6il8MqlZ2dPn2aDh06UL16dXsRv1atWmmScIO8mSgOHQZg2T3a3KRUdiMiTJ8+HV9fX5YsWcL777+vRfzczKlEYYwpYIyp4upgstL24qH8eL9eDqtUdiIitG3bln79+hEUFMTOnTt55ZVX9KlzbpZuojDGtAR2Ab/auoONMeGuDkwplXc4FvFr1qwZkydPZsWKFTz88MNujkyBc0cUo7HKg18GEJHtQK46ulBKuc+ePXuoV68eCxYsAGDIkCE899xzWsQvG3Hmm4gRkctJ3tNqW0qpOxIdHc2YMWOoXr06hw8f1sSQjTnT8LfPGNMFyGeMqQS8AGxwbViuMfb3sZyNPAvc5+5QlMrTNm/eTN++fdm1axfdunXjk08+wWXPmlF3zJkUPhioCcQD3wFRWMkix1l+bDkAvc+WdXMkSuVthw8f5uLFiyxatIg5c+ZoksjmnEkULUTk3yJS3fYaATzu6sBcpd4D9ah7tbi7w1Aqz1m1ahUzZ84EoGvXrhw4cIA2bdqkM5bKDpxJFCNTeO/1zA5EKZU7Xb16leeff55GjRoxbtw4exE/Ly8vd4emnJTqOQpjTAsgDChnjBnv0KsYVjOUUkql6X//+x/PPfccp0+f5qWXXmLMmDFaxC8HSutk9jlgN9Y5iT0O718DRrgyKJc6GwGrjkDxUHdHolSuduDAAVq3bo2vry8LFizgkUcecXdI6jalmihEZBuwzRgzS0SisjAm17pgPZJby3colflEhO3bt1O9enW8vb354YcfaNasGQUKFHB3aOoOOHOOopwxZq4xZqcx5mDCy+WRuVKolu9QKrOdPHmStm3bUrNmTXsRv5YtW2qSyAWcSRQzgK8Ag3W10zxgrgtjUkrlIPHx8UyZMgU/Pz+WLVvGf/7zH4KCgtwdlspEztxwV0RElhpjxonIEWCkMeZ3VwemlMr+RITWrVvz008/0bhxY7788kseeughd4elMpkzieKWMcYAR4wxzwGngHtcG5ZSKjuLi4sjX758GGN4/PHHad++Pf369cPaVKjcxpmmpxcBL2Ao8CgwAOjryqCUUtnX7t27qVevHgsXLgRg8ODB9O/fX5NELpZuohCRP0TkmogcF5GnRKQN8FcWxKaUykaio6N56623qFGjBkePHtVnROQhaSYKY0yIMaadMaa0rdvPGDOTHFoUUCl1ezZt2kSNGjX4v//7P7p06cK+ffto166du8NSWSTVRGGMGQvMAnoCS4wxrwMrgB1A1awJTymVHRw5coQrV67w448/8s0331C6dGl3h6SyUFrHjm2BIBG5aYy5Gzht6z6QNaEppdxp+fLlHD9+nN69e9O1a1dat27NXXfd5e6wlBuk1fQUJSI3AUTkErBfk4RSud/ly5cZMGAATZs25aOPPrIX8dMkkXeldURR2Rjzne1/A1R06EZEOrg0MqVUllu8eDHPP/88Z8+e5V//+hdvvfWWFvFTaSaKjkm6J7oyEKWUex04cIB27drh7+/PokWLqFWrlrtDUtlEWkUBf8vKQLLCqZNQ4Dps3w7bgeBgd0eklHuJCFu3bqVmzZp4e3vz008/0aRJE63PpBLJU08zP3MW4uKs/4ODoYcWkFV52IkTJ2jdujUhISH2In5hYWGaJFQyLk0UxpgwY8wBY8xhY0yKz7AwxnQxxuw1xuwxxsx2ZTwAHh5Wkli5Ep7RArIqD4qPj+fzzz/Hz8+PFStW8NFHH2kRP5Ump2+tNMYUFJFbGRjeA5gENANOApuMMYtFZK/DMA8DrwKPisjfxhitIaWUC4kILVu2ZMmSJTz22GNMmTKFSpUquTsslc2le0RhjKltjNkFHLJ1BxljPnVi2rWBwyJyVESisUqTt00yzABgkoj8DSAi5zIUvVLKKXFxcYgIxhhat27NtGnT+OWXXzRJKKc40/T0CdAKuAggIjuAxk6MVw444dB90vaeo6pAVWPMWmPMBmNMmBPTVUplwI4dO6hduzYLFiwAYODAgfTt21eL+CmnOZMo8olI0iKAcU6Ml9JaKEm6PYGHgUZAd2CqMaZEsgkZ84wxZrMxZvP58+edmLVS6tatW7zxxhvUqlWLEydO6ElqdducSRQnjDG1ATHGeBhjhgHOPAr1JPCAQ3d5rDIgSYdZJCIxIvIncAArcSQiIlNEpJaI1CpTpowTs1Yqb/vjjz+oXr06b7/9Nt27d2ffvn20bZu05Vcp5ziTKJ4HXgIqABFAHdt76dkEPGyMqWSMKQB0AxYnGeZ7bM1Ytgq1VYGjzoWulErNX3/9xfXr1/n555+ZOXMmpUqVcndIKgdz5qqnWBHpltEJi0isMWYwsBTwAKaLyB5jzGhgs4gstvVrbozZi9Wc9YqIXMzovJRSsGzZMo4fP07fvn3p3LkzrVq1okiRIu4OS+UCziSKTcaYA8C3wHcics3ZiYvIT8BPSd4b5fC/YB2tvOTsNJVSif39998MHz6c6dOnExQUxNNPP42Hh4cmCZVpnHnC3UPA20BNYJcx5ntjTIaPMJRSmS88PBxfX1/++9//MmLECDaHuySVAAAgAElEQVRs2KBF/FSmc+rObBFZJyJDgRrAVawHGiml3Gj//v107NiRe++9l40bNzJ27FgKFSrk7rBULuTMDXdexpiexpgfgI3AeaCeyyNTSiUjImzatAmAatWqsWTJEjZu3EiNGjXcHJnKzZw5otiNdaXTByJSRUReFpE/XByXUiqJ48eP88QTT/DII4/Yi/g1b96c/Pnzuzkylds5czK7sojEuzwSpVSK4uPjmTx5MiNGjEBE+OSTTwjWGvkqC6WaKIwx/xGRl4GFxpikd1TrE+6UygIiQlhYGL/++ivNmzfniy++oGLFiu4OS+UxaR1RfGv7q0+2UyqLxcbG4uHhgTGG9u3b07NnT3r16qX1mZRbpHqOQkQ22v71EZHfHF+AT9aEp1Tes23bNmrXrs38+fMBeP7553n66ac1SSi3ceZkdt8U3uuX2YEolddFRUXx+uuvExISwunTp/WGOZVtpHWOoitWfaZKxpjvHHoVBS67OjCl8pL169fTp08fDhw4QO/evfnPf/7D3Xff7e6wlALSPkexEesZFOWxnlSX4BqwzZVBKZXXnDhxgqioKJYuXUrz5s3dHY5SiaSaKGxlv/8ElmVdOErlHUuXLuXkyZP069ePzp0707p1awoXLuzusJRKJtVzFMaYVba/fxtjLjm8/jbGXMq6EJXKXS5dukTv3r0JCwtj0qRJxMXFYYzRJKGyrbROZic87rQ0UMbhldCtlMqghQsX4uvryzfffMPrr7/OunXrtIifyvbSanpKuBv7AeC0iEQbY+oDgcA3WMUBlVJO2r9/P507d6Z69eosWbJE765WOYYzl8d+j/UY1IeAmVj3UMx2aVRK5RIiwoYNGwCriN+vv/7KH3/8oUlC5SjOJIp4EYkBOgATRGQIUM61YSmV8x07dowWLVpQt25dexG/pk2b4unpTIk1pbIPZxJFrDGmM/AU8KPtPS1XqVQq4uLi+OSTT/D392f9+vVMmjRJjyBUjubMrk1fYCBWmfGjxphKwBzXhqVUzpRQxG/ZsmWEhYXxxRdfUKFCBXeHpdQdSTdRiMhuY8xQoIoxphpwWETecX1oSuUcjkX8OnXqRK9evXjyySe1PpPKFZx5wl0D4DAwDZgOHDTGPOrqwJTKKbZu3UqtWrWYN28eAM8++yxPPfWUJgmVazhzjuIj4AkReVRE6gEtgY9dG5ZS2d/NmzcZMWIEtWvXJiIiAi8vL3eHpJRLOHOOooCI7E3oEJF9xpgCLoxJqWxv7dq19OnTh0OHDtGvXz8+/PBDSpYs6e6wlHIJZxLFVmPMF8DXtu6eaFFAlcedOXOG2NhYli1bRtOmTd0djlIu5UzT03PAEeBfwL+Bo8CzrgxKqezo559/5ssvvwSgY8eO7N27V5OEyhPSTBTGmAAgDAgXkTYi0lpEPhSRqKwJTyn3u3jxIr169eKJJ57giy++sBfxK1SokLtDUypLpFU99jWs8h09gV+NMSk96U6pXEtEmDdvHj4+PsyZM4dRo0axdu1aLeKn8py0zlH0BAJF5LoxpgzwE9blsUrlCfv376dbt27UrFmTZcuWERgY6O6QlHKLtJqebonIdQAROZ/OsErlCiLC+vXrAfDx8WHZsmWsX79ek4TK09La+Fc2xnxne4UDDzl0f5fGeErlSEePHqVZs2bUq1fPXsSvSZMmWsRP5Xlp/QI6Jume6MpAlHKXhCJ+I0eOxMPDg8mTJ2sRP6UcpPXgot+yMhCl3EFEaN68OcuXL+eJJ57g888/54EHHnB3WEplK3pMrfKkmJgYPD09McbQrVs3+vbtS48ePbQ+k1IpcOkJamNMmDHmgDHmsDFmRBrDdTLGiDGmlivjUQpg06ZN1KxZk2+//RaAAQMG0LNnT00SSqXC6URhjCmYkQkbYzyAScDjgC/Q3Rjjm8JwRYGhwB8Zmb5SGXXjxg1eeeUV6tSpw6VLlyhRooS7Q1IqR3CmzHhtY8wu4JCtO8gY86kT066N9eyKoyISDcwF2qYw3BjgA0Dv9lYus2bNGoKCghg3bhz9+/dnz549hIWFuTsspXIEZ44oPgFaARcBRGQH0NiJ8coBJxy6T5LkWdvGmOrAAyLyI2kwxjxjjNlsjNl8/vx5J2atVGIRERGICMuXL+eLL76gePHi7g5JqRzDmUSRT0T+SvJenBPjpdTgK/aexuTDetbFy+lNSESmiEgtEalVpkwZJ2atFPzvf/9jypQpgFXEb8+ePTRu7Mw+jlLKkTOJ4oQxpjYgxhgPY8ww4KAT450EHK8zLA+cduguCvgDK40xx4A6wGI9oa3u1Pnz5+nZsyetWrVi6tSpxMVZ+zUFC2boNJtSysaZRPE88BJQAYjA2qA/78R4m4CHjTGVbA866gYsTugpIldEpLSIVBSRisAGoI2IbM7gZ1AKsO6JmDt3Lr6+vsyfP5+33nqLNWvWaBE/pe5QuvdRiMg5rI18hohIrDFmMLAU8ACmi8geY8xoYLOILE57CkplzP79++nRowchISFMmzYNf39/d4ekVK6QbqIwxnyJw7mFBCLyTHrjishPWFVnHd8blcqwjdKbnlJJxcfHs27dOurXr4+Pjw/Lly+nQYMGehShVCZypulpGfCb7bUWuAe45cqglHLG4cOHadq0KQ0aNLAX8WvUqJEmCaUymTNNT986dhtjvgZ+dVlESqUjLi6Ojz76iDfeeIMCBQrw5ZdfUr16dXeHpVSudTu1nioBD2Z2IEo5Q0Ro1qwZK1asoE2bNnz22WeUK1cu/RGVUrfNmXMUf/PPOYp8wCUg1bpNSrmCYxG/Hj168Oyzz9KlSxetz6RUFkjzHIWxfoVBQBnbq6SIVBaReVkRnFIAGzdupHr16syZMweA/v3707VrV00SSmWRNBOFiAgQLiJxtleyq5+UcpUbN27w8ssvU7duXa5cuUKpUqXcHZJSeZIzVz1tNMbUcHkkSjlYvXo1AQEBjB8/nmeeeYY9e/bQokULd4elVJ6U6jkKY4yniMQC9YEBxpgjwHWsGk4iIpo8lMtcuHCBfPnysXLlSkJDQ90djlJ5WlonszcCNYB2WRSLyuMWL17M6dOnee655+jQoQMtW7bU+kxKZQNpNT0ZABE5ktIri+JTecC5c+fo1q0bbdu25auvvtIifkplM2kdUZQxxryUWk8RGe+CeFQeIiLMmjWLF154gcjISMaMGcO///1vvbNaqWwmrUThAXiR8nMllLpj+/fvp1evXtSpU4epU6fi65vsSblKqWwgrURxRkRGZ1kkKk+Ij49nzZo1NGzYEB8fH1auXMmjjz6qRxFKZWPpnqNQKrMcOnSIxo0bExoaai/i17BhQ00SSmVzaSWKplkWhcrVYmNj+eCDDwgMDGTHjh1MmzZNi/gplYOk2vQkIpeyMhCVO4kIjz32GKtWraJdu3ZMmjSJ+++/391hKaUywJk7s5XKsOjoaEQEYwy9evVi3rx5fPfdd5oklMqBNFGoTLd+/XqCg4OZPXs2AH379qVz585axE+pHEoThco0kZGRDBs2jEcffZTr169TtmxZd4eklMoEt/PgIqWSWblyJX369OHYsWMMHjyYd999l6JFi7o7LKVUJtBEoTLF5cuXKViwIL///jv169d3dzhKqUykiULdtvDwcM6ePcvzzz9Pu3bteOKJJyhQoIC7w1JKZTI9R6EyLCIigs6dO9OhQwdmzpxpL+KnSUKp3EkThXKaiDBz5kx8fHxYvHgx77zzDqtXr9Y7q5XK5bTpSTlt//799OnThzp16jBt2jSqVavmkvnExMRw8uRJoqKiXDJ9pXKzQoUKUb58efLnz59p09REodIUHx/PqlWraNy4MT4+PqxatYp69eqRL5/rDkZPnjxJ0aJFqVixot57oVQGiAgXL17k5MmTVKpUKdOmq01PKlUHDhwgNDSUJk2a2Iv41a9f36VJAiAqKopSpUppklAqg4wxlCpVKtOPxjVRqGRiYmIYO3YsQUFB7NmzhxkzZmR5ET9NEkrdHlf8drTpSSUiIjRt2pTff/+dTp068emnn3Lvvfe6OyyllBvpEYUC4NatW/Yifn369GHhwoXMnz8/zyYJDw8PgoOD8ff3p3Xr1ly+fNneb8+ePTRp0oSqVavy8MMPM2bMGETE3v/nn3+mVq1a+Pj4UK1aNYYPH+6Oj3BbunfvTmBgIB999JFTw3t5ebkkjv3791O3bl0KFizIuHHjUh1ORGjSpAlXr151SRyZYcuWLQQEBFClShWGDh2aaF1J8Pfff9O+fXsCAwOpXbs2u3fvtvf76KOP8PPzw9/fn+7du9ublbp168ahQ4ey5kOISI561axZU25XiaGPSdAzxURCQ297GrnRmjVrxNvbW77++mt3hyIiInv37nV3CHLXXXfZ/+/Vq5e8/fbbIiJy48YNqVy5sixdulRERK5fvy5hYWEyceJEERHZtWuXVK5cWfbt2yciIjExMTJp0qRMjS0mJiZTp5fgzJkzUqFChQyN47icMlNERIRs3LhRXnvtNfnwww9THe7HH3+UYcOGZWjasbGxdxpehoSEhMi6deskPj5ewsLC5Keffko2zPDhw+Wtt94SEZF9+/ZJkyZNRETk5MmTUrFiRblx44aIiHTu3Fm++uorERFZuXKl9O/fP8V5pvQbAjbLbW539YgiD7t27RpDhgyhQYMGREVFcd9997k7pGSGDYNGjTL3NWxYxmKoW7cup06dAmD27Nk8+uijNG/eHIAiRYowceJE3nvvPQA++OADXn/9dfulw56engwcODDZNCMjI+nTpw8BAQEEBgaycOFCIPEe+oIFC+jduzcAvXv35qWXXqJx48a88sorVKxYMdFRTpUqVYiIiOD8+fN07NiRkJAQQkJCWLt2bbJ5R0VF2eddvXp1VqxYAUDz5s05d+4cwcHB/P7774nGiYiIoH379gQFBREUFMS6deuSfZ6mTZtSo0YNAgICWLRoEQDXr1+nZcuWBAUF4e/vz7fffgvAiBEj8PX1JTAwMMUjrnvuuYeQkJB0L/GcNWsWbdu2tXe3a9eOmjVr4ufnx5QpU+zve3l5MWrUKB555BHWr1/Pli1bCA0NpWbNmrRo0YIzZ84A8OWXXxISEkJQUBAdO3bkxo0bac4/PWfOnOHq1avUrVvXXnL/+++/Tzbc3r17adrUelZctWrVOHbsGBEREYD14K+bN28SGxvLjRs37KX6GzRowLJly4iNjb2jGJ3h0nMUxpgw4GPAA5gqIu8l6f8S0B+IBc4DfUXkL1fGpCzLly+nT58+nDhxgiFDhvDOO++4rBkhJ4uLi+O3336jX79+gNXsVLNmzUTDPPTQQ0RGRnL16lV2797Nyy+/nO50x4wZQ/Hixdm1axdgNT2k5+DBgyxbtgwPDw/i4+MJDw+nT58+/PHHH1SsWJGyZcvSo0cPXnzxRerXr8/x48dp0aIF+/btSzSdSZMmAbBr1y72799P8+bNOXjwIIsXL6ZVq1Zs37492byHDh1KaGgo4eHhxMXFERkZmah/oUKFCA8Pp1ixYly4cIE6derQpk0blixZwv3338///vc/AK5cucKlS5cIDw9n//79GGMSJbyMWrt2LV988YW9e/r06dx9993cvHmTkJAQOnbsSKlSpbh+/Tr+/v6MHj2amJgYQkNDWbRoEWXKlOHbb7/l9ddfZ/r06XTo0IEBAwYAMHLkSKZNm8aQIUMSzXPFihW8+OKLyWIpUqRIsgR66tQpypcvb+8uX768fafDUVBQEN999x3169dn48aN/PXXX5w8eZKaNWsyfPhwKlSoQOHChWnevLl9JyVfvnxUqVKFHTt2JFsnM5vLEoUxxgOYBDQDTgKbjDGLRWSvw2DbgFoicsMY8zzwAdDVVTGpf1y9epUiRYqwZs0a6tWr5+5wUjVhgnvme/PmTYKDgzl27Bg1a9akWbNmAPbzOCnJyNUmy5YtY+7cufbukiVLpjtO586d7XfBd+3aldGjR9OnTx/mzp1L165d7dPdu/efn9jVq1e5du1aokq+a9assW/8qlWrxoMPPsjBgwcpVqxYqvNevnw5M2fOBKzzN8WLF0/UX0R47bXXWL16Nfny5ePUqVNEREQQEBDA8OHD+fe//02rVq1o0KABsbGxFCpUiP79+9OyZUtatWqV7mdPzaVLlxJ9tk8++YTw8HAATpw4waFDhyhVqhQeHh507NgRsC773r17t/07jYuLsx9N7969m5EjR3L58mUiIyNp0aJFsnk2btw4xWSaEknhfERK68mIESN44YUXCA4Oth/peXp68vfff7No0SL+/PNPSpQoQefOnfnmm2948sknAevI6/Tp0zk3UQC1gcMichTAGDMXaAvY12IRWeEw/AbgSRfGk+ctWLCAiIgIBg0aRLt27WjZsmWm3r2ZmxQuXJjt27dz5coVWrVqxaRJkxg6dCh+fn6sXr060bBHjx7Fy8uLokWL4ufnx5YtWwgKCkpz+qklHMf3kl4Lf9ddd9n/r1u3LocPH+b8+fN8//33jBw5ErBukFy/fj2FCxdOc96ZbdasWZw/f54tW7aQP39+KlasSFRUFFWrVmXLli389NNPvPrqqzRv3pxRo0axceNGfvvtN+bOncvEiRNZvnz5bc3X09OT+Ph48uXLx8qVK1m2bBnr16+nSJEiNGrUyL4MCxUqZE+yIoKfnx/r169PNr3evXvz/fffExQUxIwZM1i5cmWyYTJyRFG+fHlOnjxp7z558mSKT3ksVqwYX331lT2+SpUqUalSJZYuXUqlSpUoU6YMAB06dGDdunX2RBEVFZXmd51ZXHmOohxwwqH7pO291PQDfk6phzHmGWPMZmPM5vPnz2diiHnDmTNn6NChA507d2b27Nn2In6aJNJXvHhxPvnkE8aNG0dMTAw9e/ZkzZo1LFu2DLCOPIYOHcq//vUvAF555RXeffddDh48CFgb7vHjxyebbvPmzZk4caK9O6HpqWzZsuzbt8/etJQaYwzt27fnpZdewsfHh1KlSqU43ZT2fBs2bMisWbMAqznr+PHjeHt7p7kcmjZtyuTJkwFrDzzpVUZXrlzhnnvuIX/+/KxYsYK//rJakE+fPk2RIkV48sknGT58OFu3biUyMpIrV67wxBNPMGHCBKf3zlPi7e3N0aNH7TGULFmSIkWKsH//fjZs2JDqOOfPn7cnipiYGPbs2QNY5+3uu+8+YmJi7MsoqYQjiqSvpEkC4L777qNo0aJs2LDBXivN8ZxKgsuXLxMdHQ3A1KlTadiwIcWKFaNChQps2LCBGzduICL89ttv+Pj42Mc7ePAgfn5+GVhit+l2z4Kn9wI6Y52XSOh+Cvg0lWGfxDqiKJjedPWqJ+fFx8fL9OnTpUSJElKoUCF5//33XXbFTGbKblc9iYi0atVKZs6cKSIiO3fulNDQUKlatao89NBD8tZbb0l8fLx92B9++EFq1Kgh1apVEx8fHxk+fHiy6V+7dk169eolfn5+EhgYKAsXLhQRkfnz50vlypUlNDRUBg0aJE8//bSIiDz99NMyf/78RNPYtGmTADJjxgz7e+fPn5cuXbpIQECA+Pj4yLPPPpts3jdv3pSnn35a/P39JTg4WJYvXy4iIn/++af4+fmluDzOnj0rbdq0EX9/fwkKCpJ169YlWk7nz5+XOnXqSM2aNaVfv35SrVo1+fPPP2XJkiUSEBAgQUFBUqtWLdm0aZOcPn1aQkJCJCAgQPz9/RPFn+DMmTNSrlw5KVq0qBQvXlzKlSsnV65cSTbc6NGj5csvvxQRkaioKAkLC5OAgADp1KmThIaGyooVKxLFmWDbtm3SoEEDCQwMFF9fX5kyZYqIiHz22WdSsWJFCQ0NlcGDB9uX/53YtGmT+Pn5SeXKlWXQoEH2dWXy5MkyefJkERFZt26dVKlSRby9vaV9+/Zy6dIl+/ijRo0Sb29v8fPzkyeffFKioqJExPpOQkJCUpxnZl/15MpEURdY6tD9KvBqCsM9BuwD7nFmupoonLdnzx7Jly+fNGjQQA4cOODucJyWHRKFyhlOnz4tjz32mLvDcIvx48fL1KlTU+yXky6P3QQ8bIypZIwpAHQDFjsOYIypDnwBtBGRcy6MJc+Ii4uzt/f6+vqyZs0aVq5cSdWqVd0cmVKZ77777mPAgAHZ+oY7VylRogRPP/10lszLZYlCRGKBwcBSrCOGeSKyxxgz2hjTxjbYh4AXMN8Ys90YsziVySkn7Nu3j4YNG9K0aVO2bdsGWCc9XV3ETyl36tKlS5pXbOVWffr0wdMza6owuXQuIvIT8FOS90Y5/P+YK+efV8TExPDBBx8wevRovLy8+PrrrwkODnZ3WEqpXEKLAuZwIlatmzVr1tClSxc+/fRT7rnnHneHpZTKRbRNIoeKioqyX4vfv39/wsPD+fbbbzVJKKUynSaKHGj16tUEBgbyzTffAPD000/Trl07N0ellMqtNFHkIFevXmXgwIGEhoYSGxvLAw884O6Qci0tM+7eMuOzZs0iMDCQwMBA6tWrx44dO1IcLqHpNTtf9XQnZcYPHDhAcHCw/VWsWDEm2OraDB8+/LbvaM+w272u1l2vvHofxa+//ioPPPCAGGPkxRdflMjISHeH5DLZ4T4KLTPuHFeVGV+7dq39prOffvpJateuneJwub3MuKPY2FgpW7asHDt2TEREjh07Js2aNUtxnjnpPgqViW7cuEHRokVZt24d48ePT1T3J1fLBnXGtcx41pcZr1evnr1QYp06dRLVS3KUF8qMJ/jtt9946KGHePDBBwF48MEHuXjxImfPnr2jGJ2hVz1lUyLC/PnziYiIYMiQIbRp04Ynnngiy66bVhYtM25xZ5nxadOm8fjjj6fYL7eXGS9btqx9mLlz59K9e/dE49WoUYO1a9faK+O6im51sqHTp08zcOBAFi1axKOPPsrAgQPx8PDIm0nCTXXGtcx4Yu4qM75ixQqmTZvGmjVrUuyf28uMJ4iOjmbx4sWMHTs20XgJZcZdLQ9uebIvEWH69Om8/PLL3Lp1iw8//JBhw4bZNw4q62iZ8YxxRZnxnTt30r9/f37++Wd7ddykcnuZ8QQ///wzNWrUSHSEAbmjzLjKoH379vHMM88QHBzMrl27GD58eN48ishGtMy4JavLjB8/fpwOHTrw9ddfp1mnLLeXGU8wZ86cZM1OYH1//v7+qS6fTHO7Z8Hd9cptVz3FxsbKL7/8Yu9ev369xMXFuTEi98tuVz2JaJnxrC4z3q9fPylRooQEBQVJUFCQpPa7zwtlxq9fvy533323XL58OdF0o6OjpVq1aileBZfZVz0ZccFhqCvVqlVLNm/efFvjlnyhGQ9GbWT7geqQwiFlVtuzZw/9+vXjjz/+YOvWrVSvXt3dIWUL+/btS/RwFqVSc+bMGXr16sWvv/7q7lCyXHh4OFu3bmXMmDHJ+qX0GzLGbBGRWrczL216coPo6GhGjx5N9erVOXLkCLNnz9YifkrdhrxcZjw2NtapK+wygzaAZzERoVGjRqxfv54ePXowYcIE+/NwlVIZ16VLF3eH4BadO3fOsnnpEUUWcSzi99xzz7F48WJmzZqlSUIple1posgCK1euxN/fn6+//hqAXr160bp1azdHpZRSztFE4UJXrlzh2WefpXHjxgD2W++VUionyVOJosytU9wVmzUnvX755Rf8/PyYOnUqw4cPZ+fOnYSGhmbJvJVSKjPlqURxd7RV+IsePVw+r6ioKEqWLMn69ev58MMPKVKkiMvnqTKPlhl3b5nxRYsWERgYSHBwMLVq1Uq1hMfNmzcJDQ0lLi7OJXFkhiVLluDt7U2VKlXsxSOT+uuvv2jatCmBgYE0atQo0d3c//rXv/Dz88PHxydRmfLHHnvMqRphmeJ2b8Bw1+tObrh7pE8J64Y7F4iPj5fZs2fLhAkT7O+5qhx0bpfdbrjTMuOpc1WZ8WvXrtlvTNuxY4d4e3unONzEiRMT/ebSEx8fn6U3tMbGxkrlypXlyJEjcuvWLQkMDJQ9e/YkG65Tp072Gw9/++03efLJJ0XEKrder149iY2NldjYWKlTp479JsIZM2bY18ukMvuGO708NhOcPHmS559/nh9//JGGDRsyZMgQ8uXLp+U3MsGwJcPYfta5AmzOCr43mAlhzhcbrFu3Ljt37gRSLzPeqFEjBg0alKEy40OGDGHz5s0YY3jzzTfp2LEjXl5e9sqsCxYs4Mcff2TGjBn07t2bu+++m23bthEcHEx4eDjbt2+nRIkSgFVmfO3ateTLl4/nnnuO48ePAzBhwgQeffTRRPOOiori+eefZ/PmzXh6ejJ+/HgaN26cqMz4p59+SoMGDezjRERE8Nxzz9nLZUyePJl69eol+jxt27bl77//JiYmhrfffpu2bdty/fp1unTpwsmTJ4mLi+ONN96ga9eujBgxgsWLF+Pp6Unz5s0ZN25cohgdj1SuX7+easHFWbNmMXv27DRjOHbsGI8//jiNGzdm/fr1fP/99xw4cIA333yTW7du8dBDD/HVV1/h5eXF6NGj+eGHH7h58yb16tXjiy++yFCxx6Q2btxIlSpVqFy5MgDdunVj0aJF+Pr6Jhpu79699qO4xo0b259YaYwhKiqK6OhoRISYmBh7vac2bdrQoEEDXn/99duOz1m6JbsD8fHxfPnll7zyyivExsYyfvx4hg4dSr58eapFL1fTMuMWd5QZDw8P59VXX+XcuXP28R1FR0dz9OhRKlasmGYMYFWM/eqrr/jss8+4cOECb7/9NsuWLeOuu+7i/fffZ/z48YwaNYrBgwczatQoAJ566il+/PHHZFcozpo1iw8//DBZPFWqVGHBggWJ3jt16lSiJ1GWL1+eP/74I9m4QUFBLFy4kBdeeIHw8HCuXbvGxYsXqVu3Lo0bN+a+++5DRBg8eLD9juuSJUty69YtLl68mLFSuFsAAA2sSURBVGrRxMyiieIO7N+/n4EDB9KoUSO+/PJL+16DyjwZ2fPPTFpmPDF3lBlv37497du3Z/Xq1bzxxhv2QowJLly4YD+iSisGsK44rFOnDgAbNmxg79699iOt6Oho6tatC1iVYT/44ANu3LjBpUuX8PPzS5YoevbsSc+ePVNdVkmXS1IprSfjxo1j8ODBzJgxg4YNG1KuXDk8PT05fPgw+/bts5+zaNasGatXr6Zhw4bAP2XGXZ0odNc3g2JjY1m6dCkAvr6+bNiwgWXLlmmSyGUSyoz/9ddfREdH2/fC/fz8SFprLKUy4+lJLeHcbpnxDh06AP+UGU+oaHrq1KlESSJh3pnNscz49u3bKVu2bKIy4wEBAbz66quMHj0aT09PNm7cSMeOHfn+++8JCwtLc9oNGzbkyJEjXLhwIdH7hQsXTrSMUosBEi87EaFZs2b2ZbR3716mTZtGVFQUAwcOZMGCBezatYsBAwYk+w4S5uP4HOuEV6dOnZINW758eU6cOGHvTq3M+P333893333Htm3beOeddwCrcnF4eDh16tTBy8sLLy8vHn/88URVcbXMeDa0a9cu6tWrR1hYGNu2bQMgJCTkjtowVfamZcYtWV1m/PDhw/aEtnXrVqKjo5PtNZcsWZK4uDj7xjy1GJKqU6cOa9eu5fDhw4D1mOGDBw/ap1O6dGkiIyOTNSMl6NmzZ4plxlMaPiQkhEOHDvHnn38SHR3N3Llz7c1hji5cuEB8fDwAY8eOpW/fvgBUqFCBVatWERsbS0xMDKtWrbI3PYkIZ8+etTe9uZImCifcunWLN998kxo1anDs2DHmzp2rRfzykOrVqxMUFMTcuXMpXLgwixYt4u2338bb25uAgABCQkIYPHgwAIGBgUyYMIHu3bvj4+ODv7+//XnMjkaOHMnff/+Nv78/QUFB9udWv/fee7Rq1YomTZrYn7qWmq5du/LNN9/Ym53AesLb5s2bCQwMxNfXl88//zzZeAMHDiQuLo6AgAC6du3KjBkzKFiwYJrz+vjjj1mxYgUBAQHUrFnT/vyGBD179mTz5s3UqlWLWbNm2U/m79q1i9q1axMcHMw777zDyJEjuXbtGq1atSIwMJDQ0NAUL8VduHAh/v7+BAcHM2jQIL799tsUd8iaN29uv3Q2tRiSKlOmDDNmzLBfClynTh32799PiRIlGDBgAAEBAbRr146QkJA0l4kzPD09mThxIi1atMDHx4cuXbrg5+cH/H979x8jRX3Gcfz9qaCnVWnthUY9K4hCexwKlsq1JrX0qFEQbc1FMJ5KFYwUexFrYxsba3/8YbWmwapFas0hEUs1Wk7qj8oV1BBORRREopUikWsNnFcgrWBFePrH93u4Hnu7c+ft7N3u80o22Z2ZnXme3bv57nxn5vnCTTfdRHNzMxCqN4waNYqRI0eybdu2Ayeo6+vrGTFiBGPGjDkwXnlnV9hLL71EbW1tOhfN9PZyqWI90r48dv/+/VZbW2uANTQ0WHt7e6+375LpD5fHuoFh7dq1By4lLTeNjY22fPnyrPP6+vJYP6Loxu7duw/0I8+ZM4dly5axaNEiKisrix2acy4aN24cEydO7Nc33BVKTU0NdXV1qWzLG4osWlpaqKmpYeHChQA0NDQwZcqUIkflnMvmiiuuKMtx5WfNmpXatryhyLBz505mzpzJpEmTGDRoECNGjCh2SGXLCnBljnPloBD/O95QRE8++STV1dU0NTVxww03sG7duo/dmerSU1FRQUdHhzcWzvWQmdHR0UFFRUWfrtdvuIv27t3L0KFDeeyxxw6689alq6qqira2Ntrb24sdinMDTkVFBVVVVX26zrJtKMyMxYsXs337dubOncvUqVOZPHlyWfZ19jeDBw9m+PDhxQ7DORcVtOtJ0jmS3pC0SdKPssw/TNKSOP95ScMKGU+nrVu3ct5559HQ0MDSpUsP3OjijYRzzh2sYA2FpEOAu4BzgWrgYknVXRa7EthhZicDvwF+Vah4IBxFvLvxA0aPHs3KlSuZN28eLS0tXsTPOedyKOQe8gxgk5ltNrMPgD8CF3RZ5gJgYXz+MFCnAtbD2LNrP/9c9T4TJkxgw4YNNDY2+lGEc87lUchzFMcDWzNetwETulvGzD6UtAv4HPCx6l+SrgKuii//K+mNTxBX5fLly98t4yJ+lXT5fMtMOedfzrmD55+7oFcOhWwosh0ZdL3eMckymNkCYEGfBCWtMbPxfbGugcjzL9/8yzl38Pwlrcm/VHaF7HpqA07IeF0F/Ku7ZSQNAoYA/y5gTM4553qokA3Fi8ApkoZLOhSYDjR3WaYZuDw+rwf+Zn6XlXPO9SsF63qK5xyuAZ4CDgHuM7PXJP2cUMWwGfgDsEjSJsKRxPRCxZOhT7qwBjDPv3yVc+7g+fc6f/kPeOecc7n4DQTOOedy8obCOedcTiXbUPTX8iFpSJD7dZI2SlovqUXSicWIs1Dy5Z+xXL0kk1RSl0wmyV/SRfFv4DVJi9OOsZAS/P1/QdIKSS/H/4HJxYizECTdJ2m7pA3dzJekO+Jns17S6YlW3Nuh8frzg3Dy/B/AScChwDqgussy3wPmx+fTgSXFjjvF3CcCR8Tns0sl96T5x+WOAp4FWoHxxY475e//FOBl4LPx9dBix51y/guA2fF5NbCl2HH3Yf5fB04HNnQzfzLwBOEetlrg+STrLdUjin5XPiRFeXM3sxVmtju+bCXc41Iqknz3AL8AbgXeTzO4FCTJfxZwl5ntADCz7SnHWEhJ8jfg6Ph8CAff3zVgmdmz5L4X7QLgfgtagc9IOjbfeku1ochWPuT47pYxsw+BzvIhA12S3DNdSfiFUSry5i9pHHCCmS1LM7CUJPn+RwIjJa2S1CrpnNSiK7wk+d8MNEhqAx4Hvp9OaP1CT/cPQOmOR9Fn5UMGoMR5SWoAxgNnFTSidOXMX9KnCJWKZ6QVUMqSfP+DCN1P3yAcTT4nqcbMdhY4tjQkyf9ioMnMbpf0VcK9XDVmtr/w4RVdr/Z7pXpEUc7lQ5LkjqRJwI3A+Wb2v5RiS0O+/I8CaoCVkrYQ+mmbS+iEdtK//aVmttfM3gLeIDQcpSBJ/lcCfwIws9VABaFgYDlItH/oqlQbinIuH5I399j1cg+hkSil/mnIk7+Z7TKzSjMbZmbDCOdozjezXhdM62eS/O3/mXBBA5IqCV1Rm1ONsnCS5P82UAcg6UuEhqJcxt1tBi6LVz/VArvM7J18byrJrifrv+VDCi5h7rcBRwIPxfP3b5vZ+UULug8lzL9kJcz/KeBsSRuBfcAPzayjeFH3nYT5/wD4vaS5hG6XGSXyIxFJDxK6FCvjOZifAoMBzGw+4ZzMZGATsBv4bqL1lsjn45xzrkBKtevJOedcH/GGwjnnXE7eUDjnnMvJGwrnnHM5eUPhnHMuJ28oXL8jaZ+kVzIew3IsO6y7Spk93ObKWHF0XSxtMaoX67ha0mXx+QxJx2XMu1dSdR/H+aKksQnec62kIz7ptl358obC9Ud7zGxsxmNLStu9xMxOIxSLvK2nbzaz+WZ2f3w5AzguY95MM9vYJ1F+FOfdJIvzWsAbCtdr3lC4ASEeOTwnaW18fC3LMqMlvRCPQtZLOiVOb8iYfo+kQ/Js7lng5Pjeujhuwaux1v9hcfot+mhMj1/HaTdLul5SPaGG1gNxm4fHI4HxkmZLujUj5hmSftvLOFeTUdBN0u8krVEYY+JncVojocFaIWlFnHa2pNXxc3xI0pF5tuPKnDcUrj86PKPb6dE4bTvwLTM7HZgG3JHlfVcD88xsLGFH3RZLNEwDzozT9wGX5Nn+VOBVSRVAEzDNzMYQKhnMlnQM8B1gtJmdCvwy881m9jCwhvDLf6yZ7cmY/TBwYcbracCSXsZ5DqEcR6cbzWw8cCpwlqRTzewOQi2fiWY2MZbs+AkwKX6Wa4Dr8mzHlbmSLOHhBrw9cWeZaTBwZ+yT30eoT9TVauBGSVXAI2b2pqQ64MvAi7FcyeGERiebByTtAbYQSk+PAt4ys7/H+QuBOcCdhHEs7pX0FyBxuXIza5e0OdbZeTNuY1Vcb0/i/DShREXmCGUXSbqK8H99LGFQnvVd3lsbp6+K2zmU8Lk51y1vKNxAMRfYBpxGOBI+aMAhM1ss6XlgCvCUpJmEssoLzezHCbZxSWZxQElZxyeJ9YTOIBSWmw5cA3yzB7ksAS4CXgceNTNT2GsnjpMwctstwF3AhZKGA9cDXzGzHZKaCMXuuhLwtJld3IN4XZnzric3UAwB3oljBlxK+DX9MZJOAjbH7pZmQhdMC1AvaWhc5hglHyP8dWCYpJPj60uBZ2Kf/hAze5xwojjblUf/IZQ0z+YR4NuEcRGWxGk9itPM9hK6kGpjt9XRwHvALkmfB87tJpZW4MzOnCQdISnb0ZlzB3hD4QaKu4HLJbUSup3ey7LMNGCDpFeALxKGfNxI2KH+VdJ64GlCt0xeZvY+obrmQ5JeBfYD8wk73WVxfc8Qjna6agLmd57M7rLeHcBG4EQzeyFO63Gc8dzH7cD1ZraOMA72a8B9hO6sTguAJyStMLN2whVZD8bttBI+K+e65dVjnXPO5eRHFM4553LyhsI551xO3lA455zLyRsK55xzOXlD4ZxzLidvKJxzzuXkDYVzzrmc/g+NQ2s+4rQWwAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "params = {'n_estimators': 1000,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .15}\n",
    "\n",
    "# Binarize the output\n",
    "y = label_binarize(y, classes=[1, 2, 3])\n",
    "n_classes = 3\n",
    "\n",
    "#X = scaler.fit_transform(X)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=0)\n",
    "\n",
    "classifier = OneVsRestClassifier(ensemble.GradientBoostingClassifier(**params,random_state=0))\n",
    "y_score = classifier.fit(X_train, y_train).decision_function(X_test)\n",
    "\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "colors = cycle(['blue', 'red', 'green'])\n",
    "for i, color in zip(range(n_classes), colors):\n",
    "    plt.plot(fpr[i], tpr[i], color=color,\n",
    "             label='ROC curve of class {0} (area = {1:0.2f})'\n",
    "             ''.format(i+1, roc_auc[i]))\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.xlim([-0.05, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic for multi-class data')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.savefig('ROC_GradTree.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boosted random forest on test set: 0.94\n",
      "\n",
      "0.9320985603543741\n"
     ]
    }
   ],
   "source": [
    "y = df_cardio['NSP']\n",
    "X = df_cardio.drop(['NSP','Tendency','Nzeros','Nmax','DS','DL'],axis=1)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    "\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.fit_transform(X_test)\n",
    "\n",
    "params = {'n_estimators': 1000,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .2}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "predict_train_clf = clf.predict(X_train)\n",
    "predict_test_clf = clf.predict(X_test)\n",
    "\n",
    "# Accuracy tables.\n",
    "table_train = pd.crosstab(y_train, predict_train_clf, margins=True)\n",
    "table_test = pd.crosstab(y_test, predict_test_clf, margins=True)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test)\n",
    "print('Accuracy of gradient boosted random forest on test set: {:.2f}\\n'.format(clf.score(X_test, y_test)))\n",
    "\n",
    "print(cross_val_score(clf, X_test, y_test, cv=10).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 2. 3.]\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(y_pred_clf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         1.0       0.97      0.96      0.97       339\n",
      "         2.0       0.72      0.81      0.76        48\n",
      "         3.0       0.97      0.85      0.90        39\n",
      "\n",
      "    accuracy                           0.94       426\n",
      "   macro avg       0.89      0.87      0.88       426\n",
      "weighted avg       0.94      0.94      0.94       426\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, y_pred_clf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set accuracy:\n",
      "AVG Percent Type I errors: 0.0026773761713520714\n",
      "AVG Percent Type II errors: 0.0005065856129686056\n",
      "\n",
      "Test set accuracy:\n",
      "AVG Percent Type I errors: 0.11324464039396169\n",
      "AVG Percent Type II errors: 0.12558146131154982\n"
     ]
    }
   ],
   "source": [
    "# Accuracy tables.\n",
    "table_train = pd.crosstab(y_train, predict_train_clf, margins=True)\n",
    "table_test = pd.crosstab(y_test, predict_test_clf, margins=True)\n",
    "\n",
    "train_tI_errors = ((1-(table_train.loc[1.0,1.0]/table_train.loc['All',1.0]))\n",
    "                   +(1-(table_train.loc[2.0,2.0]/table_train.loc['All',2.0]))\n",
    "                   +(1-(table_train.loc[3.0,3.0]/table_train.loc['All',3.0])))/3 \n",
    "\n",
    "train_tII_errors = ((1-(table_train.loc[1.0,1.0]/table_train.loc[1.0,'All']))\n",
    "                   +(1-(table_train.loc[2.0,2.0]/table_train.loc[2.0,'All']))\n",
    "                   +(1-(table_train.loc[3.0,3.0]/table_train.loc[3.0,'All'])))/3 \n",
    "\n",
    "test_tI_errors = ((1-(table_test.loc[1.0,1.0]/table_test.loc['All',1.0]))\n",
    "                   +(1-(table_test.loc[2.0,2.0]/table_test.loc['All',2.0]))\n",
    "                   +(1-(table_test.loc[3.0,3.0]/table_test.loc['All',3.0])))/3 \n",
    "\n",
    "test_tII_errors = ((1-(table_test.loc[1.0,1.0]/table_test.loc[1.0,'All']))\n",
    "                   +(1-(table_test.loc[2.0,2.0]/table_test.loc[2.0,'All']))\n",
    "                   +(1-(table_test.loc[3.0,3.0]/table_test.loc[3.0,'All'])))/3 \n",
    "print((\n",
    "    'Training set accuracy:\\n'\n",
    "    'AVG Percent Type I errors: {}\\n'\n",
    "    'AVG Percent Type II errors: {}\\n\\n'\n",
    "    'Test set accuracy:\\n'\n",
    "    'AVG Percent Type I errors: {}\\n'\n",
    "    'AVG Percent Type II errors: {}'\n",
    ").format(train_tI_errors, train_tII_errors, test_tI_errors, test_tII_errors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Differentials:\n",
      "Type I error difference: 0.11056726422260962\n",
      "\n",
      "Type II error difference: 0.12507487569858122\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print((\n",
    "    'Accuracy Differentials:\\n'\n",
    "    'Type I error difference: {}\\n\\n'\n",
    "    'Type II error difference: {}\\n'\n",
    ").format(test_tI_errors - train_tI_errors, test_tII_errors - train_tII_errors))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upsampling the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create training and test sets.\n",
    "offset = int(X.shape[0] * 0.8)\n",
    "\n",
    "# Put 90% of the data in the training set.\n",
    "df_train = df_cardio[:offset]\n",
    "\n",
    "# And put 10% in the test set.\n",
    "df_test = df_cardio[offset:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0    1315\n",
       "2.0    1315\n",
       "1.0    1315\n",
       "Name: NSP, dtype: int64"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Separate majority and minority classes\n",
    "df_majority = df_train[df_train.NSP==1]\n",
    "df_minority_two = df_train[df_train.NSP==2]\n",
    "df_minority_three = df_train[df_train.NSP==3]\n",
    " \n",
    "# Upsample minority class\n",
    "df_minority_two_upsampled = resample(df_minority_two, \n",
    "                                 replace=True,     # sample with replacement\n",
    "                                 n_samples=1315,    # to match majority class\n",
    "                                 random_state=123) # reproducible results\n",
    "\n",
    "df_minority_three_upsampled = resample(df_minority_three, \n",
    "                                 replace=True,     # sample with replacement\n",
    "                                 n_samples=1315,    # to match majority class\n",
    "                                 random_state=123) # reproducible results\n",
    " \n",
    "# Combine majority class with upsampled minority class\n",
    "df_upsampled = pd.concat([df_majority, df_minority_two_upsampled,df_minority_three_upsampled])\n",
    " \n",
    "# Display new class counts\n",
    "df_upsampled.NSP.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of gradient boosted random forest on test set: 0.99\n",
      "\n",
      "0.9302989237465044\n"
     ]
    }
   ],
   "source": [
    "y = df_upsampled['NSP']\n",
    "X = df_upsampled.drop(['NSP','Tendency','Nzeros','Nmax','DS','DL'],axis=1)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 465) \n",
    "                   \n",
    "X_train_sub, X_test_sub, y_train_sub, y_test_sub = train_test_split(X_train, y_train, test_size = 0.2, random_state = 465) \n",
    " \n",
    "params = {'n_estimators': 1000,\n",
    "          'max_depth': 6,\n",
    "          'loss': 'deviance',\n",
    "          'learning_rate': .2}\n",
    "\n",
    "# Initialize and fit the model.\n",
    "clf = ensemble.GradientBoostingClassifier(**params)\n",
    "clf.fit(X_train_sub, y_train_sub)\n",
    "\n",
    "predict_train_clf = clf.predict(X_train_sub)\n",
    "predict_test_clf = clf.predict(X_test_sub)\n",
    "\n",
    "# Accuracy tables.\n",
    "table_train = pd.crosstab(y_train_sub, predict_train_clf, margins=True)\n",
    "table_test = pd.crosstab(y_test_sub, predict_test_clf, margins=True)\n",
    "\n",
    "y_pred_clf = clf.predict(X_test_sub)\n",
    "print('Accuracy of gradient boosted random forest on test set: {:.2f}\\n'.format(clf.score(X_test_sub, y_test_sub)))\n",
    "\n",
    "print(cross_val_score(clf, X_test_sub, y_test_sub, cv=10).mean())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 2. 3.]\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(y_pred_clf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         1.0       1.00      0.98      0.99       211\n",
      "         2.0       0.98      0.99      0.99       212\n",
      "         3.0       1.00      1.00      1.00       209\n",
      "\n",
      "    accuracy                           0.99       632\n",
      "   macro avg       0.99      0.99      0.99       632\n",
      "weighted avg       0.99      0.99      0.99       632\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test_sub, y_pred_clf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set accuracy:\n",
      "AVG Percent Type I errors: 0.0004050222762251865\n",
      "AVG Percent Type II errors: 0.00039215686274511147\n",
      "\n",
      "Test set accuracy:\n",
      "AVG Percent Type I errors: 0.009420395284881256\n",
      "AVG Percent Type II errors: 0.009463769411904965\n"
     ]
    }
   ],
   "source": [
    "# Accuracy tables.\n",
    "table_train = pd.crosstab(y_train_sub, predict_train_clf, margins=True)\n",
    "table_test = pd.crosstab(y_test_sub, predict_test_clf, margins=True)\n",
    "\n",
    "train_tI_errors = ((1-(table_train.loc[1.0,1.0]/table_train.loc['All',1.0]))\n",
    "                   +(1-(table_train.loc[2.0,2.0]/table_train.loc['All',2.0]))\n",
    "                   +(1-(table_train.loc[3.0,3.0]/table_train.loc['All',3.0])))/3 \n",
    "\n",
    "train_tII_errors = ((1-(table_train.loc[1.0,1.0]/table_train.loc[1.0,'All']))\n",
    "                   +(1-(table_train.loc[2.0,2.0]/table_train.loc[2.0,'All']))\n",
    "                   +(1-(table_train.loc[3.0,3.0]/table_train.loc[3.0,'All'])))/3 \n",
    "\n",
    "test_tI_errors = ((1-(table_test.loc[1.0,1.0]/table_test.loc['All',1.0]))\n",
    "                   +(1-(table_test.loc[2.0,2.0]/table_test.loc['All',2.0]))\n",
    "                   +(1-(table_test.loc[3.0,3.0]/table_test.loc['All',3.0])))/3 \n",
    "\n",
    "test_tII_errors = ((1-(table_test.loc[1.0,1.0]/table_test.loc[1.0,'All']))\n",
    "                   +(1-(table_test.loc[2.0,2.0]/table_test.loc[2.0,'All']))\n",
    "                   +(1-(table_test.loc[3.0,3.0]/table_test.loc[3.0,'All'])))/3 \n",
    "print((\n",
    "    'Training set accuracy:\\n'\n",
    "    'AVG Percent Type I errors: {}\\n'\n",
    "    'AVG Percent Type II errors: {}\\n\\n'\n",
    "    'Test set accuracy:\\n'\n",
    "    'AVG Percent Type I errors: {}\\n'\n",
    "    'AVG Percent Type II errors: {}'\n",
    ").format(train_tI_errors, train_tII_errors, test_tI_errors, test_tII_errors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Differentials:\n",
      "Type I error difference: 0.009015373008656069\n",
      "\n",
      "Type II error difference: 0.009071612549159854\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print((\n",
    "    'Accuracy Differentials:\\n'\n",
    "    'Type I error difference: {}\\n\\n'\n",
    "    'Type II error difference: {}\\n'\n",
    ").format(test_tI_errors - train_tI_errors, test_tII_errors - train_tII_errors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}