classifier-ml.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:01.456862Z",
     "iopub.status.busy": "2024-04-17T20:45:01.456419Z",
     "iopub.status.idle": "2024-04-17T20:45:02.224219Z",
     "shell.execute_reply": "2024-04-17T20:45:02.223823Z"
    },
    "metadata": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import lzma\n",
    "import pickle\n",
    "import hashlib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests as r\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from json import load, dump, loads, dumps\n",
    "from joblib import hash\n",
    "from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer\n",
    "from sklearn.feature_selection import SelectKBest, f_classif, mutual_info_classif\n",
    "from sklearn.linear_model import SGDClassifier, LogisticRegression\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.preprocessing import MaxAbsScaler\n",
    "from sklearn.semi_supervised import SelfTrainingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier, plot_tree\n",
    "\n",
    "warnings.simplefilter('ignore')\n",
    "mpl.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Source Data\n",
    "\n",
    "IF source data is missing run Elasticsearch query to extract data and then save it in JSON format to `data` directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:02.226047Z",
     "iopub.status.busy": "2024-04-17T20:45:02.225831Z",
     "iopub.status.idle": "2024-04-17T20:45:18.524432Z",
     "shell.execute_reply": "2024-04-17T20:45:18.523973Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "# news_json = r.get('http://elasticsearch.storage:9200/news/scraped/_search?sort=date:desc&size=6000').json()\n",
    "# with lzma.open('./data/news.json.xz', 'wt', encoding='utf-8') as fh:\n",
    "#   fh.write(dumps(news_json['hits']['hits']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.json_normalize(news_json['hits']['hits'])\n",
    "# df = df[['_source.body', '_source.date', '_source.subject', '_source.language', '_source.categories']]\n",
    "# df.columns = ['body', 'pubdate', 'subject', 'language', 'categories']\n",
    "# df = df.loc[(df['categories'] != 'News') & \n",
    "#             (df['categories'] != 'Uncategorized') &\n",
    "#             (df['language'] == 'English')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.to_csv('./data/news.json.xz', mode='wt', encoding='utf-8', compression='xz', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.525902Z",
     "iopub.status.busy": "2024-04-17T20:45:18.525801Z",
     "iopub.status.idle": "2024-04-17T20:45:18.686235Z",
     "shell.execute_reply": "2024-04-17T20:45:18.685869Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "# b = loads(lzma.open('./data/news.json.xz', 'rt', encoding='utf-8').read())\n",
    "# df = pd.json_normalize(b)\n",
    "df = pd.read_csv('./data/news.json.xz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Common issues that we generally face during the data preparation phase:\n",
    " - Format and structure normalization\n",
    " - Detect and fix missing values\n",
    " - Duplicates removal\n",
    " - Units normalization\n",
    " - Constraints validations\n",
    " - Anomaly detection and removal\n",
    " - Study of features importance/relevance\n",
    " - Dimentional reduction, feature selection & extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.701712Z",
     "iopub.status.busy": "2024-04-17T20:45:18.701621Z",
     "iopub.status.idle": "2024-04-17T20:45:18.760891Z",
     "shell.execute_reply": "2024-04-17T20:45:18.760592Z"
    },
    "metadata": {}
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_7931b\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_7931b_level0_col0\" class=\"col_heading level0 col0\" >0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_7931b_level0_row0\" class=\"row_heading level0 row0\" >Unnamed: 0</th>\n",
       "      <td id=\"T_7931b_row0_col0\" class=\"data row0 col0\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_7931b_level0_row1\" class=\"row_heading level0 row1\" >body</th>\n",
       "      <td id=\"T_7931b_row1_col0\" class=\"data row1 col0\" >Saudi Arabia and the UN Environment Programme (UNEP) will celebrate World Environment Day on 5 June under the theme “Our Land, Our Future”. According to the Saudi Press Agency (SPA), the ceremony will be held at the King Abdulaziz International Conference Center in Riyadh with dignitaries, officials, experts, and specialists from various countries in attendance. The Ministry of Environment, Water and Agriculture (MEWA) said the event will showcase the Kingdom’s commitment to environmental protection and sustainable development, aligning with Vision 2030 goals. The focus will be on land restoration, combating desertification, and building drought resilience. Saudi Arabia, UNEP to Host World Environment Day.https://t.co/YKKGxiUsI4#SPAGOV pic.twitter.com/dlgTPWLw1I &#8212; SPAENG (@Spa_Eng) June 1, 2024 Last month, Deputy Executive Director of UNEP, Elizabeth Mrema, spoke [&#8230;]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_7931b_level0_row2\" class=\"row_heading level0 row2\" >pubdate</th>\n",
       "      <td id=\"T_7931b_row2_col0\" class=\"data row2 col0\" >2024-06-02T13:30:57.000Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_7931b_level0_row3\" class=\"row_heading level0 row3\" >subject</th>\n",
       "      <td id=\"T_7931b_row3_col0\" class=\"data row3 col0\" >Saudi Arabia and UNEP to mark World Environment Day with major event in Riyadh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_7931b_level0_row4\" class=\"row_heading level0 row4\" >language</th>\n",
       "      <td id=\"T_7931b_row4_col0\" class=\"data row4 col0\" >English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_7931b_level0_row5\" class=\"row_heading level0 row5\" >categories</th>\n",
       "      <td id=\"T_7931b_row5_col0\" class=\"data row5 col0\" >Middle East, News, Saudi Arabia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0xffff2232b390>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop_duplicates(inplace=True)\n",
    "df.head(1).T.style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.796477Z",
     "iopub.status.busy": "2024-04-17T20:45:18.796389Z",
     "iopub.status.idle": "2024-04-17T20:45:18.814222Z",
     "shell.execute_reply": "2024-04-17T20:45:18.813853Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "df['categories'] = df['categories'].str.replace(r'[^a-zA-Z_, ]+', '').replace(', ', '')\n",
    "df['categories'] = df['categories'].str.replace(r'^, ', '')\n",
    "df = df[~df.eq('').all(1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.815574Z",
     "iopub.status.busy": "2024-04-17T20:45:18.815462Z",
     "iopub.status.idle": "2024-04-17T20:45:18.839160Z",
     "shell.execute_reply": "2024-04-17T20:45:18.838830Z"
    },
    "metadata": {}
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>body</th>\n",
       "      <th>pubdate</th>\n",
       "      <th>subject</th>\n",
       "      <th>language</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>categories</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>#Culture, #Life, Books</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>#Culture, featured, Featured Culture</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>#Islam, Arabic Studies, featured</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>#Islam, featured, Featured Islam</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Afghanistan, Africa, Asia &amp; Americas</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Afghanistan, Asia &amp; Americas, Europe &amp; Russia</th>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Algeria, Article</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Algeria, Europe &amp; Russia</th>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Algeria, International Organisations</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Algeria, News</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Article, Asia &amp; Americas</th>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Article, Egypt</th>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Asia &amp; Americas, Egypt</th>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Asia &amp; Americas, Europe &amp; Russia</th>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Egypt, Europe &amp; Russia</th>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Egypt, International Organisations</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Egypt, Israel</th>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Egypt, Middle East</th>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Egypt, News</th>\n",
       "      <td>68</td>\n",
       "      <td>68</td>\n",
       "      <td>68</td>\n",
       "      <td>68</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Europe &amp; Russia, News</th>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, International Organisations, News</th>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, Middle East, News</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, News, Sudan</th>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Africa, News, Tunisia</th>\n",
       "      <td>58</td>\n",
       "      <td>58</td>\n",
       "      <td>58</td>\n",
       "      <td>58</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Article, Israel, Middle East</th>\n",
       "      <td>60</td>\n",
       "      <td>60</td>\n",
       "      <td>60</td>\n",
       "      <td>60</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Asia &amp; Americas, International Organisations, Israel</th>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Asia &amp; Americas, Iran, Middle East</th>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Asia &amp; Americas, Middle East, News</th>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Blog, News, gaza</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>China, Defence &amp; Foreign Policy, Politics &amp; Protest</th>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Europe &amp; Russia, Israel, Middle East</th>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Europe &amp; Russia, Middle East, News</th>\n",
       "      <td>96</td>\n",
       "      <td>96</td>\n",
       "      <td>96</td>\n",
       "      <td>96</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Europe &amp; Russia, News, Turkey</th>\n",
       "      <td>79</td>\n",
       "      <td>79</td>\n",
       "      <td>79</td>\n",
       "      <td>79</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hong Kong, Law &amp; Crime, Politics &amp; Protest</th>\n",
       "      <td>64</td>\n",
       "      <td>64</td>\n",
       "      <td>64</td>\n",
       "      <td>64</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Incidents</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>International Organisations, Israel, Middle East</th>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>International Organisations, Middle East, News</th>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Iran, Middle East, News</th>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Israel, Jordan, Middle East</th>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Israel, Lebanon, Middle East</th>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Middle East, News, Palestine</th>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Middle East, News, Saudi Arabia</th>\n",
       "      <td>41</td>\n",
       "      <td>41</td>\n",
       "      <td>41</td>\n",
       "      <td>41</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Opinion</th>\n",
       "      <td>61</td>\n",
       "      <td>61</td>\n",
       "      <td>61</td>\n",
       "      <td>61</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>World</th>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   Unnamed: 0  body pubdate  \\\n",
       "                                                        count count   count   \n",
       "categories                                                                    \n",
       "#Culture, #Life, Books                                      1     1       1   \n",
       "#Culture, featured, Featured Culture                        3     3       3   \n",
       "#Islam, Arabic Studies, featured                            2     2       2   \n",
       "#Islam, featured, Featured Islam                            4     4       4   \n",
       "Afghanistan, Africa, Asia & Americas                        6     6       6   \n",
       "Afghanistan, Asia & Americas, Europe & Russia              14    14      14   \n",
       "Africa                                                      5     5       5   \n",
       "Africa, Algeria, Article                                    8     8       8   \n",
       "Africa, Algeria, Europe & Russia                           30    30      30   \n",
       "Africa, Algeria, International Organisations                7     7       7   \n",
       "Africa, Algeria, News                                      12    12      12   \n",
       "Africa, Article, Asia & Americas                           28    28      28   \n",
       "Africa, Article, Egypt                                     29    29      29   \n",
       "Africa, Asia & Americas, Egypt                             37    37      37   \n",
       "Africa, Asia & Americas, Europe & Russia                    9     9       9   \n",
       "Africa, Egypt, Europe & Russia                             45    45      45   \n",
       "Africa, Egypt, International Organisations                 10    10      10   \n",
       "Africa, Egypt, Israel                                      49    49      49   \n",
       "Africa, Egypt, Middle East                                 19    19      19   \n",
       "Africa, Egypt, News                                        68    68      68   \n",
       "Africa, Europe & Russia, News                              23    23      23   \n",
       "Africa, International Organisations, News                  13    13      13   \n",
       "Africa, Middle East, News                                  11    11      11   \n",
       "Africa, News, Sudan                                        27    27      27   \n",
       "Africa, News, Tunisia                                      58    58      58   \n",
       "Article, Israel, Middle East                               60    60      60   \n",
       "Asia & Americas, International Organisations, I...         25    25      25   \n",
       "Asia & Americas, Iran, Middle East                         20    20      20   \n",
       "Asia & Americas, Middle East, News                         36    36      36   \n",
       "Blog, News, gaza                                           24    24      24   \n",
       "China, Defence & Foreign Policy, Politics & Pro...         17    17      17   \n",
       "Europe & Russia, Israel, Middle East                       99    99      99   \n",
       "Europe & Russia, Middle East, News                         96    96      96   \n",
       "Europe & Russia, News, Turkey                              79    79      79   \n",
       "Hong Kong, Law & Crime, Politics & Protest                 64    64      64   \n",
       "Incidents                                                  16    16      16   \n",
       "International Organisations, Israel, Middle East          110   110     110   \n",
       "International Organisations, Middle East, News             18    18      18   \n",
       "Iran, Middle East, News                                    52    52      52   \n",
       "Israel, Jordan, Middle East                                26    26      26   \n",
       "Israel, Lebanon, Middle East                               44    44      44   \n",
       "Middle East, News, Palestine                               22    22      22   \n",
       "Middle East, News, Saudi Arabia                            41    41      41   \n",
       "Opinion                                                    61    61      61   \n",
       "World                                                      88    88      88   \n",
       "\n",
       "                                                   subject language  \n",
       "                                                     count    count  \n",
       "categories                                                           \n",
       "#Culture, #Life, Books                                   1        1  \n",
       "#Culture, featured, Featured Culture                     3        3  \n",
       "#Islam, Arabic Studies, featured                         2        2  \n",
       "#Islam, featured, Featured Islam                         4        4  \n",
       "Afghanistan, Africa, Asia & Americas                     6        6  \n",
       "Afghanistan, Asia & Americas, Europe & Russia           14       14  \n",
       "Africa                                                   5        5  \n",
       "Africa, Algeria, Article                                 8        8  \n",
       "Africa, Algeria, Europe & Russia                        30       30  \n",
       "Africa, Algeria, International Organisations             7        7  \n",
       "Africa, Algeria, News                                   12       12  \n",
       "Africa, Article, Asia & Americas                        28       28  \n",
       "Africa, Article, Egypt                                  29       29  \n",
       "Africa, Asia & Americas, Egypt                          37       37  \n",
       "Africa, Asia & Americas, Europe & Russia                 9        9  \n",
       "Africa, Egypt, Europe & Russia                          45       45  \n",
       "Africa, Egypt, International Organisations              10       10  \n",
       "Africa, Egypt, Israel                                   49       49  \n",
       "Africa, Egypt, Middle East                              19       19  \n",
       "Africa, Egypt, News                                     68       68  \n",
       "Africa, Europe & Russia, News                           23       23  \n",
       "Africa, International Organisations, News               13       13  \n",
       "Africa, Middle East, News                               11       11  \n",
       "Africa, News, Sudan                                     27       27  \n",
       "Africa, News, Tunisia                                   58       58  \n",
       "Article, Israel, Middle East                            60       60  \n",
       "Asia & Americas, International Organisations, I...      25       25  \n",
       "Asia & Americas, Iran, Middle East                      20       20  \n",
       "Asia & Americas, Middle East, News                      36       36  \n",
       "Blog, News, gaza                                        24       24  \n",
       "China, Defence & Foreign Policy, Politics & Pro...      17       17  \n",
       "Europe & Russia, Israel, Middle East                    99       99  \n",
       "Europe & Russia, Middle East, News                      96       96  \n",
       "Europe & Russia, News, Turkey                           79       79  \n",
       "Hong Kong, Law & Crime, Politics & Protest              64       64  \n",
       "Incidents                                               16       16  \n",
       "International Organisations, Israel, Middle East       110      110  \n",
       "International Organisations, Middle East, News          18       18  \n",
       "Iran, Middle East, News                                 52       52  \n",
       "Israel, Jordan, Middle East                             26       26  \n",
       "Israel, Lebanon, Middle East                            44       44  \n",
       "Middle East, News, Palestine                            22       22  \n",
       "Middle East, News, Saudi Arabia                         41       41  \n",
       "Opinion                                                 61       61  \n",
       "World                                                   88       88  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['categories']).agg({'count'}).drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.840641Z",
     "iopub.status.busy": "2024-04-17T20:45:18.840339Z",
     "iopub.status.idle": "2024-04-17T20:45:18.883206Z",
     "shell.execute_reply": "2024-04-17T20:45:18.882857Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "df['cat_id']       = df['categories'].factorize()[0]\n",
    "df['lang_id']      = df['language'].factorize()[0]\n",
    "df['char_count']   = df['body'].apply(len)\n",
    "df['word_count']   = df['body'].apply(lambda x: len(x.split()))\n",
    "df['word_density'] = df['char_count'] / (df['word_count']+1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.884543Z",
     "iopub.status.busy": "2024-04-17T20:45:18.884450Z",
     "iopub.status.idle": "2024-04-17T20:45:18.897076Z",
     "shell.execute_reply": "2024-04-17T20:45:18.896737Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.898359Z",
     "iopub.status.busy": "2024-04-17T20:45:18.898270Z",
     "iopub.status.idle": "2024-04-17T20:45:18.909134Z",
     "shell.execute_reply": "2024-04-17T20:45:18.908828Z"
    },
    "metadata": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4915, 11)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.910232Z",
     "iopub.status.busy": "2024-04-17T20:45:18.910143Z",
     "iopub.status.idle": "2024-04-17T20:45:18.919194Z",
     "shell.execute_reply": "2024-04-17T20:45:18.918883Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "# sns.set()\n",
    "# sns.pairplot(df, height=3.5, kind=\"reg\", palette=\"husl\", diag_kind=\"auto\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:18.920309Z",
     "iopub.status.busy": "2024-04-17T20:45:18.920222Z",
     "iopub.status.idle": "2024-04-17T20:45:18.930062Z",
     "shell.execute_reply": "2024-04-17T20:45:18.929751Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "xtrain, xtest, ytrain, ytest = train_test_split(df['body'], df['categories'], test_size=0.2, random_state=42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:45:20.439579Z",
     "iopub.status.busy": "2024-04-17T20:45:20.439475Z",
     "iopub.status.idle": "2024-04-17T20:45:20.450028Z",
     "shell.execute_reply": "2024-04-17T20:45:20.449701Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "estimators = [\n",
    "      # ('AdaBoostClassifier', AdaBoostClassifier(n_estimators=100)), # 0.02\n",
    "      # ('LogisticRegression', SelfTrainingClassifier(LogisticRegression(fit_intercept=False, max_iter=1000))), # 20%\n",
    "      # ('SGDClassifier', SelfTrainingClassifier(SGDClassifier(fit_intercept=False, loss='modified_huber', max_iter=2000))), # 10%\n",
    "      ('RandomForestClassifier', SelfTrainingClassifier(RandomForestClassifier(min_samples_leaf=0.01))), # 16%\n",
    "      ('MLPClassifier', SelfTrainingClassifier(MLPClassifier(max_iter=300))), # 21%\n",
    "      ('DecisionTreeClassifier', SelfTrainingClassifier(DecisionTreeClassifier(min_samples_leaf=0.01))), # 15%\n",
    "      # ('GradientBoostingClassifier', GradientBoostingClassifier(min_samples_leaf=0.01)), # takes too long > 20 min\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "metadata": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Voting] ... (1 of 3) Processing RandomForestClassifier, total=  23.2s\n",
      "[Voting] ............ (2 of 3) Processing MLPClassifier, total= 4.0min\n",
      "[Voting] ... (3 of 3) Processing DecisionTreeClassifier, total=  41.1s\n"
     ]
    }
   ],
   "source": [
    "vote = Pipeline([\n",
    "      ('vect', TfidfVectorizer(sublinear_tf=True, encoding='latin-1', analyzer='char', lowercase=False, ngram_range=(1, 2), stop_words='english')),\n",
    "      ('tfidf', TfidfTransformer(sublinear_tf=True, use_idf=True, smooth_idf=True)),\n",
    "      ('scaler', MaxAbsScaler(copy=False)),\n",
    "      ('feature_union', FeatureUnion([\n",
    "        ('select_best', SelectKBest(score_func=mutual_info_classif, k='all')),\n",
    "        ('kmeans', KMeans(n_clusters=5, random_state=42, verbose=1)),\n",
    "      ])),\n",
    "      ('clf', VotingClassifier(estimators=estimators, voting='soft', verbose=True))])\n",
    "vote.fit(xtrain, ytrain)\n",
    "prediction = vote.predict(xtest)\n",
    "score      = vote.score(xtest, prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26.14%\n"
     ]
    }
   ],
   "source": [
    "print(f'{accuracy_score(ytest, prediction) * 100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(classification_report(ytest, prediction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-17T20:48:03.569671Z",
     "iopub.status.busy": "2024-04-17T20:48:03.569389Z",
     "iopub.status.idle": "2024-04-17T20:48:03.581000Z",
     "shell.execute_reply": "2024-04-17T20:48:03.580622Z"
    },
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "s = '''\n",
    "\n",
    "‘Guys, you’ve got to hear this,” I said. I was sitting in front of my computer one day in July 2012, with one eye on a screen of share prices and the other on a live stream of the House of Commons Treasury select committee hearings. As the Barclays share price took a graceful swan dive, I pulled my headphones out of the socket and turned up the volume so everyone could hear. My colleagues left their terminals and came around to watch BBC Parliament with me.\n",
    "\n",
    "It didn’t take long to realise what was happening. “Bob’s getting murdered,” someone said.\n",
    "\n",
    "Bob Diamond, the swashbuckling chief executive of Barclays, had been called before the committee to explain exactly what his bank had been playing at in regards to the Libor rate-fixing scandal. The day before his appearance, he had made things very much worse by seeming to accuse the deputy governor of the Bank of England of ordering him to fiddle an important benchmark, then walking back the accusation as soon as it was challenged. He was trying to turn on his legendary charm in front of a committee of angry MPs, and it wasn’t working. On our trading floor, in Mayfair, calls were coming in from all over the City. Investors needed to know what was happening and whether the damage was reparable.\n",
    "\n",
    "A couple of weeks later, the damage was done. The money was gone, Diamond was out of a job and the market, as it always does, had moved on. We were left asking ourselves: How did we get it so wrong?\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Opinion']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vote.predict([s])[0].split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with lzma.open('./data/voting_classifier.pickle.xz', 'wb') as f:\n",
    "  pickle.dump(vote, f, protocol=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dt = DecisionTreeClassifier(min_samples_leaf=1)\n",
    "# dt.fit(vote.named_steps['vect'].transform(xtrain), vote.predict(xtrain))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def serialize_and_hash(data):\n",
    "    serialized_data = pickle.dumps(data)\n",
    "    data_hash = hashlib.sha256(serialized_data).hexdigest()\n",
    "    return serialized_data, data_hash\n",
    "\n",
    "def verify_integrity(serialized_data, stored_hash):\n",
    "    calculated_hash = hashlib.sha256(serialized_data).hexdigest()\n",
    "    return calculated_hash == stored_hash\n",
    "\n",
    "# Example usage\n",
    "data = df\n",
    "serialized_data, data_hash = serialize_and_hash(data)\n",
    "\n",
    "# When loading\n",
    "# Load serialized_data and stored_hash\n",
    "with lzma.open('./data/voting_classifier.pickle.xz', 'rb') as f:\n",
    "    loaded_serialized_data = f.read()\n",
    "\n",
    "# Load the stored hash\n",
    "with lzma.open('./data/news.json.xz', 'rt') as f:\n",
    "    stored_hash = f.read()\n",
    "\n",
    "if verify_integrity(loaded_serialized_data, stored_hash):\n",
    "    loaded_data = pickle.loads(loaded_serialized_data)\n",
    "    print(\"Data integrity verified!\")\n",
    "else:\n",
    "    print(\"Data integrity verification failed!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}