impressions_exploration.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Impressions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.sparse as sps\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Random seed for reproducibility\n",
    "SEED = 42\n",
    "\n",
    "random.seed(SEED)\n",
    "os.environ['PYTHONHASHSEED'] = str(SEED)\n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "item_type (23091, 1)\n",
      "item_length (23091, 1)\n",
      "interactions (5826506, 4)\n",
      "views (3567479, 2)\n",
      "details (2259027, 2)\n"
     ]
    }
   ],
   "source": [
    "item_type = pd.read_csv(\n",
    "    \"data/data_ICM_type.csv\",\n",
    "    usecols=[\"item_id\", \"feature_id\"],\n",
    "    dtype={0: \"Int64\", 1: \"Int64\"},\n",
    ").set_index(\"item_id\")\n",
    "item_length = pd.read_csv(\n",
    "    \"data/data_ICM_length.csv\",\n",
    "    usecols=[\"item_id\", \"data\"],\n",
    "    dtype={0: \"Int64\", 1: \"Int64\"},\n",
    ").set_index(\"item_id\")\n",
    "item_length = item_length.rename(columns={\"data\": \"length\"})\n",
    "interactions = pd.read_csv(\n",
    "    \"data/interactions_and_impressions.csv\",\n",
    "    dtype={0: \"Int64\", 1: \"Int64\", 2: str, 3: \"Int64\"},\n",
    ")\n",
    "interactions = interactions.rename(\n",
    "    columns={\n",
    "        \"UserID\": \"user_id\",\n",
    "        \"ItemID\": \"item_id\",\n",
    "        \"Data\": \"data\",\n",
    "        \"Impressions\": \"impressions\",\n",
    "    }\n",
    ")\n",
    "target_users = pd.read_csv(\n",
    "    \"data/data_target_users_test.csv\",\n",
    "    usecols=[\"user_id\"],\n",
    "    dtype={0: \"Int64\"},\n",
    ")\n",
    "views = interactions[interactions[\"data\"] == 0].drop([\"data\", \"impressions\"], axis=1)\n",
    "details = interactions[interactions[\"data\"] == 1].drop([\"data\", \"impressions\"], axis=1)\n",
    "print(\"item_type\", item_type.shape)\n",
    "print(\"item_length\", item_length.shape)\n",
    "print(\"interactions\", interactions.shape)\n",
    "print(\"views\", views.shape)\n",
    "print(\"details\", details.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>views_count</th>\n",
       "      <th>details_count</th>\n",
       "      <th>length</th>\n",
       "      <th>feature_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>124</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>808</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1326</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1995</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1554635</th>\n",
       "      <td>41628</td>\n",
       "      <td>11228</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1554636</th>\n",
       "      <td>41628</td>\n",
       "      <td>15033</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1554637</th>\n",
       "      <td>41628</td>\n",
       "      <td>15181</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1554638</th>\n",
       "      <td>41628</td>\n",
       "      <td>20896</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1554639</th>\n",
       "      <td>41628</td>\n",
       "      <td>21506</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1554640 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_id  item_id  views_count  details_count  length  feature_id\n",
       "0              0       21          3.0            9.0    27.0           4\n",
       "1              0      124          1.0            0.0     3.0           1\n",
       "2              0      808          1.0            0.0     1.0           1\n",
       "3              0     1326          1.0            3.0     3.0           1\n",
       "4              0     1995          1.0            2.0    17.0           4\n",
       "...          ...      ...          ...            ...     ...         ...\n",
       "1554635    41628    11228          0.0            1.0    30.0           4\n",
       "1554636    41628    15033          0.0            1.0     1.0           3\n",
       "1554637    41628    15181          0.0            1.0     8.0           4\n",
       "1554638    41628    20896          0.0            1.0     NaN        <NA>\n",
       "1554639    41628    21506          0.0            1.0     NaN        <NA>\n",
       "\n",
       "[1554640 rows x 6 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "views[\"views_count\"] = 1\n",
    "views_count = views.groupby([\"user_id\", \"item_id\"], as_index = False)[\"views_count\"].sum()\n",
    "details[\"details_count\"] = 1\n",
    "details_count = details.groupby([\"user_id\", \"item_id\"], as_index = False)[\"details_count\"].sum()\n",
    "\n",
    "view_details_count = views_count.set_index(['user_id', 'item_id']).join(\n",
    "    details_count.set_index(['user_id', 'item_id']), on=[\"user_id\", \"item_id\"], how='outer'\n",
    ")\n",
    "view_details_count.loc[view_details_count['details_count'].isna(), 'details_count'] = 0\n",
    "view_details_count.loc[view_details_count['views_count'].isna(), 'views_count'] = 0\n",
    "view_details_count = view_details_count.join(item_length, on=\"item_id\")\n",
    "view_details_count = view_details_count.join(item_type, on=\"item_id\")\n",
    "view_details_count = view_details_count.reset_index()\n",
    "view_details_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>impressions</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>(20, 21, 22, 23, 24, 25, 26, 27, 28, 29)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5826501</th>\n",
       "      <td>41628</td>\n",
       "      <td>20448</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5826502</th>\n",
       "      <td>41628</td>\n",
       "      <td>20896</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5826503</th>\n",
       "      <td>41628</td>\n",
       "      <td>21506</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5826504</th>\n",
       "      <td>41628</td>\n",
       "      <td>22882</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5826505</th>\n",
       "      <td>41628</td>\n",
       "      <td>23757</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5826506 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_id  item_id                                        impressions  \\\n",
       "0              0       11  (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...   \n",
       "1              0       21                                                NaN   \n",
       "2              0       21                                                NaN   \n",
       "3              0       21           (20, 21, 22, 23, 24, 25, 26, 27, 28, 29)   \n",
       "4              0       21                                                NaN   \n",
       "...          ...      ...                                                ...   \n",
       "5826501    41628    20448                                                NaN   \n",
       "5826502    41628    20896                                                NaN   \n",
       "5826503    41628    21506                                                NaN   \n",
       "5826504    41628    22882                                                NaN   \n",
       "5826505    41628    23757                                                NaN   \n",
       "\n",
       "         data  \n",
       "0           1  \n",
       "1           0  \n",
       "2           0  \n",
       "3           0  \n",
       "4           1  \n",
       "...       ...  \n",
       "5826501     0  \n",
       "5826502     1  \n",
       "5826503     1  \n",
       "5826504     0  \n",
       "5826505     0  \n",
       "\n",
       "[5826506 rows x 4 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ast import literal_eval\n",
    "mask_with_impressions = ~interactions[\"impressions\"].isna()\n",
    "interactions.loc[mask_with_impressions, \"impressions\"] = interactions.loc[mask_with_impressions, \"impressions\"].apply(literal_eval)\n",
    "interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>impressions</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>(20, 21, 22, 23, 24, 25, 26, 27, 28, 29)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>(20, 21, 30, 22, 23, 24, 26, 31, 29, 32)</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>(20, 21, 22, 23, 24, 25, 26, 27, 28, 29)</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>(20, 21, 30, 22, 23, 24, 25, 27, 28, 29)</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>(33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 4...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    user_id  item_id                                        impressions  data\n",
       "0         0       11  (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...     1\n",
       "1         0       21                                                NaN     0\n",
       "2         0       21                                                NaN     0\n",
       "3         0       21           (20, 21, 22, 23, 24, 25, 26, 27, 28, 29)     0\n",
       "4         0       21                                                NaN     1\n",
       "5         0       21                                                NaN     1\n",
       "6         0       21                                                NaN     1\n",
       "7         0       21                                                NaN     1\n",
       "8         0       21                                                NaN     1\n",
       "9         0       21                                                NaN     1\n",
       "10        0       21                                                NaN     1\n",
       "11        0       21           (20, 21, 30, 22, 23, 24, 26, 31, 29, 32)     1\n",
       "12        0       21           (20, 21, 22, 23, 24, 25, 26, 27, 28, 29)     1\n",
       "13        0       22                                                NaN     1\n",
       "14        0       22                                                NaN     1\n",
       "15        0       22                                                NaN     1\n",
       "16        0       22                                                NaN     1\n",
       "17        0       22                                                NaN     1\n",
       "18        0       22                                                NaN     1\n",
       "19        0       22                                                NaN     1\n",
       "20        0       22                                                NaN     1\n",
       "21        0       22                                                NaN     1\n",
       "22        0       22                                                NaN     1\n",
       "23        0       22                                                NaN     1\n",
       "24        0       22                                                NaN     1\n",
       "25        0       22                                                NaN     1\n",
       "26        0       22                                                NaN     1\n",
       "27        0       22                                                NaN     1\n",
       "28        0       24           (20, 21, 30, 22, 23, 24, 25, 27, 28, 29)     1\n",
       "29        0       44  (33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 4...     1"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interactions.loc[interactions[\"user_id\"] == 0].head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>impressions</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5443853</th>\n",
       "      <td>36018</td>\n",
       "      <td>65</td>\n",
       "      <td>2341</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5443853</th>\n",
       "      <td>36018</td>\n",
       "      <td>65</td>\n",
       "      <td>361</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5443853</th>\n",
       "      <td>36018</td>\n",
       "      <td>65</td>\n",
       "      <td>1300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5443853</th>\n",
       "      <td>36018</td>\n",
       "      <td>65</td>\n",
       "      <td>1503</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5443853</th>\n",
       "      <td>36018</td>\n",
       "      <td>65</td>\n",
       "      <td>65</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2914990 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_id  item_id impressions  data\n",
       "0              0       11           0     1\n",
       "0              0       11           1     1\n",
       "0              0       11           2     1\n",
       "0              0       11           3     1\n",
       "0              0       11           4     1\n",
       "...          ...      ...         ...   ...\n",
       "5443853    36018       65        2341     0\n",
       "5443853    36018       65         361     0\n",
       "5443853    36018       65        1300     0\n",
       "5443853    36018       65        1503     0\n",
       "5443853    36018       65          65     0\n",
       "\n",
       "[2914990 rows x 4 columns]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exploded = interactions[mask_with_impressions].explode(\"impressions\")\n",
    "exploded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>impressions_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1605037</th>\n",
       "      <td>36018</td>\n",
       "      <td>1675</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1605038</th>\n",
       "      <td>36018</td>\n",
       "      <td>1906</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1605039</th>\n",
       "      <td>36018</td>\n",
       "      <td>2340</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1605040</th>\n",
       "      <td>36018</td>\n",
       "      <td>2341</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1605041</th>\n",
       "      <td>36018</td>\n",
       "      <td>2964</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1605042 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_id  item_id  impressions_count\n",
       "0              0        0                  1\n",
       "1              0        1                  1\n",
       "2              0        2                  1\n",
       "3              0        3                  1\n",
       "4              0        4                  1\n",
       "...          ...      ...                ...\n",
       "1605037    36018     1675                  1\n",
       "1605038    36018     1906                  1\n",
       "1605039    36018     2340                  1\n",
       "1605040    36018     2341                  1\n",
       "1605041    36018     2964                  1\n",
       "\n",
       "[1605042 rows x 3 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exploded[\"impressions_count\"] = 1\n",
    "impressions_count = exploded.groupby([\"user_id\", \"impressions\"], as_index = False)[\"impressions_count\"].sum()\n",
    "impressions_count = impressions_count.rename(columns={\"impressions\": \"item_id\"})\n",
    "impressions_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>impressions_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    user_id  item_id  impressions_count\n",
       "0         0        0                  1\n",
       "1         0        1                  1\n",
       "2         0        2                  1\n",
       "3         0        3                  1\n",
       "4         0        4                  1\n",
       "5         0        5                  1\n",
       "6         0        6                  1\n",
       "7         0        7                  1\n",
       "8         0        8                  1\n",
       "9         0        9                  1\n",
       "10        0       10                  1\n",
       "12        0       12                  1\n",
       "13        0       13                  1\n",
       "14        0       14                  1\n",
       "15        0       15                  1\n",
       "16        0       16                  1\n",
       "17        0       17                  1\n",
       "18        0       18                  1\n",
       "19        0       19                  1\n",
       "20        0       20                  5\n",
       "23        0       23                  5\n",
       "25        0       25                  4\n",
       "26        0       26                  3\n",
       "27        0       27                  3\n",
       "28        0       28                  3\n",
       "29        0       29                  4\n",
       "30        0       30                  3\n",
       "31        0       31                  1\n",
       "32        0       32                  2\n",
       "33        0       33                  1"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove impressions of seen items\n",
    "merged = pd.merge(impressions_count, view_details_count, on=[\"user_id\", \"item_id\"], how=\"left\", indicator=\"exists\")\n",
    "impressions_count = impressions_count.loc[merged[\"exists\"] != \"both\"]\n",
    "impressions_count.reset_index()\n",
    "impressions_count.head(30)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10 (tags/v3.8.10:3d8993a, May  3 2021, 11:48:03) [MSC v.1928 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4188b889b75fa04b540dc3b3d88b44a92f17011a3546136f7949774b76f654ae"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}