Main.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# %cd /content\n",
        "# !rm -rf Dataset-Obat-Entity-Matching\n",
        "# !git clone https://github.com/rivandaputra/Dataset-Obat-Entity-Matching.git"
      ],
      "metadata": {
        "id": "WGFIlnkpmVmO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# src = \"Dataset-Obat-Entity-Matching\"\n",
        "# %cd $src"
      ],
      "metadata": {
        "id": "HeNfHbfJmoGj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UcE2tXHeLODR",
        "outputId": "13d5b44f-3533-482d-b638-3bf5b99e347b"
      },
      "source": [
        "try:\n",
        "    import deepmatcher\n",
        "except:\n",
        "    !pip install -qqq deepmatcher"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.0/51.0 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m68.8/68.8 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for deepmatcher (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for fasttext (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kgBJKbJzLjyn",
        "outputId": "15411e2e-46aa-4d31-e019-9019d5571dac"
      },
      "source": [
        "!pip install torch==1.8.0 torchtext==0.9.0 --extra-index-url https://download.pytorch.org/whl/cu101\n",
        "import deepmatcher as dm\n",
        "import torch\n",
        "import os\n",
        "\n",
        "torch.cuda.is_available()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/, https://download.pytorch.org/whl/cu101\n",
            "Collecting torch==1.8.0\n",
            "  Downloading https://download.pytorch.org/whl/cu101/torch-1.8.0%2Bcu101-cp39-cp39-linux_x86_64.whl (763.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m763.6/763.6 MB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchtext==0.9.0\n",
            "  Downloading torchtext-0.9.0-cp39-cp39-manylinux1_x86_64.whl (7.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m55.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch==1.8.0) (4.5.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from torch==1.8.0) (1.22.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from torchtext==0.9.0) (4.65.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from torchtext==0.9.0) (2.27.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->torchtext==0.9.0) (3.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->torchtext==0.9.0) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->torchtext==0.9.0) (1.26.15)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->torchtext==0.9.0) (2.0.12)\n",
            "Installing collected packages: torch, torchtext\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 2.0.0+cu118\n",
            "    Uninstalling torch-2.0.0+cu118:\n",
            "      Successfully uninstalled torch-2.0.0+cu118\n",
            "  Attempting uninstall: torchtext\n",
            "    Found existing installation: torchtext 0.15.1\n",
            "    Uninstalling torchtext-0.15.1:\n",
            "      Successfully uninstalled torchtext-0.15.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchvision 0.15.1+cu118 requires torch==2.0.0, but you have torch 1.8.0+cu101 which is incompatible.\n",
            "torchdata 0.6.0 requires torch==2.0.0, but you have torch 1.8.0+cu101 which is incompatible.\n",
            "torchaudio 2.0.1+cu118 requires torch==2.0.0, but you have torch 1.8.0+cu101 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed torch-1.8.0+cu101 torchtext-0.9.0\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.zip --directory-prefix=/root/.vector_cache\n",
        "!unzip /root/.vector_cache/wiki.en.zip -d /root/.vector_cache/\n",
        "!rm /root/.vector_cache/wiki.en.vec"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TKIyCtlOqXpa",
        "outputId": "035dcd97-68ff-4fa0-aefd-d8afba0d5889"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-05-20 14:41:08--  https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.zip\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 13.249.141.108, 13.249.141.9, 13.249.141.13, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|13.249.141.108|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10356881291 (9.6G) [application/zip]\n",
            "Saving to: ‘/root/.vector_cache/wiki.en.zip’\n",
            "\n",
            "wiki.en.zip         100%[===================>]   9.65G  36.1MB/s    in 4m 7s   \n",
            "\n",
            "2023-05-20 14:45:15 (40.0 MB/s) - ‘/root/.vector_cache/wiki.en.zip’ saved [10356881291/10356881291]\n",
            "\n",
            "Archive:  /root/.vector_cache/wiki.en.zip\n",
            "  inflating: /root/.vector_cache/wiki.en.vec  \n",
            "  inflating: /root/.vector_cache/wiki.en.bin  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pandas\n",
        "import pandas as pd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SPUrfFlT77f1",
        "outputId": "c72e7cac-e53d-4a5b-e15d-1467484dee27"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (1.5.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: numpy>=1.20.3 in /usr/local/lib/python3.9/dist-packages (from pandas) (1.22.4)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the CSV file\n",
        "df = pd.read_csv('/content/Dataset.csv')"
      ],
      "metadata": {
        "id": "KoUMTz0ty-4c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# The directory where the data splits will be saved.\n",
        "split_path = os.path.join('.')"
      ],
      "metadata": {
        "id": "aRMMo3Fmykyx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Split labeled data into train, valid, and test csv files to disk, with the split ratio of 3:1:1.\n",
        "dm.data.split(df, split_path, 'train.csv', 'valid.csv', 'test.csv',\n",
        "              [3, 1, 1])"
      ],
      "metadata": {
        "id": "MgLCIQqAygoV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tNr8wGZVLptm",
        "outputId": "6eeb1acb-afde-41df-c6a5-22a7cb4f4bb6"
      },
      "source": [
        "train, validation, test = dm.data.process(\n",
        "    path='/content/',\n",
        "    train='train.csv',\n",
        "    validation='valid.csv',\n",
        "    test='test.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Reading and processing data from \"/content/train.csv\"\n",
            "0% [############################# ] 100% | ETA: 00:00:00\n",
            "Reading and processing data from \"/content/valid.csv\"\n",
            "0% [############################# ] 100% | ETA: 00:00:00\n",
            "Reading and processing data from \"/content/test.csv\"\n",
            "0% [############################# ] 100% | ETA: 00:00:00Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n",
            "\n",
            "Building vocabulary\n",
            "0% [#] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n",
            "\n",
            "Computing principal components\n",
            "0% [#] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        },
        "id": "z9DGVMdiNMPW",
        "outputId": "34b6590b-e897-4c05-a158-72438ec2e41f"
      },
      "source": [
        "train_table = train.get_raw_table()\n",
        "train_table.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     id                                         left_title  \\\n",
              "0   278  rawls 's original position and algorithmic fai...   \n",
              "1   272  rawls 's original position and algorithmic fai...   \n",
              "2  1054  the rising tide of artificial intelligence and...   \n",
              "3  1120  effective exposure amortizing for fair top-k r...   \n",
              "4   164  beyond fairness metrics : roadblocks and chall...   \n",
              "\n",
              "                                         right_title  \\\n",
              "0  effective exposure amortizing for fair top-k r...   \n",
              "1  why fairness can not be automated : bridging t...   \n",
              "2  why fairness can not be automated : bridging t...   \n",
              "3      a sociotechnical view of algorithmic fairness   \n",
              "4  achieving causal fairness through generative a...   \n",
              "\n",
              "                      left_authors                          right_authors  \\\n",
              "0                         franke u                   yang t , xu z , ai q   \n",
              "1                         franke u  wachter s , mittelstadt b , russell c   \n",
              "2           wright sa , schultz ae  wachter s , mittelstadt b , russell c   \n",
              "3             yang t , xu z , ai q   dolata m , feuerriegel s , schwabe g   \n",
              "4  chen j , storchan v , kurshan e  xu d , wu y , yuan s , zhang l , wu x   \n",
              "\n",
              "                                       left_keywords  \\\n",
              "0  algorithmic fairness , original position , vei...   \n",
              "1  algorithmic fairness , original position , vei...   \n",
              "2  artificial intelligence , automation , busines...   \n",
              "3  ccs concepts â€¢ information systems â† ’ reco...   \n",
              "4  algorithmic bias , artificial intelligence , e...   \n",
              "\n",
              "                                      right_keywords  label  \n",
              "0  ccs concepts â€¢ information systems â† ’ reco...      0  \n",
              "1  algorithm , artificial intelligence , bias , d...      0  \n",
              "2  algorithm , artificial intelligence , bias , d...      1  \n",
              "3  algorithmic fairness , literature review , pro...      0  \n",
              "4  humans and ai : ethical issues in ai , machine...      1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-cf55eff4-face-4102-8da7-23772e38b62f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>278</td>\n",
              "      <td>rawls 's original position and algorithmic fai...</td>\n",
              "      <td>effective exposure amortizing for fair top-k r...</td>\n",
              "      <td>franke u</td>\n",
              "      <td>yang t , xu z , ai q</td>\n",
              "      <td>algorithmic fairness , original position , vei...</td>\n",
              "      <td>ccs concepts â€¢ information systems â† ’ reco...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>272</td>\n",
              "      <td>rawls 's original position and algorithmic fai...</td>\n",
              "      <td>why fairness can not be automated : bridging t...</td>\n",
              "      <td>franke u</td>\n",
              "      <td>wachter s , mittelstadt b , russell c</td>\n",
              "      <td>algorithmic fairness , original position , vei...</td>\n",
              "      <td>algorithm , artificial intelligence , bias , d...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1054</td>\n",
              "      <td>the rising tide of artificial intelligence and...</td>\n",
              "      <td>why fairness can not be automated : bridging t...</td>\n",
              "      <td>wright sa , schultz ae</td>\n",
              "      <td>wachter s , mittelstadt b , russell c</td>\n",
              "      <td>artificial intelligence , automation , busines...</td>\n",
              "      <td>algorithm , artificial intelligence , bias , d...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1120</td>\n",
              "      <td>effective exposure amortizing for fair top-k r...</td>\n",
              "      <td>a sociotechnical view of algorithmic fairness</td>\n",
              "      <td>yang t , xu z , ai q</td>\n",
              "      <td>dolata m , feuerriegel s , schwabe g</td>\n",
              "      <td>ccs concepts â€¢ information systems â† ’ reco...</td>\n",
              "      <td>algorithmic fairness , literature review , pro...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>164</td>\n",
              "      <td>beyond fairness metrics : roadblocks and chall...</td>\n",
              "      <td>achieving causal fairness through generative a...</td>\n",
              "      <td>chen j , storchan v , kurshan e</td>\n",
              "      <td>xu d , wu y , yuan s , zhang l , wu x</td>\n",
              "      <td>algorithmic bias , artificial intelligence , e...</td>\n",
              "      <td>humans and ai : ethical issues in ai , machine...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cf55eff4-face-4102-8da7-23772e38b62f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-cf55eff4-face-4102-8da7-23772e38b62f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-cf55eff4-face-4102-8da7-23772e38b62f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Hybrid**"
      ],
      "metadata": {
        "id": "DJ0D6MvfgPyE"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tDAvFSG7NfiQ",
        "outputId": "cfbabd52-cb25-40b8-d4c4-909367020e13"
      },
      "source": [
        "model = dm.MatchingModel(attr_summarizer='hybrid')\n",
        "\n",
        "model.run_train(\n",
        "    train,\n",
        "    validation,\n",
        "    epochs=10,\n",
        "    batch_size=16,\n",
        "    best_save_path='hybrid_model.pth',\n",
        "    pos_neg_ratio=17)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:770: UserWarning: Using non-full backward hooks on a Module that does not take as input a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not take as input a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "* Number of trainable parameters: 7133105\n",
            "===>  TRAIN Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:760: UserWarning: Using non-full backward hooks on a Module that does not return a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_output. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not return a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/functional.py:2607: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
            "  warnings.warn(\n",
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:17\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 1 || Run Time:   79.1 | Load Time:    0.4 || F1:  43.30 | Prec:  27.63 | Rec: 100.00 || Ex/s:   9.24\n",
            "\n",
            "===>  EVAL Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:08\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 1 || Run Time:    8.3 | Load Time:    0.1 || F1:  44.16 | Prec:  28.33 | Rec: 100.00 || Ex/s:  29.02\n",
            "\n",
            "* Best F1: tensor(44.1558)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:06\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 2 || Run Time:   68.4 | Load Time:    0.3 || F1:  44.54 | Prec:  28.65 | Rec: 100.00 || Ex/s:  10.69\n",
            "\n",
            "===>  EVAL Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:09\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 2 || Run Time:    9.1 | Load Time:    0.1 || F1:  49.82 | Prec:  33.17 | Rec: 100.00 || Ex/s:  26.55\n",
            "\n",
            "* Best F1: tensor(49.8168)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:05\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 3 || Run Time:   68.5 | Load Time:    0.3 || F1:  46.98 | Prec:  30.70 | Rec: 100.00 || Ex/s:  10.67\n",
            "\n",
            "===>  EVAL Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:08\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 3 || Run Time:    8.5 | Load Time:    0.1 || F1:  51.52 | Prec:  34.69 | Rec: 100.00 || Ex/s:  28.37\n",
            "\n",
            "* Best F1: tensor(51.5152)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:06\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 4 || Run Time:   68.4 | Load Time:    0.3 || F1:  48.56 | Prec:  32.06 | Rec: 100.00 || Ex/s:  10.70\n",
            "\n",
            "===>  EVAL Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:08\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 4 || Run Time:    9.0 | Load Time:    0.1 || F1:  52.11 | Prec:  35.23 | Rec: 100.00 || Ex/s:  26.77\n",
            "\n",
            "* Best F1: tensor(52.1073)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:04\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 5 || Run Time:   66.3 | Load Time:    0.3 || F1:  49.82 | Prec:  33.17 | Rec: 100.00 || Ex/s:  11.04\n",
            "\n",
            "===>  EVAL Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:09\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 5 || Run Time:    9.1 | Load Time:    0.1 || F1:  48.75 | Prec:  32.23 | Rec: 100.00 || Ex/s:  26.60\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:04\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 6 || Run Time:   67.8 | Load Time:    0.4 || F1:  50.56 | Prec:  33.84 | Rec: 100.00 || Ex/s:  10.79\n",
            "\n",
            "===>  EVAL Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:07\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 6 || Run Time:    7.4 | Load Time:    0.1 || F1:  52.92 | Prec:  35.98 | Rec: 100.00 || Ex/s:  32.74\n",
            "\n",
            "* Best F1: tensor(52.9183)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:05\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 7 || Run Time:   67.6 | Load Time:    0.3 || F1:  54.37 | Prec:  37.34 | Rec: 100.00 || Ex/s:  10.82\n",
            "\n",
            "===>  EVAL Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:07\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 7 || Run Time:    7.9 | Load Time:    0.1 || F1:  53.01 | Prec:  36.46 | Rec:  97.06 || Ex/s:  30.74\n",
            "\n",
            "* Best F1: tensor(53.0120)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:05\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 8 || Run Time:   66.9 | Load Time:    0.3 || F1:  59.50 | Prec:  42.35 | Rec: 100.00 || Ex/s:  10.93\n",
            "\n",
            "===>  EVAL Epoch 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:08\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 8 || Run Time:    9.0 | Load Time:    0.1 || F1:  51.01 | Prec:  35.20 | Rec:  92.65 || Ex/s:  26.82\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:04\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 9 || Run Time:   66.7 | Load Time:    0.3 || F1:  63.03 | Prec:  46.01 | Rec: 100.00 || Ex/s:  10.97\n",
            "\n",
            "===>  EVAL Epoch 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:08\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 9 || Run Time:    8.7 | Load Time:    0.1 || F1:  52.50 | Prec:  36.63 | Rec:  92.65 || Ex/s:  27.68\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:01:06\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:   68.4 | Load Time:    0.3 || F1:  65.91 | Prec:  49.15 | Rec: 100.00 || Ex/s:  10.70\n",
            "\n",
            "===>  EVAL Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:07\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    7.5 | Load Time:    0.1 || F1:  54.95 | Prec:  39.61 | Rec:  89.71 || Ex/s:  32.29\n",
            "\n",
            "* Best F1: tensor(54.9550)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "Loading best model...\n",
            "Training done.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(54.9550)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WhdAyaBxRjcZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "895688ab-06e4-44ea-e1ac-246113421f7b"
      },
      "source": [
        "unlabeled = dm.data.process_unlabeled(\n",
        "    path='Unlabeled.csv',\n",
        "    trained_model=model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Reading and processing data from \"Unlabeled.csv\"\n",
            "0% [############################# ] 100% | ETA: 00:00:00"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OxvMTtQLRnye",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "outputId": "c1220c10-1ec3-4adc-8c18-21992832b21c"
      },
      "source": [
        "predictions = model.run_prediction(unlabeled)\n",
        "predictions.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===>  PREDICT Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:32\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:   35.5 | Load Time:    0.5 || F1:   0.00 | Prec:   0.00 | Rec:   0.00 || Ex/s:   0.00\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    match_score\n",
              "id             \n",
              "1      0.632310\n",
              "2      0.715031\n",
              "3      0.421817\n",
              "4      0.456334\n",
              "5      0.935682"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-29647c8c-e937-4f9f-a4b5-2bf30743a741\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.632310</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.715031</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.421817</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.456334</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.935682</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-29647c8c-e937-4f9f-a4b5-2bf30743a741')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-29647c8c-e937-4f9f-a4b5-2bf30743a741 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-29647c8c-e937-4f9f-a4b5-2bf30743a741');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTrIKkaiRqPI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 752
        },
        "outputId": "01b9a4a5-df52-4c64-8118-ceb960195137"
      },
      "source": [
        "predictions = model.run_prediction(unlabeled, output_attributes=True)\n",
        "predictions.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===>  PREDICT Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:770: UserWarning: Using non-full backward hooks on a Module that does not take as input a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not take as input a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:760: UserWarning: Using non-full backward hooks on a Module that does not return a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_output. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not return a \"\n",
            "0% [███████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:31\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:   34.2 | Load Time:    0.5 || F1:   0.00 | Prec:   0.00 | Rec:   0.00 || Ex/s:   0.00\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    match_score                                         left_title  \\\n",
              "id                                                                   \n",
              "1      0.632310  transparency and explainability of ai systems:...   \n",
              "2      0.715031  transparency and explainability of ai systems:...   \n",
              "3      0.421817  transparency and explainability of ai systems:...   \n",
              "4      0.456334  transparency and explainability of ai systems:...   \n",
              "5      0.935682  transparency and explainability of ai systems:...   \n",
              "\n",
              "                                          right_title  \\\n",
              "id                                                      \n",
              "1   artificial intelligence, intersectionality, an...   \n",
              "2          fairness & friends in the data science era   \n",
              "3       a sociotechnical view of algorithmic fairness   \n",
              "4   ai-enabled radio resource allocation in 5g for...   \n",
              "5   beyond bias and discrimination: redefining the...   \n",
              "\n",
              "                                         left_authors  \\\n",
              "id                                                      \n",
              "1   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "2   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "3   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "4   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "5   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "\n",
              "                       right_authors  \\\n",
              "id                                     \n",
              "1                bauer gr,lizotte dj   \n",
              "2   catania b,guerrini g,accinelli c   \n",
              "3   dolata m,feuerriegel s,schwabe g   \n",
              "4          elsayed m,erol-kantarci m   \n",
              "5            giovanola b,tiribelli s   \n",
              "\n",
              "                                        left_keywords  \\\n",
              "id                                                      \n",
              "1   ai systems,ethical guidelines,explainability,q...   \n",
              "2   ai systems,ethical guidelines,explainability,q...   \n",
              "3   ai systems,ethical guidelines,explainability,q...   \n",
              "4   ai systems,ethical guidelines,explainability,q...   \n",
              "5   ai systems,ethical guidelines,explainability,q...   \n",
              "\n",
              "                                       right_keywords  \n",
              "id                                                     \n",
              "1   ajph,apha,american,association,health,journal,...  \n",
              "2   data-informed automated decision system,divers...  \n",
              "3   algorithmic fairness,literature review,problem...  \n",
              "4   5g mobile communication,delays,quality of serv...  \n",
              "5   bias,discrimination,ethics of algorithms,fairn...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-38ab7cd2-ece8-444c-86ad-d696dfaec853\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.632310</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>artificial intelligence, intersectionality, an...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>bauer gr,lizotte dj</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>ajph,apha,american,association,health,journal,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.715031</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>fairness &amp; friends in the data science era</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>catania b,guerrini g,accinelli c</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>data-informed automated decision system,divers...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.421817</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>a sociotechnical view of algorithmic fairness</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>dolata m,feuerriegel s,schwabe g</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>algorithmic fairness,literature review,problem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.456334</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>ai-enabled radio resource allocation in 5g for...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>elsayed m,erol-kantarci m</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>5g mobile communication,delays,quality of serv...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.935682</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>beyond bias and discrimination: redefining the...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>giovanola b,tiribelli s</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>bias,discrimination,ethics of algorithms,fairn...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-38ab7cd2-ece8-444c-86ad-d696dfaec853')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-38ab7cd2-ece8-444c-86ad-d696dfaec853 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-38ab7cd2-ece8-444c-86ad-d696dfaec853');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NBqtSZyHSDiy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "outputId": "cdeedd30-d818-41fd-848f-90dd9270a90b"
      },
      "source": [
        "high_score_pairs = predictions[predictions['match_score'] >= 0.9].sort_values(by=['match_score'], ascending=False)\n",
        "high_score_pairs.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     match_score                                         left_title  \\\n",
              "id                                                                    \n",
              "209     0.999929  bias and fairness in machine learning and arti...   \n",
              "37      0.999897  a perspective on fairness in artificial intell...   \n",
              "210     0.999889  bias and fairness in machine learning and arti...   \n",
              "179     0.999889  bias and fairness in machine learning and arti...   \n",
              "755     0.999877  how self-perceived reputation affects fairness...   \n",
              "\n",
              "                                           right_title  \\\n",
              "id                                                       \n",
              "209  artificial intelligence and the public sectorâ...   \n",
              "37   exploring bias and fairness in artificial inte...   \n",
              "210  the rising tide of artificial intelligence and...   \n",
              "179  addressing fairness, bias, and appropriate use...   \n",
              "755  addressing fairness, bias, and appropriate use...   \n",
              "\n",
              "                       left_authors  \\\n",
              "id                                    \n",
              "209         cirillo d,rementeria mj   \n",
              "37                   bhattacharya p   \n",
              "210         cirillo d,rementeria mj   \n",
              "179         cirillo d,rementeria mj   \n",
              "755  russo pa,duradoni m,guazzini a   \n",
              "\n",
              "                                        right_authors  \\\n",
              "id                                                      \n",
              "209                       wirtz bw,weyerer jc,geyer c   \n",
              "37   khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "210                              wright sa,schultz ae   \n",
              "179                     fletcher rr,nakeshimana a,...   \n",
              "755                     fletcher rr,nakeshimana a,...   \n",
              "\n",
              "                                         left_keywords  \\\n",
              "id                                                       \n",
              "209  artificial intelligence,bias,fairness,model de...   \n",
              "37   a perspective on fairness in artificial intell...   \n",
              "210  artificial intelligence,bias,fairness,model de...   \n",
              "179  artificial intelligence,bias,fairness,model de...   \n",
              "755  artificial intelligence,fairness,reputation,ul...   \n",
              "\n",
              "                                        right_keywords  \n",
              "id                                                      \n",
              "209  ai applications,ai challenges,artificial intel...  \n",
              "37   artificial intelligence,detection and tracking...  \n",
              "210  artificial intelligence,automation,business et...  \n",
              "179  appropriate use,artificial intelligence,bias,e...  \n",
              "755  appropriate use,artificial intelligence,bias,e...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e415b761-835d-4e4c-b1c1-b91786495742\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>209</th>\n",
              "      <td>0.999929</td>\n",
              "      <td>bias and fairness in machine learning and arti...</td>\n",
              "      <td>artificial intelligence and the public sectorâ...</td>\n",
              "      <td>cirillo d,rementeria mj</td>\n",
              "      <td>wirtz bw,weyerer jc,geyer c</td>\n",
              "      <td>artificial intelligence,bias,fairness,model de...</td>\n",
              "      <td>ai applications,ai challenges,artificial intel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>0.999897</td>\n",
              "      <td>a perspective on fairness in artificial intell...</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>bhattacharya p</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>a perspective on fairness in artificial intell...</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>210</th>\n",
              "      <td>0.999889</td>\n",
              "      <td>bias and fairness in machine learning and arti...</td>\n",
              "      <td>the rising tide of artificial intelligence and...</td>\n",
              "      <td>cirillo d,rementeria mj</td>\n",
              "      <td>wright sa,schultz ae</td>\n",
              "      <td>artificial intelligence,bias,fairness,model de...</td>\n",
              "      <td>artificial intelligence,automation,business et...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>179</th>\n",
              "      <td>0.999889</td>\n",
              "      <td>bias and fairness in machine learning and arti...</td>\n",
              "      <td>addressing fairness, bias, and appropriate use...</td>\n",
              "      <td>cirillo d,rementeria mj</td>\n",
              "      <td>fletcher rr,nakeshimana a,...</td>\n",
              "      <td>artificial intelligence,bias,fairness,model de...</td>\n",
              "      <td>appropriate use,artificial intelligence,bias,e...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>755</th>\n",
              "      <td>0.999877</td>\n",
              "      <td>how self-perceived reputation affects fairness...</td>\n",
              "      <td>addressing fairness, bias, and appropriate use...</td>\n",
              "      <td>russo pa,duradoni m,guazzini a</td>\n",
              "      <td>fletcher rr,nakeshimana a,...</td>\n",
              "      <td>artificial intelligence,fairness,reputation,ul...</td>\n",
              "      <td>appropriate use,artificial intelligence,bias,e...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e415b761-835d-4e4c-b1c1-b91786495742')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e415b761-835d-4e4c-b1c1-b91786495742 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e415b761-835d-4e4c-b1c1-b91786495742');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MFOMJi_oSQ42"
      },
      "source": [
        "predictions.to_csv('unlabeled_predictions_hybrid.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**SIF**"
      ],
      "metadata": {
        "id": "_Dw3b2EdghIk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = dm.MatchingModel(attr_summarizer='sif')\n",
        "\n",
        "model.run_train(\n",
        "    train,\n",
        "    validation,\n",
        "    epochs=10,\n",
        "    batch_size=16,\n",
        "    best_save_path='sif_model.pth',\n",
        "    pos_neg_ratio=17)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZpXq54V8glzv",
        "outputId": "949b3162-f039-44f7-cc4a-58e4159d2e05"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "* Number of trainable parameters: 542402\n",
            "===>  TRAIN Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/functional.py:2607: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
            "  warnings.warn(\n",
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 1 || Run Time:    1.3 | Load Time:    0.4 || F1:  42.40 | Prec:  27.32 | Rec:  94.55 || Ex/s: 429.47\n",
            "\n",
            "===>  EVAL Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 1 || Run Time:    0.2 | Load Time:    0.1 || F1:  43.45 | Prec:  27.76 | Rec: 100.00 || Ex/s: 636.57\n",
            "\n",
            "* Best F1: tensor(43.4505)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 2 || Run Time:    1.3 | Load Time:    0.4 || F1:  43.12 | Prec:  27.48 | Rec: 100.00 || Ex/s: 417.25\n",
            "\n",
            "===>  EVAL Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 2 || Run Time:    0.3 | Load Time:    0.2 || F1:  43.45 | Prec:  27.76 | Rec: 100.00 || Ex/s: 565.09\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 3 || Run Time:    1.0 | Load Time:    0.3 || F1:  43.12 | Prec:  27.48 | Rec: 100.00 || Ex/s: 595.77\n",
            "\n",
            "===>  EVAL Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 3 || Run Time:    0.2 | Load Time:    0.1 || F1:  43.45 | Prec:  27.76 | Rec: 100.00 || Ex/s: 913.84\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 4 || Run Time:    1.0 | Load Time:    0.3 || F1:  43.12 | Prec:  27.48 | Rec: 100.00 || Ex/s: 597.31\n",
            "\n",
            "===>  EVAL Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 4 || Run Time:    0.2 | Load Time:    0.1 || F1:  43.59 | Prec:  27.87 | Rec: 100.00 || Ex/s: 877.93\n",
            "\n",
            "* Best F1: tensor(43.5897)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 5 || Run Time:    0.9 | Load Time:    0.3 || F1:  43.35 | Prec:  27.67 | Rec: 100.00 || Ex/s: 625.14\n",
            "\n",
            "===>  EVAL Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 5 || Run Time:    0.2 | Load Time:    0.1 || F1:  43.73 | Prec:  27.98 | Rec: 100.00 || Ex/s: 938.57\n",
            "\n",
            "* Best F1: tensor(43.7299)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 6 || Run Time:    0.9 | Load Time:    0.3 || F1:  44.59 | Prec:  28.69 | Rec: 100.00 || Ex/s: 627.56\n",
            "\n",
            "===>  EVAL Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 6 || Run Time:    0.2 | Load Time:    0.1 || F1:  45.03 | Prec:  29.06 | Rec: 100.00 || Ex/s: 961.28\n",
            "\n",
            "* Best F1: tensor(45.0331)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 7 || Run Time:    0.9 | Load Time:    0.2 || F1:  45.65 | Prec:  29.58 | Rec: 100.00 || Ex/s: 627.71\n",
            "\n",
            "===>  EVAL Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 7 || Run Time:    0.2 | Load Time:    0.1 || F1:  45.48 | Prec:  29.44 | Rec: 100.00 || Ex/s: 929.54\n",
            "\n",
            "* Best F1: tensor(45.4850)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 8 || Run Time:    0.9 | Load Time:    0.2 || F1:  46.17 | Prec:  30.01 | Rec: 100.00 || Ex/s: 626.89\n",
            "\n",
            "===>  EVAL Epoch 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 8 || Run Time:    0.2 | Load Time:    0.1 || F1:  45.79 | Prec:  29.69 | Rec: 100.00 || Ex/s: 927.46\n",
            "\n",
            "* Best F1: tensor(45.7912)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 9 || Run Time:    0.9 | Load Time:    0.3 || F1:  47.03 | Prec:  30.75 | Rec: 100.00 || Ex/s: 626.10\n",
            "\n",
            "===>  EVAL Epoch 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 9 || Run Time:    0.3 | Load Time:    0.1 || F1:  46.26 | Prec:  30.09 | Rec: 100.00 || Ex/s: 611.95\n",
            "\n",
            "* Best F1: tensor(46.2585)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    1.2 | Load Time:    0.4 || F1:  47.64 | Prec:  31.27 | Rec: 100.00 || Ex/s: 439.38\n",
            "\n",
            "===>  EVAL Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    0.3 | Load Time:    0.2 || F1:  46.58 | Prec:  30.36 | Rec: 100.00 || Ex/s: 583.55\n",
            "\n",
            "* Best F1: tensor(46.5753)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "Loading best model...\n",
            "Training done.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(46.5753)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "unlabeled = dm.data.process_unlabeled(\n",
        "    path='Unlabeled.csv',\n",
        "    trained_model=model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VrFYikuuhOqz",
        "outputId": "737712a3-0e4a-4798-80f2-74ce8c32d45d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Reading and processing data from \"Unlabeled.csv\"\n",
            "0% [############################# ] 100% | ETA: 00:00:00"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions = model.run_prediction(unlabeled)\n",
        "predictions.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "id": "5gqJNQIQhT1A",
        "outputId": "208c8a73-881c-4ab2-bfc8-c8605806fda9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===>  PREDICT Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    0.6 | Load Time:    0.4 || F1:   0.00 | Prec:   0.00 | Rec:   0.00 || Ex/s:   0.00\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    match_score\n",
              "id             \n",
              "1      0.854694\n",
              "2      0.909587\n",
              "3      0.896578\n",
              "4      0.725479\n",
              "5      0.910258"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ebee3ced-933e-45e4-9cc3-28badf597889\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.854694</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.909587</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.896578</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.725479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.910258</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ebee3ced-933e-45e4-9cc3-28badf597889')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ebee3ced-933e-45e4-9cc3-28badf597889 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ebee3ced-933e-45e4-9cc3-28badf597889');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions = model.run_prediction(unlabeled, output_attributes=True)\n",
        "predictions.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 683
        },
        "id": "TXyXb4ZbhY6a",
        "outputId": "c96e32a6-556e-4e37-a35f-362055822623"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===>  PREDICT Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n",
            "0% [███████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    0.6 | Load Time:    0.4 || F1:   0.00 | Prec:   0.00 | Rec:   0.00 || Ex/s:   0.00\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    match_score                                         left_title  \\\n",
              "id                                                                   \n",
              "1      0.854694  transparency and explainability of ai systems:...   \n",
              "2      0.909587  transparency and explainability of ai systems:...   \n",
              "3      0.896578  transparency and explainability of ai systems:...   \n",
              "4      0.725479  transparency and explainability of ai systems:...   \n",
              "5      0.910258  transparency and explainability of ai systems:...   \n",
              "\n",
              "                                          right_title  \\\n",
              "id                                                      \n",
              "1   artificial intelligence, intersectionality, an...   \n",
              "2          fairness & friends in the data science era   \n",
              "3       a sociotechnical view of algorithmic fairness   \n",
              "4   ai-enabled radio resource allocation in 5g for...   \n",
              "5   beyond bias and discrimination: redefining the...   \n",
              "\n",
              "                                         left_authors  \\\n",
              "id                                                      \n",
              "1   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "2   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "3   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "4   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "5   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "\n",
              "                       right_authors  \\\n",
              "id                                     \n",
              "1                bauer gr,lizotte dj   \n",
              "2   catania b,guerrini g,accinelli c   \n",
              "3   dolata m,feuerriegel s,schwabe g   \n",
              "4          elsayed m,erol-kantarci m   \n",
              "5            giovanola b,tiribelli s   \n",
              "\n",
              "                                        left_keywords  \\\n",
              "id                                                      \n",
              "1   ai systems,ethical guidelines,explainability,q...   \n",
              "2   ai systems,ethical guidelines,explainability,q...   \n",
              "3   ai systems,ethical guidelines,explainability,q...   \n",
              "4   ai systems,ethical guidelines,explainability,q...   \n",
              "5   ai systems,ethical guidelines,explainability,q...   \n",
              "\n",
              "                                       right_keywords  \n",
              "id                                                     \n",
              "1   ajph,apha,american,association,health,journal,...  \n",
              "2   data-informed automated decision system,divers...  \n",
              "3   algorithmic fairness,literature review,problem...  \n",
              "4   5g mobile communication,delays,quality of serv...  \n",
              "5   bias,discrimination,ethics of algorithms,fairn...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d8158c3c-ea32-435a-b3ab-0b75dce7b1ab\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.854694</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>artificial intelligence, intersectionality, an...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>bauer gr,lizotte dj</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>ajph,apha,american,association,health,journal,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.909587</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>fairness &amp; friends in the data science era</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>catania b,guerrini g,accinelli c</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>data-informed automated decision system,divers...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.896578</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>a sociotechnical view of algorithmic fairness</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>dolata m,feuerriegel s,schwabe g</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>algorithmic fairness,literature review,problem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.725479</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>ai-enabled radio resource allocation in 5g for...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>elsayed m,erol-kantarci m</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>5g mobile communication,delays,quality of serv...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.910258</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>beyond bias and discrimination: redefining the...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>giovanola b,tiribelli s</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>bias,discrimination,ethics of algorithms,fairn...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d8158c3c-ea32-435a-b3ab-0b75dce7b1ab')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d8158c3c-ea32-435a-b3ab-0b75dce7b1ab button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d8158c3c-ea32-435a-b3ab-0b75dce7b1ab');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "high_score_pairs = predictions[predictions['match_score'] >= 0.9].sort_values(by=['match_score'], ascending=False)\n",
        "high_score_pairs.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "fQk3m1BahdVe",
        "outputId": "e72225fe-a468-4083-ee94-3552a1cd67fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     match_score                                         left_title  \\\n",
              "id                                                                    \n",
              "554     0.994315  exploring bias and fairness in artificial inte...   \n",
              "546     0.992722  exploring bias and fairness in artificial inte...   \n",
              "626     0.992425  assessing the fairness of ai systems: ai pract...   \n",
              "534     0.992107  introduction to the special issue on ai fairne...   \n",
              "547     0.991667  exploring bias and fairness in artificial inte...   \n",
              "\n",
              "                                           right_title  \\\n",
              "id                                                       \n",
              "554  artificial intelligence and the public sectorâ...   \n",
              "546  principal fairness for human and algorithmic d...   \n",
              "626  exploring bias and fairness in artificial inte...   \n",
              "534  a survey on datasets for fairness-aware machin...   \n",
              "547  introduction to the special issue on ai fairne...   \n",
              "\n",
              "                                          left_authors  \\\n",
              "id                                                       \n",
              "554   khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "546   khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "626  madaio m,egede l,subramonyam h,wortman vaughan...   \n",
              "534                      jr lp,bansal g,melville n,...   \n",
              "547   khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "\n",
              "                                        right_authors  \\\n",
              "id                                                      \n",
              "554                       wirtz bw,weyerer jc,geyer c   \n",
              "546                                    imai k,jiang z   \n",
              "626  khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "534      le quy t,roy a,iosifidis v,zhang w,ntoutsi e   \n",
              "547                     jr lp,bansal g,melville n,...   \n",
              "\n",
              "                                         left_keywords  \\\n",
              "id                                                       \n",
              "554  artificial intelligence,detection and tracking...   \n",
              "546  artificial intelligence,detection and tracking...   \n",
              "626  ai,fairness,machine learning,software developm...   \n",
              "534  ai bias,ai fairness,ai trust,artificial intell...   \n",
              "547  artificial intelligence,detection and tracking...   \n",
              "\n",
              "                                        right_keywords  \n",
              "id                                                      \n",
              "554  ai applications,ai challenges,artificial intel...  \n",
              "546  algorithmic fairness,causal inference,potentia...  \n",
              "626  artificial intelligence,detection and tracking...  \n",
              "534  aware machine learning,benchmark datasets,bias...  \n",
              "547  ai bias,ai fairness,ai trust,artificial intell...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ce6ea402-718b-420a-aaf6-9856e6aa4f62\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>554</th>\n",
              "      <td>0.994315</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>artificial intelligence and the public sectorâ...</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>wirtz bw,weyerer jc,geyer c</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "      <td>ai applications,ai challenges,artificial intel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>546</th>\n",
              "      <td>0.992722</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>principal fairness for human and algorithmic d...</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>imai k,jiang z</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "      <td>algorithmic fairness,causal inference,potentia...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>626</th>\n",
              "      <td>0.992425</td>\n",
              "      <td>assessing the fairness of ai systems: ai pract...</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>madaio m,egede l,subramonyam h,wortman vaughan...</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>ai,fairness,machine learning,software developm...</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>534</th>\n",
              "      <td>0.992107</td>\n",
              "      <td>introduction to the special issue on ai fairne...</td>\n",
              "      <td>a survey on datasets for fairness-aware machin...</td>\n",
              "      <td>jr lp,bansal g,melville n,...</td>\n",
              "      <td>le quy t,roy a,iosifidis v,zhang w,ntoutsi e</td>\n",
              "      <td>ai bias,ai fairness,ai trust,artificial intell...</td>\n",
              "      <td>aware machine learning,benchmark datasets,bias...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>547</th>\n",
              "      <td>0.991667</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>introduction to the special issue on ai fairne...</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>jr lp,bansal g,melville n,...</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "      <td>ai bias,ai fairness,ai trust,artificial intell...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ce6ea402-718b-420a-aaf6-9856e6aa4f62')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ce6ea402-718b-420a-aaf6-9856e6aa4f62 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ce6ea402-718b-420a-aaf6-9856e6aa4f62');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions.to_csv('unlabeled_predictions_sif.csv')"
      ],
      "metadata": {
        "id": "slChRQochhaC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**RNN**"
      ],
      "metadata": {
        "id": "EsKUVidggiI4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = dm.MatchingModel(attr_summarizer='rnn')\n",
        "\n",
        "model.run_train(\n",
        "    train,\n",
        "    validation,\n",
        "    epochs=10,\n",
        "    batch_size=16,\n",
        "    best_save_path='rnn_model.pth',\n",
        "    pos_neg_ratio=17)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xVN5PKQlg7Lm",
        "outputId": "48f16ddd-257a-4694-eec1-16bb9b7cafc5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "* Number of trainable parameters: 1762802\n",
            "===>  TRAIN Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:770: UserWarning: Using non-full backward hooks on a Module that does not take as input a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not take as input a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:760: UserWarning: Using non-full backward hooks on a Module that does not return a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_output. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not return a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/functional.py:2607: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
            "  warnings.warn(\n",
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:14\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 1 || Run Time:   15.1 | Load Time:    0.3 || F1:  43.12 | Prec:  27.48 | Rec: 100.00 || Ex/s:  47.58\n",
            "\n",
            "===>  EVAL Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 1 || Run Time:    1.7 | Load Time:    0.1 || F1:  43.45 | Prec:  27.76 | Rec: 100.00 || Ex/s: 136.23\n",
            "\n",
            "* Best F1: tensor(43.4505)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 2 || Run Time:   15.2 | Load Time:    0.3 || F1:  43.12 | Prec:  27.48 | Rec: 100.00 || Ex/s:  47.31\n",
            "\n",
            "===>  EVAL Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 2 || Run Time:    1.7 | Load Time:    0.1 || F1:  43.45 | Prec:  27.76 | Rec: 100.00 || Ex/s: 134.41\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 3 || Run Time:   15.3 | Load Time:    0.3 || F1:  43.77 | Prec:  28.02 | Rec: 100.00 || Ex/s:  47.18\n",
            "\n",
            "===>  EVAL Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:02\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 3 || Run Time:    2.2 | Load Time:    0.1 || F1:  44.30 | Prec:  28.45 | Rec: 100.00 || Ex/s: 105.31\n",
            "\n",
            "* Best F1: tensor(44.2997)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 4 || Run Time:   15.9 | Load Time:    0.3 || F1:  47.09 | Prec:  30.79 | Rec: 100.00 || Ex/s:  45.24\n",
            "\n",
            "===>  EVAL Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:02\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 4 || Run Time:    2.1 | Load Time:    0.1 || F1:  46.53 | Prec:  30.45 | Rec:  98.53 || Ex/s: 112.04\n",
            "\n",
            "* Best F1: tensor(46.5278)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:14\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 5 || Run Time:   15.2 | Load Time:    0.3 || F1:  51.99 | Prec:  35.13 | Rec: 100.00 || Ex/s:  47.44\n",
            "\n",
            "===>  EVAL Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 5 || Run Time:    1.7 | Load Time:    0.1 || F1:  48.91 | Prec:  32.52 | Rec:  98.53 || Ex/s: 135.53\n",
            "\n",
            "* Best F1: tensor(48.9051)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:14\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 6 || Run Time:   15.1 | Load Time:    0.3 || F1:  56.35 | Prec:  39.22 | Rec: 100.00 || Ex/s:  47.52\n",
            "\n",
            "===>  EVAL Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 6 || Run Time:    1.7 | Load Time:    0.1 || F1:  50.00 | Prec:  33.85 | Rec:  95.59 || Ex/s: 137.09\n",
            "\n",
            "* Best F1: tensor(50.)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:14\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 7 || Run Time:   15.1 | Load Time:    0.3 || F1:  60.39 | Prec:  43.25 | Rec: 100.00 || Ex/s:  47.59\n",
            "\n",
            "===>  EVAL Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 7 || Run Time:    1.7 | Load Time:    0.1 || F1:  52.59 | Prec:  36.07 | Rec:  97.06 || Ex/s: 135.79\n",
            "\n",
            "* Best F1: tensor(52.5896)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 8 || Run Time:   15.5 | Load Time:    0.3 || F1:  63.03 | Prec:  46.01 | Rec: 100.00 || Ex/s:  46.35\n",
            "\n",
            "===>  EVAL Epoch 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:02\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 8 || Run Time:    2.7 | Load Time:    0.2 || F1:  53.23 | Prec:  36.67 | Rec:  97.06 || Ex/s:  86.42\n",
            "\n",
            "* Best F1: tensor(53.2258)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:14\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 9 || Run Time:   15.1 | Load Time:    0.3 || F1:  65.48 | Prec:  48.67 | Rec: 100.00 || Ex/s:  47.53\n",
            "\n",
            "===>  EVAL Epoch 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 9 || Run Time:    1.7 | Load Time:    0.1 || F1:  52.67 | Prec:  36.57 | Rec:  94.12 || Ex/s: 134.31\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:   15.3 | Load Time:    0.3 || F1:  66.45 | Prec:  49.75 | Rec: 100.00 || Ex/s:  47.12\n",
            "\n",
            "===>  EVAL Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    1.7 | Load Time:    0.1 || F1:  53.56 | Prec:  37.43 | Rec:  94.12 || Ex/s: 136.27\n",
            "\n",
            "* Best F1: tensor(53.5565)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "Loading best model...\n",
            "Training done.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(53.5565)"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "unlabeled = dm.data.process_unlabeled(\n",
        "    path='Unlabeled.csv',\n",
        "    trained_model=model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CfZBg-WAhPbF",
        "outputId": "1fd0210d-98f3-486d-f144-f68b3981aaf4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Reading and processing data from \"Unlabeled.csv\"\n",
            "0% [############################# ] 100% | ETA: 00:00:00"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions = model.run_prediction(unlabeled)\n",
        "predictions.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "id": "hZ5_-MfvhQAY",
        "outputId": "14fbc732-b0f6-4d4b-dc57-def2a7656047"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===>  PREDICT Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:07\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    7.8 | Load Time:    0.5 || F1:   0.00 | Prec:   0.00 | Rec:   0.00 || Ex/s:   0.00\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    match_score\n",
              "id             \n",
              "1      0.490867\n",
              "2      0.675077\n",
              "3      0.687438\n",
              "4      0.455088\n",
              "5      0.932421"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b0173d5a-bd3e-42cb-baf4-1e8dbc727bd2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.490867</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.675077</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.687438</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.455088</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.932421</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b0173d5a-bd3e-42cb-baf4-1e8dbc727bd2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b0173d5a-bd3e-42cb-baf4-1e8dbc727bd2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b0173d5a-bd3e-42cb-baf4-1e8dbc727bd2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions = model.run_prediction(unlabeled, output_attributes=True)\n",
        "predictions.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 752
        },
        "id": "nJp9s7iMhZrF",
        "outputId": "d131efdb-c184-4654-c348-00a222d653b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===>  PREDICT Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:770: UserWarning: Using non-full backward hooks on a Module that does not take as input a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not take as input a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:760: UserWarning: Using non-full backward hooks on a Module that does not return a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_output. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not return a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n",
            "0% [███████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:06\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    6.3 | Load Time:    0.4 || F1:   0.00 | Prec:   0.00 | Rec:   0.00 || Ex/s:   0.00\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    match_score                                         left_title  \\\n",
              "id                                                                   \n",
              "1      0.490867  transparency and explainability of ai systems:...   \n",
              "2      0.675077  transparency and explainability of ai systems:...   \n",
              "3      0.687438  transparency and explainability of ai systems:...   \n",
              "4      0.455088  transparency and explainability of ai systems:...   \n",
              "5      0.932421  transparency and explainability of ai systems:...   \n",
              "\n",
              "                                          right_title  \\\n",
              "id                                                      \n",
              "1   artificial intelligence, intersectionality, an...   \n",
              "2          fairness & friends in the data science era   \n",
              "3       a sociotechnical view of algorithmic fairness   \n",
              "4   ai-enabled radio resource allocation in 5g for...   \n",
              "5   beyond bias and discrimination: redefining the...   \n",
              "\n",
              "                                         left_authors  \\\n",
              "id                                                      \n",
              "1   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "2   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "3   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "4   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "5   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "\n",
              "                       right_authors  \\\n",
              "id                                     \n",
              "1                bauer gr,lizotte dj   \n",
              "2   catania b,guerrini g,accinelli c   \n",
              "3   dolata m,feuerriegel s,schwabe g   \n",
              "4          elsayed m,erol-kantarci m   \n",
              "5            giovanola b,tiribelli s   \n",
              "\n",
              "                                        left_keywords  \\\n",
              "id                                                      \n",
              "1   ai systems,ethical guidelines,explainability,q...   \n",
              "2   ai systems,ethical guidelines,explainability,q...   \n",
              "3   ai systems,ethical guidelines,explainability,q...   \n",
              "4   ai systems,ethical guidelines,explainability,q...   \n",
              "5   ai systems,ethical guidelines,explainability,q...   \n",
              "\n",
              "                                       right_keywords  \n",
              "id                                                     \n",
              "1   ajph,apha,american,association,health,journal,...  \n",
              "2   data-informed automated decision system,divers...  \n",
              "3   algorithmic fairness,literature review,problem...  \n",
              "4   5g mobile communication,delays,quality of serv...  \n",
              "5   bias,discrimination,ethics of algorithms,fairn...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b07bacdb-44d7-45f6-ac1a-ebba1fff1363\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.490867</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>artificial intelligence, intersectionality, an...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>bauer gr,lizotte dj</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>ajph,apha,american,association,health,journal,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.675077</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>fairness &amp; friends in the data science era</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>catania b,guerrini g,accinelli c</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>data-informed automated decision system,divers...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.687438</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>a sociotechnical view of algorithmic fairness</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>dolata m,feuerriegel s,schwabe g</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>algorithmic fairness,literature review,problem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.455088</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>ai-enabled radio resource allocation in 5g for...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>elsayed m,erol-kantarci m</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>5g mobile communication,delays,quality of serv...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.932421</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>beyond bias and discrimination: redefining the...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>giovanola b,tiribelli s</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>bias,discrimination,ethics of algorithms,fairn...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b07bacdb-44d7-45f6-ac1a-ebba1fff1363')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b07bacdb-44d7-45f6-ac1a-ebba1fff1363 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b07bacdb-44d7-45f6-ac1a-ebba1fff1363');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "high_score_pairs = predictions[predictions['match_score'] >= 0.9].sort_values(by=['match_score'], ascending=False)\n",
        "high_score_pairs.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 507
        },
        "id": "CLTPSidXhd_2",
        "outputId": "76218496-9c93-4a75-f863-e0fd28f75705"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     match_score                                         left_title  \\\n",
              "id                                                                    \n",
              "190     0.999999  bias and fairness in machine learning and arti...   \n",
              "554     0.999989  exploring bias and fairness in artificial inte...   \n",
              "599     0.999983  fmea-ai: ai fairness impact assessment using f...   \n",
              "321     0.999980                   an ontology for fairness metrics   \n",
              "755     0.999967  how self-perceived reputation affects fairness...   \n",
              "\n",
              "                                           right_title  \\\n",
              "id                                                       \n",
              "190  exploring bias and fairness in artificial inte...   \n",
              "554  artificial intelligence and the public sectorâ...   \n",
              "599  assessing the fairness of ai systems: ai pract...   \n",
              "321     algorithmic fairness in computational medicine   \n",
              "755  addressing fairness, bias, and appropriate use...   \n",
              "\n",
              "                                          left_authors  \\\n",
              "id                                                       \n",
              "190                            cirillo d,rementeria mj   \n",
              "554   khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "599                  li j,chignell textperiodcenteredm   \n",
              "321  franklin js,bhanot k,ghalwash m,bennett kp,mcc...   \n",
              "755                     russo pa,duradoni m,guazzini a   \n",
              "\n",
              "                                         right_authors  \\\n",
              "id                                                       \n",
              "190   khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "554                        wirtz bw,weyerer jc,geyer c   \n",
              "599  madaio m,egede l,subramonyam h,wortman vaughan...   \n",
              "321  xu j,xiao y,wang wh,ning y,shenkman ea,bian j,...   \n",
              "755                      fletcher rr,nakeshimana a,...   \n",
              "\n",
              "                                         left_keywords  \\\n",
              "id                                                       \n",
              "190  artificial intelligence,bias,fairness,model de...   \n",
              "554  artificial intelligence,detection and tracking...   \n",
              "599  artificial intelligence,ethics,human rights im...   \n",
              "321  bias,fairness metric,machine learning evaluati...   \n",
              "755  artificial intelligence,fairness,reputation,ul...   \n",
              "\n",
              "                                        right_keywords  \n",
              "id                                                      \n",
              "190  artificial intelligence,detection and tracking...  \n",
              "554  ai applications,ai challenges,artificial intel...  \n",
              "599  ai,fairness,machine learning,software developm...  \n",
              "321        algorithmic fairness,computational medicine  \n",
              "755  appropriate use,artificial intelligence,bias,e...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a8aa408e-f5fd-4058-b167-8ed9c8ca53d2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>190</th>\n",
              "      <td>0.999999</td>\n",
              "      <td>bias and fairness in machine learning and arti...</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>cirillo d,rementeria mj</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>artificial intelligence,bias,fairness,model de...</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>554</th>\n",
              "      <td>0.999989</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>artificial intelligence and the public sectorâ...</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>wirtz bw,weyerer jc,geyer c</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "      <td>ai applications,ai challenges,artificial intel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>599</th>\n",
              "      <td>0.999983</td>\n",
              "      <td>fmea-ai: ai fairness impact assessment using f...</td>\n",
              "      <td>assessing the fairness of ai systems: ai pract...</td>\n",
              "      <td>li j,chignell textperiodcenteredm</td>\n",
              "      <td>madaio m,egede l,subramonyam h,wortman vaughan...</td>\n",
              "      <td>artificial intelligence,ethics,human rights im...</td>\n",
              "      <td>ai,fairness,machine learning,software developm...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>321</th>\n",
              "      <td>0.999980</td>\n",
              "      <td>an ontology for fairness metrics</td>\n",
              "      <td>algorithmic fairness in computational medicine</td>\n",
              "      <td>franklin js,bhanot k,ghalwash m,bennett kp,mcc...</td>\n",
              "      <td>xu j,xiao y,wang wh,ning y,shenkman ea,bian j,...</td>\n",
              "      <td>bias,fairness metric,machine learning evaluati...</td>\n",
              "      <td>algorithmic fairness,computational medicine</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>755</th>\n",
              "      <td>0.999967</td>\n",
              "      <td>how self-perceived reputation affects fairness...</td>\n",
              "      <td>addressing fairness, bias, and appropriate use...</td>\n",
              "      <td>russo pa,duradoni m,guazzini a</td>\n",
              "      <td>fletcher rr,nakeshimana a,...</td>\n",
              "      <td>artificial intelligence,fairness,reputation,ul...</td>\n",
              "      <td>appropriate use,artificial intelligence,bias,e...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a8aa408e-f5fd-4058-b167-8ed9c8ca53d2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a8aa408e-f5fd-4058-b167-8ed9c8ca53d2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a8aa408e-f5fd-4058-b167-8ed9c8ca53d2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions.to_csv('unlabeled_predictions_rnn.csv')"
      ],
      "metadata": {
        "id": "5m_O5Ebbhjfa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Attention**"
      ],
      "metadata": {
        "id": "mCGQ6iMAgicI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = dm.MatchingModel(attr_summarizer='attention')\n",
        "\n",
        "model.run_train(\n",
        "    train,\n",
        "    validation,\n",
        "    epochs=10,\n",
        "    batch_size=16,\n",
        "    best_save_path='attention_model.pth',\n",
        "    pos_neg_ratio=17)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hdDsMCc2hDrR",
        "outputId": "71653ec2-55d2-482b-e6d7-96a3fc28afe8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "* Number of trainable parameters: 3429602\n",
            "===>  TRAIN Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:770: UserWarning: Using non-full backward hooks on a Module that does not take as input a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not take as input a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:760: UserWarning: Using non-full backward hooks on a Module that does not return a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_output. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not return a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/functional.py:2607: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
            "  warnings.warn(\n",
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:28\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 1 || Run Time:   28.8 | Load Time:    0.3 || F1:  43.44 | Prec:  27.94 | Rec:  97.52 || Ex/s:  25.23\n",
            "\n",
            "===>  EVAL Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:04\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 1 || Run Time:    4.8 | Load Time:    0.1 || F1:  46.26 | Prec:  30.09 | Rec: 100.00 || Ex/s:  49.43\n",
            "\n",
            "* Best F1: tensor(46.2585)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:27\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 2 || Run Time:   28.6 | Load Time:    0.3 || F1:  44.94 | Prec:  28.98 | Rec: 100.00 || Ex/s:  25.45\n",
            "\n",
            "===>  EVAL Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:03\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 2 || Run Time:    3.5 | Load Time:    0.1 || F1:  46.26 | Prec:  30.09 | Rec: 100.00 || Ex/s:  67.00\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:28\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 3 || Run Time:   29.2 | Load Time:    0.3 || F1:  46.22 | Prec:  30.06 | Rec: 100.00 || Ex/s:  24.93\n",
            "\n",
            "===>  EVAL Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:03\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 3 || Run Time:    3.4 | Load Time:    0.1 || F1:  44.30 | Prec:  28.45 | Rec: 100.00 || Ex/s:  69.32\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:28\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 4 || Run Time:   29.0 | Load Time:    0.3 || F1:  46.65 | Prec:  30.42 | Rec: 100.00 || Ex/s:  25.09\n",
            "\n",
            "===>  EVAL Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:04\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 4 || Run Time:    4.2 | Load Time:    0.1 || F1:  43.87 | Prec:  28.10 | Rec: 100.00 || Ex/s:  57.42\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:27\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 5 || Run Time:   29.0 | Load Time:    0.3 || F1:  47.14 | Prec:  30.84 | Rec: 100.00 || Ex/s:  25.06\n",
            "\n",
            "===>  EVAL Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:04\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 5 || Run Time:    4.2 | Load Time:    0.1 || F1:  44.44 | Prec:  28.57 | Rec: 100.00 || Ex/s:  56.58\n",
            "\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:28\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 6 || Run Time:   29.1 | Load Time:    0.3 || F1:  48.85 | Prec:  32.32 | Rec: 100.00 || Ex/s:  25.05\n",
            "\n",
            "===>  EVAL Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:03\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 6 || Run Time:    3.5 | Load Time:    0.1 || F1:  46.58 | Prec:  30.36 | Rec: 100.00 || Ex/s:  67.33\n",
            "\n",
            "* Best F1: tensor(46.5753)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:29\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 7 || Run Time:   29.9 | Load Time:    0.3 || F1:  49.39 | Prec:  32.79 | Rec: 100.00 || Ex/s:  24.36\n",
            "\n",
            "===>  EVAL Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:04\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 7 || Run Time:    4.4 | Load Time:    0.1 || F1:  47.72 | Prec:  31.34 | Rec: 100.00 || Ex/s:  54.27\n",
            "\n",
            "* Best F1: tensor(47.7193)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:30\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 8 || Run Time:   31.3 | Load Time:    0.3 || F1:  50.06 | Prec:  33.39 | Rec: 100.00 || Ex/s:  23.24\n",
            "\n",
            "===>  EVAL Epoch 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:03\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 8 || Run Time:    3.6 | Load Time:    0.1 || F1:  50.18 | Prec:  33.50 | Rec: 100.00 || Ex/s:  67.10\n",
            "\n",
            "* Best F1: tensor(50.1845)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:29\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 9 || Run Time:   30.6 | Load Time:    0.3 || F1:  51.33 | Prec:  34.53 | Rec: 100.00 || Ex/s:  23.82\n",
            "\n",
            "===>  EVAL Epoch 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:03\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 9 || Run Time:    3.7 | Load Time:    0.1 || F1:  51.13 | Prec:  34.34 | Rec: 100.00 || Ex/s:  64.38\n",
            "\n",
            "* Best F1: tensor(51.1278)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "===>  TRAIN Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [█████████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:29\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:   31.1 | Load Time:    0.3 || F1:  51.79 | Prec:  34.95 | Rec: 100.00 || Ex/s:  23.37\n",
            "\n",
            "===>  EVAL Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:04\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:    4.6 | Load Time:    0.1 || F1:  51.91 | Prec:  35.05 | Rec: 100.00 || Ex/s:  52.17\n",
            "\n",
            "* Best F1: tensor(51.9084)\n",
            "Saving best model...\n",
            "Done.\n",
            "---------------------\n",
            "\n",
            "Loading best model...\n",
            "Training done.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(51.9084)"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "unlabeled = dm.data.process_unlabeled(\n",
        "    path='Unlabeled.csv',\n",
        "    trained_model=model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9iw4lzAMhQdo",
        "outputId": "1b15c7c9-aac4-407a-c71d-9112f324f25d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Reading and processing data from \"Unlabeled.csv\"\n",
            "0% [############################# ] 100% | ETA: 00:00:00"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions = model.run_prediction(unlabeled)\n",
        "predictions.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "id": "cHIVBM5ShVC5",
        "outputId": "c1e4d67d-4405-441c-f2f1-967102bdb344"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===>  PREDICT Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "0% [███████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:17\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:   18.5 | Load Time:    0.5 || F1:   0.00 | Prec:   0.00 | Rec:   0.00 || Ex/s:   0.00\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    match_score\n",
              "id             \n",
              "1      0.566377\n",
              "2      0.450363\n",
              "3      0.822375\n",
              "4      0.375629\n",
              "5      0.551092"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-dc2b560a-3502-4fd5-9899-db705d4a330d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.566377</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.450363</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.822375</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.375629</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.551092</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dc2b560a-3502-4fd5-9899-db705d4a330d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-dc2b560a-3502-4fd5-9899-db705d4a330d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-dc2b560a-3502-4fd5-9899-db705d4a330d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions = model.run_prediction(unlabeled, output_attributes=True)\n",
        "predictions.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 752
        },
        "id": "xlFIST4ihaRv",
        "outputId": "4e620648-bb46-4473-b664-00b200ab8247"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "===>  PREDICT Epoch 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:770: UserWarning: Using non-full backward hooks on a Module that does not take as input a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not take as input a \"\n",
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py:760: UserWarning: Using non-full backward hooks on a Module that does not return a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_output. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not return a \"\n",
            "0% [███████] 100% | ETA: 00:00:00\n",
            "Total time elapsed: 00:00:16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished Epoch 10 || Run Time:   17.9 | Load Time:    0.5 || F1:   0.00 | Prec:   0.00 | Rec:   0.00 || Ex/s:   0.00\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    match_score                                         left_title  \\\n",
              "id                                                                   \n",
              "1      0.566377  transparency and explainability of ai systems:...   \n",
              "2      0.450363  transparency and explainability of ai systems:...   \n",
              "3      0.822375  transparency and explainability of ai systems:...   \n",
              "4      0.375629  transparency and explainability of ai systems:...   \n",
              "5      0.551092  transparency and explainability of ai systems:...   \n",
              "\n",
              "                                          right_title  \\\n",
              "id                                                      \n",
              "1   artificial intelligence, intersectionality, an...   \n",
              "2          fairness & friends in the data science era   \n",
              "3       a sociotechnical view of algorithmic fairness   \n",
              "4   ai-enabled radio resource allocation in 5g for...   \n",
              "5   beyond bias and discrimination: redefining the...   \n",
              "\n",
              "                                         left_authors  \\\n",
              "id                                                      \n",
              "1   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "2   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "3   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "4   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "5   balasubramaniam n,kauppinen m,hiekkanen k,kuja...   \n",
              "\n",
              "                       right_authors  \\\n",
              "id                                     \n",
              "1                bauer gr,lizotte dj   \n",
              "2   catania b,guerrini g,accinelli c   \n",
              "3   dolata m,feuerriegel s,schwabe g   \n",
              "4          elsayed m,erol-kantarci m   \n",
              "5            giovanola b,tiribelli s   \n",
              "\n",
              "                                        left_keywords  \\\n",
              "id                                                      \n",
              "1   ai systems,ethical guidelines,explainability,q...   \n",
              "2   ai systems,ethical guidelines,explainability,q...   \n",
              "3   ai systems,ethical guidelines,explainability,q...   \n",
              "4   ai systems,ethical guidelines,explainability,q...   \n",
              "5   ai systems,ethical guidelines,explainability,q...   \n",
              "\n",
              "                                       right_keywords  \n",
              "id                                                     \n",
              "1   ajph,apha,american,association,health,journal,...  \n",
              "2   data-informed automated decision system,divers...  \n",
              "3   algorithmic fairness,literature review,problem...  \n",
              "4   5g mobile communication,delays,quality of serv...  \n",
              "5   bias,discrimination,ethics of algorithms,fairn...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-cb4e4041-a7d8-4231-9ba8-611e30a4a9ca\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.566377</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>artificial intelligence, intersectionality, an...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>bauer gr,lizotte dj</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>ajph,apha,american,association,health,journal,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.450363</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>fairness &amp; friends in the data science era</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>catania b,guerrini g,accinelli c</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>data-informed automated decision system,divers...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.822375</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>a sociotechnical view of algorithmic fairness</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>dolata m,feuerriegel s,schwabe g</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>algorithmic fairness,literature review,problem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.375629</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>ai-enabled radio resource allocation in 5g for...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>elsayed m,erol-kantarci m</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>5g mobile communication,delays,quality of serv...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.551092</td>\n",
              "      <td>transparency and explainability of ai systems:...</td>\n",
              "      <td>beyond bias and discrimination: redefining the...</td>\n",
              "      <td>balasubramaniam n,kauppinen m,hiekkanen k,kuja...</td>\n",
              "      <td>giovanola b,tiribelli s</td>\n",
              "      <td>ai systems,ethical guidelines,explainability,q...</td>\n",
              "      <td>bias,discrimination,ethics of algorithms,fairn...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cb4e4041-a7d8-4231-9ba8-611e30a4a9ca')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-cb4e4041-a7d8-4231-9ba8-611e30a4a9ca button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-cb4e4041-a7d8-4231-9ba8-611e30a4a9ca');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "high_score_pairs = predictions[predictions['match_score'] >= 0.9].sort_values(by=['match_score'], ascending=False)\n",
        "high_score_pairs.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 489
        },
        "id": "DunTvWhNhevY",
        "outputId": "92d6f8aa-173a-411d-e3ec-5debc06c4180"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     match_score                                         left_title  \\\n",
              "id                                                                    \n",
              "246     0.998468  addressing fairness, bias, and appropriate use...   \n",
              "190     0.998406  bias and fairness in machine learning and arti...   \n",
              "234     0.998380  addressing fairness, bias, and appropriate use...   \n",
              "380     0.997740  beyond bias and discrimination: redefining the...   \n",
              "179     0.997508  bias and fairness in machine learning and arti...   \n",
              "\n",
              "                                           right_title  \\\n",
              "id                                                       \n",
              "246  bias, fairness and accountability with artific...   \n",
              "190  exploring bias and fairness in artificial inte...   \n",
              "234  exploring bias and fairness in artificial inte...   \n",
              "380  assessing the fairness of ai systems: ai pract...   \n",
              "179  addressing fairness, bias, and appropriate use...   \n",
              "\n",
              "                      left_authors  \\\n",
              "id                                   \n",
              "246  fletcher rr,nakeshimana a,...   \n",
              "190        cirillo d,rementeria mj   \n",
              "234  fletcher rr,nakeshimana a,...   \n",
              "380        giovanola b,tiribelli s   \n",
              "179        cirillo d,rementeria mj   \n",
              "\n",
              "                                         right_authors  \\\n",
              "id                                                       \n",
              "246            zhou n,zhang z,nair vn,singhal h,chen j   \n",
              "190   khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "234   khakurel ub,abdelmoumin g,bajracharya a,rawat db   \n",
              "380  madaio m,egede l,subramonyam h,wortman vaughan...   \n",
              "179                      fletcher rr,nakeshimana a,...   \n",
              "\n",
              "                                         left_keywords  \\\n",
              "id                                                       \n",
              "246  appropriate use,artificial intelligence,bias,e...   \n",
              "190  artificial intelligence,bias,fairness,model de...   \n",
              "234  appropriate use,artificial intelligence,bias,e...   \n",
              "380  bias,discrimination,ethics of algorithms,fairn...   \n",
              "179  artificial intelligence,bias,fairness,model de...   \n",
              "\n",
              "                                        right_keywords  \n",
              "id                                                      \n",
              "246                      ai,ml,algorithm,bias,fairness  \n",
              "190  artificial intelligence,detection and tracking...  \n",
              "234  artificial intelligence,detection and tracking...  \n",
              "380  ai,fairness,machine learning,software developm...  \n",
              "179  appropriate use,artificial intelligence,bias,e...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-49cdc1f7-557b-4ec3-9da8-0e5fc581ea78\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_score</th>\n",
              "      <th>left_title</th>\n",
              "      <th>right_title</th>\n",
              "      <th>left_authors</th>\n",
              "      <th>right_authors</th>\n",
              "      <th>left_keywords</th>\n",
              "      <th>right_keywords</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>246</th>\n",
              "      <td>0.998468</td>\n",
              "      <td>addressing fairness, bias, and appropriate use...</td>\n",
              "      <td>bias, fairness and accountability with artific...</td>\n",
              "      <td>fletcher rr,nakeshimana a,...</td>\n",
              "      <td>zhou n,zhang z,nair vn,singhal h,chen j</td>\n",
              "      <td>appropriate use,artificial intelligence,bias,e...</td>\n",
              "      <td>ai,ml,algorithm,bias,fairness</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>190</th>\n",
              "      <td>0.998406</td>\n",
              "      <td>bias and fairness in machine learning and arti...</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>cirillo d,rementeria mj</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>artificial intelligence,bias,fairness,model de...</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>234</th>\n",
              "      <td>0.998380</td>\n",
              "      <td>addressing fairness, bias, and appropriate use...</td>\n",
              "      <td>exploring bias and fairness in artificial inte...</td>\n",
              "      <td>fletcher rr,nakeshimana a,...</td>\n",
              "      <td>khakurel ub,abdelmoumin g,bajracharya a,rawat db</td>\n",
              "      <td>appropriate use,artificial intelligence,bias,e...</td>\n",
              "      <td>artificial intelligence,detection and tracking...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>380</th>\n",
              "      <td>0.997740</td>\n",
              "      <td>beyond bias and discrimination: redefining the...</td>\n",
              "      <td>assessing the fairness of ai systems: ai pract...</td>\n",
              "      <td>giovanola b,tiribelli s</td>\n",
              "      <td>madaio m,egede l,subramonyam h,wortman vaughan...</td>\n",
              "      <td>bias,discrimination,ethics of algorithms,fairn...</td>\n",
              "      <td>ai,fairness,machine learning,software developm...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>179</th>\n",
              "      <td>0.997508</td>\n",
              "      <td>bias and fairness in machine learning and arti...</td>\n",
              "      <td>addressing fairness, bias, and appropriate use...</td>\n",
              "      <td>cirillo d,rementeria mj</td>\n",
              "      <td>fletcher rr,nakeshimana a,...</td>\n",
              "      <td>artificial intelligence,bias,fairness,model de...</td>\n",
              "      <td>appropriate use,artificial intelligence,bias,e...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-49cdc1f7-557b-4ec3-9da8-0e5fc581ea78')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-49cdc1f7-557b-4ec3-9da8-0e5fc581ea78 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-49cdc1f7-557b-4ec3-9da8-0e5fc581ea78');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions.to_csv('unlabeled_predictions_attention.csv')"
      ],
      "metadata": {
        "id": "dowRxpsMhl02"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}