sota_log.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b0d87b52",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import lr_scheduler\n",
    "import torch.utils.data as Data\n",
    "\n",
    "from transformers import AutoModel, AutoTokenizer\n",
    "from utils import *\n",
    "\n",
    "import dgl\n",
    "\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "import numpy as np\n",
    "import os\n",
    "import shutil\n",
    "import argparse\n",
    "import sys\n",
    "import logging\n",
    "from datetime import datetime\n",
    "\n",
    "from ignite.engine import Events, create_supervised_evaluator, create_supervised_trainer, Engine\n",
    "from ignite.metrics import Accuracy, Loss, Metric\n",
    "from ignite.contrib.handlers.tqdm_logger import ProgressBar\n",
    "from ignite.utils import manual_seed\n",
    "\n",
    "from model import BertGCN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7be911cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 51\n",
    "manual_seed(seed)\n",
    "max_length = 64\n",
    "batch_size = 256\n",
    "ratio = 0.1\n",
    "nb_epochs = 5\n",
    "dataset = \"klue\"\n",
    "n_hidden = 200\n",
    "dropout = 0.5\n",
    "gcn_lr = 1e-3\n",
    "bert_lr = 1e-5\n",
    "args = [max_length, batch_size, ratio, nb_epochs, dataset, n_hidden, dropout, gcn_lr, bert_lr, seed]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7be21904",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "params:\n",
      "[64, 256, 0.1, 5, 'klue', 200, 0.5, 0.001, 1e-05, 51]\n",
      "checkpoints path: ./checkpoint/robertagcn_klue_0.1\n"
     ]
    }
   ],
   "source": [
    "ckpt_dir = './checkpoint/robertagcn_{}_{}'.format(dataset, ratio)\n",
    "os.makedirs(ckpt_dir, exist_ok=True)\n",
    "\n",
    "streamhandle = logging.StreamHandler(sys.stdout)\n",
    "streamhandle.setFormatter(logging.Formatter('%(message)s'))\n",
    "streamhandle.setLevel(logging.INFO)\n",
    "\n",
    "filehandle = logging.FileHandler(filename=os.path.join(ckpt_dir, 'training.log'), mode='w')\n",
    "filehandle.setFormatter(logging.Formatter('%(message)s'))\n",
    "filehandle.setLevel(logging.INFO)\n",
    "\n",
    "logger = logging.getLogger('training logger')\n",
    "logger.addHandler(streamhandle)\n",
    "logger.addHandler(filehandle)\n",
    "logger.setLevel(logging.INFO)\n",
    "\n",
    "cpu = torch.device('cpu')\n",
    "gpu = torch.device('cuda:0')\n",
    "\n",
    "logger.info('params:')\n",
    "logger.info(str(args))\n",
    "logger.info('checkpoints path: {}'.format(ckpt_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9759d745",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(41111, 300) (41111, 7) (9107, 300) (9107, 7) (66610, 300) (66610, 7)\n",
      "75717\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at klue/roberta-base were not used when initializing RobertaModel: ['lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.decoder.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
      "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaModel were not initialized from the model checkpoint at klue/roberta-base and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "graph information:\n",
      "Graph(num_nodes=75717, num_edges=4991863,\n",
      "      ndata_schemes={'input_ids': Scheme(shape=(64,), dtype=torch.int64), 'attention_mask': Scheme(shape=(64,), dtype=torch.int64), 'label': Scheme(shape=(), dtype=torch.int64), 'train': Scheme(shape=(), dtype=torch.float32), 'val': Scheme(shape=(), dtype=torch.float32), 'test': Scheme(shape=(), dtype=torch.float32), 'label_train': Scheme(shape=(), dtype=torch.int64), 'cls_feats': Scheme(shape=(768,), dtype=torch.float32)}\n",
      "      edata_schemes={'edge_weight': Scheme(shape=(), dtype=torch.float32)})\n"
     ]
    }
   ],
   "source": [
    "adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask, train_size, test_size = load_corpus(dataset)\n",
    "\n",
    "nb_node = features.shape[0]\n",
    "nb_train, nb_val, nb_test = train_mask.sum(), val_mask.sum(), test_mask.sum()\n",
    "nb_word = nb_node - nb_train - nb_val - nb_test\n",
    "nb_class = y_train.shape[1]\n",
    "\n",
    "\n",
    "model = BertGCN(nb_class=nb_class, ratio=ratio, n_hidden=n_hidden, dropout=dropout)\n",
    "\n",
    "corpse_file = './data/corpus/' + dataset +'_shuffle.txt'\n",
    "with open(corpse_file, 'r', encoding=\"utf-8\") as f:\n",
    "    text = f.read()\n",
    "    text = text.replace('\\\\', '')\n",
    "    text = text.split('\\n')\n",
    "\n",
    "def encode_input(text, tokenizer):\n",
    "    input = tokenizer(text, max_length=max_length, truncation=True, padding='max_length', return_tensors='pt')\n",
    "    return input.input_ids, input.attention_mask\n",
    "\n",
    "\n",
    "input_ids, attention_mask = encode_input(text, model.tokenizer)\n",
    "input_ids = torch.cat([input_ids[:-nb_test], torch.zeros((nb_word, max_length), dtype=torch.long), input_ids[-nb_test:]])\n",
    "attention_mask = torch.cat([attention_mask[:-nb_test], torch.zeros((nb_word, max_length), dtype=torch.long), attention_mask[-nb_test:]])\n",
    "\n",
    "y = y_train + y_test + y_val\n",
    "y_train = y_train.argmax(axis=1)\n",
    "y = y.argmax(axis=1)\n",
    "\n",
    "doc_mask  = train_mask + val_mask + test_mask\n",
    "\n",
    "adj_norm = normalize_adj(adj + sp.eye(adj.shape[0]))\n",
    "g = dgl.from_scipy(adj_norm.astype('float32'), eweight_name='edge_weight')\n",
    "g.ndata['input_ids'], g.ndata['attention_mask'] = input_ids, attention_mask\n",
    "g.ndata['label'], g.ndata['train'], g.ndata['val'], g.ndata['test'] = \\\n",
    "    torch.LongTensor(y), torch.FloatTensor(train_mask), torch.FloatTensor(val_mask), torch.FloatTensor(test_mask)\n",
    "g.ndata['label_train'] = torch.LongTensor(y_train)\n",
    "g.ndata['cls_feats'] = torch.zeros((nb_node, model.feat_dim))\n",
    "\n",
    "logger.info('graph information:')\n",
    "logger.info(str(g))\n",
    "\n",
    "train_idx = Data.TensorDataset(torch.arange(0, nb_train, dtype=torch.long))\n",
    "val_idx = Data.TensorDataset(torch.arange(nb_train, nb_train + nb_val, dtype=torch.long))\n",
    "test_idx = Data.TensorDataset(torch.arange(nb_node-nb_test, nb_node, dtype=torch.long))\n",
    "doc_idx = Data.ConcatDataset([train_idx, val_idx, test_idx])\n",
    "\n",
    "idx_loader_train = Data.DataLoader(train_idx, batch_size=batch_size, shuffle=True)\n",
    "idx_loader_val = Data.DataLoader(val_idx, batch_size=batch_size)\n",
    "idx_loader_test = Data.DataLoader(test_idx, batch_size=batch_size)\n",
    "idx_loader = Data.DataLoader(doc_idx, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d3c5cdc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class F1Score(Metric):\n",
    "\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        self.f1 = 0\n",
    "        self.count = 0\n",
    "        super().__init__(*args, **kwargs)\n",
    "\n",
    "    def update(self, output):\n",
    "        y_pred, y = output[0].detach(), output[1].detach()\n",
    "\n",
    "        _, predicted = torch.max(y_pred, 1)\n",
    "        f = f1_score(y.cpu(), predicted.cpu(), average='micro')\n",
    "        self.f1 += f\n",
    "        self.count += 1\n",
    "\n",
    "    def reset(self):\n",
    "        self.f1 = 0\n",
    "        self.count = 0\n",
    "        super(F1Score, self).reset()\n",
    "\n",
    "    def compute(self):\n",
    "        return self.f1 / self.count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "33756dd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_feature():\n",
    "    global model, g, doc_mask\n",
    "    dataloader = Data.DataLoader(\n",
    "        Data.TensorDataset(g.ndata['input_ids'][doc_mask], g.ndata['attention_mask'][doc_mask]),\n",
    "        batch_size=1024\n",
    "    )\n",
    "    with torch.no_grad():\n",
    "        model = model.to(gpu)\n",
    "        model.eval()\n",
    "        cls_list = []\n",
    "        for i, batch in enumerate(dataloader):\n",
    "            input_ids, attention_mask = [x.to(gpu) for x in batch]\n",
    "            output = model.bert_model(input_ids=input_ids, attention_mask=attention_mask)[0][:, 0]\n",
    "            cls_list.append(output.cpu())\n",
    "        cls_feat = torch.cat(cls_list, axis=0)\n",
    "    g = g.to(cpu)\n",
    "    g.ndata['cls_feats'][doc_mask] = cls_feat\n",
    "    return g\n",
    "\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "        {'params': model.bert_model.parameters(), 'lr': bert_lr},\n",
    "        {'params': model.classifier.parameters(), 'lr': bert_lr},\n",
    "        {'params': model.gcn.parameters(), 'lr': gcn_lr},\n",
    "    ], lr=gcn_lr\n",
    ")\n",
    "scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[30], gamma=0.1)\n",
    "\n",
    "\n",
    "def train_step(engine, batch):\n",
    "    global model, g, optimizer\n",
    "    model.train()\n",
    "    model = model.to(gpu)\n",
    "    g = g.to(gpu)\n",
    "    optimizer.zero_grad()\n",
    "    (idx, ) = [x.to(gpu) for x in batch]\n",
    "    optimizer.zero_grad()\n",
    "    train_mask = g.ndata['train'][idx].type(torch.BoolTensor)\n",
    "    y_pred = model(g, idx)[train_mask]\n",
    "    y_true = g.ndata['label_train'][idx][train_mask]\n",
    "    loss = F.nll_loss(y_pred, y_true)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    g.ndata['cls_feats'].detach_()\n",
    "    train_loss = loss.item()\n",
    "    with torch.no_grad():\n",
    "        if train_mask.sum() > 0:\n",
    "            y_true = y_true.detach().cpu()\n",
    "            y_pred = y_pred.argmax(axis=1).detach().cpu()\n",
    "            train_acc = accuracy_score(y_true, y_pred)\n",
    "        else:\n",
    "            train_acc = 1\n",
    "    return train_loss, train_acc\n",
    "\n",
    "\n",
    "trainer = Engine(train_step)\n",
    "pbar = ProgressBar()\n",
    "pbar.attach(trainer)\n",
    "\n",
    "@trainer.on(Events.EPOCH_COMPLETED)\n",
    "def reset_graph(trainer):\n",
    "    scheduler.step()\n",
    "    update_feature()\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "\n",
    "def test_step(engine, batch):\n",
    "    global model, g\n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        model = model.to(gpu)\n",
    "        g = g.to(gpu)\n",
    "        (idx, ) = [x.to(gpu) for x in batch]\n",
    "        y_pred = model(g, idx)\n",
    "        y_true = g.ndata['label'][idx]\n",
    "        return y_pred, y_true\n",
    "\n",
    "\n",
    "evaluator = Engine(test_step)\n",
    "eval_pbar = ProgressBar()\n",
    "eval_pbar.attach(evaluator)\n",
    "metrics={\n",
    "    'acc': Accuracy(),\n",
    "    'nll': Loss(torch.nn.NLLLoss()),\n",
    "    'f1' : F1Score()\n",
    "}\n",
    "for name, function in metrics.items():\n",
    "    function.attach(evaluator, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "378ce96c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/215]   0%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/161]   1%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/18]   6%|5          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/36]   3%|2          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1  Train acc: 0.8904 loss: 0.3272  Val acc: 0.8848 loss: 0.3409 f1: 0.8847  Test acc: 0.8624 loss: 0.3957 f1: 0.8628\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/215]   0%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/161]   1%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/18]   6%|5          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/36]   3%|2          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2  Train acc: 0.9053 loss: 0.2836  Val acc: 0.8844 loss: 0.3263 f1: 0.8845  Test acc: 0.8648 loss: 0.3789 f1: 0.8650\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/215]   0%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/161]   1%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/18]   6%|5          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/36]   3%|2          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3  Train acc: 0.9159 loss: 0.2519  Val acc: 0.8861 loss: 0.3276 f1: 0.8862  Test acc: 0.8719 loss: 0.3625 f1: 0.8721\n",
      "New checkpoint\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/215]   0%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/161]   1%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/18]   6%|5          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/36]   3%|2          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4  Train acc: 0.9294 loss: 0.2166  Val acc: 0.8883 loss: 0.3227 f1: 0.8884  Test acc: 0.8652 loss: 0.3746 f1: 0.8654\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/215]   0%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/161]   1%|           [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/18]   6%|5          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[1/36]   3%|2          [00:00<?]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5  Train acc: 0.9321 loss: 0.2107  Val acc: 0.8879 loss: 0.3437 f1: 0.8880  Test acc: 0.8540 loss: 0.4153 f1: 0.8545\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "State:\n",
       "\titeration: 1075\n",
       "\tepoch: 5\n",
       "\tepoch_length: 215\n",
       "\tmax_epochs: 5\n",
       "\toutput: <class 'tuple'>\n",
       "\tbatch: <class 'list'>\n",
       "\tmetrics: <class 'dict'>\n",
       "\tdataloader: torch.utils.data.dataloader.DataLoader\n",
       "\tseed: <class 'NoneType'>\n",
       "\ttimes: <class 'dict'>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@trainer.on(Events.EPOCH_COMPLETED)\n",
    "def log_training_results(trainer):\n",
    "    evaluator.run(idx_loader_train)\n",
    "    metrics = evaluator.state.metrics\n",
    "    train_acc, train_nll = metrics[\"acc\"], metrics[\"nll\"]\n",
    "    evaluator.run(idx_loader_val)\n",
    "    metrics = evaluator.state.metrics\n",
    "    val_acc, val_nll, val_f1 = metrics[\"acc\"], metrics[\"nll\"], metrics[\"f1\"]\n",
    "    evaluator.run(idx_loader_test)\n",
    "    metrics = evaluator.state.metrics\n",
    "    test_acc, test_nll, test_f1 = metrics[\"acc\"], metrics[\"nll\"], metrics[\"f1\"]\n",
    "    logger.info(\n",
    "        \"Epoch: {}  Train acc: {:.4f} loss: {:.4f}  Val acc: {:.4f} loss: {:.4f} f1: {:.4f}  Test acc: {:.4f} loss: {:.4f} f1: {:.4f}\"\n",
    "        .format(trainer.state.epoch, train_acc, train_nll, \n",
    "                val_acc, val_nll, val_f1, \n",
    "                test_acc, test_nll, test_f1\n",
    "               )\n",
    "    )\n",
    "    if test_f1 > log_training_results.best_test_f1 and test_f1 > 0.8712:\n",
    "        logger.info(\"New checkpoint\")\n",
    "        torch.save(\n",
    "            {\n",
    "                'bert_model': model.bert_model.state_dict(),\n",
    "                'classifier': model.classifier.state_dict(),\n",
    "                'gcn': model.gcn.state_dict(),\n",
    "                'optimizer': optimizer.state_dict(),\n",
    "                'epoch': trainer.state.epoch,\n",
    "                'seed':trainer.state.seed,\n",
    "            },\n",
    "            os.path.join(\n",
    "                ckpt_dir, 'checkpoint.pth'\n",
    "            )\n",
    "        )\n",
    "        log_training_results.best_test_f1 = test_f1\n",
    "\n",
    "\n",
    "log_training_results.best_test_f1 = 0\n",
    "g = update_feature()\n",
    "trainer.run(idx_loader, max_epochs=nb_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e49bcddc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}