keras-model-qps-benchmarking-v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c1c154a8",
   "metadata": {
    "papermill": {
     "duration": 0.007646,
     "end_time": "2024-05-15T07:14:54.843978",
     "exception": false,
     "start_time": "2024-05-15T07:14:54.836332",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Benchmark Queries per second(QPS) or Transaction per second(TPS)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fd6571d",
   "metadata": {
    "papermill": {
     "duration": 0.006978,
     "end_time": "2024-05-15T07:14:54.858318",
     "exception": false,
     "start_time": "2024-05-15T07:14:54.851340",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "v1.6: 15 May\n",
    "* Added option to filter models list to run selected models behcmark and to save time\n",
    "\n",
    "v1.5: 14 May  2024\n",
    "* Modified code to run Keras cv attention models first than tf keras. So if any issue in that it will be detected early.\n",
    "* reduced batch size and number of iteration to save time.\n",
    "* Skipped detection models from benchmark\n",
    "\n",
    "v1.0: 14 May  2024\n",
    "\n",
    "* Added support to check model QPS (Queries per second) or we can say Transactions per second\n",
    "* Used tf dataset\n",
    "* tested on v2.15.0  keras, tf_keras and tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd252978",
   "metadata": {
    "papermill": {
     "duration": 0.006777,
     "end_time": "2024-05-15T07:14:54.872353",
     "exception": false,
     "start_time": "2024-05-15T07:14:54.865576",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## <font color=\"#000080\" size=\"+3\">1. Setup</font>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "455ed9f2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:14:54.888592Z",
     "iopub.status.busy": "2024-05-15T07:14:54.887868Z",
     "iopub.status.idle": "2024-05-15T07:15:07.313103Z",
     "shell.execute_reply": "2024-05-15T07:15:07.312009Z"
    },
    "papermill": {
     "duration": 12.435847,
     "end_time": "2024-05-15T07:15:07.315484",
     "exception": false,
     "start_time": "2024-05-15T07:14:54.879637",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: tensorflow\r\n",
      "Version: 2.15.0\r\n",
      "Summary: TensorFlow is an open source machine learning framework for everyone.\r\n",
      "Home-page: https://www.tensorflow.org/\r\n",
      "Author: Google Inc.\r\n",
      "Author-email: packages@tensorflow.org\r\n",
      "License: Apache 2.0\r\n",
      "Location: /opt/conda/lib/python3.10/site-packages\r\n",
      "Requires: absl-py, astunparse, flatbuffers, gast, google-pasta, grpcio, h5py, keras, libclang, ml-dtypes, numpy, opt-einsum, packaging, protobuf, setuptools, six, tensorboard, tensorflow-estimator, tensorflow-io-gcs-filesystem, termcolor, typing-extensions, wrapt\r\n",
      "Required-by: explainable-ai-sdk, tensorflow-cloud, tensorflow-decision-forests, tensorflow-serving-api, tensorflow-text, tf_keras, witwidget\r\n"
     ]
    }
   ],
   "source": [
    "!pip show tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4f1e52a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:15:07.331906Z",
     "iopub.status.busy": "2024-05-15T07:15:07.331603Z",
     "iopub.status.idle": "2024-05-15T07:15:22.268797Z",
     "shell.execute_reply": "2024-05-15T07:15:22.267698Z"
    },
    "papermill": {
     "duration": 14.948101,
     "end_time": "2024-05-15T07:15:22.271155",
     "exception": false,
     "start_time": "2024-05-15T07:15:07.323054",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tf_keras==2.15.0\r\n",
      "  Downloading tf_keras-2.15.0-py3-none-any.whl.metadata (1.6 kB)\r\n",
      "Downloading tf_keras-2.15.0-py3-none-any.whl (1.7 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m37.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hInstalling collected packages: tf_keras\r\n",
      "  Attempting uninstall: tf_keras\r\n",
      "    Found existing installation: tf_keras 2.15.1\r\n",
      "    Uninstalling tf_keras-2.15.1:\r\n",
      "      Successfully uninstalled tf_keras-2.15.1\r\n",
      "Successfully installed tf_keras-2.15.0\r\n"
     ]
    }
   ],
   "source": [
    "!pip install tf_keras==\"2.15.0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2aaaebfd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:15:22.289203Z",
     "iopub.status.busy": "2024-05-15T07:15:22.288644Z",
     "iopub.status.idle": "2024-05-15T07:15:37.964084Z",
     "shell.execute_reply": "2024-05-15T07:15:37.963189Z"
    },
    "papermill": {
     "duration": 15.687123,
     "end_time": "2024-05-15T07:15:37.966559",
     "exception": false,
     "start_time": "2024-05-15T07:15:22.279436",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting keras-cv-attention-models==1.4.1\r\n",
      "  Downloading keras_cv_attention_models-1.4.1-py3-none-any.whl.metadata (188 kB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m188.3/188.3 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: pillow in /opt/conda/lib/python3.10/site-packages (from keras-cv-attention-models==1.4.1) (9.5.0)\r\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from keras-cv-attention-models==1.4.1) (4.66.1)\r\n",
      "Collecting ftfy (from keras-cv-attention-models==1.4.1)\r\n",
      "  Downloading ftfy-6.2.0-py3-none-any.whl.metadata (7.3 kB)\r\n",
      "Requirement already satisfied: regex in /opt/conda/lib/python3.10/site-packages (from keras-cv-attention-models==1.4.1) (2023.12.25)\r\n",
      "Requirement already satisfied: tensorflow-datasets in /opt/conda/lib/python3.10/site-packages (from keras-cv-attention-models==1.4.1) (4.9.4)\r\n",
      "Requirement already satisfied: tensorflow in /opt/conda/lib/python3.10/site-packages (from keras-cv-attention-models==1.4.1) (2.15.0)\r\n",
      "Requirement already satisfied: wcwidth<0.3.0,>=0.2.12 in /opt/conda/lib/python3.10/site-packages (from ftfy->keras-cv-attention-models==1.4.1) (0.2.13)\r\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (1.4.0)\r\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (1.6.3)\r\n",
      "Requirement already satisfied: flatbuffers>=23.5.26 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (23.5.26)\r\n",
      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (0.5.4)\r\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (0.2.0)\r\n",
      "Requirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (3.10.0)\r\n",
      "Requirement already satisfied: libclang>=13.0.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (16.0.6)\r\n",
      "Requirement already satisfied: ml-dtypes~=0.2.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (0.2.0)\r\n",
      "Requirement already satisfied: numpy<2.0.0,>=1.23.5 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (1.26.4)\r\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (3.3.0)\r\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (21.3)\r\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (3.20.3)\r\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (69.0.3)\r\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (1.16.0)\r\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (2.4.0)\r\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (4.9.0)\r\n",
      "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (1.14.1)\r\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (0.35.0)\r\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (1.51.1)\r\n",
      "Requirement already satisfied: tensorboard<2.16,>=2.15 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (2.15.1)\r\n",
      "Requirement already satisfied: tensorflow-estimator<2.16,>=2.15.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow->keras-cv-attention-models==1.4.1) (2.15.0)\r\n",
      "Collecting keras<2.16,>=2.15.0 (from tensorflow->keras-cv-attention-models==1.4.1)\r\n",
      "  Downloading keras-2.15.0-py3-none-any.whl.metadata (2.4 kB)\r\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.10/site-packages (from tensorflow-datasets->keras-cv-attention-models==1.4.1) (8.1.7)\r\n",
      "Requirement already satisfied: dm-tree in /opt/conda/lib/python3.10/site-packages (from tensorflow-datasets->keras-cv-attention-models==1.4.1) (0.1.8)\r\n",
      "Requirement already satisfied: etils>=0.9.0 in /opt/conda/lib/python3.10/site-packages (from etils[enp,epath,etree]>=0.9.0->tensorflow-datasets->keras-cv-attention-models==1.4.1) (1.6.0)\r\n",
      "Requirement already satisfied: promise in /opt/conda/lib/python3.10/site-packages (from tensorflow-datasets->keras-cv-attention-models==1.4.1) (2.3)\r\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from tensorflow-datasets->keras-cv-attention-models==1.4.1) (5.9.3)\r\n",
      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow-datasets->keras-cv-attention-models==1.4.1) (2.31.0)\r\n",
      "Requirement already satisfied: tensorflow-metadata in /opt/conda/lib/python3.10/site-packages (from tensorflow-datasets->keras-cv-attention-models==1.4.1) (0.14.0)\r\n",
      "Requirement already satisfied: toml in /opt/conda/lib/python3.10/site-packages (from tensorflow-datasets->keras-cv-attention-models==1.4.1) (0.10.2)\r\n",
      "Requirement already satisfied: array-record>=0.5.0 in /opt/conda/lib/python3.10/site-packages (from tensorflow-datasets->keras-cv-attention-models==1.4.1) (0.5.0)\r\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.10/site-packages (from astunparse>=1.6.0->tensorflow->keras-cv-attention-models==1.4.1) (0.42.0)\r\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from etils[enp,epath,etree]>=0.9.0->tensorflow-datasets->keras-cv-attention-models==1.4.1) (2024.2.0)\r\n",
      "Requirement already satisfied: importlib_resources in /opt/conda/lib/python3.10/site-packages (from etils[enp,epath,etree]>=0.9.0->tensorflow-datasets->keras-cv-attention-models==1.4.1) (6.1.1)\r\n",
      "Requirement already satisfied: zipp in /opt/conda/lib/python3.10/site-packages (from etils[enp,epath,etree]>=0.9.0->tensorflow-datasets->keras-cv-attention-models==1.4.1) (3.17.0)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->tensorflow-datasets->keras-cv-attention-models==1.4.1) (3.3.2)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->tensorflow-datasets->keras-cv-attention-models==1.4.1) (3.6)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->tensorflow-datasets->keras-cv-attention-models==1.4.1) (1.26.18)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->tensorflow-datasets->keras-cv-attention-models==1.4.1) (2024.2.2)\r\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (2.26.1)\r\n",
      "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (1.2.0)\r\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (3.5.2)\r\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (0.7.2)\r\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (3.0.2)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->tensorflow->keras-cv-attention-models==1.4.1) (3.1.1)\r\n",
      "Requirement already satisfied: googleapis-common-protos in /opt/conda/lib/python3.10/site-packages (from tensorflow-metadata->tensorflow-datasets->keras-cv-attention-models==1.4.1) (1.62.0)\r\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (4.2.4)\r\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (0.3.0)\r\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (4.9)\r\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (1.3.1)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.10/site-packages (from werkzeug>=1.0.1->tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (2.1.3)\r\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /opt/conda/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (0.5.1)\r\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.10/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow->keras-cv-attention-models==1.4.1) (3.2.2)\r\n",
      "Downloading keras_cv_attention_models-1.4.1-py3-none-any.whl (796 kB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m796.3/796.3 kB\u001b[0m \u001b[31m33.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading ftfy-6.2.0-py3-none-any.whl (54 kB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.4/54.4 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading keras-2.15.0-py3-none-any.whl (1.7 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m69.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hInstalling collected packages: keras, ftfy, keras-cv-attention-models\r\n",
      "  Attempting uninstall: keras\r\n",
      "    Found existing installation: keras 3.2.1\r\n",
      "    Uninstalling keras-3.2.1:\r\n",
      "      Successfully uninstalled keras-3.2.1\r\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "tensorflow-decision-forests 1.8.1 requires wurlitzer, which is not installed.\u001b[0m\u001b[31m\r\n",
      "\u001b[0mSuccessfully installed ftfy-6.2.0 keras-2.15.0 keras-cv-attention-models-1.4.1\r\n"
     ]
    }
   ],
   "source": [
    "# !pip install keras_cv_attention_models\n",
    "!pip install keras-cv-attention-models==1.4.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "997646ba",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:15:37.987993Z",
     "iopub.status.busy": "2024-05-15T07:15:37.987689Z",
     "iopub.status.idle": "2024-05-15T07:15:37.992283Z",
     "shell.execute_reply": "2024-05-15T07:15:37.991482Z"
    },
    "papermill": {
     "duration": 0.017324,
     "end_time": "2024-05-15T07:15:37.994143",
     "exception": false,
     "start_time": "2024-05-15T07:15:37.976819",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# set environment variable values before importing tensorflow and keras\n",
    "import os\n",
    "\n",
    "os.environ[\"KECAM_BACKEND\"] = \"tensorflow\"\n",
    "os.environ[\"TF_USE_LEGACY_KERAS\"] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f2740682",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:15:38.014791Z",
     "iopub.status.busy": "2024-05-15T07:15:38.013976Z",
     "iopub.status.idle": "2024-05-15T07:15:50.314372Z",
     "shell.execute_reply": "2024-05-15T07:15:50.313251Z"
    },
    "papermill": {
     "duration": 12.312883,
     "end_time": "2024-05-15T07:15:50.316640",
     "exception": false,
     "start_time": "2024-05-15T07:15:38.003757",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting nb_utils\r\n",
      "  Downloading nb_utils-0.1.1-py3-none-any.whl.metadata (510 bytes)\r\n",
      "Downloading nb_utils-0.1.1-py3-none-any.whl (24 kB)\r\n",
      "Installing collected packages: nb_utils\r\n",
      "Successfully installed nb_utils-0.1.1\r\n"
     ]
    }
   ],
   "source": [
    "!pip install nb_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ded482b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:15:50.339659Z",
     "iopub.status.busy": "2024-05-15T07:15:50.339336Z",
     "iopub.status.idle": "2024-05-15T07:16:05.818236Z",
     "shell.execute_reply": "2024-05-15T07:16:05.817334Z"
    },
    "papermill": {
     "duration": 15.492352,
     "end_time": "2024-05-15T07:16:05.820637",
     "exception": false,
     "start_time": "2024-05-15T07:15:50.328285",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tf_tools\r\n",
      "  Cloning https://github.com/Nivratti/tf_tools.git to /tmp/pip-install-lnuwrkpz/tf-tools_627bc976c94e4b73aa335587c6ef74dc\r\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/Nivratti/tf_tools.git /tmp/pip-install-lnuwrkpz/tf-tools_627bc976c94e4b73aa335587c6ef74dc\r\n",
      "  Resolved https://github.com/Nivratti/tf_tools.git to commit 9b53934b0f7194d192b3b6c8b412c0f24fa1a3a2\r\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\r\n",
      "\u001b[?25hBuilding wheels for collected packages: tf_tools\r\n",
      "  Building wheel for tf_tools (setup.py) ... \u001b[?25l-\b \bdone\r\n",
      "\u001b[?25h  Created wheel for tf_tools: filename=tf_tools-1.1.0-py3-none-any.whl size=27074 sha256=49b348b6c89bc53b672a0bb5f25c3859f6861b288d3bed5c8615ad05295e9fa3\r\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-spov0l4r/wheels/47/2f/43/d3226af5507f8cb43d9bd1655817b7954246fb1929d95953d7\r\n",
      "Successfully built tf_tools\r\n",
      "Installing collected packages: tf_tools\r\n",
      "Successfully installed tf_tools-1.1.0\r\n"
     ]
    }
   ],
   "source": [
    "# !pip  uninstall -y tf_tools\n",
    "!pip install git+https://github.com/Nivratti/tf_tools.git#egg=tf_tools"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25bfa3ac",
   "metadata": {
    "papermill": {
     "duration": 0.01044,
     "end_time": "2024-05-15T07:16:05.842007",
     "exception": false,
     "start_time": "2024-05-15T07:16:05.831567",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## <font color=\"#000080\" size=\"+3\">2. Import libraries</font>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2513574f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:16:05.864646Z",
     "iopub.status.busy": "2024-05-15T07:16:05.864350Z",
     "iopub.status.idle": "2024-05-15T07:16:17.350157Z",
     "shell.execute_reply": "2024-05-15T07:16:17.349152Z"
    },
    "papermill": {
     "duration": 11.49985,
     "end_time": "2024-05-15T07:16:17.352390",
     "exception": false,
     "start_time": "2024-05-15T07:16:05.852540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-05-15 07:16:07.425630: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-05-15 07:16:07.425754: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-05-15 07:16:07.536665: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "import tf_keras\n",
    "from tf_keras.applications import *\n",
    "from tf_keras.preprocessing import image\n",
    "from tf_keras.utils import get_file\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import inspect \n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5492091f",
   "metadata": {
    "papermill": {
     "duration": 0.010525,
     "end_time": "2024-05-15T07:16:17.374053",
     "exception": false,
     "start_time": "2024-05-15T07:16:17.363528",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## <font color=\"#000080\" size=\"+3\">3. Useful functions</font>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1ac8edb5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:16:17.397720Z",
     "iopub.status.busy": "2024-05-15T07:16:17.396220Z",
     "iopub.status.idle": "2024-05-15T07:16:17.408171Z",
     "shell.execute_reply": "2024-05-15T07:16:17.407380Z"
    },
    "papermill": {
     "duration": 0.025426,
     "end_time": "2024-05-15T07:16:17.410022",
     "exception": false,
     "start_time": "2024-05-15T07:16:17.384596",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "## useful functions\n",
    "\n",
    "def find_max_batch_size(model, input_size):\n",
    "    \"\"\"\n",
    "    Finds the maximum batch size that can fit into GPU memory without causing a ResourceExhaustedError.\n",
    "\n",
    "    Args:\n",
    "        model (tf.keras.Model): The model for which to find the maximum batch size.\n",
    "        input_size (int): The height and width of the input images. Assumes square images.\n",
    "\n",
    "    Returns:\n",
    "        int: The maximum batch size that can be processed by the model without memory overflow.\n",
    "    \"\"\"\n",
    "    batch_size = 2\n",
    "    max_batch_size = 0\n",
    "\n",
    "    while True:\n",
    "        try:\n",
    "            # Create dummy data\n",
    "            data = np.random.rand(batch_size, input_size, input_size, 3).astype('float32')\n",
    "            # Predict\n",
    "            preds = model.predict(data)\n",
    "            max_batch_size = batch_size\n",
    "            batch_size *= 2  # Double the batch size\n",
    "        except tf.errors.ResourceExhaustedError:\n",
    "            break  # Out of memory\n",
    "        finally:\n",
    "            tf_keras.backend.clear_session()\n",
    "    return max_batch_size\n",
    "\n",
    "def benchmark_model(\n",
    "        model, strategy, input_shape, \n",
    "        warm_up_batches=10, num_batches=100, \n",
    "        batch_size=32\n",
    "    ):\n",
    "    \"\"\"\n",
    "    Benchmarks a TensorFlow model to estimate its queries per second (QPS) under a specific distribution strategy.\n",
    "\n",
    "    Args:\n",
    "        model (tf.keras.Model): The model to benchmark.\n",
    "        strategy (tf.distribute.Strategy): The distribution strategy under which the benchmark is performed.\n",
    "        input_shape (tuple): The shape of the input data (excluding the batch size).\n",
    "        warm_up_batches (int, optional): Number of warm-up batches to run before timing to stabilize performance metrics.\n",
    "        num_batches (int, optional): Number of batches to process for benchmarking.\n",
    "        batch_size (int, optional): Number of samples per batch.\n",
    "\n",
    "    Returns:\n",
    "        tuple: A tuple containing the estimated queries per second (QPS) and the total time taken for the benchmark (in seconds).\n",
    "    \"\"\"\n",
    "    input_data = np.random.random((batch_size, *input_shape)).astype('float32')\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(input_data).repeat().batch(batch_size)\n",
    "\n",
    "    # Warm-up\n",
    "    with strategy.scope():\n",
    "        print(f\"performing warmup...\")\n",
    "        for images in dataset.take(warm_up_batches):\n",
    "            # _ = model.predict_on_batch(images)\n",
    "            _ = model(images)  # To make it work for keras-cv_attention_models\n",
    "\n",
    "    start_time = time.time()\n",
    "    with strategy.scope():\n",
    "        print(f\"Benchmarking...\")\n",
    "        for batch in dataset.take(num_batches):\n",
    "            _ = model(batch)\n",
    "            \n",
    "    total_time = time.time() - start_time\n",
    "\n",
    "    total_queries = num_batches * batch_size\n",
    "    qps = total_queries / total_time\n",
    "    return qps, total_time"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4737843d",
   "metadata": {
    "papermill": {
     "duration": 0.010323,
     "end_time": "2024-05-15T07:16:17.431455",
     "exception": false,
     "start_time": "2024-05-15T07:16:17.421132",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## <font color=\"#000080\" size=\"+3\">4. Device statergy</font>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f0aaf0fd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:16:17.453833Z",
     "iopub.status.busy": "2024-05-15T07:16:17.453296Z",
     "iopub.status.idle": "2024-05-15T07:16:18.216980Z",
     "shell.execute_reply": "2024-05-15T07:16:18.215966Z"
    },
    "papermill": {
     "duration": 0.777273,
     "end_time": "2024-05-15T07:16:18.219224",
     "exception": false,
     "start_time": "2024-05-15T07:16:17.441951",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-05-15 07:16:17.695\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mtf_tools.device_strategy\u001b[0m:\u001b[36mget_device_strategy\u001b[0m:\u001b[36m41\u001b[0m - \u001b[1mMultiple GPUs are available, using MirroredStrategy. Count: 2\u001b[0m\n",
      "\u001b[32m2024-05-15 07:16:18.213\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mtf_tools.device_strategy\u001b[0m:\u001b[36mget_device_strategy\u001b[0m:\u001b[36m45\u001b[0m - \u001b[1mAs num_cores_to_use is set to 1. using only ['/gpu:0']\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "from tf_tools.device_strategy import get_device_strategy\n",
    "# specify only one core\n",
    "strategy, device_type = get_device_strategy(num_cores_to_use=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb410198",
   "metadata": {
    "papermill": {
     "duration": 0.010556,
     "end_time": "2024-05-15T07:16:18.240962",
     "exception": false,
     "start_time": "2024-05-15T07:16:18.230406",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## <font color=\"#000080\" size=\"+3\">5. Configuration</font>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ba726fc8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:16:18.264148Z",
     "iopub.status.busy": "2024-05-15T07:16:18.263554Z",
     "iopub.status.idle": "2024-05-15T07:16:18.268166Z",
     "shell.execute_reply": "2024-05-15T07:16:18.267339Z"
    },
    "papermill": {
     "duration": 0.018056,
     "end_time": "2024-05-15T07:16:18.269945",
     "exception": false,
     "start_time": "2024-05-15T07:16:18.251889",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Model Benchmarking Parameters\n",
    "batch_size = 8 # Number of samples processed per batch ex. 16\n",
    "warm_up_batches = 2  # Number of initial batches to run to stabilize performance metrics ex. 10\n",
    "num_batches = 10  # Total number of batches to process for benchmarking ex. 100\n",
    "input_height = 64  # Height of the input images\n",
    "input_width = 640  # Width of the input images\n",
    "input_shape = (input_height, input_width, 3)  # Shape of the input data (H, W, C)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba09dca9",
   "metadata": {
    "papermill": {
     "duration": 0.010504,
     "end_time": "2024-05-15T07:16:18.291054",
     "exception": false,
     "start_time": "2024-05-15T07:16:18.280550",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## <font color=\"#000080\" size=\"+3\">6. Keras cv attention models QPS checker</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2cd8f43d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:16:18.314388Z",
     "iopub.status.busy": "2024-05-15T07:16:18.313822Z",
     "iopub.status.idle": "2024-05-15T07:16:18.475382Z",
     "shell.execute_reply": "2024-05-15T07:16:18.474687Z"
    },
    "papermill": {
     "duration": 0.175354,
     "end_time": "2024-05-15T07:16:18.477345",
     "exception": false,
     "start_time": "2024-05-15T07:16:18.301991",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from keras_cv_attention_models import efficientformer\n",
    "from keras_cv_attention_models import efficientvit\n",
    "# from keras_cv_attention_models import coatnet\n",
    "# from keras_cv_attention_models import convnext\n",
    "# from keras_cv_attention_models import davit\n",
    "# from keras_cv_attention_models import fasternet\n",
    "# from keras_cv_attention_models import eva02\n",
    "# from keras_cv_attention_models import gcvit\n",
    "# from keras_cv_attention_models import hiera\n",
    "# from keras_cv_attention_models import maxvit\n",
    "# from keras_cv_attention_models import mobilevit\n",
    "# from keras_cv_attention_models import tinyvit\n",
    "# from keras_cv_attention_models import volo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "79edf367",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:16:18.500084Z",
     "iopub.status.busy": "2024-05-15T07:16:18.499588Z",
     "iopub.status.idle": "2024-05-15T07:16:40.013072Z",
     "shell.execute_reply": "2024-05-15T07:16:40.011958Z"
    },
    "papermill": {
     "duration": 21.52698,
     "end_time": "2024-05-15T07:16:40.015202",
     "exception": false,
     "start_time": "2024-05-15T07:16:18.488222",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to import or process keras_cv_attention_models.keras_core_functional: No module named 'keras_core'\n",
      "Failed to import or process keras_cv_attention_models.efficientnet.convert_effnetv2_model: No module named 'effnetv2_configs'\n",
      "Failed to import or process keras_cv_attention_models.imagenet.tensorrt_engine: No module named 'tensorrt'\n",
      "\n",
      "Total 437 functions found decorated with @register_model\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Automatically find all model functions import it and test\n",
    "import importlib\n",
    "from tqdm import tqdm\n",
    "\n",
    "from tf_tools.utils.function_finder_inspector import find_decorated_functions\n",
    "\n",
    "package_name = 'keras_cv_attention_models'\n",
    "decorator_name = 'register_model'\n",
    "decorated_functions = find_decorated_functions(package_name, decorator_name)\n",
    "print(f\"\\nTotal {len(decorated_functions)} functions found decorated with @{decorator_name}\\n\")\n",
    "\n",
    "# for module_name, func_name, params in decorated_functions:\n",
    "#     print(f\"{module_name}.{func_name} with parameters: {params} \\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7cf80757",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:16:40.040467Z",
     "iopub.status.busy": "2024-05-15T07:16:40.039842Z",
     "iopub.status.idle": "2024-05-15T07:16:40.061080Z",
     "shell.execute_reply": "2024-05-15T07:16:40.059930Z"
    },
    "papermill": {
     "duration": 0.035778,
     "end_time": "2024-05-15T07:16:40.063175",
     "exception": false,
     "start_time": "2024-05-15T07:16:40.027397",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 437/437 [00:00<00:00, 65658.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " After filtering total 40 functions remaining\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# filter models list\n",
    "decorated_functions_filtered = []\n",
    "\n",
    "## optional\n",
    "## names to keep -- to perform selected models benchmark only to save time\n",
    "## Note: You can give partial name to select many matching names ex. Beit instead of BeitLargePatch16\n",
    "name_filters = [\n",
    "    \"BeitLargePatch16\", \"BeitV2LargePatch16\",\n",
    "    \"CAFormerB36\", \"ConvFormerB36\",\n",
    "    \"CoAtNet3\", \"CoAtNet4\",\n",
    "    \"ConvNeXtXlarge\", \"ConvNeXtV2Large\",\n",
    "    \"CotNetSE152D\",\n",
    "    \"DaViT_L\", \"DaViT_H\",\n",
    "    \"DINOv2_ViT_Large14\",\n",
    "    \"EfficientFormerL7\", \"EfficientFormerV2L\",\n",
    "    \"EfficientNetV1L2\", \"EfficientNetV2XL\",\n",
    "    \"EfficientViT_L3\",\n",
    "    \"EvaGiantPatch14\", \n",
    "    \"EVA02BasePatch14\", \"EVA02LargePatch14\",\n",
    "    \"FastViT_MA36\",\n",
    "    \"FlexiViTLarge\",\n",
    "    \"GCViT_Large\",\n",
    "    \"GPViT_L4\",\n",
    "    \"HieraHuge\",\n",
    "    \"HorNetLargeGF\",\n",
    "    \"IFormerLarge\",\n",
    "    \"InceptionNeXtBase\",\n",
    "    \"MaxViT_XLarge\",\n",
    "    \"MetaTransformerLargePatch14\",\n",
    "    \"MLPMixerH14\",\n",
    "    \"MogaNetLarge\",\n",
    "    \"ResMLP_B24\",\n",
    "    \"ResNest269\",\n",
    "    \"SwinTransformerV2Large\",\n",
    "    \"TinyViT_21M\",\n",
    "    \"UniformerLarge64\",\n",
    "    \"VOLO_d5\",\n",
    "]\n",
    "\n",
    "for module_name, func_name, params in tqdm(decorated_functions):\n",
    "    if name_filters:\n",
    "        if any(nf.lower().strip() in func_name.lower() for nf in name_filters):\n",
    "            # keep only name matched models\n",
    "            decorated_functions_filtered.append(\n",
    "                (module_name, func_name, params) # append tuple\n",
    "            )\n",
    "        else:\n",
    "            continue # skip if name not matching\n",
    "    else:\n",
    "        # if no any name filtering specified then remove only object detection models\n",
    "        skip_models_patterns = [\"YOLO\", \"EfficientDet\"]\n",
    "        if any(m.lower() in func_name.lower() for m in skip_models_patterns):\n",
    "            continue # skip that\n",
    "        else:\n",
    "            decorated_functions_filtered.append(\n",
    "                (module_name, func_name, params) # append tuple\n",
    "            )\n",
    "            \n",
    "print(f\"\\n After filtering total {len(decorated_functions_filtered)} functions remaining\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "31684e59",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:16:40.087758Z",
     "iopub.status.busy": "2024-05-15T07:16:40.087480Z",
     "iopub.status.idle": "2024-05-15T07:27:08.776536Z",
     "shell.execute_reply": "2024-05-15T07:27:08.775199Z"
    },
    "papermill": {
     "duration": 628.704093,
     "end_time": "2024-05-15T07:27:08.779174",
     "exception": false,
     "start_time": "2024-05-15T07:16:40.075081",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/40 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing HorNetLargeGF\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▎         | 1/40 [00:16<10:45, 16.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:13.36614717263459, duration: 5.985270023345947\n",
      "\n",
      "Testing EfficientViT_L3\n",
      "performing warmup...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1715757422.084866      24 device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 2/40 [00:25<07:40, 12.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:31.390625929666594, duration: 2.5485315322875977\n",
      "\n",
      "Testing MogaNetLarge\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 3/40 [00:59<13:35, 22.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:5.495959128175274, duration: 14.556149005889893\n",
      "\n",
      "Testing FastViT_MA36\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 4/40 [01:20<13:01, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:13.39694504556376, duration: 5.971510648727417\n",
      "\n",
      "Testing FlexiViTLarge\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▎        | 5/40 [01:31<10:21, 17.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:17.896064532525067, duration: 4.470256567001343\n",
      "\n",
      "Testing DINOv2_ViT_Large14\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 6/40 [01:42<08:47, 15.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:17.085693932528315, duration: 4.682279825210571\n",
      "\n",
      "Testing BeitLargePatch16\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 7/40 [01:54<07:49, 14.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:15.613684976253815, duration: 5.12371039390564\n",
      "\n",
      "Testing BeitV2LargePatch16\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 8/40 [02:05<07:05, 13.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:15.793658048218187, duration: 5.065324306488037\n",
      "\n",
      "Testing EvaGiantPatch14\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▎       | 9/40 [02:24<07:49, 15.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:8.37826775139792, duration: 9.548513174057007\n",
      "\n",
      "Testing EVA02BasePatch14\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 10/40 [02:32<06:24, 12.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:26.220598713414592, duration: 3.0510363578796387\n",
      "\n",
      "Testing EVA02LargePatch14\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 11/40 [02:45<06:13, 12.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:13.49731402725518, duration: 5.927105188369751\n",
      "\n",
      "Testing MetaTransformerLargePatch14\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 12/40 [02:55<05:35, 11.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:17.91625058934276, duration: 4.465219974517822\n",
      "\n",
      "Testing UniformerLarge64\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▎      | 13/40 [03:14<06:21, 14.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:9.770390338869836, duration: 8.188004493713379\n",
      "\n",
      "Testing EfficientFormerV2L\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 14/40 [03:29<06:17, 14.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:12.439395890744406, duration: 6.431180477142334\n",
      "\n",
      "Testing EfficientFormerL7\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 15/40 [03:38<05:20, 12.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:22.794743295800004, duration: 3.5095810890197754\n",
      "\n",
      "Testing ResMLP_B24\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 16/40 [03:46<04:33, 11.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:17.72615841521288, duration: 4.513104200363159\n",
      "\n",
      "Testing MLPMixerH14\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▎     | 17/40 [03:57<04:21, 11.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:15.699824933812105, duration: 5.095598220825195\n",
      "\n",
      "Testing HieraHuge\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 18/40 [04:16<04:56, 13.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:9.802426994053503, duration: 8.16124415397644\n",
      "\n",
      "Testing SwinTransformerV2Large_window12\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 19/40 [04:34<05:10, 14.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:10.562042038174017, duration: 7.574292898178101\n",
      "\n",
      "Testing SwinTransformerV2Large_window16\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 20/40 [04:50<05:03, 15.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:10.499536047515344, duration: 7.619384288787842\n",
      "\n",
      "Testing SwinTransformerV2Large_window24\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▎    | 21/40 [05:05<04:50, 15.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:10.896032039909734, duration: 7.342122316360474\n",
      "\n",
      "Testing CAFormerB36\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 22/40 [05:20<04:31, 15.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:14.112454381365643, duration: 5.6687517166137695\n",
      "\n",
      "Testing ConvFormerB36\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▊    | 23/40 [05:31<03:58, 14.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:16.499955694496393, duration: 4.8484978675842285\n",
      "\n",
      "Testing EfficientNetV2XL\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 24/40 [05:55<04:27, 16.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:8.350084675597016, duration: 9.580741167068481\n",
      "\n",
      "Testing EfficientNetV1L2\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▎   | 25/40 [06:18<04:39, 18.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:8.128073377578932, duration: 9.842430830001831\n",
      "\n",
      "Testing MaxViT_XLarge\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 26/40 [06:48<05:11, 22.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:5.875325358981365, duration: 13.616267204284668\n",
      "\n",
      "Testing TinyViT_21M\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 27/40 [06:54<03:45, 17.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:31.95360739526886, duration: 2.503629684448242\n",
      "\n",
      "Testing GCViT_Large\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 28/40 [07:12<03:30, 17.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:11.023039840873952, duration: 7.257526159286499\n",
      "\n",
      "Testing GPViT_L4\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▎  | 29/40 [07:24<02:54, 15.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:14.782542497610434, duration: 5.4117889404296875\n",
      "\n",
      "Testing InceptionNeXtBase\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 30/40 [07:33<02:16, 13.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:22.18702214111198, duration: 3.6057114601135254\n",
      "\n",
      "Testing CoAtNet3\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 31/40 [07:42<01:51, 12.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:20.65964840383582, duration: 3.8722827434539795\n",
      "\n",
      "Testing CoAtNet4\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 32/40 [07:57<01:46, 13.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:11.661714944423542, duration: 6.8600544929504395\n",
      "\n",
      "Testing ResNest269\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▎ | 33/40 [08:35<02:23, 20.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:5.116387394557255, duration: 15.636032581329346\n",
      "\n",
      "Testing CotNetSE152D\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 34/40 [09:03<02:16, 22.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:7.8517781951995635, duration: 10.188774824142456\n",
      "\n",
      "Testing ConvNeXtV2Large\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 35/40 [09:16<01:38, 19.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:14.646021428554905, duration: 5.462234258651733\n",
      "\n",
      "Testing ConvNeXtXlarge\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 36/40 [09:23<01:04, 16.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:25.02353579634323, duration: 3.1969902515411377\n",
      "\n",
      "Testing IFormerLarge\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▎| 37/40 [09:44<00:52, 17.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:8.779139688876988, duration: 9.112510204315186\n",
      "\n",
      "Testing DaViT_H\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 38/40 [09:57<00:32, 16.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:14.074373343176404, duration: 5.684089660644531\n",
      "\n",
      "Testing DaViT_L\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 39/40 [10:10<00:15, 15.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:14.26710444522113, duration: 5.607304573059082\n",
      "\n",
      "Testing VOLO_d5\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 40/40 [10:28<00:00, 15.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:10.44864217822207, duration: 7.656497240066528\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#########################################\n",
    "\n",
    "results_kecam = []\n",
    "\n",
    "for module_name, func_name, params in tqdm(decorated_functions_filtered):\n",
    "    try:  \n",
    "        model_name = func_name\n",
    "        print(f\"Testing {model_name}\")\n",
    "        \n",
    "        # Import the module and function\n",
    "        module = importlib.import_module(module_name)\n",
    "        function = getattr(module, func_name)\n",
    "\n",
    "        # Parameters\n",
    "        params = {\n",
    "            \"input_shape\": input_shape,\n",
    "            \"num_classes\": 1000,\n",
    "            \"pretrained\": None, # we don't need pretrained weights here\n",
    "        }\n",
    "\n",
    "        # Call the function with dynamic parameters\n",
    "        model = function(**params)\n",
    "\n",
    "        ## Use the model as needed, e.g., print summary\n",
    "        #print(model.summary())\n",
    "        \n",
    "        # get tps\n",
    "        # max_batch_size = find_max_batch_size(model, input_size)\n",
    "        qps, duration = benchmark_model(\n",
    "            model, strategy, input_shape, \n",
    "            warm_up_batches=warm_up_batches, num_batches=num_batches, \n",
    "            batch_size=batch_size\n",
    "        )\n",
    "        print(f\"qps:{qps}, duration: {duration}\\n\")\n",
    "        results_kecam.append((model_name, batch_size, qps, duration))\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"Error in {module_name}.{func_name}.. {e}\")\n",
    "        \n",
    "    finally:\n",
    "        # clear the session\n",
    "        tf_keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "97035af6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:27:08.835728Z",
     "iopub.status.busy": "2024-05-15T07:27:08.835381Z",
     "iopub.status.idle": "2024-05-15T07:27:08.862642Z",
     "shell.execute_reply": "2024-05-15T07:27:08.861351Z"
    },
    "papermill": {
     "duration": 0.055688,
     "end_time": "2024-05-15T07:27:08.864876",
     "exception": false,
     "start_time": "2024-05-15T07:27:08.809188",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                              Model  Max Batch Size        QPS   Time (s)\n",
      "26                      TinyViT_21M               8  31.953607   2.503630\n",
      "1                   EfficientViT_L3               8  31.390626   2.548532\n",
      "9                  EVA02BasePatch14               8  26.220599   3.051036\n",
      "35                   ConvNeXtXlarge               8  25.023536   3.196990\n",
      "14                EfficientFormerL7               8  22.794743   3.509581\n",
      "29                InceptionNeXtBase               8  22.187022   3.605711\n",
      "30                         CoAtNet3               8  20.659648   3.872283\n",
      "11      MetaTransformerLargePatch14               8  17.916251   4.465220\n",
      "4                     FlexiViTLarge               8  17.896065   4.470257\n",
      "15                       ResMLP_B24               8  17.726158   4.513104\n",
      "5                DINOv2_ViT_Large14               8  17.085694   4.682280\n",
      "22                    ConvFormerB36               8  16.499956   4.848498\n",
      "7                BeitV2LargePatch16               8  15.793658   5.065324\n",
      "16                      MLPMixerH14               8  15.699825   5.095598\n",
      "6                  BeitLargePatch16               8  15.613685   5.123710\n",
      "28                         GPViT_L4               8  14.782542   5.411789\n",
      "34                  ConvNeXtV2Large               8  14.646021   5.462234\n",
      "38                          DaViT_L               8  14.267104   5.607305\n",
      "21                      CAFormerB36               8  14.112454   5.668752\n",
      "37                          DaViT_H               8  14.074373   5.684090\n",
      "10                EVA02LargePatch14               8  13.497314   5.927105\n",
      "3                      FastViT_MA36               8  13.396945   5.971511\n",
      "0                     HorNetLargeGF               8  13.366147   5.985270\n",
      "13               EfficientFormerV2L               8  12.439396   6.431180\n",
      "31                         CoAtNet4               8  11.661715   6.860054\n",
      "27                      GCViT_Large               8  11.023040   7.257526\n",
      "20  SwinTransformerV2Large_window24               8  10.896032   7.342122\n",
      "18  SwinTransformerV2Large_window12               8  10.562042   7.574293\n",
      "19  SwinTransformerV2Large_window16               8  10.499536   7.619384\n",
      "39                          VOLO_d5               8  10.448642   7.656497\n",
      "17                        HieraHuge               8   9.802427   8.161244\n",
      "12                 UniformerLarge64               8   9.770390   8.188004\n",
      "36                     IFormerLarge               8   8.779140   9.112510\n",
      "8                   EvaGiantPatch14               8   8.378268   9.548513\n",
      "23                 EfficientNetV2XL               8   8.350085   9.580741\n",
      "24                 EfficientNetV1L2               8   8.128073   9.842431\n",
      "33                     CotNetSE152D               8   7.851778  10.188775\n",
      "25                    MaxViT_XLarge               8   5.875325  13.616267\n",
      "2                      MogaNetLarge               8   5.495959  14.556149\n",
      "32                       ResNest269               8   5.116387  15.636033\n",
      "Saved result to file..\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Convert results to DataFrame\n",
    "df_kecam = pd.DataFrame(results_kecam, columns=['Model', 'Max Batch Size', 'QPS', 'Time (s)'])\n",
    "\n",
    "# sort records\n",
    "df_kecam = df_kecam.sort_values(by=\"QPS\", ascending=False)\n",
    "\n",
    "# Display the results in a table\n",
    "print(df_kecam)\n",
    "\n",
    "# save result in file\n",
    "df_kecam.to_csv('results_kecam.csv', index=False)  # Set index=False to avoid saving the index as a separate column\n",
    "print(\"Saved result to file..\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54629b23",
   "metadata": {
    "papermill": {
     "duration": 0.024044,
     "end_time": "2024-05-15T07:27:08.915246",
     "exception": false,
     "start_time": "2024-05-15T07:27:08.891202",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## <font color=\"#000080\" size=\"+3\">7. List all keras pretrained models and check QPS</font>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c2b29e24",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:27:08.969812Z",
     "iopub.status.busy": "2024-05-15T07:27:08.969098Z",
     "iopub.status.idle": "2024-05-15T07:27:08.978308Z",
     "shell.execute_reply": "2024-05-15T07:27:08.977443Z"
    },
    "papermill": {
     "duration": 0.036903,
     "end_time": "2024-05-15T07:27:08.980696",
     "exception": false,
     "start_time": "2024-05-15T07:27:08.943793",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total models: 71\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "import tf_keras\n",
    "from tf_keras.applications import *\n",
    "from tf_keras.preprocessing import image\n",
    "from tf_keras.utils import get_file\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import inspect \n",
    "from tqdm import tqdm\n",
    "\n",
    "# # List of models to test\n",
    "# model_list = [MobileNetV2, Xception, ResNet50, VGG16, VGG19, InceptionV3]\n",
    "\n",
    "# Get all model-creating functions without filtering\n",
    "model_funcs = {\n",
    "    name: obj for name, obj in inspect.getmembers(tf.keras.applications, inspect.isfunction)\n",
    "}\n",
    "model_list = []\n",
    "for model_name, model_cls in model_funcs.items():\n",
    "    model_list.append(model_cls)\n",
    "\n",
    "print(f\"Total models: {len(model_list)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ebf56233",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:27:09.035926Z",
     "iopub.status.busy": "2024-05-15T07:27:09.035071Z",
     "iopub.status.idle": "2024-05-15T07:27:09.044330Z",
     "shell.execute_reply": "2024-05-15T07:27:09.043452Z"
    },
    "papermill": {
     "duration": 0.037708,
     "end_time": "2024-05-15T07:27:09.046861",
     "exception": false,
     "start_time": "2024-05-15T07:27:09.009153",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total models after filtering: 15\n"
     ]
    }
   ],
   "source": [
    "filtered_model_list = []\n",
    "\n",
    "## optional\n",
    "## names to keep -- to perform selected models benchmark only to save time\n",
    "name_filters = [\n",
    "    \"VGG16\", \"VGG19\",\n",
    "    \"ResNet50\", \"ResNet101\",\n",
    "    \"ResNet50V2\", \"ResNet152V2\",\n",
    "    \"InceptionResNetV2\",\n",
    "    \"MobileNetV2\",\n",
    "    \"DenseNet201\",\n",
    "    \"NASNetLarge\",\n",
    "    \"EfficientNetB2\", \"EfficientNetB7\",\n",
    "    \"EfficientNetV2L\",\n",
    "    \"ConvNeXtXLarge\",\n",
    "]\n",
    "\n",
    "if name_filters:\n",
    "    for model_class in model_list:\n",
    "        model_name = model_class.__name__\n",
    "        if any(nf.lower().strip() in model_name.lower() for nf in name_filters):\n",
    "            filtered_model_list.append(model_class)\n",
    "else:\n",
    "    filtered_model_list = model_list # if no filtering\n",
    "    \n",
    "print(f\"Total models after filtering: {len(filtered_model_list)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b5915793",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:27:09.153525Z",
     "iopub.status.busy": "2024-05-15T07:27:09.152542Z",
     "iopub.status.idle": "2024-05-15T07:30:12.163856Z",
     "shell.execute_reply": "2024-05-15T07:30:12.162769Z"
    },
    "papermill": {
     "duration": 183.042892,
     "end_time": "2024-05-15T07:30:12.166303",
     "exception": false,
     "start_time": "2024-05-15T07:27:09.123411",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/15 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "Testing ConvNeXtXLarge\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 1/15 [00:14<03:21, 14.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:21.936058070380014, duration: 3.646963357925415\n",
      "\n",
      "================================================================================\n",
      "Testing DenseNet201\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 2/15 [00:33<03:43, 17.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:8.395874839570519, duration: 9.528488874435425\n",
      "\n",
      "================================================================================\n",
      "Testing EfficientNetB2\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 3/15 [00:43<02:45, 13.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:16.54807807254508, duration: 4.83439826965332\n",
      "\n",
      "================================================================================\n",
      "Testing EfficientNetB7\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 4/15 [01:06<03:12, 17.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:6.875055717988738, duration: 11.636269330978394\n",
      "\n",
      "================================================================================\n",
      "Testing EfficientNetV2L\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 5/15 [01:34<03:34, 21.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:5.551856516486695, duration: 14.409594297409058\n",
      "\n",
      "================================================================================\n",
      "Testing InceptionResNetV2\n",
      "Error in InceptionResNetV2.. Input size must be at least 75x75; Received: input_shape=(64, 640, 3)\n",
      "================================================================================\n",
      "Testing MobileNetV2\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 7/15 [01:39<01:34, 11.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:30.98382488611601, duration: 2.5819923877716064\n",
      "\n",
      "================================================================================\n",
      "Testing NASNetLarge\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 8/15 [02:10<01:56, 16.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:5.371611681115114, duration: 14.893109321594238\n",
      "\n",
      "================================================================================\n",
      "Testing ResNet101\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 9/15 [02:21<01:31, 15.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:14.920223365594259, duration: 5.361850023269653\n",
      "\n",
      "================================================================================\n",
      "Testing ResNet101V2\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 10/15 [02:31<01:09, 13.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:16.03575860266942, duration: 4.988850355148315\n",
      "\n",
      "================================================================================\n",
      "Testing ResNet152V2\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 11/15 [02:47<00:56, 14.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:10.574415569685758, duration: 7.565429925918579\n",
      "\n",
      "================================================================================\n",
      "Testing ResNet50\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 12/15 [02:53<00:35, 11.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:29.033460558948786, duration: 2.755441427230835\n",
      "\n",
      "================================================================================\n",
      "Testing ResNet50V2\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 13/15 [02:58<00:19,  9.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:30.844178693541465, duration: 2.593682289123535\n",
      "\n",
      "================================================================================\n",
      "Testing VGG16\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 14/15 [03:01<00:08,  8.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:166.56456689004716, duration: 0.48029422760009766\n",
      "\n",
      "================================================================================\n",
      "Testing VGG19\n",
      "performing warmup...\n",
      "Benchmarking...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [03:02<00:00, 12.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qps:130.37986625686298, duration: 0.6135916709899902\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Main testing loop\n",
    "results = []\n",
    "for model_class in tqdm(filtered_model_list):\n",
    "    model_name = model_class.__name__\n",
    "    print(\"=\" * 80)\n",
    "    print(f\"Testing {model_name}\")\n",
    "    \n",
    "    try:\n",
    "        with strategy.scope():\n",
    "            model = model_class(\n",
    "                weights=None, input_shape=input_shape\n",
    "            )\n",
    "\n",
    "        # max_batch_size = find_max_batch_size(model, input_size)\n",
    "        qps, duration = benchmark_model(\n",
    "            model, strategy, input_shape, \n",
    "            warm_up_batches=warm_up_batches, num_batches=num_batches, \n",
    "            batch_size=batch_size\n",
    "        )\n",
    "        print(f\"qps:{qps}, duration: {duration}\\n\")\n",
    "        results.append((model_name, batch_size, qps, duration))\n",
    "        \n",
    "    except Exception as e:\n",
    "        print(f\"Error in {model_name}.. {e}\")\n",
    "        \n",
    "    finally:\n",
    "        # clear the session\n",
    "        tf_keras.backend.clear_session()\n",
    "    \n",
    "# # Print results\n",
    "# for model_name, batch_size, qps, dur in results:\n",
    "#     print(f\"Model: {model_name}, Max Batch Size: {batch_size}, QPS: {qps:.2f}, Time: {dur:.2f}s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "82be85d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-15T07:30:12.226673Z",
     "iopub.status.busy": "2024-05-15T07:30:12.226360Z",
     "iopub.status.idle": "2024-05-15T07:30:12.237770Z",
     "shell.execute_reply": "2024-05-15T07:30:12.236763Z"
    },
    "papermill": {
     "duration": 0.043314,
     "end_time": "2024-05-15T07:30:12.239751",
     "exception": false,
     "start_time": "2024-05-15T07:30:12.196437",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              Model  Max Batch Size         QPS   Time (s)\n",
      "12            VGG16               8  166.564567   0.480294\n",
      "13            VGG19               8  130.379866   0.613592\n",
      "5       MobileNetV2               8   30.983825   2.581992\n",
      "11       ResNet50V2               8   30.844179   2.593682\n",
      "10         ResNet50               8   29.033461   2.755441\n",
      "0    ConvNeXtXLarge               8   21.936058   3.646963\n",
      "2    EfficientNetB2               8   16.548078   4.834398\n",
      "8       ResNet101V2               8   16.035759   4.988850\n",
      "7         ResNet101               8   14.920223   5.361850\n",
      "9       ResNet152V2               8   10.574416   7.565430\n",
      "1       DenseNet201               8    8.395875   9.528489\n",
      "3    EfficientNetB7               8    6.875056  11.636269\n",
      "4   EfficientNetV2L               8    5.551857  14.409594\n",
      "6       NASNetLarge               8    5.371612  14.893109\n",
      "Saved result to file..\n"
     ]
    }
   ],
   "source": [
    "# Convert results to DataFrame\n",
    "df = pd.DataFrame(results, columns=['Model', 'Max Batch Size', 'QPS', 'Time (s)'])\n",
    "\n",
    "# sort records\n",
    "df = df.sort_values(by=\"QPS\", ascending=False)\n",
    "\n",
    "# Display the results in a table\n",
    "print(df)\n",
    "\n",
    "# save result in file\n",
    "df.to_csv('results_tf_keras.csv', index=False)  # Set index=False to avoid saving the index as a separate column\n",
    "print(\"Saved result to file..\")"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [],
   "dockerImageVersionId": 30698,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 923.506737,
   "end_time": "2024-05-15T07:30:15.682600",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-05-15T07:14:52.175863",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}