pixtral12BDemo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "A100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b3c20a5ee56d4163afcced408f074ba4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_88570c81ffa8404abdd0d413e3aa0a34",
              "IPY_MODEL_fe3a5e53bb5a4ab79faf1faf03f7a64f",
              "IPY_MODEL_d7c5a5ab4a6141a68c4259fd08ce893e"
            ],
            "layout": "IPY_MODEL_c310e16674b34f8981aedffebc7549b1"
          }
        },
        "88570c81ffa8404abdd0d413e3aa0a34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_57d55a02f35243e19cf3d1f0827f32db",
            "placeholder": "​",
            "style": "IPY_MODEL_ce2ce8675fe14a8796ecda8ff194a7c9",
            "value": ""
          }
        },
        "fe3a5e53bb5a4ab79faf1faf03f7a64f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_25f9d746ebad456eba8cc502a2779771",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6cb5b82af87b4c6dbecbc22fc0f94a9c",
            "value": 1
          }
        },
        "d7c5a5ab4a6141a68c4259fd08ce893e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c3452e4c2454d7a8879ad551011a6e5",
            "placeholder": "​",
            "style": "IPY_MODEL_c6c4e2e2a93b4093b1c276478aa25d83",
            "value": "Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:12&lt;00:00, 12.32s/it]\n"
          }
        },
        "c310e16674b34f8981aedffebc7549b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57d55a02f35243e19cf3d1f0827f32db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce2ce8675fe14a8796ecda8ff194a7c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "25f9d746ebad456eba8cc502a2779771": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6cb5b82af87b4c6dbecbc22fc0f94a9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3c3452e4c2454d7a8879ad551011a6e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c6c4e2e2a93b4093b1c276478aa25d83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install vllm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tdPG7MtXXVws",
        "outputId": "841e9cb5-9126-49f2-e5c2-b3c277fde724"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting vllm\n",
            "  Downloading vllm-0.6.1-cp38-abi3-manylinux1_x86_64.whl.metadata (2.3 kB)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from vllm) (5.9.5)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from vllm) (0.2.0)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (1.26.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from vllm) (2.32.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from vllm) (4.66.5)\n",
            "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from vllm) (9.0.0)\n",
            "Requirement already satisfied: transformers>=4.43.2 in /usr/local/lib/python3.10/dist-packages (from vllm) (4.44.2)\n",
            "Requirement already satisfied: tokenizers>=0.19.1 in /usr/local/lib/python3.10/dist-packages (from vllm) (0.19.1)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from vllm) (3.20.3)\n",
            "Collecting fastapi (from vllm)\n",
            "  Using cached fastapi-0.114.1-py3-none-any.whl.metadata (27 kB)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from vllm) (3.10.5)\n",
            "Collecting openai>=1.40.0 (from vllm)\n",
            "  Using cached openai-1.44.1-py3-none-any.whl.metadata (22 kB)\n",
            "Collecting uvicorn[standard] (from vllm)\n",
            "  Using cached uvicorn-0.30.6-py3-none-any.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: pydantic>=2.8 in /usr/local/lib/python3.10/dist-packages (from vllm) (2.9.1)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from vllm) (10.4.0)\n",
            "Requirement already satisfied: prometheus-client>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (0.20.0)\n",
            "Collecting prometheus-fastapi-instrumentator>=7.0.0 (from vllm)\n",
            "  Using cached prometheus_fastapi_instrumentator-7.0.0-py3-none-any.whl.metadata (13 kB)\n",
            "Requirement already satisfied: tiktoken>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (0.7.0)\n",
            "Collecting lm-format-enforcer==0.10.6 (from vllm)\n",
            "  Using cached lm_format_enforcer-0.10.6-py3-none-any.whl.metadata (16 kB)\n",
            "Collecting outlines<0.1,>=0.0.43 (from vllm)\n",
            "  Using cached outlines-0.0.46-py3-none-any.whl.metadata (15 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.10 in /usr/local/lib/python3.10/dist-packages (from vllm) (4.12.2)\n",
            "Requirement already satisfied: filelock>=3.10.4 in /usr/local/lib/python3.10/dist-packages (from vllm) (3.16.0)\n",
            "Collecting partial-json-parser (from vllm)\n",
            "  Using cached partial_json_parser-0.2.1.1.post4-py3-none-any.whl.metadata (6.2 kB)\n",
            "Requirement already satisfied: pyzmq in /usr/local/lib/python3.10/dist-packages (from vllm) (24.0.1)\n",
            "Collecting msgspec (from vllm)\n",
            "  Using cached msgspec-0.18.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.9 kB)\n",
            "Collecting gguf==0.9.1 (from vllm)\n",
            "  Using cached gguf-0.9.1-py3-none-any.whl.metadata (3.3 kB)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.10/dist-packages (from vllm) (8.4.0)\n",
            "Requirement already satisfied: mistral-common>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (1.4.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from vllm) (6.0.2)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from vllm) (0.8.0)\n",
            "Collecting ray>=2.9 (from vllm)\n",
            "  Using cached ray-2.35.0-cp310-cp310-manylinux2014_x86_64.whl.metadata (16 kB)\n",
            "Collecting nvidia-ml-py (from vllm)\n",
            "  Using cached nvidia_ml_py-12.560.30-py3-none-any.whl.metadata (8.6 kB)\n",
            "Requirement already satisfied: torch==2.4.0 in /usr/local/lib/python3.10/dist-packages (from vllm) (2.4.0+cu121)\n",
            "Requirement already satisfied: torchvision==0.19 in /usr/local/lib/python3.10/dist-packages (from vllm) (0.19.0+cu121)\n",
            "Collecting xformers==0.0.27.post2 (from vllm)\n",
            "  Using cached xformers-0.0.27.post2-cp310-cp310-manylinux2014_x86_64.whl.metadata (1.0 kB)\n",
            "Collecting vllm-flash-attn==2.6.1 (from vllm)\n",
            "  Using cached vllm_flash_attn-2.6.1-cp310-cp310-manylinux1_x86_64.whl.metadata (476 bytes)\n",
            "Collecting interegular>=0.3.2 (from lm-format-enforcer==0.10.6->vllm)\n",
            "  Using cached interegular-0.3.3-py37-none-any.whl.metadata (3.0 kB)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from lm-format-enforcer==0.10.6->vllm) (24.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch==2.4.0->vllm) (1.13.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.4.0->vllm) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch==2.4.0->vllm) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch==2.4.0->vllm) (2024.6.1)\n",
            "Requirement already satisfied: jsonschema<5.0.0,>=4.21.1 in /usr/local/lib/python3.10/dist-packages (from mistral-common>=1.4.0->vllm) (4.23.0)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.40.0->vllm) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.40.0->vllm) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai>=1.40.0->vllm)\n",
            "  Using cached httpx-0.27.2-py3-none-any.whl.metadata (7.1 kB)\n",
            "Collecting jiter<1,>=0.4.0 (from openai>=1.40.0->vllm)\n",
            "  Using cached jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.40.0->vllm) (1.3.1)\n",
            "Collecting lark (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached lark-1.2.2-py3-none-any.whl.metadata (1.8 kB)\n",
            "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from outlines<0.1,>=0.0.43->vllm) (1.6.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.10/dist-packages (from outlines<0.1,>=0.0.43->vllm) (2.2.1)\n",
            "Collecting diskcache (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached diskcache-5.6.3-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from outlines<0.1,>=0.0.43->vllm) (0.60.0)\n",
            "Requirement already satisfied: referencing in /usr/local/lib/python3.10/dist-packages (from outlines<0.1,>=0.0.43->vllm) (0.35.1)\n",
            "Collecting datasets (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached datasets-3.0.0-py3-none-any.whl.metadata (19 kB)\n",
            "Collecting pycountry (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached pycountry-24.6.1-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting pyairports (from outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached pyairports-2.1.1-py3-none-any.whl.metadata (1.7 kB)\n",
            "Collecting starlette<1.0.0,>=0.30.0 (from prometheus-fastapi-instrumentator>=7.0.0->vllm)\n",
            "  Using cached starlette-0.38.5-py3-none-any.whl.metadata (6.0 kB)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.8->vllm) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.8->vllm) (2.23.3)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (8.1.7)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.0.8)\n",
            "Requirement already satisfied: aiosignal in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.3.1)\n",
            "Requirement already satisfied: frozenlist in /usr/local/lib/python3.10/dist-packages (from ray>=2.9->vllm) (1.4.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (3.8)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->vllm) (2024.8.30)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken>=0.6.0->vllm) (2024.5.15)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.19.1->vllm) (0.24.6)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.43.2->vllm) (0.4.5)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (2.4.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (24.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (1.11.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->vllm) (4.0.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata->vllm) (3.20.1)\n",
            "Collecting h11>=0.8 (from uvicorn[standard]->vllm)\n",
            "  Using cached h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
            "Collecting httptools>=0.5.0 (from uvicorn[standard]->vllm)\n",
            "  Using cached httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\n",
            "Collecting python-dotenv>=0.13 (from uvicorn[standard]->vllm)\n",
            "  Using cached python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]->vllm)\n",
            "  Using cached uvloop-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
            "Collecting watchfiles>=0.13 (from uvicorn[standard]->vllm)\n",
            "  Using cached watchfiles-0.24.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
            "Collecting websockets>=10.4 (from uvicorn[standard]->vllm)\n",
            "  Using cached websockets-13.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.40.0->vllm) (1.2.2)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai>=1.40.0->vllm)\n",
            "  Using cached httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema<5.0.0,>=4.21.1->mistral-common>=1.4.0->vllm) (2023.12.1)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema<5.0.0,>=4.21.1->mistral-common>=1.4.0->vllm) (0.20.0)\n",
            "Collecting pyarrow>=15.0.0 (from datasets->outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets->outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->outlines<0.1,>=0.0.43->vllm) (2.1.4)\n",
            "Collecting xxhash (from datasets->outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting multiprocess (from datasets->outlines<0.1,>=0.0.43->vllm)\n",
            "  Using cached multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch==2.4.0->vllm) (2.1.5)\n",
            "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->outlines<0.1,>=0.0.43->vllm) (0.43.0)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch==2.4.0->vllm) (1.3.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->outlines<0.1,>=0.0.43->vllm) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->outlines<0.1,>=0.0.43->vllm) (2024.1)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->outlines<0.1,>=0.0.43->vllm) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets->outlines<0.1,>=0.0.43->vllm) (1.16.0)\n",
            "Downloading vllm-0.6.1-cp38-abi3-manylinux1_x86_64.whl (170.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m170.6/170.6 MB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hUsing cached gguf-0.9.1-py3-none-any.whl (49 kB)\n",
            "Using cached lm_format_enforcer-0.10.6-py3-none-any.whl (43 kB)\n",
            "Using cached vllm_flash_attn-2.6.1-cp310-cp310-manylinux1_x86_64.whl (75.9 MB)\n",
            "Using cached xformers-0.0.27.post2-cp310-cp310-manylinux2014_x86_64.whl (20.8 MB)\n",
            "Using cached openai-1.44.1-py3-none-any.whl (373 kB)\n",
            "Using cached outlines-0.0.46-py3-none-any.whl (101 kB)\n",
            "Using cached prometheus_fastapi_instrumentator-7.0.0-py3-none-any.whl (19 kB)\n",
            "Using cached ray-2.35.0-cp310-cp310-manylinux2014_x86_64.whl (65.0 MB)\n",
            "Using cached fastapi-0.114.1-py3-none-any.whl (94 kB)\n",
            "Using cached msgspec-0.18.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (210 kB)\n",
            "Using cached nvidia_ml_py-12.560.30-py3-none-any.whl (40 kB)\n",
            "Using cached partial_json_parser-0.2.1.1.post4-py3-none-any.whl (9.9 kB)\n",
            "Using cached h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "Using cached httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "Using cached httpx-0.27.2-py3-none-any.whl (76 kB)\n",
            "Using cached httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "Using cached interegular-0.3.3-py37-none-any.whl (23 kB)\n",
            "Using cached jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (318 kB)\n",
            "Using cached python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
            "Using cached starlette-0.38.5-py3-none-any.whl (71 kB)\n",
            "Using cached uvloop-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "Using cached watchfiles-0.24.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (425 kB)\n",
            "Using cached websockets-13.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (157 kB)\n",
            "Using cached datasets-3.0.0-py3-none-any.whl (474 kB)\n",
            "Using cached diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "Using cached lark-1.2.2-py3-none-any.whl (111 kB)\n",
            "Using cached pyairports-2.1.1-py3-none-any.whl (371 kB)\n",
            "Using cached pycountry-24.6.1-py3-none-any.whl (6.3 MB)\n",
            "Using cached uvicorn-0.30.6-py3-none-any.whl (62 kB)\n",
            "Using cached dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "Using cached pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (39.9 MB)\n",
            "Using cached multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "Using cached xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "Installing collected packages: pyairports, nvidia-ml-py, xxhash, websockets, uvloop, python-dotenv, pycountry, pyarrow, partial-json-parser, msgspec, lark, jiter, interegular, httptools, h11, gguf, diskcache, dill, watchfiles, uvicorn, starlette, multiprocess, httpcore, xformers, vllm-flash-attn, prometheus-fastapi-instrumentator, lm-format-enforcer, httpx, fastapi, ray, openai, datasets, outlines, vllm\n",
            "  Attempting uninstall: pyarrow\n",
            "    Found existing installation: pyarrow 14.0.2\n",
            "    Uninstalling pyarrow-14.0.2:\n",
            "      Successfully uninstalled pyarrow-14.0.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 17.0.0 which is incompatible.\n",
            "ibis-framework 8.0.0 requires pyarrow<16,>=2, but you have pyarrow 17.0.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed datasets-3.0.0 dill-0.3.8 diskcache-5.6.3 fastapi-0.114.1 gguf-0.9.1 h11-0.14.0 httpcore-1.0.5 httptools-0.6.1 httpx-0.27.2 interegular-0.3.3 jiter-0.5.0 lark-1.2.2 lm-format-enforcer-0.10.6 msgspec-0.18.6 multiprocess-0.70.16 nvidia-ml-py-12.560.30 openai-1.44.1 outlines-0.0.46 partial-json-parser-0.2.1.1.post4 prometheus-fastapi-instrumentator-7.0.0 pyairports-2.1.1 pyarrow-17.0.0 pycountry-24.6.1 python-dotenv-1.0.1 ray-2.35.0 starlette-0.38.5 uvicorn-0.30.6 uvloop-0.20.0 vllm-0.6.1 vllm-flash-attn-2.6.1 watchfiles-0.24.0 websockets-13.0.1 xformers-0.0.27.post2 xxhash-3.5.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade mistral_common"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 775
        },
        "id": "BSCw25MnX-Ie",
        "outputId": "aaa8b52a-caf8-4b7c-e95a-f82ad9f44ae9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting mistral_common\n",
            "  Using cached mistral_common-1.4.0-py3-none-any.whl.metadata (4.4 kB)\n",
            "Requirement already satisfied: jsonschema<5.0.0,>=4.21.1 in /usr/local/lib/python3.10/dist-packages (from mistral_common) (4.23.0)\n",
            "Collecting pillow<11.0.0,>=10.3.0 (from mistral_common)\n",
            "  Using cached pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (9.2 kB)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.6.1 in /usr/local/lib/python3.10/dist-packages (from mistral_common) (2.9.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from mistral_common) (2.32.3)\n",
            "Collecting sentencepiece==0.2.0 (from mistral_common)\n",
            "  Using cached sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.7 kB)\n",
            "Collecting tiktoken<0.8.0,>=0.7.0 (from mistral_common)\n",
            "  Using cached tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: typing-extensions<5.0.0,>=4.11.0 in /usr/local/lib/python3.10/dist-packages (from mistral_common) (4.12.2)\n",
            "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema<5.0.0,>=4.21.1->mistral_common) (24.2.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema<5.0.0,>=4.21.1->mistral_common) (2023.12.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema<5.0.0,>=4.21.1->mistral_common) (0.35.1)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema<5.0.0,>=4.21.1->mistral_common) (0.20.0)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.6.1->mistral_common) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.6.1->mistral_common) (2.23.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.0.0->mistral_common) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.0.0->mistral_common) (3.8)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.0.0->mistral_common) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.0.0->mistral_common) (2024.8.30)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.8.0,>=0.7.0->mistral_common) (2024.5.15)\n",
            "Using cached mistral_common-1.4.0-py3-none-any.whl (6.0 MB)\n",
            "Using cached sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "Using cached pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl (4.5 MB)\n",
            "Using cached tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "Installing collected packages: sentencepiece, pillow, tiktoken, mistral_common\n",
            "  Attempting uninstall: sentencepiece\n",
            "    Found existing installation: sentencepiece 0.1.99\n",
            "    Uninstalling sentencepiece-0.1.99:\n",
            "      Successfully uninstalled sentencepiece-0.1.99\n",
            "  Attempting uninstall: pillow\n",
            "    Found existing installation: Pillow 9.4.0\n",
            "    Uninstalling Pillow-9.4.0:\n",
            "      Successfully uninstalled Pillow-9.4.0\n",
            "Successfully installed mistral_common-1.4.0 pillow-10.4.0 sentencepiece-0.2.0 tiktoken-0.7.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL"
                ]
              },
              "id": "828273f90da34a24a2be8e6aca3e2cd0"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm import LLM\n",
        "from vllm.sampling_params import SamplingParams\n",
        "\n",
        "model_name = \"mistralai/Pixtral-12B-2409\"\n",
        "\n",
        "sampling_params = SamplingParams(max_tokens=8192)\n",
        "\n",
        "llm = LLM(model=model_name, tokenizer_mode=\"mistral\",max_model_len=70000)\n",
        "\n",
        "prompt = \"Describe this image in one sentence.\"\n",
        "image_url = \"https://picsum.photos/id/237/200/300\"\n",
        "\n",
        "messages = [\n",
        "    {\n",
        "        \"role\": \"user\",\n",
        "        \"content\": [{\"type\": \"text\", \"text\": prompt}, {\"type\": \"image_url\", \"image_url\": {\"url\": image_url}}]\n",
        "    },\n",
        "]\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 644,
          "referenced_widgets": [
            "b3c20a5ee56d4163afcced408f074ba4",
            "88570c81ffa8404abdd0d413e3aa0a34",
            "fe3a5e53bb5a4ab79faf1faf03f7a64f",
            "d7c5a5ab4a6141a68c4259fd08ce893e",
            "c310e16674b34f8981aedffebc7549b1",
            "57d55a02f35243e19cf3d1f0827f32db",
            "ce2ce8675fe14a8796ecda8ff194a7c9",
            "25f9d746ebad456eba8cc502a2779771",
            "6cb5b82af87b4c6dbecbc22fc0f94a9c",
            "3c3452e4c2454d7a8879ad551011a6e5",
            "c6c4e2e2a93b4093b1c276478aa25d83"
          ]
        },
        "id": "4h1bYNB1Xvug",
        "outputId": "6ffdd715-cf12-4aca-8e3d-faead68227a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-12 09:06:45 importing.py:10] Triton not installed; certain GPU-related functions will not be available.\n",
            "INFO 09-12 09:06:54 config.py:1646] Downcasting torch.float32 to torch.float16.\n",
            "WARNING 09-12 09:06:54 arg_utils.py:890] Chunked prefill is enabled by default for models with max_model_len > 32K. Currently, chunked prefill might not work with some features or models. If you encounter any issues, please disable chunked prefill by setting --enable-chunked-prefill=False.\n",
            "INFO 09-12 09:06:54 config.py:1006] Chunked prefill is enabled with max_num_batched_tokens=4096.\n",
            "INFO 09-12 09:06:54 llm_engine.py:232] Initializing an LLM engine (v0.6.1) with config: model='mistralai/Pixtral-12B-2409', speculative_config=None, tokenizer='mistralai/Pixtral-12B-2409', skip_tokenizer_init=False, tokenizer_mode=mistral, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=70000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=mistralai/Pixtral-12B-2409, use_v2_block_manager=False, num_scheduler_steps=1, enable_prefix_caching=False, use_async_output_proc=True)\n",
            "INFO 09-12 09:06:57 model_runner.py:997] Starting to load model mistralai/Pixtral-12B-2409...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.\n",
            "  @torch.library.impl_abstract(\"xformers_flash::flash_fwd\")\n",
            "/usr/local/lib/python3.10/dist-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch.\n",
            "  @torch.library.impl_abstract(\"xformers_flash::flash_bwd\")\n",
            "WARNING:xformers:A matching Triton is not available, some optimizations will not be enabled\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/xformers/__init__.py\", line 57, in _is_triton_available\n",
            "    import triton  # noqa\n",
            "ModuleNotFoundError: No module named 'triton'\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-12 09:06:58 weight_utils.py:242] Using model weights format ['*.safetensors']\n",
            "INFO 09-12 09:06:59 weight_utils.py:287] No model.safetensors.index.json found in remote.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]\n"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b3c20a5ee56d4163afcced408f074ba4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 09-12 09:07:13 model_runner.py:1008] Loading model weights took 23.6259 GB\n",
            "INFO 09-12 09:07:16 gpu_executor.py:122] # GPU blocks: 4499, # CPU blocks: 1638\n",
            "INFO 09-12 09:07:19 model_runner.py:1309] Capturing the model for CUDA graphs. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI.\n",
            "INFO 09-12 09:07:19 model_runner.py:1313] CUDA graphs can take additional 1~3 GiB memory per GPU. If you are running out of memory, consider decreasing `gpu_memory_utilization` or enforcing eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.\n",
            "INFO 09-12 09:07:44 model_runner.py:1428] Graph capturing finished in 25 secs.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'vllm_model' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-9e6141676ca8>\u001b[0m in \u001b[0;36m<cell line: 20>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m ]\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvllm_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msampling_params\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msampling_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'vllm_model' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "outputs = llm.chat(messages, sampling_params=sampling_params)\n",
        "\n",
        "print('\\n'+ outputs[0].outputs[0].text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "it_DE-p6fild",
        "outputId": "63099231-50c8-444b-bd17-9a7a2fb98e93"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processed prompts: 100%|██████████| 1/1 [00:00<00:00,  1.04it/s, est. speed input: 288.81 toks/s, output: 42.75 toks/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "The image depicts a black dog with a اear bat up, lying on a patterned background resembling a wooden or rope texture. The dog appears to be looking off to the side with a relaxed expression.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = \"Extract text from this image\"\n",
        "image_url = \"https://forum.ui.vision/uploads/db8324/original/2X/c/cb453dbf3890b72f41731b69e47a10cbb510e8a6.png\"\n",
        "\n",
        "messages = [\n",
        "    {\n",
        "        \"role\": \"user\",\n",
        "        \"content\": [{\"type\": \"text\", \"text\": prompt}, {\"type\": \"image_url\", \"image_url\": {\"url\": image_url}}]\n",
        "    },\n",
        "]\n",
        "\n",
        "outputs = llm.chat(messages, sampling_params=sampling_params)\n",
        "\n",
        "print('\\n'+outputs[0].outputs[0].text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "akeIZ4MRaxmB",
        "outputId": "f8a8658a-0a01-49f6-8b5b-513fde046ac2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processed prompts: 100%|██████████| 1/1 [00:01<00:00,  1.19s/it, est. speed input: 1209.03 toks/s, output: 38.54 toks/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Dear User,\n",
            "Handwrytten uses robotic\n",
            "handwriting machines that use\n",
            "an actual pen to write your\n",
            "message. The results are virtually\n",
            "indistinguishable from actual\n",
            "handwriting.\n",
            "Try it today!\n",
            "The Robot\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = \"Describe this image\"\n",
        "image_url = \"https://dynamic-media-cdn.tripadvisor.com/media/photo-o/24/99/4f/cd/caption.jpg?w=1200&h=700&s=1&cx=885&cy=590&chk=v1_e3d8683e7000393d4844\"\n",
        "\n",
        "messages = [\n",
        "    {\n",
        "        \"role\": \"user\",\n",
        "        \"content\": [{\"type\": \"text\", \"text\": prompt}, {\"type\": \"image_url\", \"image_url\": {\"url\": image_url}}]\n",
        "    },\n",
        "]\n",
        "\n",
        "outputs = llm.chat(messages, sampling_params=sampling_params)\n",
        "\n",
        "print('\\n'+outputs[0].outputs[0].text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "13e4mjX8gV8Q",
        "outputId": "e9886501-d5d3-404c-b15e-b632b7721be7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processed prompts: 100%|██████████| 1/1 [00:05<00:00,  5.83s/it, est. speed input: 424.70 toks/s, output: 43.89 toks/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "The image depicts a bustling urban environment, likely in a European city, judging by the architectural style. The scene is dominated by beautifully ornate buildings with historic and intricate designs, playing a pivotal role in the image's aesthetic. These buildings feature multiple stories, domed tops, and detailed facades, hinting at a blend of classical and Art Nouveau influences.\n",
            "\n",
            "On the left side of the image, a prominent building corner is labeled \"Royal Air Maroc,\" which indicates the presence of a business or office location. Another storefront near the center displays \"B meas.\" Similarly, there are other shops with French or French-influenced names.\n",
            "\n",
            "The street below is lively with numerous people walking, crisscrossing the thoroughfare, signifying this might be a popular shopping or tourist area. There is a streetclock posting the time around 10:10, helping us establish the time context of the scene.\n",
            "\n",
            "The traffic is also significant, as visible by several cars and a large tour bus on the left driving towards the camera. The presence of classic street lamps reinforces the old-world charm of the area.\n",
            "\n",
            "Overall, the image captures the essence of a vibrant European cityscape, characterized by historical architecture, active pedestrian and vehicle movement, and commercial activity.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    }
  ]
}