IMBD_MOVIE_REVIEW_WEBSCRAPER.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RljOfuIDhB-T"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import requests\n",
        "from bs4 import BeautifulSoup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YDWGsg3UhB-Z",
        "outputId": "0b942413-9376-439b-b3a1-2cdcaa447c97"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "     Rating                                             Review\n",
            "0      6/10  Margot does the best with what she's given, bu...\n",
            "1      7/10  The first reason to go see it:It's good fun. I...\n",
            "2      6/10  Before making Barbie (2023), Greta Gerwig sing...\n",
            "3     10/10  As much as it pains me to give a movie called ...\n",
            "4      6/10  As a woman that grew up with Barbie, I was ver...\n",
            "...     ...                                                ...\n",
            "1225   9/10  It strikes generational emotional strings, the...\n",
            "1226  10/10  This movie moved me deeply. I am a 20-year-old...\n",
            "1227   9/10  I definitely went in with doubts, as one does ...\n",
            "1228   8/10  Very good felt like it dragged a but at the en...\n",
            "1229   9/10  Wasn't sure how it would go however so much fu...\n",
            "\n",
            "[1230 rows x 2 columns]\n"
          ]
        }
      ],
      "source": [
        "url = (\n",
        "    \"https://www.imdb.com/title/tt1517268/reviews/_ajax?ref_=undefined&paginationKey={}\"\n",
        ")\n",
        "key = \"\"\n",
        "data = {\"Rating\": [], \"Review\": []}\n",
        "\n",
        "while True:\n",
        "    response = requests.get(url.format(key))\n",
        "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
        "    # Find the pagination key\n",
        "    pagination_key = soup.find(\"div\", class_=\"load-more-data\")\n",
        "    if not pagination_key:\n",
        "        break\n",
        "\n",
        "    # Update the `key` variable in-order to scrape more reviews\n",
        "    key = pagination_key[\"data-key\"]\n",
        "    for Rating, review in zip(\n",
        "        soup.find_all(class_=\"rating-other-user-rating\"), soup.find_all(class_=\"text show-more__control\")\n",
        "    ):\n",
        "        data[\"Rating\"].append(Rating.get_text(strip=True))\n",
        "        data[\"Review\"].append(review.get_text())\n",
        "\n",
        "df = pd.DataFrame(data)\n",
        "print(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7d-wZvkthB-b"
      },
      "outputs": [],
      "source": [
        "df['Rate'] = df[\"Rating\"].str.split('/').str[0]\n",
        "df.drop(['Rating'], axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HBUvXT_nhB-c"
      },
      "outputs": [],
      "source": [
        "df.to_excel('barbie.xlsx')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gn2BhazKhB-d"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}