DL4D_14_Processing_Language.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KNtUhWhI6EP_"
      },
      "source": [
        "# Processing Language"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ziZeCZZp6EQJ"
      },
      "source": [
        "## Defining understanding as tokenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "8QDKUGFO6EQK"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "\n",
        "texts = [\"My dog gets along with cats\", \n",
        "         \"That cat is vicious\",\n",
        "         \"My dog is happy when it is lunch\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jAj4vgfw6EQO"
      },
      "source": [
        "## Putting all the documents into a bag"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "LCyYzQHu6EQQ",
        "outputId": "46d0a432-9547-490b-b556-de006a60ee79",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 14 unique words\n"
          ]
        }
      ],
      "source": [
        "unique_words = set(word.lower() for phrase in texts for \n",
        "                   word in phrase.split(\" \"))\n",
        "print(f\"There are {len(unique_words)} unique words\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "GkhEH4JW6EQU"
      },
      "outputs": [],
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "vocabulary_size = len(unique_words) + 1\n",
        "tokenizer = Tokenizer(num_words=vocabulary_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "A4BJPaPh6EQW",
        "outputId": "ded90231-6e66-4a62-fb15-3b92ec4c1eac",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{1: 'is', 2: 'my', 3: 'dog', 4: 'gets', 5: 'along', 6: 'with', 7: 'cats', 8: 'that', 9: 'cat', 10: 'vicious', 11: 'happy', 12: 'when', 13: 'it', 14: 'lunch'}\n"
          ]
        }
      ],
      "source": [
        "tokenizer.fit_on_texts(texts)\n",
        "print(tokenizer.index_word)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "ML_wEueW6EQY",
        "outputId": "b0ee9b5e-47e9-4e88-8a03-b19a578e49b6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0. 0. 1. 1. 1. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 1. 0. 0. 0. 0. 0. 0. 1. 1. 1. 0. 0. 0. 0.]\n",
            " [0. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 1. 1. 1. 1.]]\n"
          ]
        }
      ],
      "source": [
        "print(tokenizer.texts_to_matrix(texts))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "FijXIFg46EQa",
        "outputId": "3d897ce9-22d0-40ad-ec85-764e656c1c51",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.  0.  0.7 0.7 0.9 0.9 0.9 0.9 0.  0.  0.  0.  0.  0.  0. ]\n",
            " [0.  0.7 0.  0.  0.  0.  0.  0.  0.9 0.9 0.9 0.  0.  0.  0. ]\n",
            " [0.  1.2 0.7 0.7 0.  0.  0.  0.  0.  0.  0.  0.9 0.9 0.9 0.9]]\n"
          ]
        }
      ],
      "source": [
        "print(np.round(tokenizer.texts_to_matrix(texts, mode='tfidf'), 1))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "f3ida87K6EQc",
        "outputId": "4b664912-01fb-41f1-f660-4ca40a48c4bf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[2, 3, 4, 5, 6, 7], [8, 9, 1, 10], [2, 3, 1, 11, 12, 13, 1, 14]]\n"
          ]
        }
      ],
      "source": [
        "print(tokenizer.texts_to_sequences(texts))"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}