Text_Preprocessing.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PHa3XrRYwZyn",
        "outputId": "2853ed09-eaf8-4ac6-d459-4248178b3854"
      },
      "outputs": [],
      "source": [
        "# pip install autocorrect"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3bSO42CDNTR7",
        "outputId": "9b116fa3-2e60-4cb9-ddf7-5fda8822acb4"
      },
      "outputs": [],
      "source": [
        "# pip install --user -U nltk"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "QbteUreAI5zg"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import json\n",
        "import numpy as np\n",
        "from collections import defaultdict\n",
        "from pathlib import Path\n",
        "import pandas as pd\n",
        "import string\n",
        "import re\n",
        "from nltk.corpus import stopwords\n",
        "from autocorrect import Speller\n",
        "from nltk.stem import WordNetLemmatizer "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iwjm4YxgwZyq",
        "outputId": "9ed444bf-b5e2-4bbe-bf87-167010d3cd16"
      },
      "outputs": [],
      "source": [
        "import nltk\n",
        "# nltk.download('stopwords')\n",
        "# nltk.download('punkt')\n",
        "# nltk.download('wordnet')\n",
        "# nltk.download('omw-1.4')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9bvEfIfwwZyr",
        "outputId": "13e33ad4-e56e-41ee-835f-1ed3f001d165"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/var/folders/c9/z73mt59x7xndbcx483qrvg500000gn/T/ipykernel_13253/3652881321.py:1: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
            "  pd.set_option('display.max_colwidth', -1)\n"
          ]
        }
      ],
      "source": [
        "pd.set_option('display.max_colwidth', -1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lXDzkW0CwvJd"
      },
      "outputs": [],
      "source": [
        "data = pd.read_csv(\"sample_reviews.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        },
        "id": "BTtyYiA0bGUv",
        "outputId": "9c1c4df2-9c95-4488-8011-ff3be1e3827c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>l3Wk_mvAog6XANIuGQ9C7Q</td>\n",
              "      <td>ZbqSHbgCjzVAqaa7NKWn5A</td>\n",
              "      <td>EQ-TZ2eeD_E0BHuvoaeG5Q</td>\n",
              "      <td>Locals recommended Milktooth, and it's an amaz...</td>\n",
              "      <td>['Beer', 'Wine &amp; Spirits', 'Cafes', 'Coffee &amp; ...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>vBK79c3_1Ff_oqkh5VpfGg</td>\n",
              "      <td>Ohhrhu1RkqfVciIVx_W5HQ</td>\n",
              "      <td>nRKndeZLQ3eDL10UMwS2rQ</td>\n",
              "      <td>HOLY SMOKES!\\n\\nactual pumpkin pie mixed in wi...</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Food', 'Local F...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0               review_id                 user_id  \\\n",
              "0           0  l3Wk_mvAog6XANIuGQ9C7Q  ZbqSHbgCjzVAqaa7NKWn5A   \n",
              "1           1  vBK79c3_1Ff_oqkh5VpfGg  Ohhrhu1RkqfVciIVx_W5HQ   \n",
              "\n",
              "              business_id                                               text  \\\n",
              "0  EQ-TZ2eeD_E0BHuvoaeG5Q  Locals recommended Milktooth, and it's an amaz...   \n",
              "1  nRKndeZLQ3eDL10UMwS2rQ  HOLY SMOKES!\\n\\nactual pumpkin pie mixed in wi...   \n",
              "\n",
              "                                            category  n_classes  \n",
              "0  ['Beer', 'Wine & Spirits', 'Cafes', 'Coffee & ...          8  \n",
              "1  ['Ice Cream & Frozen Yogurt', 'Food', 'Local F...          4  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Size of the dataset: (570442, 7)\n"
          ]
        }
      ],
      "source": [
        "display(data.head(2))\n",
        "print(\"Size of the dataset:\",data.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "str"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "type(data.text.loc[0])"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "FqI0WQzawZyx"
      },
      "source": [
        "### **Text Preprocessing**\n",
        "1. Case folding\n",
        "2. Removing newlines\n",
        "3. Removing links\n",
        "4. Removing numbers\n",
        "5. Removing extra whitespaces\n",
        "6. Removing emoji \n",
        "7. Tokenization + stopwords \n",
        "8. Lemmatizaion\n",
        "9. Removing punctuation\n",
        "10. Removing repeated characters\n",
        "11. Spelling correction\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RWAqtzS1wZy5"
      },
      "source": [
        "#### **Case-folding**\n",
        "All the text of \"text\" column converted into lower case "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "ZXh22F-GwZy5",
        "outputId": "7464a69a-9915-4813-d9ee-87031bdaf596"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>l3Wk_mvAog6XANIuGQ9C7Q</td>\n",
              "      <td>ZbqSHbgCjzVAqaa7NKWn5A</td>\n",
              "      <td>EQ-TZ2eeD_E0BHuvoaeG5Q</td>\n",
              "      <td>locals recommended milktooth, and it's an amaz...</td>\n",
              "      <td>['Beer', 'Wine &amp; Spirits', 'Cafes', 'Coffee &amp; ...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>vBK79c3_1Ff_oqkh5VpfGg</td>\n",
              "      <td>Ohhrhu1RkqfVciIVx_W5HQ</td>\n",
              "      <td>nRKndeZLQ3eDL10UMwS2rQ</td>\n",
              "      <td>holy smokes!\\n\\nactual pumpkin pie mixed in wi...</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Food', 'Local F...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>z0osLHDvXvzfm57D4DmD2Q</td>\n",
              "      <td>xVKE_HJ2pwUtTdLbL3pnCg</td>\n",
              "      <td>S2Ho8yLxhKAa26pBAm6rxA</td>\n",
              "      <td>service was crappy, and food was mediocre.  i ...</td>\n",
              "      <td>['Cajun/Creole', 'Seafood', 'Restaurants', 'Br...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>elqRpX9T3YwL07uLNtN3Bg</td>\n",
              "      <td>-sryo4gDYxbZ1T5Bz4l5Bw</td>\n",
              "      <td>ltBBYdNzkeKdCNPDAsxwAA</td>\n",
              "      <td>i at least have to give this restaurant two st...</td>\n",
              "      <td>['Cocktail Bars', 'Nightlife', 'Gastropubs', '...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>pHwbdway4yeI-dSSmZA7-Q</td>\n",
              "      <td>qEEk0PuoH1dVa619t8fgpw</td>\n",
              "      <td>PY9GRfzr4nTZeINf346QOw</td>\n",
              "      <td>we checked in around 2:30 pm.  check-in was qu...</td>\n",
              "      <td>['Event Planning &amp; Services', 'Casinos', 'Beau...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0               review_id                 user_id  \\\n",
              "0           0  l3Wk_mvAog6XANIuGQ9C7Q  ZbqSHbgCjzVAqaa7NKWn5A   \n",
              "1           1  vBK79c3_1Ff_oqkh5VpfGg  Ohhrhu1RkqfVciIVx_W5HQ   \n",
              "2           2  z0osLHDvXvzfm57D4DmD2Q  xVKE_HJ2pwUtTdLbL3pnCg   \n",
              "3           3  elqRpX9T3YwL07uLNtN3Bg  -sryo4gDYxbZ1T5Bz4l5Bw   \n",
              "4           4  pHwbdway4yeI-dSSmZA7-Q  qEEk0PuoH1dVa619t8fgpw   \n",
              "\n",
              "              business_id                                               text  \\\n",
              "0  EQ-TZ2eeD_E0BHuvoaeG5Q  locals recommended milktooth, and it's an amaz...   \n",
              "1  nRKndeZLQ3eDL10UMwS2rQ  holy smokes!\\n\\nactual pumpkin pie mixed in wi...   \n",
              "2  S2Ho8yLxhKAa26pBAm6rxA  service was crappy, and food was mediocre.  i ...   \n",
              "3  ltBBYdNzkeKdCNPDAsxwAA  i at least have to give this restaurant two st...   \n",
              "4  PY9GRfzr4nTZeINf346QOw  we checked in around 2:30 pm.  check-in was qu...   \n",
              "\n",
              "                                            category  n_classes  \n",
              "0  ['Beer', 'Wine & Spirits', 'Cafes', 'Coffee & ...          8  \n",
              "1  ['Ice Cream & Frozen Yogurt', 'Food', 'Local F...          4  \n",
              "2  ['Cajun/Creole', 'Seafood', 'Restaurants', 'Br...          4  \n",
              "3  ['Cocktail Bars', 'Nightlife', 'Gastropubs', '...          8  \n",
              "4  ['Event Planning & Services', 'Casinos', 'Beau...          8  "
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.text = data.text.str.lower()\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P6DsWNFbV2fa"
      },
      "source": [
        "#### **Remove newlines**\n",
        "This function will remove all the occurrences of newlines, tabs, and combinations like: `\\\\n`, `\\\\`. \n",
        " all the occurrences of `\\n,\\\\n,\\t,\\\\` are replaced with a space."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "yIEO9obOWiik"
      },
      "outputs": [],
      "source": [
        "def remove_newlines_tabs(text):\n",
        "    formatted = text.replace('\\\\n', ' ').replace('\\n', ' ').replace('\\t',' ').replace('\\\\', ' ').replace('. com', '.com')\n",
        "    return formatted"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "ItUBADQ-WiZG"
      },
      "outputs": [],
      "source": [
        "data.text = data.text.apply(lambda text: remove_newlines_tabs(text))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 719
        },
        "id": "cg4cbN8uW0pl",
        "outputId": "d392a956-2d29-4470-f10f-aa61bf7ce3c8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>l3Wk_mvAog6XANIuGQ9C7Q</td>\n",
              "      <td>ZbqSHbgCjzVAqaa7NKWn5A</td>\n",
              "      <td>EQ-TZ2eeD_E0BHuvoaeG5Q</td>\n",
              "      <td>locals recommended milktooth, and it's an amaz...</td>\n",
              "      <td>['Beer', 'Wine &amp; Spirits', 'Cafes', 'Coffee &amp; ...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>vBK79c3_1Ff_oqkh5VpfGg</td>\n",
              "      <td>Ohhrhu1RkqfVciIVx_W5HQ</td>\n",
              "      <td>nRKndeZLQ3eDL10UMwS2rQ</td>\n",
              "      <td>holy smokes!  actual pumpkin pie mixed in with...</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Food', 'Local F...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0               review_id                 user_id  \\\n",
              "0           0  l3Wk_mvAog6XANIuGQ9C7Q  ZbqSHbgCjzVAqaa7NKWn5A   \n",
              "1           1  vBK79c3_1Ff_oqkh5VpfGg  Ohhrhu1RkqfVciIVx_W5HQ   \n",
              "\n",
              "              business_id                                               text  \\\n",
              "0  EQ-TZ2eeD_E0BHuvoaeG5Q  locals recommended milktooth, and it's an amaz...   \n",
              "1  nRKndeZLQ3eDL10UMwS2rQ  holy smokes!  actual pumpkin pie mixed in with...   \n",
              "\n",
              "                                            category  n_classes  \n",
              "0  ['Beer', 'Wine & Spirits', 'Cafes', 'Coffee & ...          8  \n",
              "1  ['Ice Cream & Frozen Yogurt', 'Food', 'Local F...          4  "
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.head(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VQebnm8xcLPa"
      },
      "source": [
        "#### **Removing links**\n",
        "\n",
        "Since punctuation is not removed yet, we can notice there are observations containing URLs in the text, as we can see below. \n",
        "We will remove the URLs from the text."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "P_b8Xs0tUXAN",
        "outputId": "214613e5-ac40-47f9-81a8-3626f83879b4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Size of the dataset with links: (563, 7)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1025</th>\n",
              "      <td>1025</td>\n",
              "      <td>AyELQAUQjjccm1pWHwLKYw</td>\n",
              "      <td>DihTsliUxbqpoA9W3uRDBA</td>\n",
              "      <td>EtKSTHV5Qx_Q7Aur9o4kQQ</td>\n",
              "      <td>sorry yelpers, but i thought this jose garces ...</td>\n",
              "      <td>['Bars', 'Nightlife', 'Whiskey Bars', 'Burgers...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2158</th>\n",
              "      <td>2158</td>\n",
              "      <td>dXj8xfsfXapb4zgH-rQK3A</td>\n",
              "      <td>k6GurG0wmHVyiZGzmxs8eQ</td>\n",
              "      <td>I6L0Zxi5Ww0zEWSAVgngeQ</td>\n",
              "      <td>i like the other cafe beignet location better ...</td>\n",
              "      <td>['Sandwiches', 'Breakfast &amp; Brunch', 'Cajun/Cr...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      Unnamed: 0               review_id                 user_id  \\\n",
              "1025        1025  AyELQAUQjjccm1pWHwLKYw  DihTsliUxbqpoA9W3uRDBA   \n",
              "2158        2158  dXj8xfsfXapb4zgH-rQK3A  k6GurG0wmHVyiZGzmxs8eQ   \n",
              "\n",
              "                 business_id  \\\n",
              "1025  EtKSTHV5Qx_Q7Aur9o4kQQ   \n",
              "2158  I6L0Zxi5Ww0zEWSAVgngeQ   \n",
              "\n",
              "                                                   text  \\\n",
              "1025  sorry yelpers, but i thought this jose garces ...   \n",
              "2158  i like the other cafe beignet location better ...   \n",
              "\n",
              "                                               category  n_classes  \n",
              "1025  ['Bars', 'Nightlife', 'Whiskey Bars', 'Burgers...          6  \n",
              "2158  ['Sandwiches', 'Breakfast & Brunch', 'Cajun/Cr...          7  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "data_links = data[data['text'].str.contains(r\"www\\.[A-Za-z]*\\.com\", regex = True)] #data containing links\n",
        "print(\"Size of the dataset with links:\", data_links.shape)\n",
        "display(data_links.head(2))\n",
        "\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "563 ovservations contain URLs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "_90EFNtZdO3T"
      },
      "outputs": [],
      "source": [
        "def remove_links(text):\n",
        "  https = re.sub(r'http\\S+', ' ', text)\n",
        "  com = re.sub(r\"www\\.[A-Za-z]*\\.com\", \" \", https)\n",
        "  return com"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "M-Xtyp0bdpw3"
      },
      "outputs": [],
      "source": [
        "data['text']= data['text'].apply(remove_links)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e_ImrQNmTzL0"
      },
      "source": [
        "As we can see, there are no more observations where text contains links:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49
        },
        "id": "yY8ARlYUTvNq",
        "outputId": "d77047bc-5863-427b-cec3-f790bdc5cd50"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [Unnamed: 0, review_id, user_id, business_id, text, category, n_classes]\n",
              "Index: []"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data[data['text'].str.contains(r\"www\\.[A-Za-z]*\\.com\", regex = True)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49
        },
        "id": "o8fiOwPHUt43",
        "outputId": "19c67191-465d-4301-bedc-7c6c7f7e6b0e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [Unnamed: 0, review_id, user_id, business_id, text, category, n_classes]\n",
              "Index: []"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data[data['text'].str.contains(r\"http\\S+\", regex = True)]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yG6Nv5irS4_0"
      },
      "source": [
        "Anyway, there are still websites written just in the form ending with `.com` . As we can see below:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 716
        },
        "id": "LQMPFH7bgQFP",
        "outputId": "1027991c-c5b9-4e1c-d5fc-23f4ddf832ac"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(594, 7)\n"
          ]
        }
      ],
      "source": [
        "data_com = data[data['text'].str.contains(r\"[A-Za-z]*\\.com[^a-zA-Z]\", regex = True)]  #consider after .com, any non alphanumeric symbol. \n",
        "print(data_com.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>683</th>\n",
              "      <td>683</td>\n",
              "      <td>BfJsxwDZ9pruhNRfjljT0w</td>\n",
              "      <td>mkQz8aihI1V19sYH8JQy-A</td>\n",
              "      <td>8uF-bhJFgT4Tn6DTb27viA</td>\n",
              "      <td>coming from a nyc sweets-hater - these donuts are divine. luxuriously extravagant and very creative. i had a banana pudding donut with a house brew. others on display were thai iced tea, pb&amp;j and milk &amp; cereal. so cute.   taste is absolutely bomb.com. will make it a pit-stop next time i'm in no.</td>\n",
              "      <td>['Food', 'Donuts', 'Burgers', 'American (Traditional)', 'Coffee &amp; Tea', 'Restaurants']</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>760</th>\n",
              "      <td>760</td>\n",
              "      <td>2mxuPqNLpEzg1LGPEMG4Fg</td>\n",
              "      <td>gqDbe1YwS4EhuCCNSNhpxg</td>\n",
              "      <td>8uF-bhJFgT4Tn6DTb27viA</td>\n",
              "      <td>bomb.com! i forget what i ordered but it was similar to a cuban sandwich on donut bread. my friend ordered a cinnamon sugar donut while she waited for her actual food and that was delicious. i ate everything because it was amazing and deserves all the hype it has received. i need to go back and try those stuffed donuts.</td>\n",
              "      <td>['Food', 'Donuts', 'Burgers', 'American (Traditional)', 'Coffee &amp; Tea', 'Restaurants']</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Unnamed: 0               review_id                 user_id  \\\n",
              "683  683         BfJsxwDZ9pruhNRfjljT0w  mkQz8aihI1V19sYH8JQy-A   \n",
              "760  760         2mxuPqNLpEzg1LGPEMG4Fg  gqDbe1YwS4EhuCCNSNhpxg   \n",
              "\n",
              "                business_id  \\\n",
              "683  8uF-bhJFgT4Tn6DTb27viA   \n",
              "760  8uF-bhJFgT4Tn6DTb27viA   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                  text  \\\n",
              "683  coming from a nyc sweets-hater - these donuts are divine. luxuriously extravagant and very creative. i had a banana pudding donut with a house brew. others on display were thai iced tea, pb&j and milk & cereal. so cute.   taste is absolutely bomb.com. will make it a pit-stop next time i'm in no.                            \n",
              "760  bomb.com! i forget what i ordered but it was similar to a cuban sandwich on donut bread. my friend ordered a cinnamon sugar donut while she waited for her actual food and that was delicious. i ate everything because it was amazing and deserves all the hype it has received. i need to go back and try those stuffed donuts.   \n",
              "\n",
              "                                                                                   category  \\\n",
              "683  ['Food', 'Donuts', 'Burgers', 'American (Traditional)', 'Coffee & Tea', 'Restaurants']   \n",
              "760  ['Food', 'Donuts', 'Burgers', 'American (Traditional)', 'Coffee & Tea', 'Restaurants']   \n",
              "\n",
              "     n_classes  \n",
              "683  6          \n",
              "760  6          "
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data_com.head(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vSp8ix6aTNI6"
      },
      "source": [
        "We will remove the websites written in this form as well."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "WgZEomMlgiXl"
      },
      "outputs": [],
      "source": [
        "def remove_com(text):\n",
        " c = re.sub(r'[A-Za-z]*\\.com[^a-zA-Z]', ' ', text)\n",
        " return c"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/var/folders/c9/z73mt59x7xndbcx483qrvg500000gn/T/ipykernel_28329/1957766861.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data_com['text'] = data_com['text'].apply(remove_com)\n"
          ]
        }
      ],
      "source": [
        "data_com['text'] = data_com['text'].apply(remove_com)\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "As we can see, websites ending with \".com\" were removed."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>683</th>\n",
              "      <td>683</td>\n",
              "      <td>BfJsxwDZ9pruhNRfjljT0w</td>\n",
              "      <td>mkQz8aihI1V19sYH8JQy-A</td>\n",
              "      <td>8uF-bhJFgT4Tn6DTb27viA</td>\n",
              "      <td>coming from a nyc sweets-hater - these donuts are divine. luxuriously extravagant and very creative. i had a banana pudding donut with a house brew. others on display were thai iced tea, pb&amp;j and milk &amp; cereal. so cute.   taste is absolutely   will make it a pit-stop next time i'm in no.</td>\n",
              "      <td>['Food', 'Donuts', 'Burgers', 'American (Traditional)', 'Coffee &amp; Tea', 'Restaurants']</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>760</th>\n",
              "      <td>760</td>\n",
              "      <td>2mxuPqNLpEzg1LGPEMG4Fg</td>\n",
              "      <td>gqDbe1YwS4EhuCCNSNhpxg</td>\n",
              "      <td>8uF-bhJFgT4Tn6DTb27viA</td>\n",
              "      <td>i forget what i ordered but it was similar to a cuban sandwich on donut bread. my friend ordered a cinnamon sugar donut while she waited for her actual food and that was delicious. i ate everything because it was amazing and deserves all the hype it has received. i need to go back and try those stuffed donuts.</td>\n",
              "      <td>['Food', 'Donuts', 'Burgers', 'American (Traditional)', 'Coffee &amp; Tea', 'Restaurants']</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Unnamed: 0               review_id                 user_id  \\\n",
              "683  683         BfJsxwDZ9pruhNRfjljT0w  mkQz8aihI1V19sYH8JQy-A   \n",
              "760  760         2mxuPqNLpEzg1LGPEMG4Fg  gqDbe1YwS4EhuCCNSNhpxg   \n",
              "\n",
              "                business_id  \\\n",
              "683  8uF-bhJFgT4Tn6DTb27viA   \n",
              "760  8uF-bhJFgT4Tn6DTb27viA   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                          text  \\\n",
              "683  coming from a nyc sweets-hater - these donuts are divine. luxuriously extravagant and very creative. i had a banana pudding donut with a house brew. others on display were thai iced tea, pb&j and milk & cereal. so cute.   taste is absolutely   will make it a pit-stop next time i'm in no.                            \n",
              "760    i forget what i ordered but it was similar to a cuban sandwich on donut bread. my friend ordered a cinnamon sugar donut while she waited for her actual food and that was delicious. i ate everything because it was amazing and deserves all the hype it has received. i need to go back and try those stuffed donuts.   \n",
              "\n",
              "                                                                                   category  \\\n",
              "683  ['Food', 'Donuts', 'Burgers', 'American (Traditional)', 'Coffee & Tea', 'Restaurants']   \n",
              "760  ['Food', 'Donuts', 'Burgers', 'American (Traditional)', 'Coffee & Tea', 'Restaurants']   \n",
              "\n",
              "     n_classes  \n",
              "683  6          \n",
              "760  6          "
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data_com.head(2) #bomb.com removed"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "The function is applied to the entire dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "17aUEcs8SVW4"
      },
      "outputs": [],
      "source": [
        "data['text'] = data['text'].apply(remove_com)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w6Skf8ZjVLm2"
      },
      "source": [
        "The websites in the form `.com` are removed:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49
        },
        "id": "zywJ03WsSXoG",
        "outputId": "be786e80-c9f6-4aff-bea7-6f3ec8dc20e4"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [Unnamed: 0, review_id, user_id, business_id, text, category, n_classes]\n",
              "Index: []"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data[data['text'].str.contains(r\"[A-Za-z]*\\.com[^a-zA-Z]\", regex = True)] "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mFiGnm7twZy1"
      },
      "source": [
        "#### **Remove numbers**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "4i7KZF3zLDjb"
      },
      "outputs": [],
      "source": [
        "data['text']= data['text'].apply(lambda x: re.sub(r'\\d+', '', x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Pw51mJDFwZy2",
        "outputId": "8ce474ea-8c84-423b-80b8-47986d93e881",
        "scrolled": true
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>l3Wk_mvAog6XANIuGQ9C7Q</td>\n",
              "      <td>ZbqSHbgCjzVAqaa7NKWn5A</td>\n",
              "      <td>EQ-TZ2eeD_E0BHuvoaeG5Q</td>\n",
              "      <td>locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.</td>\n",
              "      <td>['Beer', 'Wine &amp; Spirits', 'Cafes', 'Coffee &amp; Tea', 'Restaurants', 'Food', 'Breakfast &amp; Brunch', 'American (New)']</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>vBK79c3_1Ff_oqkh5VpfGg</td>\n",
              "      <td>Ohhrhu1RkqfVciIVx_W5HQ</td>\n",
              "      <td>nRKndeZLQ3eDL10UMwS2rQ</td>\n",
              "      <td>holy smokes!  actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat.   the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>z0osLHDvXvzfm57D4DmD2Q</td>\n",
              "      <td>xVKE_HJ2pwUtTdLbL3pnCg</td>\n",
              "      <td>S2Ho8yLxhKAa26pBAm6rxA</td>\n",
              "      <td>service was crappy, and food was mediocre.  i wish i would have picked some other place for my last dinner in town.</td>\n",
              "      <td>['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast &amp; Brunch']</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>elqRpX9T3YwL07uLNtN3Bg</td>\n",
              "      <td>-sryo4gDYxbZ1T5Bz4l5Bw</td>\n",
              "      <td>ltBBYdNzkeKdCNPDAsxwAA</td>\n",
              "      <td>i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée.  he made it very apparent and direct that he could not help me with that situation and i would have to flag down my  waitress so she could bill me for the four dollar avocado.</td>\n",
              "      <td>['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>pHwbdway4yeI-dSSmZA7-Q</td>\n",
              "      <td>qEEk0PuoH1dVa619t8fgpw</td>\n",
              "      <td>PY9GRfzr4nTZeINf346QOw</td>\n",
              "      <td>we checked in around : pm.  check-in was quick and easy with complimentary valet parking (always appreciated).  the peppermill is huge and reminded us of the caesars palace in las vegas.  they have around eight restaurants on site offering almost any type of food you could want.  the casino area is huge, rivaling almost anything you could find in las vegas.  one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from  cents to $.  talk about false advertising!  we were in the tuscany tower on the th floor.  our room had a nice view of the huge pool area.  our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode.  there were dual sinks and lots of counter space.  plenty of towels were supplied.  the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table.  the thermostat did a good job controlling the temperature and keeping it comfortable.  staff were all courteous and helpful.  a very nice place and we enjoyed our stay.</td>\n",
              "      <td>['Event Planning &amp; Services', 'Casinos', 'Beauty &amp; Spas', 'Venues &amp; Event Spaces', 'Arts &amp; Entertainment', 'Day Spas', 'Hotels', 'Hotels &amp; Travel']</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0               review_id                 user_id  \\\n",
              "0  0           l3Wk_mvAog6XANIuGQ9C7Q  ZbqSHbgCjzVAqaa7NKWn5A   \n",
              "1  1           vBK79c3_1Ff_oqkh5VpfGg  Ohhrhu1RkqfVciIVx_W5HQ   \n",
              "2  2           z0osLHDvXvzfm57D4DmD2Q  xVKE_HJ2pwUtTdLbL3pnCg   \n",
              "3  3           elqRpX9T3YwL07uLNtN3Bg  -sryo4gDYxbZ1T5Bz4l5Bw   \n",
              "4  4           pHwbdway4yeI-dSSmZA7-Q  qEEk0PuoH1dVa619t8fgpw   \n",
              "\n",
              "              business_id  \\\n",
              "0  EQ-TZ2eeD_E0BHuvoaeG5Q   \n",
              "1  nRKndeZLQ3eDL10UMwS2rQ   \n",
              "2  S2Ho8yLxhKAa26pBAm6rxA   \n",
              "3  ltBBYdNzkeKdCNPDAsxwAA   \n",
              "4  PY9GRfzr4nTZeINf346QOw   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                text  \\\n",
              "0  locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
              "1  holy smokes!  actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat.   the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
              "2  service was crappy, and food was mediocre.  i wish i would have picked some other place for my last dinner in town.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "3  i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée.  he made it very apparent and direct that he could not help me with that situation and i would have to flag down my  waitress so she could bill me for the four dollar avocado.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "4  we checked in around : pm.  check-in was quick and easy with complimentary valet parking (always appreciated).  the peppermill is huge and reminded us of the caesars palace in las vegas.  they have around eight restaurants on site offering almost any type of food you could want.  the casino area is huge, rivaling almost anything you could find in las vegas.  one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from  cents to $.  talk about false advertising!  we were in the tuscany tower on the th floor.  our room had a nice view of the huge pool area.  our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode.  there were dual sinks and lots of counter space.  plenty of towels were supplied.  the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table.  the thermostat did a good job controlling the temperature and keeping it comfortable.  staff were all courteous and helpful.  a very nice place and we enjoyed our stay.   \n",
              "\n",
              "                                                                                                                                              category  \\\n",
              "0  ['Beer', 'Wine & Spirits', 'Cafes', 'Coffee & Tea', 'Restaurants', 'Food', 'Breakfast & Brunch', 'American (New)']                                    \n",
              "1  ['Ice Cream & Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']                                                                                     \n",
              "2  ['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast & Brunch']                                                                                      \n",
              "3  ['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']                                       \n",
              "4  ['Event Planning & Services', 'Casinos', 'Beauty & Spas', 'Venues & Event Spaces', 'Arts & Entertainment', 'Day Spas', 'Hotels', 'Hotels & Travel']   \n",
              "\n",
              "   n_classes  \n",
              "0  8          \n",
              "1  4          \n",
              "2  4          \n",
              "3  8          \n",
              "4  8          "
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.head()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "7Ju2gxohwZy3"
      },
      "source": [
        "#### **Remove whitespaces**\n",
        "To remove extrawhitespaces"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "I275MtuzwZy4"
      },
      "outputs": [],
      "source": [
        "def remove_whitespaces(text):\n",
        "    return \" \".join(text.split())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "aQykX8Glt3nM"
      },
      "outputs": [],
      "source": [
        "data['text'] = data['text'].apply(lambda x: remove_whitespaces(x))"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "htkMNzRwo_qT"
      },
      "source": [
        "#### Removing emojis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "qydeTMPApfhn"
      },
      "outputs": [],
      "source": [
        "def remove_emoji(string):\n",
        "    emoji_pattern = re.compile(\"[\"\n",
        "                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                               u\"\\U00002500-\\U00002BEF\"  # chinese char\n",
        "                               u\"\\U00002702-\\U000027B0\"\n",
        "                               u\"\\U00002702-\\U000027B0\"\n",
        "                               u\"\\U000024C2-\\U0001F251\"\n",
        "                               u\"\\U0001f926-\\U0001f937\"\n",
        "                               u\"\\U00010000-\\U0010ffff\"\n",
        "                               u\"\\u2640-\\u2642\"\n",
        "                               u\"\\u2600-\\u2B55\"\n",
        "                               u\"\\u200d\"\n",
        "                               u\"\\u23cf\"\n",
        "                               u\"\\u23e9\"\n",
        "                               u\"\\u231a\"\n",
        "                               u\"\\ufe0f\"  # dingbats\n",
        "                               u\"\\u3030\"\n",
        "                               \"]+\", flags=re.UNICODE)\n",
        "    try:\n",
        "         emoji_pattern.sub(r' ', string)\n",
        "         return emoji_pattern.sub(r' ', string)\n",
        "    except:\n",
        "        print(string)\n",
        "        return string\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LotEDv_ZsUvm"
      },
      "source": [
        "The function is applied to the dataset we are working with:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "yqXWOnN-rdbT"
      },
      "outputs": [],
      "source": [
        "data['text'] = data['text'].apply(lambda x: remove_emoji(x))"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Tokenization + stopwords removal"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "In the following steps, we will use the SpaCy library to tokenize the text, aplly stopwords and puctuation removal, storing the result in a new column."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {},
      "outputs": [],
      "source": [
        "import spacy\n",
        "#load the spaCy English language model\n",
        "nlp=spacy.load('en_core_web_sm')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'spacy.tokens.doc.Doc'>\n"
          ]
        }
      ],
      "source": [
        "#doc = nlp(str(data['text'])) #convert our text data into doc object with tokens\n",
        "#print(type(doc))"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "To start processing the input text, we construct a Doc object.\n",
        "A Doc object is a sequence of Token objects representing a lexical token. Each Token object has information about a particular piece — typically one word — of text.\n",
        "\n",
        "SpaCy stores a list of stop words for the English language:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of stopwords: 326\n"
          ]
        }
      ],
      "source": [
        "spacy_stopwords = spacy.lang.en.stop_words.STOP_WORDS\n",
        "print(\"Number of stopwords:\", len(spacy_stopwords))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "during\n",
            "former\n",
            "meanwhile\n",
            "whither\n",
            "anyone\n",
            "while\n",
            "besides\n",
            "one\n",
            "alone\n",
            "only\n"
          ]
        }
      ],
      "source": [
        "for stop_word in list(spacy_stopwords)[:10]:\n",
        " print(stop_word)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {},
      "outputs": [],
      "source": [
        "def process_text(text):\n",
        "    # Process the text using SpaCy\n",
        "    doc = nlp(str(text))\n",
        "    \n",
        "    # Remove stop words\n",
        "    tokens = [token.text for token in doc if not token.is_stop and not token.is_punct and token.text != \"€\" and token.text != \"$\"]\n",
        "    \n",
        "    # Join the tokens into a single string\n",
        "    processed_text = ' '.join(tokens)\n",
        "    \n",
        "    return processed_text\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {},
      "outputs": [],
      "source": [
        "import time"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {},
      "outputs": [],
      "source": [
        "from pandarallel import pandarallel #to parallelize operations on dataframe for efficiency improving"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO: Pandarallel will run on 4 workers.\n",
            "INFO: Pandarallel will use standard multiprocessing data transfer (pipe) to transfer data between the main process and workers.\n"
          ]
        }
      ],
      "source": [
        "pandarallel.initialize( #use 4-core to speed up the process\n",
        "    nb_workers=4,\n",
        "    progress_bar=True\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "29777b2d0bb649fbb88bce1c1e9eb6b9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=142611), Label(value='0 / 142611')…"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CPU times: user 45.4 s, sys: 9.63 s, total: 55 s\n",
            "Wall time: 1h 22min 9s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "data['token'] = data['text'].parallel_apply(process_text)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {},
      "outputs": [],
      "source": [
        "#data.to_csv(\"data_token.csv\", index=False) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "      <th>token</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>l3Wk_mvAog6XANIuGQ9C7Q</td>\n",
              "      <td>ZbqSHbgCjzVAqaa7NKWn5A</td>\n",
              "      <td>EQ-TZ2eeD_E0BHuvoaeG5Q</td>\n",
              "      <td>locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.</td>\n",
              "      <td>['Beer', 'Wine &amp; Spirits', 'Cafes', 'Coffee &amp; Tea', 'Restaurants', 'Food', 'Breakfast &amp; Brunch', 'American (New)']</td>\n",
              "      <td>8</td>\n",
              "      <td>locals recommended milktooth amazing jewel indianapolis glade chance experience</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>vBK79c3_1Ff_oqkh5VpfGg</td>\n",
              "      <td>Ohhrhu1RkqfVciIVx_W5HQ</td>\n",
              "      <td>nRKndeZLQ3eDL10UMwS2rQ</td>\n",
              "      <td>holy smokes! actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat. the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']</td>\n",
              "      <td>4</td>\n",
              "      <td>holy smokes actual pumpkin pie mixed frozen custard kidding huge sweep nation treat best chunks pie crust like finding diamond inside diamond super yummy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>z0osLHDvXvzfm57D4DmD2Q</td>\n",
              "      <td>xVKE_HJ2pwUtTdLbL3pnCg</td>\n",
              "      <td>S2Ho8yLxhKAa26pBAm6rxA</td>\n",
              "      <td>service was crappy, and food was mediocre. i wish i would have picked some other place for my last dinner in town.</td>\n",
              "      <td>['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast &amp; Brunch']</td>\n",
              "      <td>4</td>\n",
              "      <td>service crappy food mediocre wish picked place dinner town</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>elqRpX9T3YwL07uLNtN3Bg</td>\n",
              "      <td>-sryo4gDYxbZ1T5Bz4l5Bw</td>\n",
              "      <td>ltBBYdNzkeKdCNPDAsxwAA</td>\n",
              "      <td>i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée. he made it very apparent and direct that he could not help me with that situation and i would have to flag down my waitress so she could bill me for the four dollar avocado.</td>\n",
              "      <td>['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']</td>\n",
              "      <td>8</td>\n",
              "      <td>restaurant stars decent food dinner meeting spending entrées appetizers flag assistant cook guy bright red baggy pants watching tv avocado tuna entrée apparent direct help situation flag waitress bill dollar avocado</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>pHwbdway4yeI-dSSmZA7-Q</td>\n",
              "      <td>qEEk0PuoH1dVa619t8fgpw</td>\n",
              "      <td>PY9GRfzr4nTZeINf346QOw</td>\n",
              "      <td>we checked in around : pm. check-in was quick and easy with complimentary valet parking (always appreciated). the peppermill is huge and reminded us of the caesars palace in las vegas. they have around eight restaurants on site offering almost any type of food you could want. the casino area is huge, rivaling almost anything you could find in las vegas. one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from cents to $. talk about false advertising! we were in the tuscany tower on the th floor. our room had a nice view of the huge pool area. our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode. there were dual sinks and lots of counter space. plenty of towels were supplied. the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table. the thermostat did a good job controlling the temperature and keeping it comfortable. staff were all courteous and helpful. a very nice place and we enjoyed our stay.</td>\n",
              "      <td>['Event Planning &amp; Services', 'Casinos', 'Beauty &amp; Spas', 'Venues &amp; Event Spaces', 'Arts &amp; Entertainment', 'Day Spas', 'Hotels', 'Hotels &amp; Travel']</td>\n",
              "      <td>8</td>\n",
              "      <td>checked pm check quick easy complimentary valet parking appreciated peppermill huge reminded caesars palace las vegas restaurants site offering type food want casino area huge rivaling find las vegas thing noticed slot machines noted penny machines actually started play realized minimum bets cents talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciated huge bathroom area large shower shower heads tv large jetted bathtub phone commode dual sinks lots counter space plenty towels supplied roomy main area king bed night stands closet tv mini fridge dresser drawers safe table computer writing complimentary bottles water nicely upholstered chairs table thermostat good job controlling temperature keeping comfortable staff courteous helpful nice place enjoyed stay</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0               review_id                 user_id  \\\n",
              "0  0           l3Wk_mvAog6XANIuGQ9C7Q  ZbqSHbgCjzVAqaa7NKWn5A   \n",
              "1  1           vBK79c3_1Ff_oqkh5VpfGg  Ohhrhu1RkqfVciIVx_W5HQ   \n",
              "2  2           z0osLHDvXvzfm57D4DmD2Q  xVKE_HJ2pwUtTdLbL3pnCg   \n",
              "3  3           elqRpX9T3YwL07uLNtN3Bg  -sryo4gDYxbZ1T5Bz4l5Bw   \n",
              "4  4           pHwbdway4yeI-dSSmZA7-Q  qEEk0PuoH1dVa619t8fgpw   \n",
              "\n",
              "              business_id  \\\n",
              "0  EQ-TZ2eeD_E0BHuvoaeG5Q   \n",
              "1  nRKndeZLQ3eDL10UMwS2rQ   \n",
              "2  S2Ho8yLxhKAa26pBAm6rxA   \n",
              "3  ltBBYdNzkeKdCNPDAsxwAA   \n",
              "4  PY9GRfzr4nTZeINf346QOw   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                text  \\\n",
              "0  locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
              "1  holy smokes! actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat. the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "2  service was crappy, and food was mediocre. i wish i would have picked some other place for my last dinner in town.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "3  i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée. he made it very apparent and direct that he could not help me with that situation and i would have to flag down my waitress so she could bill me for the four dollar avocado.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "4  we checked in around : pm. check-in was quick and easy with complimentary valet parking (always appreciated). the peppermill is huge and reminded us of the caesars palace in las vegas. they have around eight restaurants on site offering almost any type of food you could want. the casino area is huge, rivaling almost anything you could find in las vegas. one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from cents to $. talk about false advertising! we were in the tuscany tower on the th floor. our room had a nice view of the huge pool area. our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode. there were dual sinks and lots of counter space. plenty of towels were supplied. the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table. the thermostat did a good job controlling the temperature and keeping it comfortable. staff were all courteous and helpful. a very nice place and we enjoyed our stay.   \n",
              "\n",
              "                                                                                                                                              category  \\\n",
              "0  ['Beer', 'Wine & Spirits', 'Cafes', 'Coffee & Tea', 'Restaurants', 'Food', 'Breakfast & Brunch', 'American (New)']                                    \n",
              "1  ['Ice Cream & Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']                                                                                     \n",
              "2  ['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast & Brunch']                                                                                      \n",
              "3  ['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']                                       \n",
              "4  ['Event Planning & Services', 'Casinos', 'Beauty & Spas', 'Venues & Event Spaces', 'Arts & Entertainment', 'Day Spas', 'Hotels', 'Hotels & Travel']   \n",
              "\n",
              "   n_classes  \\\n",
              "0  8           \n",
              "1  4           \n",
              "2  4           \n",
              "3  8           \n",
              "4  8           \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   token  \n",
              "0  locals recommended milktooth amazing jewel indianapolis glade chance experience                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \n",
              "1  holy smokes actual pumpkin pie mixed frozen custard kidding huge sweep nation treat best chunks pie crust like finding diamond inside diamond super yummy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "2  service crappy food mediocre wish picked place dinner town                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
              "3  restaurant stars decent food dinner meeting spending entrées appetizers flag assistant cook guy bright red baggy pants watching tv avocado tuna entrée apparent direct help situation flag waitress bill dollar avocado                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "4  checked pm check quick easy complimentary valet parking appreciated peppermill huge reminded caesars palace las vegas restaurants site offering type food want casino area huge rivaling find las vegas thing noticed slot machines noted penny machines actually started play realized minimum bets cents talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciated huge bathroom area large shower shower heads tv large jetted bathtub phone commode dual sinks lots counter space plenty towels supplied roomy main area king bed night stands closet tv mini fridge dresser drawers safe table computer writing complimentary bottles water nicely upholstered chairs table thermostat good job controlling temperature keeping comfortable staff courteous helpful nice place enjoyed stay  "
            ]
          },
          "execution_count": 60,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bBj8X8_6T8Sj"
      },
      "source": [
        "#### **SpaCy Lemmatizer**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {},
      "outputs": [],
      "source": [
        "def lemma_spacy(text): # input: tokens, tipo stringa\n",
        "    doc = nlp(text)\n",
        "    # lemmatization\n",
        "    lemmas = [token.lemma_ for token in doc]\n",
        "    # join lemmas into a single string\n",
        "    lemma_text = ' '.join(lemmas)\n",
        "    \n",
        "    return lemma_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "badd4bb9b79e448ab151989b1934bf40",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=142611), Label(value='0 / 142611')…"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CPU times: user 24 s, sys: 5.26 s, total: 29.2 s\n",
            "Wall time: 36min 6s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "data['lemma'] = data['token'].parallel_apply(lemma_spacy) \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {},
      "outputs": [],
      "source": [
        "#data.to_csv(\"data_preprocessed.csv\",index=False)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "WordNet Lemmatizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "wordnet_lemmatizer = WordNetLemmatizer()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [],
      "source": [
        "def lemmatizer(text):\n",
        "  lemm_text = [wordnet_lemmatizer.lemmatize(word) for word in text]\n",
        "  return lemm_text\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Remaing preprocessing "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "data = pd.read_csv(\"data_preprocessed.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(570442, 9)"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "      <th>token</th>\n",
              "      <th>lemma</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>l3Wk_mvAog6XANIuGQ9C7Q</td>\n",
              "      <td>ZbqSHbgCjzVAqaa7NKWn5A</td>\n",
              "      <td>EQ-TZ2eeD_E0BHuvoaeG5Q</td>\n",
              "      <td>locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.</td>\n",
              "      <td>['Beer', 'Wine &amp; Spirits', 'Cafes', 'Coffee &amp; Tea', 'Restaurants', 'Food', 'Breakfast &amp; Brunch', 'American (New)']</td>\n",
              "      <td>8</td>\n",
              "      <td>locals recommended milktooth amazing jewel indianapolis glade chance experience</td>\n",
              "      <td>local recommend milktooth amazing jewel indianapolis grade chance experience</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>vBK79c3_1Ff_oqkh5VpfGg</td>\n",
              "      <td>Ohhrhu1RkqfVciIVx_W5HQ</td>\n",
              "      <td>nRKndeZLQ3eDL10UMwS2rQ</td>\n",
              "      <td>holy smokes! actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat. the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']</td>\n",
              "      <td>4</td>\n",
              "      <td>holy smokes actual pumpkin pie mixed frozen custard kidding huge sweep nation treat best chunks pie crust like finding diamond inside diamond super yummy</td>\n",
              "      <td>holy smoke actual pumpkin pie mix frozen custard kid huge sweep nation treat well chunk pie crust like find diamond inside diamond super dummy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>z0osLHDvXvzfm57D4DmD2Q</td>\n",
              "      <td>xVKE_HJ2pwUtTdLbL3pnCg</td>\n",
              "      <td>S2Ho8yLxhKAa26pBAm6rxA</td>\n",
              "      <td>service was crappy, and food was mediocre. i wish i would have picked some other place for my last dinner in town.</td>\n",
              "      <td>['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast &amp; Brunch']</td>\n",
              "      <td>4</td>\n",
              "      <td>service crappy food mediocre wish picked place dinner town</td>\n",
              "      <td>service crappy food mediocre wish pick place dinner town</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>elqRpX9T3YwL07uLNtN3Bg</td>\n",
              "      <td>-sryo4gDYxbZ1T5Bz4l5Bw</td>\n",
              "      <td>ltBBYdNzkeKdCNPDAsxwAA</td>\n",
              "      <td>i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée. he made it very apparent and direct that he could not help me with that situation and i would have to flag down my waitress so she could bill me for the four dollar avocado.</td>\n",
              "      <td>['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']</td>\n",
              "      <td>8</td>\n",
              "      <td>restaurant stars decent food dinner meeting spending entrées appetizers flag assistant cook guy bright red baggy pants watching tv avocado tuna entrée apparent direct help situation flag waitress bill dollar avocado</td>\n",
              "      <td>restaurant star decent food dinner meeting spending entryées appetite flag assistant cook guy bright red buggy part watch tv avocado tuna entryée apparent direct help situation flag waitress bill dollar avocado</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>pHwbdway4yeI-dSSmZA7-Q</td>\n",
              "      <td>qEEk0PuoH1dVa619t8fgpw</td>\n",
              "      <td>PY9GRfzr4nTZeINf346QOw</td>\n",
              "      <td>we checked in around : pm. check-in was quick and easy with complimentary valet parking (always appreciated). the peppermill is huge and reminded us of the caesars palace in las vegas. they have around eight restaurants on site offering almost any type of food you could want. the casino area is huge, rivaling almost anything you could find in las vegas. one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from cents to $. talk about false advertising! we were in the tuscany tower on the th floor. our room had a nice view of the huge pool area. our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode. there were dual sinks and lots of counter space. plenty of towels were supplied. the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table. the thermostat did a good job controlling the temperature and keeping it comfortable. staff were all courteous and helpful. a very nice place and we enjoyed our stay.</td>\n",
              "      <td>['Event Planning &amp; Services', 'Casinos', 'Beauty &amp; Spas', 'Venues &amp; Event Spaces', 'Arts &amp; Entertainment', 'Day Spas', 'Hotels', 'Hotels &amp; Travel']</td>\n",
              "      <td>8</td>\n",
              "      <td>checked pm check quick easy complimentary valet parking appreciated peppermill huge reminded caesars palace las vegas restaurants site offering type food want casino area huge rivaling find las vegas thing noticed slot machines noted penny machines actually started play realized minimum bets cents talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciated huge bathroom area large shower shower heads tv large jetted bathtub phone commode dual sinks lots counter space plenty towels supplied roomy main area king bed night stands closet tv mini fridge dresser drawers safe table computer writing complimentary bottles water nicely upholstered chairs table thermostat good job controlling temperature keeping comfortable staff courteous helpful nice place enjoyed stay</td>\n",
              "      <td>check pm check quick easy complimentary valet parking appreciate peppermill huge remind cyears palace las vegas restaurant site offer type food want casino area huge revealing find las vegas thing notice slot machine note penny machine actually start play realize minimum bet cent talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciate huge bathroom area large shower shower head tv large netted bathtub phone common dual sink lot counter space plenty towel supply room main area king bed night stand closet tv mini fridge dressed drawer safe table computer write complimentary bottle water nicely upholstery chair table thermostat good job control temperature keep comfortable staff courteous helpful nice place enjoy stay</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>570437</th>\n",
              "      <td>oVH6oK8G_FTYiaglrIk09A</td>\n",
              "      <td>piIljcy0fhEW6IhB1eyrFA</td>\n",
              "      <td>oBNrLz4EDhiscSlbOl8uAw</td>\n",
              "      <td>please come hungry!!!! every time i've been in i've never been disappointed. the mimosas are perfect and i can never seem to order just one. my go to are the eggs benedict's served on soft fluffy biscuits. the portions are generous and filling.</td>\n",
              "      <td>['Restaurants', 'American (Traditional)', 'American (New)', 'Cafes', 'Breakfast &amp; Brunch']</td>\n",
              "      <td>5</td>\n",
              "      <td>come hungry time disappointed mimosas perfect order eggs benedict served soft fluffy biscuits portions generous filling</td>\n",
              "      <td>come hungry time disappointed mimosas perfect order eggs benedict serve soft fluffy biscuit portion generous filling</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>570438</th>\n",
              "      <td>X5R98ygOtbhryDiKA-J2qQ</td>\n",
              "      <td>LHWtjTG7e1NzNPYUbUo-9w</td>\n",
              "      <td>rgeuy1qbw6Z8B6CSVANHIA</td>\n",
              "      <td>i've been to the other federal donuts location multiple times. i love donuts, and coffee and this place does both very well. the south philly location is so small, and i often find myself cramped while in there. however, the donuts are always amazing, which keeps me coming back for more and more. i recently visited the center city location - and found that they had a better handle on crowd control, which i appreciated. the donuts were just as good as what i had tried in the past - specifically, i had the green tea sesame which i could have bathed in vats of. i ate all the little crumbly pieces that had fallen off into the bag when i got home. that's how good they are. both hot and fancy, really in any variety are always amazing. try them all!</td>\n",
              "      <td>['Donuts', 'Sandwiches', 'Soul Food', 'Food', 'Coffee &amp; Tea', 'Restaurants', 'Chicken Wings']</td>\n",
              "      <td>7</td>\n",
              "      <td>federal donuts location multiple times love donuts coffee place south philly location small find cramped donuts amazing keeps coming recently visited center city location found better handle crowd control appreciated donuts good tried past specifically green tea sesame bathed vats ate little crumbly pieces fallen bag got home good hot fancy variety amazing try</td>\n",
              "      <td>federal dont location multiple time love dont coffee place south philly location small find camp dont amazing keep come recently visit center city location find well handle crowd control appreciate dont good try past specifically green tea sesame bath vat eat little rumble piece fall bag get home good hot fancy variety amazing try</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>570439</th>\n",
              "      <td>MVg4YUQeEhCA7Z7RsBJSVg</td>\n",
              "      <td>7-7A0Avj47slLGV7yBFc8w</td>\n",
              "      <td>ytynqOUb3hjKeJfRj5Tshw</td>\n",
              "      <td>i was so excited about all the food i saw, but unfortunately this place closes so early!!!!!!!!!!!!!!!! after being a tourist and sightseeing, by the time i got there at or , things started closing. the hours of operation is definitely the biggest con ever!!!</td>\n",
              "      <td>['Candy Stores', 'Shopping', 'Department Stores', 'Fast Food', 'Beer', 'Wine &amp; Spirits', 'Fruits &amp; Veggies', 'Chinese', 'Food', 'Ice Cream &amp; Frozen Yogurt', 'Desserts', 'Seafood', 'Health Markets', 'Bagels', 'Cheese Shops', 'Shopping Centers', 'Chocolatiers &amp; Shops', 'Meat Shops', 'Public Markets', 'Food Court', 'Wineries', 'Local Flavor', 'Ethnic Food', 'Restaurants', 'Specialty Food', 'Arts &amp; Entertainment', 'Juice Bars &amp; Smoothies', 'Seafood Markets', 'Farmers Market', 'Coffee &amp; Tea', 'Bakeries', 'Food Stands', 'Dinner Theater', 'Sporting Goods', 'Grocery', 'Fashion']</td>\n",
              "      <td>36</td>\n",
              "      <td>excited food saw unfortunately place closes early tourist sightseeing time got things started closing hours operation definitely biggest con</td>\n",
              "      <td>excited food see unfortunately place close early tourist sightseeing time get thing start close hour operation definitely big con</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>570440</th>\n",
              "      <td>KlHxcAifUF5zDKpJCBrRsw</td>\n",
              "      <td>7ziWZULyiZv2TesYNMFf4g</td>\n",
              "      <td>qQO7ErS_RAN4Vs1uX0L55Q</td>\n",
              "      <td>ice cream! ice cream sodas, sundaes!! wanted to share a sundae-the great pumpkin one but was told it has huge scoops of ice cream plus toppings.... my friend and i are sorta full so we just each got a small scoop of ice cream (which was still pretty big!) i had the coconut based pumpkin ice cream, which was creamy, coconuty, pumpkin-y, and delicious! i am glad they have nondairy options, although im not lactose intolerant. friend got maple walnut which was robust and delicious as well! fun ice cream shop to chill!</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Coffee &amp; Tea', 'Food', 'Desserts']</td>\n",
              "      <td>4</td>\n",
              "      <td>ice cream ice cream sodas sundaes wanted share sundae great pumpkin told huge scoops ice cream plus toppings friend sorta got small scoop ice cream pretty big coconut based pumpkin ice cream creamy coconuty pumpkin y delicious glad nondairy options m lactose intolerant friend got maple walnut robust delicious fun ice cream shop chill</td>\n",
              "      <td>ice cream ice cream soda sunday want share sunday great pumpkin tell huge scoop ice cream plus topping friend sorta get small scoop ice cream pretty big coconut base pumpkin ice cream creamy coconut pumpkin y delicious glad nondairy option m lacrosse intolerant friend get maple walnut robust delicious fun ice cream shop chill</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>570441</th>\n",
              "      <td>RwcKOdEuLRHNJe4M9-qpqg</td>\n",
              "      <td>6JehEvdoCvZPJ_XIxnzIIw</td>\n",
              "      <td>VAeEXLbEcI9Emt9KGYq9aA</td>\n",
              "      <td>located in the 'walking district' in nashville, it's was a bit out of the way for us, but we were on a mission to experience the proclaimed 'greatest burger in nashville'. the menu includes several burgers, wursthen, and chicken sandwich options. the waitress told us that the farm burger is the most popular. i for one have never had a burger with an egg and the thought of it isn't very appealing. i ended up ordering the 'pharmacy burger', which is a less cheesier version of their 'cheese burger' that my husband ordered. they loaded the mustard, but that really wasn't what i didn't like about it. it really wasn't that i disliked the burger, it just wasn't anywhere near one of the best burgers that i've had. the extra toppings at fifty cents each are pretty skimpy too. i was a fan of the wasabi aioli though. i also loved the tater tots though. i highly recommend ordering tater tots over the french fries. the beer cheese dipping sauce was an extra dollar and was borderline disgusting. stick with wasabi aioli. i'm thinking that the toppings and sauces are possibly what the hype about this place is, otherwise, i just don't get it. to describe this place in one word... overhyped.</td>\n",
              "      <td>['Burgers', 'Sandwiches', 'Food', 'Beer', 'Wine &amp; Spirits', 'American (Traditional)', 'Bars', 'Nightlife', 'Restaurants']</td>\n",
              "      <td>9</td>\n",
              "      <td>located walking district nashville bit way mission experience proclaimed greatest burger nashville menu includes burgers wursthen chicken sandwich options waitress told farm burger popular burger egg thought appealing ended ordering pharmacy burger cheesier version cheese burger husband ordered loaded mustard like disliked burger near best burgers extra toppings cents pretty skimpy fan wasabi aioli loved tater tots highly recommend ordering tater tots french fries beer cheese dipping sauce extra dollar borderline disgusting stick wasabi aioli thinking toppings sauces possibly hype place describe place word overhyped</td>\n",
              "      <td>locate walk district nashville bit way mission experience proclaim great burger nashville menu include burger burthen chicken sandwich option waitress tell farm burger popular burger egg think appeal end order pharmacy burger cheese version cheese burger husband order loaded mustard like dislike burger near good burger extra topping cent pretty simply fan asahi violin love later tot highly recommend order later tot french fry beer cheese dipping sauce extra dollar borderline disgusting stick asahi violin think topping sauce possibly hype place describe place word overhype</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>570438 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                     review_id                 user_id  \\\n",
              "0       l3Wk_mvAog6XANIuGQ9C7Q  ZbqSHbgCjzVAqaa7NKWn5A   \n",
              "1       vBK79c3_1Ff_oqkh5VpfGg  Ohhrhu1RkqfVciIVx_W5HQ   \n",
              "2       z0osLHDvXvzfm57D4DmD2Q  xVKE_HJ2pwUtTdLbL3pnCg   \n",
              "3       elqRpX9T3YwL07uLNtN3Bg  -sryo4gDYxbZ1T5Bz4l5Bw   \n",
              "4       pHwbdway4yeI-dSSmZA7-Q  qEEk0PuoH1dVa619t8fgpw   \n",
              "...                        ...                     ...   \n",
              "570437  oVH6oK8G_FTYiaglrIk09A  piIljcy0fhEW6IhB1eyrFA   \n",
              "570438  X5R98ygOtbhryDiKA-J2qQ  LHWtjTG7e1NzNPYUbUo-9w   \n",
              "570439  MVg4YUQeEhCA7Z7RsBJSVg  7-7A0Avj47slLGV7yBFc8w   \n",
              "570440  KlHxcAifUF5zDKpJCBrRsw  7ziWZULyiZv2TesYNMFf4g   \n",
              "570441  RwcKOdEuLRHNJe4M9-qpqg  6JehEvdoCvZPJ_XIxnzIIw   \n",
              "\n",
              "                   business_id  \\\n",
              "0       EQ-TZ2eeD_E0BHuvoaeG5Q   \n",
              "1       nRKndeZLQ3eDL10UMwS2rQ   \n",
              "2       S2Ho8yLxhKAa26pBAm6rxA   \n",
              "3       ltBBYdNzkeKdCNPDAsxwAA   \n",
              "4       PY9GRfzr4nTZeINf346QOw   \n",
              "...                        ...   \n",
              "570437  oBNrLz4EDhiscSlbOl8uAw   \n",
              "570438  rgeuy1qbw6Z8B6CSVANHIA   \n",
              "570439  ytynqOUb3hjKeJfRj5Tshw   \n",
              "570440  qQO7ErS_RAN4Vs1uX0L55Q   \n",
              "570441  VAeEXLbEcI9Emt9KGYq9aA   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     text  \\\n",
              "0       locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
              "1       holy smokes! actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat. the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "2       service was crappy, and food was mediocre. i wish i would have picked some other place for my last dinner in town.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "3       i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée. he made it very apparent and direct that he could not help me with that situation and i would have to flag down my waitress so she could bill me for the four dollar avocado.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "4       we checked in around : pm. check-in was quick and easy with complimentary valet parking (always appreciated). the peppermill is huge and reminded us of the caesars palace in las vegas. they have around eight restaurants on site offering almost any type of food you could want. the casino area is huge, rivaling almost anything you could find in las vegas. one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from cents to $. talk about false advertising! we were in the tuscany tower on the th floor. our room had a nice view of the huge pool area. our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode. there were dual sinks and lots of counter space. plenty of towels were supplied. the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table. the thermostat did a good job controlling the temperature and keeping it comfortable. staff were all courteous and helpful. a very nice place and we enjoyed our stay.   \n",
              "...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   ...   \n",
              "570437  please come hungry!!!! every time i've been in i've never been disappointed. the mimosas are perfect and i can never seem to order just one. my go to are the eggs benedict's served on soft fluffy biscuits. the portions are generous and filling.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "570438  i've been to the other federal donuts location multiple times. i love donuts, and coffee and this place does both very well. the south philly location is so small, and i often find myself cramped while in there. however, the donuts are always amazing, which keeps me coming back for more and more. i recently visited the center city location - and found that they had a better handle on crowd control, which i appreciated. the donuts were just as good as what i had tried in the past - specifically, i had the green tea sesame which i could have bathed in vats of. i ate all the little crumbly pieces that had fallen off into the bag when i got home. that's how good they are. both hot and fancy, really in any variety are always amazing. try them all!                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
              "570439  i was so excited about all the food i saw, but unfortunately this place closes so early!!!!!!!!!!!!!!!! after being a tourist and sightseeing, by the time i got there at or , things started closing. the hours of operation is definitely the biggest con ever!!!                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "570440  ice cream! ice cream sodas, sundaes!! wanted to share a sundae-the great pumpkin one but was told it has huge scoops of ice cream plus toppings.... my friend and i are sorta full so we just each got a small scoop of ice cream (which was still pretty big!) i had the coconut based pumpkin ice cream, which was creamy, coconuty, pumpkin-y, and delicious! i am glad they have nondairy options, although im not lactose intolerant. friend got maple walnut which was robust and delicious as well! fun ice cream shop to chill!                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
              "570441  located in the 'walking district' in nashville, it's was a bit out of the way for us, but we were on a mission to experience the proclaimed 'greatest burger in nashville'. the menu includes several burgers, wursthen, and chicken sandwich options. the waitress told us that the farm burger is the most popular. i for one have never had a burger with an egg and the thought of it isn't very appealing. i ended up ordering the 'pharmacy burger', which is a less cheesier version of their 'cheese burger' that my husband ordered. they loaded the mustard, but that really wasn't what i didn't like about it. it really wasn't that i disliked the burger, it just wasn't anywhere near one of the best burgers that i've had. the extra toppings at fifty cents each are pretty skimpy too. i was a fan of the wasabi aioli though. i also loved the tater tots though. i highly recommend ordering tater tots over the french fries. the beer cheese dipping sauce was an extra dollar and was borderline disgusting. stick with wasabi aioli. i'm thinking that the toppings and sauces are possibly what the hype about this place is, otherwise, i just don't get it. to describe this place in one word... overhyped.                                                                                                                                                            \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 category  \\\n",
              "0       ['Beer', 'Wine & Spirits', 'Cafes', 'Coffee & Tea', 'Restaurants', 'Food', 'Breakfast & Brunch', 'American (New)']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "1       ['Ice Cream & Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "2       ['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast & Brunch']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
              "3       ['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "4       ['Event Planning & Services', 'Casinos', 'Beauty & Spas', 'Venues & Event Spaces', 'Arts & Entertainment', 'Day Spas', 'Hotels', 'Hotels & Travel']                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "...                                                                                                                                                     ...                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "570437  ['Restaurants', 'American (Traditional)', 'American (New)', 'Cafes', 'Breakfast & Brunch']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          \n",
              "570438  ['Donuts', 'Sandwiches', 'Soul Food', 'Food', 'Coffee & Tea', 'Restaurants', 'Chicken Wings']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
              "570439  ['Candy Stores', 'Shopping', 'Department Stores', 'Fast Food', 'Beer', 'Wine & Spirits', 'Fruits & Veggies', 'Chinese', 'Food', 'Ice Cream & Frozen Yogurt', 'Desserts', 'Seafood', 'Health Markets', 'Bagels', 'Cheese Shops', 'Shopping Centers', 'Chocolatiers & Shops', 'Meat Shops', 'Public Markets', 'Food Court', 'Wineries', 'Local Flavor', 'Ethnic Food', 'Restaurants', 'Specialty Food', 'Arts & Entertainment', 'Juice Bars & Smoothies', 'Seafood Markets', 'Farmers Market', 'Coffee & Tea', 'Bakeries', 'Food Stands', 'Dinner Theater', 'Sporting Goods', 'Grocery', 'Fashion']   \n",
              "570440  ['Ice Cream & Frozen Yogurt', 'Coffee & Tea', 'Food', 'Desserts']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
              "570441  ['Burgers', 'Sandwiches', 'Food', 'Beer', 'Wine & Spirits', 'American (Traditional)', 'Bars', 'Nightlife', 'Restaurants']                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
              "\n",
              "        n_classes  \\\n",
              "0       8           \n",
              "1       4           \n",
              "2       4           \n",
              "3       8           \n",
              "4       8           \n",
              "...    ..           \n",
              "570437  5           \n",
              "570438  7           \n",
              "570439  36          \n",
              "570440  4           \n",
              "570441  9           \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        token  \\\n",
              "0       locals recommended milktooth amazing jewel indianapolis glade chance experience                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "1       holy smokes actual pumpkin pie mixed frozen custard kidding huge sweep nation treat best chunks pie crust like finding diamond inside diamond super yummy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "2       service crappy food mediocre wish picked place dinner town                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "3       restaurant stars decent food dinner meeting spending entrées appetizers flag assistant cook guy bright red baggy pants watching tv avocado tuna entrée apparent direct help situation flag waitress bill dollar avocado                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "4       checked pm check quick easy complimentary valet parking appreciated peppermill huge reminded caesars palace las vegas restaurants site offering type food want casino area huge rivaling find las vegas thing noticed slot machines noted penny machines actually started play realized minimum bets cents talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciated huge bathroom area large shower shower heads tv large jetted bathtub phone commode dual sinks lots counter space plenty towels supplied roomy main area king bed night stands closet tv mini fridge dresser drawers safe table computer writing complimentary bottles water nicely upholstered chairs table thermostat good job controlling temperature keeping comfortable staff courteous helpful nice place enjoyed stay   \n",
              "...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       ...   \n",
              "570437  come hungry time disappointed mimosas perfect order eggs benedict served soft fluffy biscuits portions generous filling                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "570438  federal donuts location multiple times love donuts coffee place south philly location small find cramped donuts amazing keeps coming recently visited center city location found better handle crowd control appreciated donuts good tried past specifically green tea sesame bathed vats ate little crumbly pieces fallen bag got home good hot fancy variety amazing try                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "570439  excited food saw unfortunately place closes early tourist sightseeing time got things started closing hours operation definitely biggest con                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
              "570440  ice cream ice cream sodas sundaes wanted share sundae great pumpkin told huge scoops ice cream plus toppings friend sorta got small scoop ice cream pretty big coconut based pumpkin ice cream creamy coconuty pumpkin y delicious glad nondairy options m lactose intolerant friend got maple walnut robust delicious fun ice cream shop chill                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "570441  located walking district nashville bit way mission experience proclaimed greatest burger nashville menu includes burgers wursthen chicken sandwich options waitress told farm burger popular burger egg thought appealing ended ordering pharmacy burger cheesier version cheese burger husband ordered loaded mustard like disliked burger near best burgers extra toppings cents pretty skimpy fan wasabi aioli loved tater tots highly recommend ordering tater tots french fries beer cheese dipping sauce extra dollar borderline disgusting stick wasabi aioli thinking toppings sauces possibly hype place describe place word overhyped                                                                                                                                                                                                                         \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             lemma  \n",
              "0       local recommend milktooth amazing jewel indianapolis grade chance experience                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "1       holy smoke actual pumpkin pie mix frozen custard kid huge sweep nation treat well chunk pie crust like find diamond inside diamond super dummy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "2       service crappy food mediocre wish pick place dinner town                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
              "3       restaurant star decent food dinner meeting spending entryées appetite flag assistant cook guy bright red buggy part watch tv avocado tuna entryée apparent direct help situation flag waitress bill dollar avocado                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          \n",
              "4       check pm check quick easy complimentary valet parking appreciate peppermill huge remind cyears palace las vegas restaurant site offer type food want casino area huge revealing find las vegas thing notice slot machine note penny machine actually start play realize minimum bet cent talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciate huge bathroom area large shower shower head tv large netted bathtub phone common dual sink lot counter space plenty towel supply room main area king bed night stand closet tv mini fridge dressed drawer safe table computer write complimentary bottle water nicely upholstery chair table thermostat good job control temperature keep comfortable staff courteous helpful nice place enjoy stay  \n",
              "...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            ...  \n",
              "570437  come hungry time disappointed mimosas perfect order eggs benedict serve soft fluffy biscuit portion generous filling                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \n",
              "570438  federal dont location multiple time love dont coffee place south philly location small find camp dont amazing keep come recently visit center city location find well handle crowd control appreciate dont good try past specifically green tea sesame bath vat eat little rumble piece fall bag get home good hot fancy variety amazing try                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "570439  excited food see unfortunately place close early tourist sightseeing time get thing start close hour operation definitely big con                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
              "570440  ice cream ice cream soda sunday want share sunday great pumpkin tell huge scoop ice cream plus topping friend sorta get small scoop ice cream pretty big coconut base pumpkin ice cream creamy coconut pumpkin y delicious glad nondairy option m lacrosse intolerant friend get maple walnut robust delicious fun ice cream shop chill                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "570441  locate walk district nashville bit way mission experience proclaim great burger nashville menu include burger burthen chicken sandwich option waitress tell farm burger popular burger egg think appeal end order pharmacy burger cheese version cheese burger husband order loaded mustard like dislike burger near good burger extra topping cent pretty simply fan asahi violin love later tot highly recommend order later tot french fry beer cheese dipping sauce extra dollar borderline disgusting stick asahi violin think topping sauce possibly hype place describe place word overhype                                                                                                                                                                                                                          \n",
              "\n",
              "[570438 rows x 8 columns]"
            ]
          },
          "execution_count": 65,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.drop(columns=['Unnamed: 0'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {},
      "outputs": [],
      "source": [
        "data = data.dropna()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(570438, 9)"
            ]
          },
          "execution_count": 58,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.shape"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Punctuation removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {},
      "outputs": [],
      "source": [
        "punct = list(string.punctuation)\n",
        "punct.append(['€','¯'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Punctuation to remove: !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ € ¯ \n"
          ]
        }
      ],
      "source": [
        "print(\"Punctuation to remove:\", string.punctuation, \"€ ¯ \")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "metadata": {},
      "outputs": [],
      "source": [
        "def remove_punctuation(text):\n",
        "    punctuationfree=\"\".join([w for w in text if w not in punct])\n",
        "    return punctuationfree"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {},
      "outputs": [],
      "source": [
        "data['lemma']= data['lemma'].apply(remove_punctuation)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Removing repeated characters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "metadata": {},
      "outputs": [],
      "source": [
        "def removing_character_repetition(text):\n",
        "    pattern = re.compile(r\"([A-Za-z])\\1{1,}\", re.DOTALL) #matching for all case alphabets\n",
        "    formatted = pattern.sub(r\"\\1\\1\", text) #limits to two characters\n",
        "    return formatted"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {},
      "outputs": [],
      "source": [
        "data['lemma'] = data['lemma'].apply(removing_character_repetition)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Spelling correction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [],
      "source": [
        "def spelling_correction(text):\n",
        " spell = Speller(lang='en')\n",
        " corrected_text = spell(text)\n",
        " return corrected_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/pandarallel/data_types/series.py:16: FutureWarning: The behavior of `series[i:j]` with an integer-dtype index is deprecated. In a future version, this will be treated as *label-based* indexing, consistent with e.g. `series[i]` lookups. To retain the old behavior, use `series.iloc[i:j]`. To get the future behavior, use `series.loc[i:j]`.\n",
            "  yield data[chunk_]\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b39b747af8944113936fd289bd257024",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=142610), Label(value='0 / 142610')…"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CPU times: user 1min 59s, sys: 25.2 s, total: 2min 24s\n",
            "Wall time: 6h 24min 9s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "data['lemma'] = data['lemma'].parallel_apply(spelling_correction)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "      <th>token</th>\n",
              "      <th>lemma</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>l3Wk_mvAog6XANIuGQ9C7Q</td>\n",
              "      <td>ZbqSHbgCjzVAqaa7NKWn5A</td>\n",
              "      <td>EQ-TZ2eeD_E0BHuvoaeG5Q</td>\n",
              "      <td>locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.</td>\n",
              "      <td>['Beer', 'Wine &amp; Spirits', 'Cafes', 'Coffee &amp; Tea', 'Restaurants', 'Food', 'Breakfast &amp; Brunch', 'American (New)']</td>\n",
              "      <td>8</td>\n",
              "      <td>locals recommended milktooth amazing jewel indianapolis glade chance experience</td>\n",
              "      <td>local recommend milktooth amazing jewel indianapolis grade chance experience</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>vBK79c3_1Ff_oqkh5VpfGg</td>\n",
              "      <td>Ohhrhu1RkqfVciIVx_W5HQ</td>\n",
              "      <td>nRKndeZLQ3eDL10UMwS2rQ</td>\n",
              "      <td>holy smokes! actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat. the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']</td>\n",
              "      <td>4</td>\n",
              "      <td>holy smokes actual pumpkin pie mixed frozen custard kidding huge sweep nation treat best chunks pie crust like finding diamond inside diamond super yummy</td>\n",
              "      <td>holy smoke actual pumpkin pie mix frozen custard kid huge sweep nation treat well chunk pie crust like find diamond inside diamond super dummy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>z0osLHDvXvzfm57D4DmD2Q</td>\n",
              "      <td>xVKE_HJ2pwUtTdLbL3pnCg</td>\n",
              "      <td>S2Ho8yLxhKAa26pBAm6rxA</td>\n",
              "      <td>service was crappy, and food was mediocre. i wish i would have picked some other place for my last dinner in town.</td>\n",
              "      <td>['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast &amp; Brunch']</td>\n",
              "      <td>4</td>\n",
              "      <td>service crappy food mediocre wish picked place dinner town</td>\n",
              "      <td>service crappy food mediocre wish pick place dinner town</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>elqRpX9T3YwL07uLNtN3Bg</td>\n",
              "      <td>-sryo4gDYxbZ1T5Bz4l5Bw</td>\n",
              "      <td>ltBBYdNzkeKdCNPDAsxwAA</td>\n",
              "      <td>i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée. he made it very apparent and direct that he could not help me with that situation and i would have to flag down my waitress so she could bill me for the four dollar avocado.</td>\n",
              "      <td>['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']</td>\n",
              "      <td>8</td>\n",
              "      <td>restaurant stars decent food dinner meeting spending entrées appetizers flag assistant cook guy bright red baggy pants watching tv avocado tuna entrée apparent direct help situation flag waitress bill dollar avocado</td>\n",
              "      <td>restaurant star decent food dinner meeting spending entryées appetite flag assistant cook guy bright red buggy part watch tv avocado tuna entryée apparent direct help situation flag waitress bill dollar avocado</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>pHwbdway4yeI-dSSmZA7-Q</td>\n",
              "      <td>qEEk0PuoH1dVa619t8fgpw</td>\n",
              "      <td>PY9GRfzr4nTZeINf346QOw</td>\n",
              "      <td>we checked in around : pm. check-in was quick and easy with complimentary valet parking (always appreciated). the peppermill is huge and reminded us of the caesars palace in las vegas. they have around eight restaurants on site offering almost any type of food you could want. the casino area is huge, rivaling almost anything you could find in las vegas. one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from cents to $. talk about false advertising! we were in the tuscany tower on the th floor. our room had a nice view of the huge pool area. our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode. there were dual sinks and lots of counter space. plenty of towels were supplied. the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table. the thermostat did a good job controlling the temperature and keeping it comfortable. staff were all courteous and helpful. a very nice place and we enjoyed our stay.</td>\n",
              "      <td>['Event Planning &amp; Services', 'Casinos', 'Beauty &amp; Spas', 'Venues &amp; Event Spaces', 'Arts &amp; Entertainment', 'Day Spas', 'Hotels', 'Hotels &amp; Travel']</td>\n",
              "      <td>8</td>\n",
              "      <td>checked pm check quick easy complimentary valet parking appreciated peppermill huge reminded caesars palace las vegas restaurants site offering type food want casino area huge rivaling find las vegas thing noticed slot machines noted penny machines actually started play realized minimum bets cents talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciated huge bathroom area large shower shower heads tv large jetted bathtub phone commode dual sinks lots counter space plenty towels supplied roomy main area king bed night stands closet tv mini fridge dresser drawers safe table computer writing complimentary bottles water nicely upholstered chairs table thermostat good job controlling temperature keeping comfortable staff courteous helpful nice place enjoyed stay</td>\n",
              "      <td>check pm check quick easy complimentary valet parking appreciate peppermill huge remind cyears palace las vegas restaurant site offer type food want casino area huge revealing find las vegas thing notice slot machine note penny machine actually start play realize minimum bet cent talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciate huge bathroom area large shower shower head tv large netted bathtub phone common dual sink lot counter space plenty towel supply room main area king bed night stand closet tv mini fridge dressed drawer safe table computer write complimentary bottle water nicely upholstery chair table thermostat good job control temperature keep comfortable staff courteous helpful nice place enjoy stay</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0               review_id                 user_id  \\\n",
              "0  0           l3Wk_mvAog6XANIuGQ9C7Q  ZbqSHbgCjzVAqaa7NKWn5A   \n",
              "1  1           vBK79c3_1Ff_oqkh5VpfGg  Ohhrhu1RkqfVciIVx_W5HQ   \n",
              "2  2           z0osLHDvXvzfm57D4DmD2Q  xVKE_HJ2pwUtTdLbL3pnCg   \n",
              "3  3           elqRpX9T3YwL07uLNtN3Bg  -sryo4gDYxbZ1T5Bz4l5Bw   \n",
              "4  4           pHwbdway4yeI-dSSmZA7-Q  qEEk0PuoH1dVa619t8fgpw   \n",
              "\n",
              "              business_id  \\\n",
              "0  EQ-TZ2eeD_E0BHuvoaeG5Q   \n",
              "1  nRKndeZLQ3eDL10UMwS2rQ   \n",
              "2  S2Ho8yLxhKAa26pBAm6rxA   \n",
              "3  ltBBYdNzkeKdCNPDAsxwAA   \n",
              "4  PY9GRfzr4nTZeINf346QOw   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                text  \\\n",
              "0  locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
              "1  holy smokes! actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat. the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "2  service was crappy, and food was mediocre. i wish i would have picked some other place for my last dinner in town.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "3  i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée. he made it very apparent and direct that he could not help me with that situation and i would have to flag down my waitress so she could bill me for the four dollar avocado.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "4  we checked in around : pm. check-in was quick and easy with complimentary valet parking (always appreciated). the peppermill is huge and reminded us of the caesars palace in las vegas. they have around eight restaurants on site offering almost any type of food you could want. the casino area is huge, rivaling almost anything you could find in las vegas. one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from cents to $. talk about false advertising! we were in the tuscany tower on the th floor. our room had a nice view of the huge pool area. our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode. there were dual sinks and lots of counter space. plenty of towels were supplied. the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table. the thermostat did a good job controlling the temperature and keeping it comfortable. staff were all courteous and helpful. a very nice place and we enjoyed our stay.   \n",
              "\n",
              "                                                                                                                                              category  \\\n",
              "0  ['Beer', 'Wine & Spirits', 'Cafes', 'Coffee & Tea', 'Restaurants', 'Food', 'Breakfast & Brunch', 'American (New)']                                    \n",
              "1  ['Ice Cream & Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']                                                                                     \n",
              "2  ['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast & Brunch']                                                                                      \n",
              "3  ['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']                                       \n",
              "4  ['Event Planning & Services', 'Casinos', 'Beauty & Spas', 'Venues & Event Spaces', 'Arts & Entertainment', 'Day Spas', 'Hotels', 'Hotels & Travel']   \n",
              "\n",
              "   n_classes  \\\n",
              "0  8           \n",
              "1  4           \n",
              "2  4           \n",
              "3  8           \n",
              "4  8           \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   token  \\\n",
              "0  locals recommended milktooth amazing jewel indianapolis glade chance experience                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "1  holy smokes actual pumpkin pie mixed frozen custard kidding huge sweep nation treat best chunks pie crust like finding diamond inside diamond super yummy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "2  service crappy food mediocre wish picked place dinner town                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "3  restaurant stars decent food dinner meeting spending entrées appetizers flag assistant cook guy bright red baggy pants watching tv avocado tuna entrée apparent direct help situation flag waitress bill dollar avocado                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "4  checked pm check quick easy complimentary valet parking appreciated peppermill huge reminded caesars palace las vegas restaurants site offering type food want casino area huge rivaling find las vegas thing noticed slot machines noted penny machines actually started play realized minimum bets cents talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciated huge bathroom area large shower shower heads tv large jetted bathtub phone commode dual sinks lots counter space plenty towels supplied roomy main area king bed night stands closet tv mini fridge dresser drawers safe table computer writing complimentary bottles water nicely upholstered chairs table thermostat good job controlling temperature keeping comfortable staff courteous helpful nice place enjoyed stay   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        lemma  \n",
              "0  local recommend milktooth amazing jewel indianapolis grade chance experience                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "1  holy smoke actual pumpkin pie mix frozen custard kid huge sweep nation treat well chunk pie crust like find diamond inside diamond super dummy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "2  service crappy food mediocre wish pick place dinner town                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
              "3  restaurant star decent food dinner meeting spending entryées appetite flag assistant cook guy bright red buggy part watch tv avocado tuna entryée apparent direct help situation flag waitress bill dollar avocado                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          \n",
              "4  check pm check quick easy complimentary valet parking appreciate peppermill huge remind cyears palace las vegas restaurant site offer type food want casino area huge revealing find las vegas thing notice slot machine note penny machine actually start play realize minimum bet cent talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciate huge bathroom area large shower shower head tv large netted bathtub phone common dual sink lot counter space plenty towel supply room main area king bed night stand closet tv mini fridge dressed drawer safe table computer write complimentary bottle water nicely upholstery chair table thermostat good job control temperature keep comfortable staff courteous helpful nice place enjoy stay  "
            ]
          },
          "execution_count": 66,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "metadata": {},
      "outputs": [],
      "source": [
        "data.to_csv(\"data_preprocessed2.csv\", index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [],
      "source": [
        "# data = pd.read_csv(\"data_preprocessed2.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [],
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "wordnet_lemmatizer = WordNetLemmatizer()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {},
      "outputs": [],
      "source": [
        "def lemmatizer(text):\n",
        "  lemm_text = [wordnet_lemmatizer.lemmatize(word) for word in text.split()]\n",
        "  return lemm_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {},
      "outputs": [],
      "source": [
        "data['lemma_text'] = data['token'].apply(lemmatizer) #wordnet lemmatizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>n_classes</th>\n",
              "      <th>token</th>\n",
              "      <th>lemma</th>\n",
              "      <th>lemma_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>l3Wk_mvAog6XANIuGQ9C7Q</td>\n",
              "      <td>ZbqSHbgCjzVAqaa7NKWn5A</td>\n",
              "      <td>EQ-TZ2eeD_E0BHuvoaeG5Q</td>\n",
              "      <td>locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.</td>\n",
              "      <td>['Beer', 'Wine &amp; Spirits', 'Cafes', 'Coffee &amp; Tea', 'Restaurants', 'Food', 'Breakfast &amp; Brunch', 'American (New)']</td>\n",
              "      <td>8</td>\n",
              "      <td>locals recommended milktooth amazing jewel indianapolis glade chance experience</td>\n",
              "      <td>local recommend milktooth amazing jewel indianapolis grade chance experience</td>\n",
              "      <td>[local, recommended, milktooth, amazing, jewel, indianapolis, glade, chance, experience]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>vBK79c3_1Ff_oqkh5VpfGg</td>\n",
              "      <td>Ohhrhu1RkqfVciIVx_W5HQ</td>\n",
              "      <td>nRKndeZLQ3eDL10UMwS2rQ</td>\n",
              "      <td>holy smokes! actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat. the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.</td>\n",
              "      <td>['Ice Cream &amp; Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']</td>\n",
              "      <td>4</td>\n",
              "      <td>holy smokes actual pumpkin pie mixed frozen custard kidding huge sweep nation treat best chunks pie crust like finding diamond inside diamond super yummy</td>\n",
              "      <td>holy smoke actual pumpkin pie mix frozen custard kid huge sweep nation treat well chunk pie crust like find diamond inside diamond super dummy</td>\n",
              "      <td>[holy, smoke, actual, pumpkin, pie, mixed, frozen, custard, kidding, huge, sweep, nation, treat, best, chunk, pie, crust, like, finding, diamond, inside, diamond, super, yummy]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>z0osLHDvXvzfm57D4DmD2Q</td>\n",
              "      <td>xVKE_HJ2pwUtTdLbL3pnCg</td>\n",
              "      <td>S2Ho8yLxhKAa26pBAm6rxA</td>\n",
              "      <td>service was crappy, and food was mediocre. i wish i would have picked some other place for my last dinner in town.</td>\n",
              "      <td>['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast &amp; Brunch']</td>\n",
              "      <td>4</td>\n",
              "      <td>service crappy food mediocre wish picked place dinner town</td>\n",
              "      <td>service crappy food mediocre wish pick place dinner town</td>\n",
              "      <td>[service, crappy, food, mediocre, wish, picked, place, dinner, town]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>elqRpX9T3YwL07uLNtN3Bg</td>\n",
              "      <td>-sryo4gDYxbZ1T5Bz4l5Bw</td>\n",
              "      <td>ltBBYdNzkeKdCNPDAsxwAA</td>\n",
              "      <td>i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée. he made it very apparent and direct that he could not help me with that situation and i would have to flag down my waitress so she could bill me for the four dollar avocado.</td>\n",
              "      <td>['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']</td>\n",
              "      <td>8</td>\n",
              "      <td>restaurant stars decent food dinner meeting spending entrées appetizers flag assistant cook guy bright red baggy pants watching tv avocado tuna entrée apparent direct help situation flag waitress bill dollar avocado</td>\n",
              "      <td>restaurant star decent food dinner meeting spending entryées appetite flag assistant cook guy bright red buggy part watch tv avocado tuna entryée apparent direct help situation flag waitress bill dollar avocado</td>\n",
              "      <td>[restaurant, star, decent, food, dinner, meeting, spending, entrées, appetizer, flag, assistant, cook, guy, bright, red, baggy, pant, watching, tv, avocado, tuna, entrée, apparent, direct, help, situation, flag, waitress, bill, dollar, avocado]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>pHwbdway4yeI-dSSmZA7-Q</td>\n",
              "      <td>qEEk0PuoH1dVa619t8fgpw</td>\n",
              "      <td>PY9GRfzr4nTZeINf346QOw</td>\n",
              "      <td>we checked in around : pm. check-in was quick and easy with complimentary valet parking (always appreciated). the peppermill is huge and reminded us of the caesars palace in las vegas. they have around eight restaurants on site offering almost any type of food you could want. the casino area is huge, rivaling almost anything you could find in las vegas. one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from cents to $. talk about false advertising! we were in the tuscany tower on the th floor. our room had a nice view of the huge pool area. our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode. there were dual sinks and lots of counter space. plenty of towels were supplied. the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table. the thermostat did a good job controlling the temperature and keeping it comfortable. staff were all courteous and helpful. a very nice place and we enjoyed our stay.</td>\n",
              "      <td>['Event Planning &amp; Services', 'Casinos', 'Beauty &amp; Spas', 'Venues &amp; Event Spaces', 'Arts &amp; Entertainment', 'Day Spas', 'Hotels', 'Hotels &amp; Travel']</td>\n",
              "      <td>8</td>\n",
              "      <td>checked pm check quick easy complimentary valet parking appreciated peppermill huge reminded caesars palace las vegas restaurants site offering type food want casino area huge rivaling find las vegas thing noticed slot machines noted penny machines actually started play realized minimum bets cents talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciated huge bathroom area large shower shower heads tv large jetted bathtub phone commode dual sinks lots counter space plenty towels supplied roomy main area king bed night stands closet tv mini fridge dresser drawers safe table computer writing complimentary bottles water nicely upholstered chairs table thermostat good job controlling temperature keeping comfortable staff courteous helpful nice place enjoyed stay</td>\n",
              "      <td>check pm check quick easy complimentary valet parking appreciate peppermill huge remind cyears palace las vegas restaurant site offer type food want casino area huge revealing find las vegas thing notice slot machine note penny machine actually start play realize minimum bet cent talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciate huge bathroom area large shower shower head tv large netted bathtub phone common dual sink lot counter space plenty towel supply room main area king bed night stand closet tv mini fridge dressed drawer safe table computer write complimentary bottle water nicely upholstery chair table thermostat good job control temperature keep comfortable staff courteous helpful nice place enjoy stay</td>\n",
              "      <td>[checked, pm, check, quick, easy, complimentary, valet, parking, appreciated, peppermill, huge, reminded, caesar, palace, la, vega, restaurant, site, offering, type, food, want, casino, area, huge, rivaling, find, la, vega, thing, noticed, slot, machine, noted, penny, machine, actually, started, play, realized, minimum, bet, cent, talk, false, advertising, tuscany, tower, th, floor, room, nice, view, huge, pool, area, room, large, featured, free, wifi, likewise, appreciated, huge, bathroom, area, large, shower, shower, head, tv, large, jetted, bathtub, phone, commode, dual, sink, lot, counter, space, plenty, towel, supplied, roomy, main, area, king, bed, night, stand, closet, tv, mini, fridge, dresser, drawer, safe, table, computer, ...]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0               review_id                 user_id  \\\n",
              "0  0           l3Wk_mvAog6XANIuGQ9C7Q  ZbqSHbgCjzVAqaa7NKWn5A   \n",
              "1  1           vBK79c3_1Ff_oqkh5VpfGg  Ohhrhu1RkqfVciIVx_W5HQ   \n",
              "2  2           z0osLHDvXvzfm57D4DmD2Q  xVKE_HJ2pwUtTdLbL3pnCg   \n",
              "3  3           elqRpX9T3YwL07uLNtN3Bg  -sryo4gDYxbZ1T5Bz4l5Bw   \n",
              "4  4           pHwbdway4yeI-dSSmZA7-Q  qEEk0PuoH1dVa619t8fgpw   \n",
              "\n",
              "              business_id  \\\n",
              "0  EQ-TZ2eeD_E0BHuvoaeG5Q   \n",
              "1  nRKndeZLQ3eDL10UMwS2rQ   \n",
              "2  S2Ho8yLxhKAa26pBAm6rxA   \n",
              "3  ltBBYdNzkeKdCNPDAsxwAA   \n",
              "4  PY9GRfzr4nTZeINf346QOw   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                text  \\\n",
              "0  locals recommended milktooth, and it's an amazing jewel of indianapolis. i'm glade i had the chance to experience this.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
              "1  holy smokes! actual pumpkin pie mixed in with the frozen custard......are you kidding me? why hasn't this become a huge sweep the nation treat. the best part was the chunks of pie crust. like finding a diamond inside another diamond. super yummy.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "2  service was crappy, and food was mediocre. i wish i would have picked some other place for my last dinner in town.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
              "3  i at least have to give this restaurant two stars due to the decent food. but while on a dinner meeting and after spending $ on entrées and appetizers, i had to flag down the assistant cook, (the guy in bright red baggy pants watching tv), to see if i could get more avocado for my tuna entrée. he made it very apparent and direct that he could not help me with that situation and i would have to flag down my waitress so she could bill me for the four dollar avocado.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
              "4  we checked in around : pm. check-in was quick and easy with complimentary valet parking (always appreciated). the peppermill is huge and reminded us of the caesars palace in las vegas. they have around eight restaurants on site offering almost any type of food you could want. the casino area is huge, rivaling almost anything you could find in las vegas. one thing we noticed is that many of the slot machines were noted as being penny machines, but when you actually started to play them you realized they had minimum bets of anywhere from cents to $. talk about false advertising! we were in the tuscany tower on the th floor. our room had a nice view of the huge pool area. our room was large and featured free wifi (likewise always appreciated), a huge bathroom area with large shower (with two shower heads), tv, a large jetted bathtub and a phone next to the commode. there were dual sinks and lots of counter space. plenty of towels were supplied. the roomy main area had a king bed with night stands, closet, tv, mini fridge, dresser with two drawers, a safe, table for computer or writing, complimentary bottles of water, and three nicely upholstered chairs and table. the thermostat did a good job controlling the temperature and keeping it comfortable. staff were all courteous and helpful. a very nice place and we enjoyed our stay.   \n",
              "\n",
              "                                                                                                                                              category  \\\n",
              "0  ['Beer', 'Wine & Spirits', 'Cafes', 'Coffee & Tea', 'Restaurants', 'Food', 'Breakfast & Brunch', 'American (New)']                                    \n",
              "1  ['Ice Cream & Frozen Yogurt', 'Food', 'Local Flavor', 'Desserts']                                                                                     \n",
              "2  ['Cajun/Creole', 'Seafood', 'Restaurants', 'Breakfast & Brunch']                                                                                      \n",
              "3  ['Cocktail Bars', 'Nightlife', 'Gastropubs', 'Sports Bars', 'Restaurants', 'American (New)', 'Lounges', 'Bars']                                       \n",
              "4  ['Event Planning & Services', 'Casinos', 'Beauty & Spas', 'Venues & Event Spaces', 'Arts & Entertainment', 'Day Spas', 'Hotels', 'Hotels & Travel']   \n",
              "\n",
              "   n_classes  \\\n",
              "0  8           \n",
              "1  4           \n",
              "2  4           \n",
              "3  8           \n",
              "4  8           \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   token  \\\n",
              "0  locals recommended milktooth amazing jewel indianapolis glade chance experience                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "1  holy smokes actual pumpkin pie mixed frozen custard kidding huge sweep nation treat best chunks pie crust like finding diamond inside diamond super yummy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "2  service crappy food mediocre wish picked place dinner town                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
              "3  restaurant stars decent food dinner meeting spending entrées appetizers flag assistant cook guy bright red baggy pants watching tv avocado tuna entrée apparent direct help situation flag waitress bill dollar avocado                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "4  checked pm check quick easy complimentary valet parking appreciated peppermill huge reminded caesars palace las vegas restaurants site offering type food want casino area huge rivaling find las vegas thing noticed slot machines noted penny machines actually started play realized minimum bets cents talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciated huge bathroom area large shower shower heads tv large jetted bathtub phone commode dual sinks lots counter space plenty towels supplied roomy main area king bed night stands closet tv mini fridge dresser drawers safe table computer writing complimentary bottles water nicely upholstered chairs table thermostat good job controlling temperature keeping comfortable staff courteous helpful nice place enjoyed stay   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        lemma  \\\n",
              "0  local recommend milktooth amazing jewel indianapolis grade chance experience                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
              "1  holy smoke actual pumpkin pie mix frozen custard kid huge sweep nation treat well chunk pie crust like find diamond inside diamond super dummy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
              "2  service crappy food mediocre wish pick place dinner town                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "3  restaurant star decent food dinner meeting spending entryées appetite flag assistant cook guy bright red buggy part watch tv avocado tuna entryée apparent direct help situation flag waitress bill dollar avocado                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
              "4  check pm check quick easy complimentary valet parking appreciate peppermill huge remind cyears palace las vegas restaurant site offer type food want casino area huge revealing find las vegas thing notice slot machine note penny machine actually start play realize minimum bet cent talk false advertising tuscany tower th floor room nice view huge pool area room large featured free wifi likewise appreciate huge bathroom area large shower shower head tv large netted bathtub phone common dual sink lot counter space plenty towel supply room main area king bed night stand closet tv mini fridge dressed drawer safe table computer write complimentary bottle water nicely upholstery chair table thermostat good job control temperature keep comfortable staff courteous helpful nice place enjoy stay   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    lemma_text  \n",
              "0  [local, recommended, milktooth, amazing, jewel, indianapolis, glade, chance, experience]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
              "1  [holy, smoke, actual, pumpkin, pie, mixed, frozen, custard, kidding, huge, sweep, nation, treat, best, chunk, pie, crust, like, finding, diamond, inside, diamond, super, yummy]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
              "2  [service, crappy, food, mediocre, wish, picked, place, dinner, town]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "3  [restaurant, star, decent, food, dinner, meeting, spending, entrées, appetizer, flag, assistant, cook, guy, bright, red, baggy, pant, watching, tv, avocado, tuna, entrée, apparent, direct, help, situation, flag, waitress, bill, dollar, avocado]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
              "4  [checked, pm, check, quick, easy, complimentary, valet, parking, appreciated, peppermill, huge, reminded, caesar, palace, la, vega, restaurant, site, offering, type, food, want, casino, area, huge, rivaling, find, la, vega, thing, noticed, slot, machine, noted, penny, machine, actually, started, play, realized, minimum, bet, cent, talk, false, advertising, tuscany, tower, th, floor, room, nice, view, huge, pool, area, room, large, featured, free, wifi, likewise, appreciated, huge, bathroom, area, large, shower, shower, head, tv, large, jetted, bathtub, phone, commode, dual, sink, lot, counter, space, plenty, towel, supplied, roomy, main, area, king, bed, night, stand, closet, tv, mini, fridge, dresser, drawer, safe, table, computer, ...]  "
            ]
          },
          "execution_count": 44,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {},
      "outputs": [],
      "source": [
        "data.to_csv(\"data_preprocessed3.csv\")"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "RWAqtzS1wZy5",
        "P6DsWNFbV2fa",
        "VQebnm8xcLPa",
        "mFiGnm7twZy1",
        "7Ju2gxohwZy3",
        "8jXQMfSewZy2",
        "htkMNzRwo_qT",
        "xaRAnyujfnkS",
        "l3l-twtEOHvf"
      ],
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.1"
    },
    "vscode": {
      "interpreter": {
        "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}