Vine_Review_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Vine_Review_Analysis.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Opstor0EXzfD"
      },
      "source": [
        "**Determine if there is any bias in the reviews**\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tGlie-CTgRHu",
        "outputId": "5c9bb6fc-4848-47f7-92c1-e9e112467a7a"
      },
      "source": [
        "import os\r\n",
        "# Find the latest version of spark 3.0  from http://www-us.apache.org/dist/spark/ and enter as the spark version\r\n",
        "# For example:\r\n",
        "# spark_version = 'spark-3.0.2'\r\n",
        "spark_version = 'spark-3.0.2'\r\n",
        "os.environ['SPARK_VERSION']=spark_version\r\n",
        "\r\n",
        "# Install Spark and Java\r\n",
        "!apt-get update\r\n",
        "!apt-get install openjdk-11-jdk-headless -qq > /dev/null\r\n",
        "!wget -q http://www-us.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-hadoop2.7.tgz\r\n",
        "!tar xf $SPARK_VERSION-bin-hadoop2.7.tgz\r\n",
        "!pip install -q findspark\r\n",
        "\r\n",
        "# Set Environment Variables\r\n",
        "import os\r\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-11-openjdk-amd64\"\r\n",
        "os.environ[\"SPARK_HOME\"] = f\"/content/{spark_version}-bin-hadoop2.7\"\r\n",
        "\r\n",
        "# Start a SparkSession\r\n",
        "import findspark\r\n",
        "findspark.init()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [1 InRelease 14.2 kB/88.7\r                                                                               \rGet:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [1 InRelease 86.6 kB/88.7\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [1 InRelease 88.7 kB/88.7\r                                                                               \rIgn:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "\r                                                                               \r0% [Waiting for headers] [Waiting for headers] [Waiting for headers]\r0% [2 InRelease gpgv 3,626 B] [Waiting for headers] [Waiting for headers] [Wait\r                                                                               \rGet:4 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "Ign:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release [697 B]\n",
            "Hit:7 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:8 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:9 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release.gpg [836 B]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:11 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:12 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,396 kB]\n",
            "Get:15 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [1,964 kB]\n",
            "Hit:16 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Ign:18 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Packages\n",
            "Get:18 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Packages [602 kB]\n",
            "Get:19 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,745 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,163 kB]\n",
            "Get:21 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [893 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,394 kB]\n",
            "Fetched 11.4 MB in 4s (3,096 kB/s)\n",
            "Reading package lists... Done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0HUnBO8dhuEF",
        "outputId": "9a368fd6-c8d0-42a1-cfd3-4134c652c525"
      },
      "source": [
        "# Download the Postgres driver that will allow Spark to interact with Postgres.\r\n",
        "!wget https://jdbc.postgresql.org/download/postgresql-42.2.16.jar"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-03-13 21:38:51--  https://jdbc.postgresql.org/download/postgresql-42.2.16.jar\n",
            "Resolving jdbc.postgresql.org (jdbc.postgresql.org)... 72.32.157.228, 2001:4800:3e1:1::228\n",
            "Connecting to jdbc.postgresql.org (jdbc.postgresql.org)|72.32.157.228|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1002883 (979K) [application/java-archive]\n",
            "Saving to: ‘postgresql-42.2.16.jar’\n",
            "\n",
            "postgresql-42.2.16. 100%[===================>] 979.38K  5.50MB/s    in 0.2s    \n",
            "\n",
            "2021-03-13 21:38:51 (5.50 MB/s) - ‘postgresql-42.2.16.jar’ saved [1002883/1002883]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "45EM5gtWgaal"
      },
      "source": [
        "from pyspark.sql import SparkSession\r\n",
        "spark = SparkSession.builder.appName(\"determining_bias\").config(\"spark.driver.extraClassPath\",\"/content/postgresql-42.2.16.jar\").getOrCreate()"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MMrC3ikSh1nD",
        "outputId": "fb3e8c8d-c623-4479-bbfd-ef5d0c6c1519"
      },
      "source": [
        "# Load dataset into a dataframe.\r\n",
        "from pyspark import SparkFiles\r\n",
        "url = \"https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Video_Games_v1_00.tsv.gz\"\r\n",
        "spark.sparkContext.addFile(url)\r\n",
        "df = spark.read.option(\"encoding\", \"UTF-8\").csv(SparkFiles.get(\"\"), sep=\"\\t\", header=True, inferSchema=True)\r\n",
        "df.show(10)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|     review_headline|         review_body|review_date|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|         US|   12039526| RTIS3L2M1F5SM|B001CXYMFS|     737716809|Thrustmaster T-Fl...|     Video Games|          5|            0|          0|   N|                Y|an amazing joysti...|Used this for Eli...| 2015-08-31|\n",
            "|         US|    9636577| R1ZV7R40OLHKD|B00M920ND6|     569686175|Tonsee 6 buttons ...|     Video Games|          5|            0|          0|   N|                Y|Definitely a sile...|Loved it,  I didn...| 2015-08-31|\n",
            "|         US|    2331478|R3BH071QLH8QMC|B0029CSOD2|      98937668|Hidden Mysteries:...|     Video Games|          1|            0|          1|   N|                Y|            One Star|poor quality work...| 2015-08-31|\n",
            "|         US|   52495923|R127K9NTSXA2YH|B00GOOSV98|      23143350|GelTabz Performan...|     Video Games|          3|            0|          0|   N|                Y|good, but could b...|nice, but tend to...| 2015-08-31|\n",
            "|         US|   14533949|R32ZWUXDJPW27Q|B00Y074JOM|     821342511|Zero Suit Samus a...|     Video Games|          4|            0|          0|   N|                Y|   Great but flawed.|Great amiibo, gre...| 2015-08-31|\n",
            "|         US|    2377552|R3AQQ4YUKJWBA6|B002UBI6W6|     328764615|Psyclone Recharge...|     Video Games|          1|            0|          0|   N|                Y|            One Star|The remote consta...| 2015-08-31|\n",
            "|         US|   17521011|R2F0POU5K6F73F|B008XHCLFO|      24234603|Protection for yo...|     Video Games|          5|            0|          0|   N|                Y|              A Must|I have a 2012-201...| 2015-08-31|\n",
            "|         US|   19676307|R3VNR804HYSMR6|B00BRA9R6A|     682267517|   Nerf 3DS XL Armor|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|Perfect, kids lov...| 2015-08-31|\n",
            "|         US|     224068| R3GZTM72WA2QH|B009EPWJLA|     435241890|One Piece: Pirate...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|            Excelent| 2015-08-31|\n",
            "|         US|   48467989| RNQOY62705W1K|B0000AV7GB|     256572651|Playstation 2 Dan...|     Video Games|          4|            0|          0|   N|                Y|          Four Stars|Slippery but expe...| 2015-08-31|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "only showing top 10 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_RGEjLeRh-FG",
        "outputId": "6e2d7456-e399-4872-ad83-70be95173b03"
      },
      "source": [
        "# Recreate vine df\r\n",
        "vine_df = df.select([\"review_id\", \"star_rating\", \"helpful_votes\", \"total_votes\", \"vine\", \"verified_purchase\"])\r\n",
        "\r\n",
        "# Show vine df\r\n",
        "vine_df.show(5)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| RTIS3L2M1F5SM|          5|            0|          0|   N|                Y|\n",
            "| R1ZV7R40OLHKD|          5|            0|          0|   N|                Y|\n",
            "|R3BH071QLH8QMC|          1|            0|          1|   N|                Y|\n",
            "|R127K9NTSXA2YH|          3|            0|          0|   N|                Y|\n",
            "|R32ZWUXDJPW27Q|          4|            0|          0|   N|                Y|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Yu5f3JxiZwI",
        "outputId": "68eaba27-1816-4f2a-e260-4b66a47659f7"
      },
      "source": [
        "# Filter vine df to only include reviews with more than 20 total votes (more likely to be helpful)\r\n",
        "high_votes_df = vine_df.filter(\"total_votes>=20\")\r\n",
        "high_votes_df.show(5)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| R4PKAZRQJJX14|          1|           21|         34|   N|                N|\n",
            "|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|\n",
            "|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|\n",
            "|R3EZ0EPYLDA34S|          1|           14|         31|   N|                Y|\n",
            "|R2FJ94555FZH32|          2|           55|         60|   N|                N|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0ZyQe3I9X8nl",
        "outputId": "6f4ed14b-c982-42e4-9e7c-4a4ff0aa2fa6"
      },
      "source": [
        "# Filter df with high counts to only include reviews where helpful votes are 50% or more of the votes\r\n",
        "helpful_reviews_df = high_votes_df.filter(\"(helpful_votes / total_votes) >= 0.5\")\r\n",
        "helpful_reviews_df.show()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| R4PKAZRQJJX14|          1|           21|         34|   N|                N|\n",
            "|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|\n",
            "|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|\n",
            "|R2FJ94555FZH32|          2|           55|         60|   N|                N|\n",
            "|R1U3AR67RE273L|          1|           51|         65|   N|                Y|\n",
            "|R3PZOXA5X1U8KW|          4|           31|         36|   N|                N|\n",
            "| R6KTC1OPIOIIG|          2|           19|         34|   N|                Y|\n",
            "|R36O341WWXXKNP|          5|           28|         31|   N|                N|\n",
            "|R10LZVBLQHBVJ0|          2|          151|        198|   N|                N|\n",
            "|R1VR5GLGY1GE7N|          1|           49|         51|   N|                Y|\n",
            "|R2OWDAR0EDEPTF|          1|           23|         28|   N|                N|\n",
            "|R29BOS5HMAY1LO|          5|           88|        110|   N|                N|\n",
            "| R8WD0136CETA7|          2|           99|        119|   N|                N|\n",
            "|R16621F16PQN5A|          3|           24|         31|   N|                Y|\n",
            "|R1LXBPG1M9DDOC|          1|          102|        106|   N|                N|\n",
            "| RNEMGSUKJ4535|          2|           21|         25|   N|                Y|\n",
            "|R3R2UKBFKP1IQ4|          1|           19|         23|   N|                Y|\n",
            "|R2T90XM5G2DBZA|          3|           15|         22|   N|                Y|\n",
            "| R5ZMK0Y0KRBRZ|          1|           79|         91|   N|                N|\n",
            "|R12E2W0XOV3I2S|          1|           24|         30|   N|                Y|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ovVhxkFlwdMG",
        "outputId": "7275a5e9-ea7f-4456-92c2-8b674d6ad108"
      },
      "source": [
        "# Number of reviews in helpful reviews df\r\n",
        "helpful_reviews_df.count()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "40565"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "544gqV8jws3a",
        "outputId": "fbbd8676-eea7-4d59-dd90-48dab19de04c"
      },
      "source": [
        "# Filter helpful reviews to only include reviews written as part of Vine program and show count\r\n",
        "vine_reviews = helpful_reviews_df.filter(\"vine == 'Y'\")\r\n",
        "print(vine_reviews.count())\r\n",
        "vine_reviews.show(5)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "94\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|R3KKUSGFZWSUIY|          5|           56|         63|   Y|                N|\n",
            "|R10FO5UKKVZBK2|          3|           23|         23|   Y|                N|\n",
            "| RM4KSGEOR7MU1|          5|           19|         24|   Y|                N|\n",
            "| RG7VRMYLEXD23|          4|           22|         26|   Y|                N|\n",
            "|R11O4YSCPSNL6L|          3|           20|         26|   Y|                N|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A4Cr5ablxNpA",
        "outputId": "8428e118-4e64-41b9-effc-d9a04ea3157b"
      },
      "source": [
        "# Filter helpful reviews to only include reviews not written as part of Vine program and show count\r\n",
        "nonvine_reviews = helpful_reviews_df.filter(\"vine == 'N'\")\r\n",
        "print(nonvine_reviews.count())\r\n",
        "nonvine_reviews.show(5)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "40471\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| R4PKAZRQJJX14|          1|           21|         34|   N|                N|\n",
            "|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|\n",
            "|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|\n",
            "|R2FJ94555FZH32|          2|           55|         60|   N|                N|\n",
            "|R1U3AR67RE273L|          1|           51|         65|   N|                Y|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YnEoYGmIxYjj"
      },
      "source": [
        "# Get counts for 5 star reviews\r\n",
        "total_reviews_count = helpful_reviews_df.count()\r\n",
        "total_5star_reviews = helpful_reviews_df.filter(\"star_rating = 5\").count()\r\n",
        "vine_5star_reviews = vine_reviews.filter(\"star_rating = 5\").count()\r\n",
        "vine_5star_reviews_percentage = (vine_5star_reviews / vine_reviews.count()) * 100\r\n",
        "nonvine_5star_reviews = nonvine_reviews.filter(\"star_rating = 5\").count()\r\n",
        "nonvine_5star_reviews_percentage = (nonvine_5star_reviews / nonvine_reviews.count()) * 100"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kQInBWtkz3Z3",
        "outputId": "a65915c4-bc4b-43de-a71a-6778340fdf47"
      },
      "source": [
        "# Display results\r\n",
        "print(f\"The total number of helpful reviews is: {total_reviews_count}.\")\r\n",
        "print(f\"The number of 5 star reviews in the helpful dataset is: {total_5star_reviews}.\")\r\n",
        "#print(f\"The number of 5 star reviews that were written as part of the vine program is: {vine_5star_reviews}.\")\r\n",
        "print(f\"The percentage of vine reviews that received 5 stars is: {vine_5star_reviews_percentage:.2f}%.\")\r\n",
        "#print(f\"The number of 5 star reviews that were written outside the vine program is: {nonvine_5star_reviews}.\")\r\n",
        "print(f\"The percentage of reviews outside the vine program that received 5 stars is: {nonvine_5star_reviews_percentage:.2f}%.\")"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The total number of helpful reviews is: 40565.\n",
            "The number of 5 star reviews in the helpful dataset is: 15711.\n",
            "The percentage of vine reviews that received 5 stars is: 51.06%.\n",
            "The percentage of reviews outside the vine program that received 5 stars is: 38.70%.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}