spark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init('D:/spark-2.3.1-bin-hadoop2.7')\n",
    "from pyspark import SparkContext\n",
    "from pyspark.streaming import StreamingContext\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.sql.functions import desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = SparkContext()\n",
    "ssc = StreamingContext(sc, 10 )\n",
    "sqlContext = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "socket_stream = ssc.socketTextStream(\"127.0.0.1\", 5556)\n",
    "lines = socket_stream.window( 20 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import namedtuple\n",
    "fields = (\"tag\", \"count\" )\n",
    "Tweet = namedtuple('tweets', fields )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Use Parenthesis for multiple lines or use \\.\n",
    "( lines.flatMap( lambda text: text.split( \" \" ) ) #Splits to a list\n",
    "  .filter( lambda word: word.lower().startswith(\"#\") ) # Checks for hashtag calls\n",
    "  .map( lambda word: ( word.lower(), 1 ) ) # Lower cases the word\n",
    "  .reduceByKey( lambda a, b: a + b ) # Reduces\n",
    "  .map( lambda rec: Tweet( rec[0], rec[1] ) ) # Stores in a Tweet Object\n",
    "  .foreachRDD( lambda rdd: rdd.toDF().sort( desc(\"count\") ) # Sorts Them in a DF\n",
    "  .limit(10).registerTempTable(\"tweets\") ) ) # Registers to a table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas\n",
    "# Only works for Jupyter Notebooks!\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ssc.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArEAAAHjCAYAAAA0dEdMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xu8XWV97/vPF4IbAwEUqUURghSJoJBgAnIRgtKN3VqbHlFUhI2Wsr1SpNLDPu6NbHo8ULG2Ki3uqDR4oy0Xq9XtpZtbkHsCi3DHFm3lwCmmROQmCvmdP+ZIXS5X1loJa2WuZ67P+/XKa475jGc84zdGeMGXZz5jzlQVkiRJUks263cBkiRJ0oYyxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzZnV7wI09Z73vOfV3Llz+12GJEnSuFauXLm6qnYYr58hdgaYO3cuK1as6HcZkiRJ40ryzxPp53ICSZIkNccQK0mSpOa4nGAGuPO+f+MVp3y+32VIkqRGrTz72H6X8CuciZUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQO4YkZyZZnGRJklO7tiuSLByl7xvW9dmA8ZclOXKcPsclecGGVS5JkjTYDLFj2x+4HjgUuGqsjlX1tao6awpqOA4wxEqSJA1jiB1FkrOTrAIWAdcCxwPnJjmt6/L2JNckuS3Jft0xxyU5p9veLcl1SW5MckaSR7v2JDknyR1JvgH82rBzntb1vy3J0q7vkcBC4EtJhpI8e7R+m+7OSJIkTQ+G2FFU1Sn0gusyekF2VVXtXVVndF22qqoDgfcA540yxCeAT1TVIuD+Ye2/C+wBvBz4feDAYfvOqapFVfUy4NnA66vqImAFcHRVza+qJ0brNzlXLUmS1A5D7PotAIaAecAdI/ZdAFBVy4Ftkmw3Yv8BwIXd9peHtR8CXFBVT1fV/cBlw/YdluT6JLcCrwb2Wk9dE+qX5IQkK5KseOrxR8a8UEmSpNbM6ncB002S+fRmYHcCVgOze80ZohdOAWrEYSPfj+VX+ibZEvhLYGFV/TDJ6cCWG9sPoKqWAksBtvr1XTekPkmSpGnPmdgRqmqoquYD9wB70pstPWLYx/kARwEkORh4uKoeHjHMdcAbu+23DGtfDrwlyeZJdgQO69rXBdHVSbYGhn9jwSPAnAn0kyRJmjGciR1Fkh2ANVW1Nsm8qhq5nGBNkmuAbYB3jjLEScAXk/wh8A1gXcj9Cr0lALfSC8lXAlTVj5N8pmv/AXDjsLGWAZ9O8gS9meD19ZMkSZoxUuUnzZMtyWzgiaqqJG8B3lpVv9Overb69V1r3jH/o1+nlyRJjVt59rGb7FxJVlbVr3wn/0jOxE6NVwDndF9/9WNGn62VJEnSRjLEToGqugrYp991SJIkDSof7JIkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOPzs7A7x0p+1Zcfax/S5DkiRp0jgTK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xx87mAF+9sDt/MsZL+93GZIkTZmdT7u13yVoE3MmVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHENsnyU5M8niJEuSnNq1nZHk8HGOOynJ7E1TpSRJ0vRiiO2//YHrgUOBqwCq6rSq+t/jHHcSYIiVJEkz0qx+FzBTJTkbOALYFbgW2A14TZKLgBcDX6+qi5K8BvgYvb+rG4F3A/8FeAFweZLVVXVYP65BkiSpX5yJ7ZOqOgU4HlgGLAJWVdXeVXXGuj5Jtuz2H1VVL6cXZN9dVZ8E7gcOW1+ATXJCkhVJVjz02NNTezGSJEmbmCG2vxYAQ8A84I5R9u8BfL+q7unenw8cMpGBq2ppVS2sqoXP3WrzSSlWkiRpunA5QR8kmU9vhnUnYDW9ta1JMgQcMLzrpq9OkiRp+nMmtg+qaqiq5gP3AHsClwFHVNX8qnpiWNe7gLlJfqN7fwxwZbf9CDBnU9UsSZI0nRhi+yTJDsCaqloLzKuqX1lOUFU/Bd4BXJjkVmAt8Olu91Lgm0ku31Q1S5IkTRcuJ+iTqvoR8Lpu+5Uj9h03bPtSemtnRx7/KeBTU1ulJEnS9ORMrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNmdXvAjT1nrXjXux82op+lyFJkjRpnImVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1By/J3YGuOvBuzjoUwf1uwxJ09jV77+63yVI0gZxJlaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1JxNEmKTnJlkcZIlSU6dwvOcnuSD69l3zTMYd1mSIze+sgmfZ6NrlCRJmkk21Uzs/sD1wKHAVZvonL+kqg7sx3mHSzJrrP3ToUZJkqQWTGmITXJ2klXAIuBa4Hjg3CSnJbkiyZ8lWZ7kziSLklyS5HtJ/u9hY/xdkpVJbk9ywrD21ya5KcktSS4ddto9u7HvTXLisP6Pdq+Lu/0XJbkryZeSpNv3iiRXduf7dpIdR7mmUfsk+f0kN3b1XJxkdte+LMnHk1wO/Ek3W3zeM6jxP3Vt303yySRff+Z/U5IkSW2Z0hBbVafQC67L6AXZVVW1d1Wd0XX5WVUdAnwa+CrwXuBlwHFJtu/6vLOqXgEsBE5Msn2SHYDPAG+sqn2ANw077TzgCGA/4MNJthiltAXAScCewIuBg7p+nwKO7M53HvCR4QeN0+eSqlrU1XMn8HvDDn0JcHhV/eEzrHFL4H8Cv1VVBwM7jHKcJEnSwBvz4+1JsgAYohfc7hix72vd663A7VX1AECSe4EXAf9GL7j+btfvRcDu9MLb8qr6PkBVPTRszG9U1ZPAk0keBJ4P3DfivDdU1X3duYaAucCP6QXof+gmPTcHHhhx3B5j9HlZN4O8HbA18O1hx11YVU9PQo2PAveuu27gAuAERtHNWp8A8KznPGu0LpIkSc2ashCbZD69GdidgNXA7F5zhoADum5Pdq9rh22vez8ryWLgcOCAqno8yRXAlkCAWs+ph4/zNKNf42h9Qi9IHzBK/3+/rDH6LAOWVNUtSY4DFg/b99gk1jghVbUUWAqw9c5br+9eSZIkNWnKlhNU1VBVzQfuofeR+GXAEVU1v6qemOAw2wJrugA7D3hl134tcGiSXQGSPHcSSr4b2CHJAd2YWyTZawP6zAEe6JYGHD0J9YzmLuDFSeZ274+aovNIkiRNa1O6nKBbu7qmqtYmmVdVI5cTjOdbwLu6h8PuBq4DqKofdR+XX5JkM+BB4DefSa1V9bPua7Q+mWRbevfmz4HbJ9jnv9P7BoZ/prc8Ys4zqWc9NT6R5D3At5KsBm6Y7HNIkiS1IFV+0tySJFtX1aPdtxX8BfC9qvqzsY7Zeueta59T9tk0BUpq0tXvv7rfJUgSAElWVtXC8fr5i13t+f1uXfHt9JZb/M8+1yNJkrTJbYpvJ9Ak6mZdx5x5lSRJGnTOxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHn52dAeb92jyufv/V/S5DkiRp0jgTK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xx87mAEeufturjzk0H6XIW20Q5df2e8SJEnTjDOxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmDHyITXJmksVJliQ5tWu7IsnCUfoel+ScjTjH3CS3bUD/HyR53gb0X5jkkxtalyRJ0qAa+BAL7A9cDxwKXNXnWjZKVa2oqhP7XYckSdJ0MbAhNsnZSVYBi4BrgeOBc5Oc1nV5e5JrktyWZL9Rjt8lyaVJVnWvO3ftz0/ylSS3dH8OHHHci5PcnGRRks2TfCzJrd047x/W9f1Jbur2zeuO3a+r6ebudY+ufXGSr3fbpyc5r5tNvjeJ4VaSJM04Axtiq+oUesF1Gb0gu6qq9q6qM7ouW1XVgcB7gPNGGeIc4PNVtTfwJWDdx/mfBK6sqn2AfYHb1x3Qhc6LgXdU1Y3ACcCuwIJh46yzuqr2Bc4FPti13QUcUlULgNOA/2c9lzcPOALYD/hwki1GdkhyQpIVSVY8/POfr2cYSZKkNs3qdwFTbAEwRC/03TFi3wUAVbU8yTZJthux/wDg/+i2vwB8tNt+NXBsd+zTwMNJngPsAHwVeGNVrQu2hwOfrqqnuv4PDRv/ku515bDzbAucn2R3oIBfCaedb1TVk8CTSR4Eng/cN7xDVS0FlgLsMWdOrWccSZKkJg1kiE0yn94M7E7AamB2rzlD9MIp9ELicOMFvfH2Pwz8EDiIX8zOZozjnuxen+YXfw9/DFxeVb+bZC5wxTjHjjxekiRpRhjI5QRVNVRV84F7gD2By4Ajqmp+VT3RdTsKIMnBwMNV9fCIYa4B3tJtHw18t9u+FHh3d+zmSbbp2n8GLAGOTfK2ru07wLuSzOr6P3ec0rcF/t9u+7gJXq4kSdKMM5AhFiDJDsCaqloLzKuqkcsJ1iS5Bvg08HujDHEi8I7u4bBjgD/o2v8AOCzJrfSWAuy17oCqegx4PfCBJL8DfBb4F2BVkluAtzG2jwJnJrka2HziVytJkjSzpMrlkoNujzlzaumCfftdhrTRDl1+Zb9LkCRtIklWVtWvfJ//SAM7EytJkqTBZYiVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzZnV7wI09ebssQeHLr+y32VIkiRNGmdiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXH74mdAR6872HO+cO/73cZ0kZ735/+dr9LkCRNM87ESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaMxAhNsmZSRYnWZLk1FH2z01y2zhjzE3ytkmu6w2j1SNJkqRnZiBCLLA/cD1wKHDVRo4xF5jUEFtVX6uqsyZzTEmSJDUeYpOcnWQVsAi4FjgeODfJaUlekeSWJNcC7x12zNwkVyW5qftzYLfrLOBVSYaSfGB9/ZLsmGR51++2JK/q2l/b9bslyaVd23FJzum2d0hycZIbuz8Hde2nJzkvyRVJ7k1y4rBaj02yqhvzC2ONI0mSNJPM6ncBz0RVnZLkQuAY4GTgiqpaFw5XAe+vqiuTnD3ssAeB36yqnybZHbgAWAicCnywql7fHT97Pf3eBny7qj6SZHNgdpIdgM8Ah1TV95M8d5RyPwH8WVV9N8nOwLeBl3b75gGHAXOAu5OcC7wE+BBwUFWtHjbmWONIkiTNCE2H2M4CYIheELwDIMm2wHZVdWXX5wvAb3XbWwDnJJkPPE0vLI5mff1uBM5LsgXwd1U1lGQxsLyqvg9QVQ+NMt7hwJ5J1r3fJsmcbvsbVfUk8GSSB4HnA68GLqqq1SPGHHWcqnpk+MmSnACcAPCcOTus5xIlSZLa1GyI7cLlMmAnYDUwu9ecIXqBtdZz6AeAfwX2obec4qcb0q+qlic5BHgd8IVulvfHY5xvnc2AA6rqiRHXAfDksKan6f29ZD1jjjrOSFW1FFgKsPOv7z5ebZIkSU1pdk1sVQ1V1XzgHmBP4DLgiKqaX1UPAA8nObjrfvSwQ7cFHqiqtfSWIWzetT9C7+P8Mfsl2QV4sKo+A3wO2JfeetxDk+za9RltOcF3gPete9OF8LFcCrw5yfYjxtzQcSRJkgZOsyEWeg85AWu6oDmvqu4YtvsdwF90D3YNn7X8S+A/J7mO3hKBx7r2VcBT3UNUHxij32JgKMnNwBuBT1TVj+h9dH9JkluAvxml3BOBhd2DWncA7xrr2qrqduAjwJXdmB/fmHEkSZIGUar8pHnQ7fzru9cfHf3x8TtK09T7/vS3+12CJGkTSbKyqhaO16/pmVhJkiTNTIZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5swar0OSk0dpfhhYWVVDk1+SJEmSNLZxQyywsPvz99371wE3Au9KcmFVfXSqitPk+LWdtuV9f/rb/S5DkiRp0kwkxG4P7FtVjwIk+TBwEXAIsBIwxEqSJGmTmsia2J2Bnw17/3Ngl6p6AnhySqqSJEmSxjCRmdgvA9cl+Wr3/reBC5JsBdwxZZVJkiRJ6zFuiK2qP07yTeAgIMC7qmpFt/voqSxOkiRJGs1EZmKpqhVJ/gXYEiDJzlX1L1NamSRJkrQe466JTfKGJN8Dvg9c2b1+c6oLkyRJktZnIg92/THwSuCeqtoVOBy4ekqrkiRJksYwkRD786r6N2CzJJtV1eXA/CmuS5IkSVqviayJ/XGSrYHlwJeSPEjva7bUiAe+/0985O1H9rsMaaN96IsX9bsESdI0M5EQewvwOPABet9GsC2w9VQWJUmSJI1lIiH2sKpaC6wFzgdIsmpKq5IkSZLGsN4Qm+TdwHuA3UaE1jn4YJckSZL6aKyZ2C/T+yqtM4FTh7U/UlUPTWlVkiRJ0hjWG2Kr6mHgYeCtm64cSZIkaXwT+YotSZIkaVoxxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiF2gpKcmWRxkiVJTu3a5iUZSnJzkt2SXDOBcU5KMnsjazguyTkbc6wkSdIgMcRO3P7A9cChwFVd2xLgq1W1oKr+qaoOnMA4JwEbFWIlSZLUY4gdR5Kzk6wCFgHXAscD5yY5jV4gPT7J5V3fR7vXxUmuSHJRkruSfCk9JwIvAC4fdsx/THJtkpuSXJhk6659UZJrktyS5IYkc7qSXpDkW0m+l+Sjm/RmSJIkTROz+l3AdFdVpyS5EDgGOBm4oqoOAkiyGfBoVX1slEMXAHsB9wNXAwdV1SeTnAwcVlWrkzwP+G/A4VX1WJL/Ezg5yVnA3wBHVdWNSbYBnujGnd+N/SRwd5JPVdUPR548yQnACQDbzn72JN0NSZKk6cEQOzELgCFgHnDHBI+5oaruA0gyBMwFvjuizyuBPYGrkwA8i95s7x7AA1V1I0BV/aQbB+DSqnq4e38HsAvwKyG2qpYCSwFeuP1zaoI1S5IkNcEQO4Yk84FlwE7AanprWdOF0gPGOfzJYdtPM/q9DvAPVfXWEefdG1hf8JzIuJIkSQPNNbFjqKqhqpoP3ENvxvQy4Iiqml9VT4x99Ho9Aqxb33odcFCS3wBIMjvJS4C76K19XdS1z0liWJUkSeoYjMaRZAdgTVWtTTKvqia6nGB9lgLfTPJAVR2W5DjggiT/odv/36rqniRHAZ9K8mx662EPf4bnlSRJGhipcrnkoHvh9s+p9/zWa/pdhrTRPvTFi/pdgiRpE0mysqoWjtfP5QSSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpObP6XYCm3o677uZvz0uSpIHiTKwkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1Bx/7GAG+OkDj3DnRy7rdxnSRnvph17d7xIkSdOMM7GSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYicoyZlJFidZkuTUjRxjSZI9h72/IsnCyatSkiRpZjDETtz+wPXAocBVGznGEmDPcXtJkiRpTIbYcSQ5O8kqYBFwLXA8cG6S07qZ1D9Pck2S25Lsl2SzJN9LskN3/GZJ/jHJIcAbgLOTDCXZrTvFm5LckOSeJK/qjtkyyV8luTXJzUkO69qPS3JJkm915/joJr8hkiRJ04AhdhxVdQq94LqMXpBdVVV7V9UZXZetqupA4D3AeVW1FvgicHS3/3DglqpaDnwNOKWq5lfVP3X7Z1XVfsBJwIe7tvd253458Fbg/CRbdvvmA0cBLweOSvKiqbhuSZKk6cwQOzELgCFgHnDHiH0XAHQhdZsk2wHnAcd2+98J/NUYY1/Sva4E5nbbBwNf6Ma9C/hn4CXdvkur6uGq+mlXyy6jDZrkhCQrkqx46LEfT+QaJUmSmjGr3wVMZ0nm05uB3QlYDczuNWcIOKDrViMOq6r6YZJ/TfJqemtpj2b9nuxen+YXfx+ZQP+Rx4wsYimwFOBlL9xjZI2SJElNcyZ2DFU1VFXzgXvoPZB1GXBEtxzgia7bUQBJDgYerqqHu/bP0ltW8LdV9XTX9ggwZwKnXk4XfJO8BNgZuHsSLkmSJGkgGGLH0T2gtaZb6zqvqkYuJ1iT5Brg08DvDWv/GrA1v7yU4K+BU7qHtXZj/f4S2DzJrcDfAMdV1ZNj9JckSZpRXE4wjqr6EfC6bvuVo3S5uKr+6yjt+9B7oOuuYWNdzS9/xdbiYftW062J7da7HjdKLcvoLW9Y9/71E70OSZKkQWKInQLdjyG8m7HXwkqSJGkjGWKfgapavJ72s4CzNm01kiRJM4drYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkprjz87OAFvuOIeXfujV/S5DkiRp0jgTK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xx87mAHuv/9+Tj/99H6XIW00//mVJI3kTKwkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmz+l2AflmSM4FvA9sB86rqrDH6vgt4vKo+v6nqkyRJmg6ciZ1+9geuBw4FrhqrY1V92gArSZJmImdip4kkZwNHALsC1wK7Aa9JchHwAHAC8CzgH4FjqurxJKcDj1bVx/pTtSRJUn84EztNVNUpwPHAMmARsKqq9q6qM4BLqmpRVe0D3An83njjJTkhyYokKx5//PGpLF2SJGmTM8ROLwuAIWAecMew9pcluSrJrcDRwF7jDVRVS6tqYVUtnD179tRUK0mS1CcuJ5gGksynNwO7E7AamN1rzhBwQLdvSVXdkuQ4YHFfCpUkSZomnImdBqpqqKrmA/cAewKXAUdU1fyqegKYAzyQZAt6M7GSJEkzmjOx00SSHYA1VbU2ybyqGr6c4L/T+8aCfwZupRdqJUmSZixD7DRRVT8CXtdtv3LEvnOBc0c55vRNUpwkSdI043ICSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1JxUVb9r0BRbuHBhrVixot9lSJIkjSvJyqpaOF4/Z2IlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpObM6ncBmnpr1tzJ3164X7/LGFhvftMN/S5BkqQZx5lYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzmgixSc5MsjjJkiSndm1nJDl8nON+kOR5G3Ce7ZK855nWO8FzPdq9zk3yto04fpPVKkmSNN00EWKB/YHrgUOBqwCq6rSq+t+TfJ7tgFGDYZLNJ/lc68wFNijEdrWst1ZJkqRBN61DbJKzk6wCFgHXAscD5yY5LcmyJEd2/V6T5OYktyY5L8l/GDbM+5Pc1O2b1/U/vet3RZJ7k5zY9T0L2C3JUHfuxUkuT/Jl4Nbu2L9LsjLJ7UlO6NreneSjw+o+Lsmnuu2Tk9zW/TlplMs8C3hVd84PdDOzV3U135TkwG6ckbX8Uq2TdMslSZKaMKvfBYylqk5JciFwDHAycEVVHQSQZFn3uiWwDHhNVd2T5PPAu4E/74ZZXVX7dh+9f5BeEAaYBxwGzAHuTnIucCrwsqqa3429GNiva/t+d9w7q+qhJM8GbkxyMXARvZD9R12fo4CPJHkF8A56M8kBrk9yZVXdPOwyTwU+WFWv7845G/jNqvppkt2BC4CFXd9/ryXJ3OG1SpIkzSTTeia2swAYohc67xhl/x7A96vqnu79+cAhw/Zf0r2upPfR/TrfqKonq2o18CDw/PWc/4ZhARbgxCS3ANcBLwJ2r6ofAfcmeWWS7buargYOBr5SVY9V1aNdLa8a53q3AD6T5FbgQmDPMWpZryQnJFmRZMVPfvLURA6RJElqxrSdiU0yn94M607AamB2rzmi2G0TAAAKy0lEQVRDwAHDu44z1JPd69P88vU+OWx75L7hHhtW02LgcOCAqno8yRXAlt3uvwHeDNxFL7hWkvFqG80HgH8F9qH3Pxk/Ha2W8VTVUmApwG67bVUbUYckSdK0NW1nYqtqqPuo/B56s5GXAUdU1fyqemJY17uAuUl+o3t/DHDlRp72EXrLC9ZnW2BNF2DnAa8ctu8SYAnwVnqBFmA5sCTJ7CRbAb9L92DaGOfcFnigqtZ217K+B8rGq1WSJGlgTdsQC5BkB3qhcS0wr6pGLieoqvopvXWnF3Yfwa8FPr0x56uqfwOu7h7CGu1hqW8Bs7qHzf6Y3pKCdceuobfcYZequqFru4nebPIN9L5d4bMj1sMCrAKeSnJLkg8Afwn85yTXAS9hPbOvE6hVkiRpYKWqzU+ak/w98PGqurzftUx3u+22VZ151l79LmNgvflNN/S7BEmSBkaSlVW1cLx+03omdn2SnEdvjex3+12LJEmSNr1p+2DXWKrqnf2uQZIkSf3T5EysJEmSZjZDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmtPkz85qwzznOS/lzW+6od9lSJIkTRpnYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5vhjBzPAHWt+wj4XfbvfZQysW448ot8lSJI04zgTK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiEWSHJmksVJliQ5tWu7IsnCSRj7s0n2fOZVSpIkaR1DbM/+wPXAocBVkzlwVR1fVXdM5piSJEkz3YwOsUnOTrIKWARcCxwPnJvktK7L25Nck+S2JPt1x5ye5IPDxrgtydwkWyX5RpJburajuv3/PqOb5NEkH+n6XJfk+V37DkkuTnJj9+egrv3QJEPdn5uTzEmyY5LlXdttSV61yW6YJEnSNDGjQ2xVnUIvuC6jF2RXVdXeVXVG12WrqjoQeA9w3jjDvRa4v6r2qaqXAd8apc9WwHVVtQ+wHPj9rv0TwJ9V1SLgjcBnu/YPAu+tqvnAq4AngLcB3+7a9gGGRismyQlJViRZ8dRPHh6ndEmSpLbM6BDbWUAvCM4DRn7sfwFAVS0Htkmy3Rjj3AocnuRPkryqqkZLjj8Dvt5trwTmdtuHA+ckGQK+1p1rDnA18PEkJwLbVdVTwI3AO5KcDry8qh4ZrZiqWlpVC6tq4axtth2jbEmSpPbM6ncB/ZJkPr0Z2J2A1cDsXnOGgAO6bjXisAKe4pfD/5YAVXVPklcA/wk4M8l3hs3orvPzqlo35tP84v5vBhxQVU+M6H9Wkm90Y16X5PCqWp7kEOB1wBeSnF1Vn9/Q65ckSWrZjJ2Jraqh7iP5e4A9gcuAI6pq/rAwuW5d68HAw93s6g+Afbv2fYFdu+0XAI9X1ReBj63rM0HfAd637k0XsEmyW1XdWlV/AqwA5iXZBXiwqj4DfG4DzyNJkjQQZuxMLPQeqALWVNXaJPNG+RaBNUmuAbYB3tm1XQwc283Y3kgvBAO8HDg7yVrg58C7N6CUE4G/6B4ym0Vvvey7gJOSHEZv1vYO4JvAW4BTkvwceBQ4doMuWpIkaQDkF59ua1DN3u0ltfuffKrfZQysW448ot8lSJI0MJKsrKpxv6t/xi4nkCRJUrsMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc2Z1e8CNPX2fM42rDjyiH6XIUmSNGmciZUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmpqn7XoCmW5BHg7n7XMcCeB6zudxEDzns89bzHU8v7O/W8x1NrU97fXapqh/E6+e0EM8PdVbWw30UMqiQrvL9Ty3s89bzHU8v7O/W8x1NrOt5flxNIkiSpOYZYSZIkNccQOzMs7XcBA877O/W8x1PPezy1vL9Tz3s8tabd/fXBLkmSJDXHmVhJkiQ1xxArSZKk5hhiB1iS1ya5O8k/Jjm13/UMmiTnJXkwyW39rmVQJXlRksuT3Jnk9iR/0O+aBkmSLZPckOSW7v7+j37XNIiSbJ7k5iRf73ctgyjJD5LcmmQoyYp+1zOIkmyX5KIkd3X/Pj6g3zWBa2IHVpLNgXuA3wTuA24E3lpVd/S1sAGS5BDgUeDzVfWyftcziJLsCOxYVTclmQOsBJb4z/HkSBJgq6p6NMkWwHeBP6iq6/pc2kBJcjKwENimql7f73oGTZIfAAuryh86mCJJzgeuqqrPJnkWMLuqftzvupyJHVz7Af9YVfdW1c+AvwZ+p881DZSqWg481O86BllVPVBVN3XbjwB3Ai/sb1WDo3oe7d5u0f1xZmMSJdkJeB3w2X7XIm2MJNsAhwCfA6iqn02HAAuG2EH2QuCHw97fh//xV8OSzAUWANf3t5LB0n3UPQQ8CPxDVXl/J9efA38ErO13IQOsgO8kWZnkhH4XM4BeDPwI+KtuWcxnk2zV76LAEDvIMkqbMyxqUpKtgYuBk6rqJ/2uZ5BU1dNVNR/YCdgviUtjJkmS1wMPVtXKftcy4A6qqn2B3wLe2y310uSZBewLnFtVC4DHgGnxnI0hdnDdB7xo2PudgPv7VIu00bq1mhcDX6qqS/pdz6DqPh68Anhtn0sZJAcBb+jWbP418OokX+xvSYOnqu7vXh8EvkJvOZ0mz33AfcM+pbmIXqjtO0Ps4LoR2D3Jrt0i7LcAX+tzTdIG6R48+hxwZ1V9vN/1DJokOyTZrtt+NnA4cFd/qxocVfVfq2qnqppL79/Bl1XV2/tc1kBJslX30CfdR9z/EfAbYyZRVf1/wA+T7NE1vQaYFg/Xzup3AZoaVfVUkvcB3wY2B86rqtv7XNZASXIBsBh4XpL7gA9X1ef6W9XAOQg4Bri1W7cJ8H9V1f/qY02DZEfg/O7bTDYD/raq/BooteT5wFd6/7/LLODLVfWt/pY0kN4PfKmbFLsXeEef6wH8ii1JkiQ1yOUEkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUlTKslJSWb3uw5Jg8Wv2JIkTanuF6sWVtXqftciaXA4EytJIsmxSVYluSXJF5LskuTSru3SJDt3/ZYlOXLYcY92r4uTXJHkoiR3JflSek4EXgBcnuTy/lydpEHkL3ZJ0gyXZC/gQ8BBVbU6yXOB84HPV9X5Sd4JfBJYMs5QC4C9gPuBq7vxPpnkZOAwZ2IlTSZnYiVJrwYuWhcyq+oh4ADgy93+LwAHT2CcG6rqvqpaCwwBc6egVkkCDLGSJAgw3gMS6/Y/RfffjvR+sP5Zw/o8OWz7afy0T9IUMsRKki4F3pxke4BuOcE1wFu6/UcD3+22fwC8otv+HWCLCYz/CDBnsoqVJPD/kiVpxquq25N8BLgyydPAzcCJwHlJTgF+BLyj6/4Z4KtJbqAXfh+bwCmWAt9M8kBVHTb5VyBpJvIrtiRJktQclxNIkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJas7/D9PzIcybrQI9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "while count < 10:\n",
    "    \n",
    "    time.sleep( 3 )\n",
    "    top_10_tweets = sqlContext.sql( 'Select tag, count from tweets' )\n",
    "    top_10_df = top_10_tweets.toPandas()\n",
    "    #top_10_df.sort(desc(\"count\"))\n",
    "    #print(top_10_df)\n",
    "    display.clear_output(wait=True)\n",
    "    plt.figure( figsize = ( 10, 8 ) )\n",
    "    sns.barplot( x=\"count\", y=\"keyword\", data=top_10_df)\n",
    "    plt.show()\n",
    "    count = count + 1\n",
    "    print (count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ssc.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}