three_Bigrams.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['austen-emma.txt',\n",
       " 'austen-persuasion.txt',\n",
       " 'austen-sense.txt',\n",
       " 'bible-kjv.txt',\n",
       " 'blake-poems.txt',\n",
       " 'bryant-stories.txt',\n",
       " 'burgess-busterbrown.txt',\n",
       " 'carroll-alice.txt',\n",
       " 'chesterton-ball.txt',\n",
       " 'chesterton-brown.txt',\n",
       " 'chesterton-thursday.txt',\n",
       " 'edgeworth-parents.txt',\n",
       " 'melville-moby_dick.txt',\n",
       " 'milton-paradise.txt',\n",
       " 'shakespeare-caesar.txt',\n",
       " 'shakespeare-hamlet.txt',\n",
       " 'shakespeare-macbeth.txt',\n",
       " 'whitman-leaves.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.corpus.gutenberg.fileids( ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk import FreqDist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['austen-emma.txt', 'austen-persuasion.txt', 'austen-sense.txt', 'bible-kjv.txt', 'blake-poems.txt', 'bryant-stories.txt', 'burgess-busterbrown.txt', 'carroll-alice.txt', 'chesterton-ball.txt', 'chesterton-brown.txt', 'chesterton-thursday.txt', 'edgeworth-parents.txt', 'melville-moby_dick.txt', 'milton-paradise.txt', 'shakespeare-caesar.txt', 'shakespeare-hamlet.txt', 'shakespeare-macbeth.txt', 'whitman-leaves.txt']\n"
     ]
    }
   ],
   "source": [
    "print(nltk.corpus.gutenberg.fileids( )) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file0 = nltk.corpus.gutenberg.fileids( ) [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emmatext = nltk.corpus.gutenberg.raw(file0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "emmatext = nltk.corpus.gutenberg.raw('austen-emma.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emmatokens = nltk.word_tokenize(emmatext) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emmawords = [w.lower( ) for w in emmatokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "191673"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(emmawords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emmawords1 = nltk.corpus.gutenberg.words('austen-emma.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emmawordslowercase = [w.lower() for w in emmawords1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "191673"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(emmawords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "192427"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(emmawordslowercase)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[',\n",
       " 'emma',\n",
       " 'by',\n",
       " 'jane',\n",
       " 'austen',\n",
       " '1816',\n",
       " ']',\n",
       " 'volume',\n",
       " 'i',\n",
       " 'chapter',\n",
       " 'i',\n",
       " 'emma',\n",
       " 'woodhouse',\n",
       " ',',\n",
       " 'handsome',\n",
       " ',',\n",
       " 'clever',\n",
       " ',',\n",
       " 'and',\n",
       " 'rich',\n",
       " ',',\n",
       " 'with',\n",
       " 'a',\n",
       " 'comfortable',\n",
       " 'home',\n",
       " 'and',\n",
       " 'happy',\n",
       " 'disposition',\n",
       " ',',\n",
       " 'seemed']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emmawords[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[',\n",
       " 'emma',\n",
       " 'by',\n",
       " 'jane',\n",
       " 'austen',\n",
       " '1816',\n",
       " ']',\n",
       " 'volume',\n",
       " 'i',\n",
       " 'chapter',\n",
       " 'i',\n",
       " 'emma',\n",
       " 'woodhouse',\n",
       " ',',\n",
       " 'handsome',\n",
       " ',',\n",
       " 'clever',\n",
       " ',',\n",
       " 'and',\n",
       " 'rich',\n",
       " ',',\n",
       " 'with',\n",
       " 'a',\n",
       " 'comfortable',\n",
       " 'home',\n",
       " 'and',\n",
       " 'happy',\n",
       " 'disposition',\n",
       " ',',\n",
       " 'seemed']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emmawordslowercase[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['to',\n",
       " 'unite',\n",
       " 'some',\n",
       " 'of',\n",
       " 'the',\n",
       " 'best',\n",
       " 'blessings',\n",
       " 'of',\n",
       " 'existence',\n",
       " ';',\n",
       " 'and',\n",
       " 'had',\n",
       " 'lived',\n",
       " 'nearly',\n",
       " 'twenty-one',\n",
       " 'years',\n",
       " 'in',\n",
       " 'the',\n",
       " 'world',\n",
       " 'with']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emmawords[30:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['to',\n",
       " 'unite',\n",
       " 'some',\n",
       " 'of',\n",
       " 'the',\n",
       " 'best',\n",
       " 'blessings',\n",
       " 'of',\n",
       " 'existence',\n",
       " ';',\n",
       " 'and',\n",
       " 'had',\n",
       " 'lived',\n",
       " 'nearly',\n",
       " 'twenty',\n",
       " '-',\n",
       " 'one',\n",
       " 'years',\n",
       " 'in',\n",
       " 'the']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emmawordslowercase[30:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[',\n",
       " 'emma',\n",
       " 'by',\n",
       " 'jane',\n",
       " 'austen',\n",
       " '1816',\n",
       " ']',\n",
       " 'volume',\n",
       " 'i',\n",
       " 'chapter',\n",
       " 'i',\n",
       " 'emma',\n",
       " 'woodhouse',\n",
       " ',',\n",
       " 'handsome',\n",
       " ',',\n",
       " 'clever',\n",
       " ',',\n",
       " 'and',\n",
       " 'rich',\n",
       " ',',\n",
       " 'with',\n",
       " 'a',\n",
       " 'comfortable',\n",
       " 'home',\n",
       " 'and',\n",
       " 'happy',\n",
       " 'disposition',\n",
       " ',',\n",
       " 'seemed',\n",
       " 'to',\n",
       " 'unite',\n",
       " 'some',\n",
       " 'of',\n",
       " 'the',\n",
       " 'best',\n",
       " 'blessings',\n",
       " 'of',\n",
       " 'existence',\n",
       " ';',\n",
       " 'and',\n",
       " 'had',\n",
       " 'lived',\n",
       " 'nearly',\n",
       " 'twenty-one',\n",
       " 'years',\n",
       " 'in',\n",
       " 'the',\n",
       " 'world',\n",
       " 'with',\n",
       " 'very',\n",
       " 'little',\n",
       " 'to',\n",
       " 'distress',\n",
       " 'or',\n",
       " 'vex',\n",
       " 'her',\n",
       " '.',\n",
       " 'she',\n",
       " 'was',\n",
       " 'the',\n",
       " 'youngest',\n",
       " 'of',\n",
       " 'the',\n",
       " 'two',\n",
       " 'daughters',\n",
       " 'of',\n",
       " 'a',\n",
       " 'most',\n",
       " 'affectionate',\n",
       " ',',\n",
       " 'indulgent',\n",
       " 'father',\n",
       " ';',\n",
       " 'and',\n",
       " 'had',\n",
       " ',',\n",
       " 'in',\n",
       " 'consequence',\n",
       " 'of',\n",
       " 'her',\n",
       " 'sister',\n",
       " \"'s\",\n",
       " 'marriage',\n",
       " ',',\n",
       " 'been',\n",
       " 'mistress',\n",
       " 'of',\n",
       " 'his',\n",
       " 'house',\n",
       " 'from',\n",
       " 'a',\n",
       " 'very',\n",
       " 'early',\n",
       " 'period',\n",
       " '.',\n",
       " 'her',\n",
       " 'mother',\n",
       " 'had',\n",
       " 'died',\n",
       " 'too',\n",
       " 'long',\n",
       " 'ago',\n",
       " 'for',\n",
       " 'her',\n",
       " 'to',\n",
       " 'have',\n",
       " 'more',\n",
       " 'than',\n",
       " 'an']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emmawords[:110]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ndist = FreqDist(emmawords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nitems = ndist.most_common(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ", 12016\n",
      ". 6355\n",
      "the 5198\n",
      "to 5179\n",
      "and 4875\n",
      "of 4284\n",
      "i 3164\n",
      "a 3124\n",
      "-- 3100\n",
      "it 2500\n",
      "'' 2452\n",
      "her 2448\n",
      "was 2396\n",
      "; 2353\n",
      "she 2336\n",
      "not 2279\n",
      "in 2173\n",
      "be 1970\n",
      "you 1962\n",
      "he 1806\n",
      "that 1804\n",
      "`` 1735\n",
      "had 1623\n",
      "but 1439\n",
      "as 1436\n",
      "for 1346\n",
      "have 1319\n",
      "is 1241\n",
      "with 1215\n",
      "very 1202\n"
     ]
    }
   ],
   "source": [
    "for item in nitems:\n",
    "      print (item[0], item[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emmatextText = nltk.Text(emmawords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbEAAAEWCAYAAADoyannAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAH21JREFUeJzt3Xm4XFWZ7/HvLwSIECBAoqBCDigO\ngBrgSCMGE7XFCbnQSoNiQ5xQW+yOiDY82BC5ei84261tcKAPKnBBlBboVkCaoIAMCSQGRCRiQGSG\nDoMCQvLeP9banH2Kms5cC36f56nn7Fp77bXetfauerN37VQpIjAzMyvRlMkOwMzMbKScxMzMrFhO\nYmZmViwnMTMzK5aTmJmZFctJzMzMiuUkZlYj6SeSDh1lGwskXTrKNq6XNH80bYylsZiXEfS5SNL3\nJ7JPK4+TmBVL0mpJfz2WbUbEmyPilLFss05Sn6SQ9HB+3CXpPElvaIhjp4hYMl5xDNd4zYukAUl/\nyXNxv6QLJb1kBO2M+bFgZXASM5scMyJiOvAK4ELgbEkLJisYSVMnq2/gc3kung/cDQxMYixWGCcx\ne1qStI+k5ZLWSLpc0stz+Qvyv/h3zc+fK+ne6tKdpCWS3l9r5wOSbpD0kKRf17Y7StLvauX7jyTO\niLgzIr4KLAJOlDQlt//kmYWk3SUtlfRgPnP7Ui6vzuoOk3S7pDskfbwW+5RanPdJOlPSFg3bvk/S\nrcB/S5om6fu57hpJV0t6TuO85HY/JekWSXdL+q6kzRraPVTSrXluj+lyLv4MnAbs3Gy9pH3zZdY1\nOZ6X5vLvAdsC5+Yzuk8Odz9YuZzE7GknJ5qTgQ8CWwInAedI2jAifgf8E3CqpI2AfwcGml26k3QA\nKbkcAmwK7Avcl1f/DtgL2Az4NPB9SVuPIuwfAc8GXtxk3VeBr0bEpsALgDMb1r8W2AHYGziqdlnt\nH4D9gHnAc4H/Ab7esO084KXAG4FD83i2Ic3bh4BHmsSzID9eC2wPTAe+1lBnbh7L64Fjq4TTjqTp\nwMHAtU3WvQg4HVgIzAL+i5S0NoiIvwNuBd4WEdMj4nOd+rKnDycxezr6AHBSRFwZEWvzZzmPAXsA\nRMS3gJuAK4GtgVZnCu8nXeq6OpJVEXFLbuMHEXF7RKyLiDNye7uPIubb898tmqx7HHihpJkR8XBE\nXNGw/tMR8aeIWElKyu/M5R8EjomI2yLiMVJCfkfDpcNFedtHcj9bAi/M87YsIh5sEs/BwJci4uaI\neBg4Gjiood1PR8QjEbECWEG6bNrKkZLWAKtICXFBkzoHAv8ZERdGxOPAF4BnAXu2adeeAZzE7Olo\nNvDxfNlpTX6D3IZ0NlL5Fumy1b/mN/hmtiGdcT2FpENqlyvX5LZmjiLm5+W/9zdZ9z7gRcBv8iW+\nfRrW/6G2fAuD45xN+qytivEGYC3wnBbbfg84H/h/+fLk5ySt3ySe5+Z+6n1ObWj3ztryn0nJqZUv\nRMSMiNgqIvbNZ8tt+4yIdTn25zWpa88gTmL2dPQH4LP5jbF6bBQRp8OTl62+AnwHWFR9TtSinRc0\nFkqaTUqChwNbRsQM4DpAo4h5f9JNDTc2roiImyLinaTLjScCZ0nauFZlm9rytgye1f0BeHPDPEyL\niD/Wm6/183hEfDoidiSd4exDupTa6HZSgqz3+QRwV5djHYkhfUoSadzVWPxzHM9QTmJWuvXzDQnV\nYyopwXxI0l8p2VjSWyVtkrf5KrAsIt4P/CewuEXb3yZd6tott/PCnMA2Jr1p3gMg6T20uBmhE0nP\nkXQ4cBxwdD7DaKzzbkmz8ro1uXhtrco/S9pI0k7Ae4Azcvli4LM5ZiTNkvS/2sTyWkkvk7Qe8CDp\n8uLaJlVPBz4mabv8D4L/A5wREU8MZ+zDdCbwVkmvz2eHHyddIr48r7+L9PmcPcM4iVnp/ot080H1\nWBQRS0mfi32NdDPDKvLnLPlN/E2kmxYAjgB2lXRwY8MR8QPgs6Q75h4C/gPYIiJ+DXwR+CXpzfNl\nwGXDjHuNpD8BK4G3AAdExMkt6r4JuF7Sw6QEfFBEPFpbf0ke40WkS3MX5PKvAucAF0h6CLgC+Ks2\nMW0FnEVKYDfkdpv9Z+OTSZcefw78HngU+Gj74Y5ORNwIvBv4V+Be4G2kGzn+kqv8X+BT+dLpkeMZ\ni/UW+UcxzcokqY+URNYf57Mgs57lMzEzMyuWk5iZmRXLlxPNzKxYPhMzM7NiTeaXfhZl5syZ0dfX\nN9lhmJkVZdmyZfdGxKzxat9JrEt9fX0sXbp0ssMwMyuKpFs61xo5X040M7NiOYmZmVmxnMTMzKxY\nTmJmZlYsJzEzMyuWk5iZmRXLSczMzIrlJGZmZsVyEjMzs2I5iZmZWbGcxMzMrFhOYmZmViwnMTMz\nK5aTmJmZFctJzMzMiuUkZmZmxXISMzOzYjmJmZlZsZzEzMysWE5iZmZWLCcxMzMrlpOYmZkVy0nM\nzMyK5SRmZmbFchIzM7NiOYmZmVmxnMTMzKxYTmJmZlYsJzEzMyuWk5iZmRXLSczMzIrlJGZmZsVy\nEjMzs2I5iZmZWbGcxMzMrFhOYmZmViwnMTMzK5aTmJmZFctJzMzMiuUkZmZmxXISMzOzYjmJmZlZ\nsZzEzMysWE5iZmZWLCcxMzMrlpOYmZkVy0nMzMyK5SRmZmbFchIzM7NiOYmZmVmxnMQmyNSpMH9+\nevT1wZQpsGhRekAqmz8/PZ8xIz2q55CWq78STJuW/kpp276+VLfqY8aMwfLq+bRp6XnVdl/fYGxV\nDNOmpeV6v63irtqu2qm3W/Vfjacay9Spg9tWj2nThvZRH3993qp+q3E2tl31t2hRarMqq4+n/rca\nz7RpQ/us2qu3W39MmTK0Xn1dY9+N+7QeZzVnjX1XbVVl1d9qfzTOdb2PavyN+6LZPE6bNnTu6/u1\n3jcMnaP6WCrVcVHNbX3uK1W71TiqearmWhq6P+vzXD9+q2NUGlw/derQ+a3iqI+jflw2vjaqY7Mq\nqx5VrPUxV/M2bVrapnoNVttXdadMGeyvOs7q+7Bet3FOqtd34zzVt28cYxVD476q5rd+LDTOR/24\nq/qs1ktD93WvUURMdgxF6O/vj6VLl454e6n1uoju1neqNxLN2h5Jf+MR21hpHE+vxDrcOCYr/nb9\nVW8fjcdO4/p2dcY6prHqYyzbaWyv2Tx10099e5j442AkJC2LiP6xjWZQT56JSbxb4iqJ5RInSawn\n8bDEiRLLJH4msbvEEombJfbN2y2Q+A+JcyV+L3G4xBES10pcIbFFrvcBiaslVkj8UGKjyR2xmZmN\nRM8lMYmXAgcCr45gDrAWOBjYGFgSwW7AQ8BngDcA+wPH15rYGXgXsDvwWeDPEewC/BI4JNf5UQSv\njOAVwA3A+5rHosMkLZW09J577hnjkZqZ2WhNnewAmng9sBtwdT5VfhZwN/AX4Ke5zkrgsQgel1gJ\n9NW2vziCh4CHJB4Azq1t8/K8vLPEZ4AZwHTg/GaBRMQ3gW9Cupw4FoMzM7Ox04tJTMApERw9pFAc\nGUGVSNYBjwFEsE4aMo7Hasvras/XMTjeAWC/CFZILADmj+UAzMxsYvRiErsI+LHElyO4O3+OtckY\n97EJcIfE+qRLlX8c4/afYr31YO7ctLx6Ndx6Kxx77OD62bMH7wz6yldS2Zw5g3cizZs3+PeSS2DD\nDeGxxwa3BViwAJYsScvLlw/e9dTXl54/+ihstRWsWZPaXr16MLaqnTvvTMvHHTe032ZxDwyktqt2\nNttssN01a1L/Cxak8VRj+cxn4FOfSttW7rwT9thjsI81awbHf+mlg/NWvytryZLBuaraHhhI/QGc\ncAIcddRgP9V46n8HBtJ4Nthg8M6xOXMG+6r6qMcKaZvXvGawXn39ggVD+27cpwsXDo1zs80Gt636\nruaz2k/Llw+uq/ZHfa7rfSxcmNZX81LN58MPP3UeTzgh/a3mviofGBjaN6TjrZqjStUXpBsMquOi\nmuPGeZs9e3DfzpsHV1yR5gnSXF9yyeC45s+H448fnOf68Vsdo/X5uPRSmD59cH6rOKp5qF4P1XFZ\nqV4bCxemY/P5z3/qXZXz5g3Ow8KFg/MG8MQTsHZtGtttt6XtqzE++CBsumnqrzrO6jHX56NxTk44\nIb2+G+epvn3jGFevTjFUdyhWFi5M81s/FqZPHzof9ddxX1/qs3qveOCBwX3ai3ry7kSJA4GjSZ/Z\nPQ58BPhZBNPz+kXAwxF8IT9/OILp+ayqP4LDc/nq/Pze+jqJDwOfBG4hXWbcJIIF7WIa7d2JZmbP\nRON9d2JPJrFe5CRmZjZ8z8hb7M3MzLrhJGZmZsVyEjMzs2I5iZmZWbGcxMzMrFhOYmZmViwnMTMz\nK5aTmJmZFctJzMzMiuUkZmZmxXISMzOzYjmJmZlZsZzEzMysWE5iZmZWLCcxMzMrlpOYmZkVy0nM\nzMyK5SRmZmbFchIzM7NiOYmZmVmxnMTMzKxYTmJmZlYsJzEzMyuWk5iZmRXLSczMzIrlJGZmZsVy\nEjMzs2I5iZmZWbGcxMzMrFhOYmZmViwnMTMzK5aTmJmZFctJzMzMiuUkZmZmxXISMzOzYjmJmZlZ\nsZzEzMysWE5iZmZWLCcxMzMrlpOYmZkVy0nMzMyK5SRmZmbFchIzM7NiOYmZmVmxnMTMzKxYk5LE\nJA6QuEHi4mFs0yfxri7qzZc4r8W61RIzhxPrWFm0aOjz+fO7r9vp+Xjptp9m9VrFXC9ftKj5PIym\n37HU1ze2MUz0fh2P+SmlzXrbrea53Wtwok3Ua7obk/V+M1KKiInvVPwUODGiuyQmMRWYCxwZwT4d\n6s5vVU9iNdAfwb3Djbm/vz+WLl063M3qfVOf6sbnw6nbbtux1G0/zeq1irleLqW/nbYdbXwj1U37\nw4lhovfreMxPKW3W24bOx+Jk6+VYRhubpGUR0T/6yJrr6kxM4hCJX0mskPiexGyJi3LZRRLb5noD\nEt+QuFjiZol5Eifns66BXOdYUkJaLPF5iWkS/y6xUuJaidfmegskfiBxLnABcAKwl8RyiY/lM7Nf\nSFyTH3vWQt5U4myJX0sslp46Tol3S1yV2ztJYr1RzaSZmU24qZ0qSOwEHAO8OoJ7JbYATgG+G8Ep\nEu8F/gXYL2+yOfA6YF/gXODVwPuBqyXmRHC8xOtIZ0tLJT4OEMHLJF4CXCDxotzWq4CXR3B/4xmW\nxEbAGyJ4VGIH4HSgyva7AzsCtwA/Bf4GOKs2ppcCB+YxPS7xb8DBwHeHjl2HAYcBbLvtth0n08zM\nJlY3Z2KvA86qLsFFcD8puZyW13+PdGZVOTeCAFYCd0WwMoJ1wPVAX5P25+Y2iOA3pMRTJbELc3/N\nrA98S2Il8ANS0qpcFcHNEawlJbe5Ddu+HtiNlFiX5+fbN3YQEd+MiP6I6J81a1aLMMzMbLJ0PBMD\nBHS6Ilpf/1j+u662XD1v1p/atPunNus+BtwFvIKUjB9tEU+z5wJOieDoNu2bmVmP6+ZM7CLgbyW2\nBMiXEy8HDsrrDwYuHUUMP89tkC8jbgvc2KTeQ8AmteebAXfks7y/gyGfae0usV3+LOzAJvFdBLxD\n4tnVmCRmj2IMHR133NDn8+Z1X7fT8/HSbT/N6rWKuV5+3HHN52E0/Y6l2V0cEcOJYaL363jMTylt\n1ttuNc/tXoMTbaJe092YrPebkerq7kSJQ4FPAGuBa4FFwMnATOAe4D0R3Jpv3jgvgrMk+vLyzrmN\n+rolDH4mNg1YTLq89wRwRAQXSywg3Ul4eN5+fdLnWzMhtQX8EPgzcDHw0Qim58/Ojs1xvYyUJP8+\ngnX1uxMlDgSOJiXyx4GPRHBFqzkY7d2JZmbPRON9d+Kk3GJfIicxM7Ph64lb7M3MzHqRk5iZmRXL\nSczMzIrlJGZmZsVyEjMzs2I5iZmZWbGcxMzMrFhOYmZmViwnMTMzK5aTmJmZFctJzMzMiuUkZmZm\nxXISMzOzYjmJmZlZsZzEzMysWE5iZmZWLCcxMzMrlpOYmZkVy0nMzMyK5SRmZmbFchIzM7NiOYmZ\nmVmxnMTMzKxYTmJmZlYsJzEzMyuWk5iZmRXLSczMzIrlJGZmZsVyEjMzs2I5iZmZWbGcxMzMrFhO\nYmZmViwnMTMzK5aTmJmZFctJzMzMiuUkZmZmxXISMzOzYjmJmZlZsZzEzMysWE5iZmZWLCcxMzMr\nlpOYmZkVy0nMzMyK5SRmZmbFchIzM7Ni9WQSk+iTuG6y4xhLixYNXa4/r5s/v3OdTm339T11fb29\n+t9WfXTbd7O6jf1UY2q3fTexNI6zWd+t2m7crlP7lfnz2++7dv23irfbbZq10aydZv1UcTfug/q+\nqNbXx1hvq91+62bep0x5almjZvNb77cx3uHG0WoMnfZj/XU4HJ2O5cY2G8fbKp562+3qtTvOWo23\n2XFVf964j3qNImKyY3gKiT7gvAh2nuxYKv39/bF06dIRby9BNdVS+tts6qt67eoMt+2qrCrvpo96\nm8Ppv3EM9b/ttq/H16r9xnEOp+3G7Tq13277Tu01ttvYR6txNmuv1b5tXG7VT2OdZu01qzecsbVa\n3zjO0c5vu33XLo5Wx1SrfurPh/M6qLdbb7tdTK2Oh27mqlPc7ep2Oq5aPR8JScsion9kW3fWk2di\n2XoS35K4XuICiWdJfEDiaokVEj+U2AhAYkBiscQvJH4rsU8uXyDxY4mfStwocVwu/98S/1h1JPFZ\niX+YnGGamdlI9XIS2wH4egQ7AWuAtwM/iuCVEbwCuAF4X61+HzAPeCuwWGJaLt8dOBiYAxwg0Q98\nBzgUQGIKcBBwamMAkg6TtFTS0nvuuWcchmhmZqPRy0ns9xEsz8vLSElq53y2tZKUmHaq1T8zgnUR\n3ATcDLwkl18YwX0RPAL8CJgbwWrgPoldgL2BayO4rzGAiPhmRPRHRP+sWbPGY4xmZjYKUyc7gDYe\nqy2vBZ4FDAD7RbBCYgEwv1an8YptdCj/NrAA2Ao4edTRmpnZhOvlJNbMJsAdEuuTzsT+WFt3gMQp\nwHbA9sCNwC7AGyS2AB4B9gPem+ufDRwPrA+8a7wDP+645suN5s3rXKdT2wMDrdfXn7frY6T9N2u/\nGtNI+2oWbzdjaBdbp/Yr8+YNvXusm/aarRuLeIez3Bh3tQ/q+6Kqu2TJYN16W+32WzfjOP74zts0\nm98lS5rH0G7ftdJqDJ3240heh93Wb9xf9fG2a6ebsnbH2XCPpXp8ne4wnkxF3J0ocSQwHbgL+CRw\nC7AS2CSCBRIDwP8A/cBzgCMiOC+frb0F2Bh4IXBaBJ+u9bMYWBPBUZ1iGu3diWZmz0TjfXdiT56J\n5c+sdq49/0Jt9TdabHZZBB9rUn53BIc3FuYbOvYADhhFqGZmNol6+caOcSOxI7AKuCjfCGJmZgXq\nyTOx4YpgQYvyAdLNII3lvyZ9bmZmZgV7Rp6JmZnZ04OTmJmZFctJzMzMiuUkZmZmxXISMzOzYjmJ\nmZlZsZzEzMysWE5iZmZWLCcxMzMrlpOYmZkVy0nMzMyK5SRmZmbFchIzM7NiOYmZmVmxnMTMzKxY\nTmJmZlYsJzEzMyuWk5iZmRXLSczMzIrlJGZmZsVyEjMzs2I5iZmZWbGcxMzMrFhOYmZmViwnMTMz\nK5aTmJmZFctJzMzMiuUkZmZmxXISMzOzYjmJmZlZsZzEzMysWE5iZmZWLCcxMzMrlpOYmZkVy0nM\nzMyK5SRmZmbFchIzM7NiOYmZmVmxnMTMzKxYTmJmZlYsJzEzMyuWk5iZmRXLSczMzIrlJGZmZsVy\nEjMzs2I5iZmZWbGcxMzMrFiKiMmOoQiS7gFuGUUTM4F7xyicsdbLsYHjG61ejq+XYwPHN1ozgY0j\nYtZ4deAkNkEkLY2I/smOo5lejg0c32j1cny9HBs4vtGaiPh8OdHMzIrlJGZmZsVyEps435zsANro\n5djA8Y1WL8fXy7GB4xutcY/Pn4mZmVmxfCZmZmbFchIzM7NiOYmNM0lvknSjpFWSjhrHfraRdLGk\nGyRdL+kfc/kiSX+UtDw/3lLb5ugc142S3tgpZknbSbpS0k2SzpC0wTBjXC1pZY5jaS7bQtKFuc0L\nJW2eyyXpX3IMv5K0a62dQ3P9myQdWivfLbe/Km+rYcT24tocLZf0oKSFkzl/kk6WdLek62pl4z5f\nrfroMr7PS/pNjuFsSTNyeZ+kR2rzuHikcbQba4fYxn1fStowP1+V1/cNY+7OqMW2WtLyyZi7XK/V\n+0nPHH9Pigg/xukBrAf8Dtge2ABYAew4Tn1tDeyalzcBfgvsCCwCjmxSf8ccz4bAdjnO9drFDJwJ\nHJSXFwMfHmaMq4GZDWWfA47Ky0cBJ+bltwA/AQTsAVyZy7cAbs5/N8/Lm+d1VwGvytv8BHjzKPbb\nncDsyZw/4DXArsB1EzlfrfroMr69gal5+cRafH31eg3tDCuOVmPtIrZx35fA3wOL8/JBwBndzl3D\n+i8Cx07G3HV4P+mZ4+/JWEfyIveju0feQefXnh8NHD1Bff8YeEObF+6QWIDzc7xNY84H2r0MvkEN\nqddlTKt5ahK7Edg6L28N3JiXTwLe2VgPeCdwUq38pFy2NfCbWvmQesOMc2/gsrw8qfNHwxvYRMxX\nqz66ia9h3f7Aqe3qjSSOVmPtYu7GfV9W2+blqbmehjN3uf0/ADtM1tw16at6P+mp4y8ifDlxnD2P\ndDBWbstl4ypfwtgFuDIXHZ5P8U+unZq3iq1V+ZbAmoh4oqF8OAK4QNIySYflsudExB0A+e+zRxjf\n8/JyY/lIHAScXnveK/MHEzNfrfoYrveS/oVd2U7StZIukbRXLe7hxjGa19V478snt8nrH8j1h2Mv\n4K6IuKlWNmlz1/B+0nPHn5PY+Gr2mcy4/p8GSdOBHwILI+JB4BvAC4A5wB2kyxTtYhtu+XC8OiJ2\nBd4MfETSa9rUnYz4yJ9t7Av8IBf10vy101PxSDoGeAI4NRfdAWwbEbsARwCnSdp0hHGMNPaJ2Jdj\nMa/vZOg/oiZt7pq8nwy33XE//pzExtdtwDa1588Hbh+vziStTzrgTo2IHwFExF0RsTYi1gHfAnbv\nEFur8nuBGZKmjnQsEXF7/ns3cHaO5S5JW+f4twbuHmF8t+XlxvLhejNwTUTclWPtmfnLJmK+WvXR\nlfzh/T7AwZGvCUXEYxFxX15eRvqs6UUjjGNEr6sJ2pdPbpPXbwbc3ym2St7mb4AzanFPytw1ez8Z\nQbvjfvw5iY2vq4EdlO5k2oB0meqc8ego39nzHeCGiPhSrXzrWrX9gepuqHOAg5TuptoO2IH0QWvT\nmPOb0cXAO/L2h5Kuk3cb38aSNqmWSZ87XZfjOLRJm+cAh+S7nvYAHsiXFs4H9pa0eb4ctDfp84g7\ngIck7ZHn4pDhxFcz5F/BvTJ/NRMxX6366EjSm4B/AvaNiD/XymdJWi8vb0+ar5tHGEersXaKbSL2\nZT3mdwD/XSXyLv016bOiJy+1TcbctXo/GUG743/8dfpAz4/RPUh37fyW9K+nY8axn7mk0/FfAcvz\n4y3A94CVufwcah/iAsfkuG6kdidfq5hJd2ldBawiXW7bcBjxbU+6u2sFcH3VLunzgouAm/LfLXK5\ngK/nGFYC/bW23ptjWAW8p1beT3pj+h3wNVp8oN4mxo2A+4DNamWTNn+kZHoH8DjpX67vm4j5atVH\nl/GtIn0GUh2D1Z16b8/7fQVwDfC2kcbRbqwdYhv3fQlMy89X5fXbdzt3uXwA+FBD3Qmduw7vJz1z\n/FUPf+2UmZkVy5cTzcysWE5iZmZWLCcxMzMrlpOYmZkVy0nMzMyK5SRmNoYkfVnSwtrz8yV9u/b8\ni5KOGEX7iyQd2WLdYUrfIP8bSVdJmltbt5fSt5Evl/QspW+bv17S54fZf5+kd400frOx5iRmNrYu\nB/YEkDQFmAnsVFu/J3BZNw1V/8G1y7r7AB8E5kbES4APkb6eaKtc5WDgCxExJyIeyXV3jYhPdNtH\n1gc4iVnPcBIzG1uXkZMYKXldR/pmgs0lbQi8FLg2f7PB5yVdp/SbSgcCSJqv9DtOp5H+0yiSjlH6\nTaufAS9u0e8/AZ+IiHsBIuIa4BTSd1S+H/hb4FhJp0o6B9gYuFLSgZIOyHGskPTz3Od6Ob6rlb4w\n94O5nxOAvfIZ3cfGcuLMRmJq5ypm1q2IuF3SE5K2JSWzX5K+nftVpG80/1VE/EXS20lfRPsK0tna\n1VUCIX2n384R8XtJu5G+7mgX0uv1GmBZk653alK+FDg0Iv45X1o8LyLOApD0cETMycsrgTdGxB+V\nf8SS9A0XD0TEK3PyvUzSBaTfdzoyIvYZ3UyZjQ0nMbOxV52N7Ql8iZTE9iQlsctznbnA6RGxlvSF\np5cArwQeBK6KiN/nensBZ0f+HsJ8FtUt0d03g18GDEg6E6i+6HVv4OWSqu8H3Iz0nX1/GUb/ZuPO\nlxPNxl71udjLSJcTryCdidU/D2v2UxSVPzU87yYR/RrYraFs11zeVkR8CPgU6dvGl0vaMsf30fwZ\n2pyI2C4iLugiDrMJ5SRmNvYuI/0Uyf2RfvrjfmAGKZH9Mtf5OXBg/uxpFunn6q9q0tbPgf3zHYWb\nAG9r0efngBNzAkLSHGAB8G+dgpX0goi4MiKOJf3MyDakbx//sNLPcSDpRUq/PvAQ6efqzXqCLyea\njb2VpM+5Tmsom17deEH6PbVXkb6ZPIBPRsSdkl5SbygirpF0BulbxG8BftGsw4g4R9LzgMslBSnZ\nvDu6+IkS4POSdiCdfV2UY/oV6U7Ea/JPZdwD7JfLn5C0AhiIiC930b7ZuPG32JuZWbF8OdHMzIrl\nJGZmZsVyEjMzs2I5iZmZWbGcxMzMrFhOYmZmViwnMTMzK9b/B+lkqSF9IxGaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1727037ecc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emmatextText.dispersion_plot(['emma', 'comfortable', 'happy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pattern = re.compile('^[^a-z]+$')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nonAlphaMatch = pattern.match('**')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matched non-alphabetical\n"
     ]
    }
   ],
   "source": [
    "if nonAlphaMatch: print ('matched non-alphabetical')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def alpha_filter(w):\n",
    "  # pattern to match a word of non-alphabetical characters\n",
    "    pattern = re.compile('^[^a-z]+$')\n",
    "    if (pattern.match(w)):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "alphaemmawords = [w for w in emmawords if not alpha_filter(w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "161456"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alphaemmawords[:110]\n",
    "len(alphaemmawords)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "156935"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alphaemmawords1 = [w for w in emmawords if w.isalpha()]\n",
    "len(alphaemmawords1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['emma',\n",
       " 'by',\n",
       " 'jane',\n",
       " 'austen',\n",
       " 'volume',\n",
       " 'i',\n",
       " 'chapter',\n",
       " 'i',\n",
       " 'emma',\n",
       " 'woodhouse',\n",
       " 'handsome',\n",
       " 'clever',\n",
       " 'and',\n",
       " 'rich',\n",
       " 'with',\n",
       " 'a',\n",
       " 'comfortable',\n",
       " 'home',\n",
       " 'and',\n",
       " 'happy',\n",
       " 'disposition',\n",
       " 'seemed',\n",
       " 'to',\n",
       " 'unite',\n",
       " 'some',\n",
       " 'of',\n",
       " 'the',\n",
       " 'best',\n",
       " 'blessings',\n",
       " 'of',\n",
       " 'existence',\n",
       " 'and',\n",
       " 'had',\n",
       " 'lived',\n",
       " 'nearly',\n",
       " 'years',\n",
       " 'in',\n",
       " 'the',\n",
       " 'world',\n",
       " 'with',\n",
       " 'very',\n",
       " 'little',\n",
       " 'to',\n",
       " 'distress',\n",
       " 'or',\n",
       " 'vex',\n",
       " 'her',\n",
       " 'she',\n",
       " 'was',\n",
       " 'the',\n",
       " 'youngest',\n",
       " 'of',\n",
       " 'the',\n",
       " 'two',\n",
       " 'daughters',\n",
       " 'of',\n",
       " 'a',\n",
       " 'most',\n",
       " 'affectionate',\n",
       " 'indulgent',\n",
       " 'father',\n",
       " 'and',\n",
       " 'had',\n",
       " 'in',\n",
       " 'consequence',\n",
       " 'of',\n",
       " 'her',\n",
       " 'sister',\n",
       " 'marriage',\n",
       " 'been',\n",
       " 'mistress',\n",
       " 'of',\n",
       " 'his',\n",
       " 'house',\n",
       " 'from',\n",
       " 'a',\n",
       " 'very',\n",
       " 'early',\n",
       " 'period',\n",
       " 'her',\n",
       " 'mother',\n",
       " 'had',\n",
       " 'died',\n",
       " 'too',\n",
       " 'long',\n",
       " 'ago',\n",
       " 'for',\n",
       " 'her',\n",
       " 'to',\n",
       " 'have',\n",
       " 'more',\n",
       " 'than',\n",
       " 'an',\n",
       " 'indistinct',\n",
       " 'remembrance',\n",
       " 'of',\n",
       " 'her',\n",
       " 'caresses',\n",
       " 'and',\n",
       " 'her',\n",
       " 'place',\n",
       " 'had',\n",
       " 'been',\n",
       " 'supplied',\n",
       " 'by',\n",
       " 'an',\n",
       " 'excellent',\n",
       " 'woman',\n",
       " 'as',\n",
       " 'governess']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alphaemmawords1[:110]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "179"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "len(stopwords)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the is a stopword\n"
     ]
    }
   ],
   "source": [
    "word = 'the'\n",
    "if word in stopwords:\n",
    "    print (word + ' is a stopword')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74093"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stoppedemmawords = [w for w in alphaemmawords if w not in stopwords]\n",
    "len(stoppedemmawords)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('mr.', 1089)\n",
      "(\"'s\", 866)\n",
      "('emma', 855)\n",
      "('could', 836)\n",
      "('would', 818)\n",
      "('mrs.', 668)\n",
      "('miss', 597)\n",
      "('must', 566)\n",
      "('harriet', 496)\n",
      "('much', 484)\n",
      "('said', 483)\n",
      "('one', 447)\n",
      "('every', 434)\n",
      "('weston', 430)\n",
      "('thing', 394)\n",
      "('think', 383)\n",
      "('elton', 378)\n",
      "('well', 375)\n",
      "('knightley', 373)\n",
      "('little', 359)\n",
      "('never', 358)\n",
      "('know', 335)\n",
      "('might', 325)\n",
      "('good', 313)\n",
      "('say', 310)\n",
      "('woodhouse', 308)\n",
      "('jane', 299)\n",
      "('quite', 282)\n",
      "('time', 275)\n",
      "('great', 263)\n"
     ]
    }
   ],
   "source": [
    "emmadist = FreqDist(stoppedemmawords)\n",
    "emmaitems = emmadist.most_common(30)\n",
    "for item in emmaitems:\n",
    "    print(item) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "179"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "len(stopwords)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "180"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords.append('http')\n",
    "len(stopwords)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Display first 50 Stopwords:\n",
      "['â€™s', 'a', \"a's\", 'able', 'about', 'above', 'according', 'accordingly', 'across', 'actually', 'after', 'afterwards', 'again', 'against', \"ain't\", 'all', 'allow', 'allows', 'almost', 'alone']\n"
     ]
    }
   ],
   "source": [
    "fstop = open('Smart.English.stop', 'r')\n",
    "stoptext = fstop.read()\n",
    "fstop.close()\n",
    "stopwords = nltk.word_tokenize(stoptext)\n",
    "print (\"Display first 50 Stopwords:\")\n",
    "print (stopwords[:20])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}