WikiData_Helper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import os\n",
    "import math\n",
    "import collections\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import data_helper as dh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Take Mohamed's tokenized Wikipedia sentences and create data files\n",
    "\n",
    "1. Generate a vocabulary\n",
    "2. Specify it to a particular size\n",
    "3. Replace oovs\n",
    "4. Parse with spacy\n",
    "5. Create sdp from each pair of noun phrase heads in a sentence\n",
    "   - Include the depedency it is a head of\n",
    "6. Replace sequence pairs with indices\n",
    "7. Write out vocab files and data files, line by line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def noun_chunk_to_head_noun(chunk):\n",
    "    \"\"\"Given a chunk, find the noun who's head is outside the chunk. This is the head noun\"\"\"\n",
    "    chunk_set = set(list(chunk))\n",
    "    for token in chunk:\n",
    "        if token.head not in chunk_set:\n",
    "            return token\n",
    "    print(\"No head noun found in chunk... %r\" % chunk.text)\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def sentence_to_chunk_pairs(sentence):\n",
    "    \"\"\"Iterate over  sentence generating n choose 2 noun phrase heads\"\"\"\n",
    "    chunk_pairs = []\n",
    "    noun_chunks = list(sentence.noun_chunks)\n",
    "    for i, chunk1 in enumerate(noun_chunks[:-1]):\n",
    "        head1 = noun_chunk_to_head_noun(chunk1)\n",
    "        if not head1:\n",
    "            continue # don't let bad noun chunks in\n",
    "        for chunk2 in noun_chunks[i+1:]:\n",
    "            head2 = noun_chunk_to_head_noun(chunk2)\n",
    "            if not head2:\n",
    "                continue # don't let bad noun chunks in\n",
    "            chunk_pairs.append((head1, head2))\n",
    "    return chunk_pairs\n",
    "\n",
    "# test = sentences[1]\n",
    "# chunks = list(test.noun_chunks)\n",
    "# pairs = sentence_to_chunk_pairs(test)\n",
    "# print(test)\n",
    "# print(chunks)\n",
    "# print(len(pairs), pairs)\n",
    "# assert len(pairs) == math.factorial(len(chunks))/float(2*math.factorial(len(chunks) - 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dependency_path_to_root(token):\n",
    "    \"\"\"Traverse up the dependency tree. Include the token we are tracing\"\"\"\n",
    "    dep_path = [token]\n",
    "    while token.head is not token:\n",
    "        dep_path.append(token.head)\n",
    "        token = token.head\n",
    "    # dep_path.append(token.head) # add the root node\n",
    "    return dep_path\n",
    "\n",
    "def find_common_ancestor(e1_path, e2_path):\n",
    "    \"\"\"Loop through both dep paths and return common ancestor\"\"\"\n",
    "    for t1 in e1_path:\n",
    "        for t2 in e2_path:\n",
    "            if t1.idx ==  t2.idx:\n",
    "                return t1\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[a composer, Artur Cimirro, the composer, different languages, Franz Liszt, Leopold Godowsky, Ferrucio Busoni]\n",
      "[composer , Cimirro , composer , languages , Liszt , Godowsky , Busoni ]\n",
      "[(composer , Cimirro ), (composer , composer ), (composer , languages ), (composer , Liszt ), (composer , Godowsky ), (composer , Busoni ), (Cimirro , composer ), (Cimirro , languages ), (Cimirro , Liszt ), (Cimirro , Godowsky ), (Cimirro , Busoni ), (composer , languages ), (composer , Liszt ), (composer , Godowsky ), (composer , Busoni ), (languages , Liszt ), (languages , Godowsky ), (languages , Busoni ), (Liszt , Godowsky ), (Liszt , Busoni ), (Godowsky , Busoni )]\n"
     ]
    }
   ],
   "source": [
    "test = dh.nlp(u'As a composer , Artur Cimirro is strongly influenced by the composer / pianists of different languages such as Franz Liszt , Leopold Godowsky , Ferrucio Busoni , Kaikhosru Shapurji Sorabji .')\n",
    "# test = dh.nlp(u\"This sentence has multiple nouns, like apple, orange, and banana\")\n",
    "print(list(test.noun_chunks))\n",
    "print([noun_chunk_to_head_noun(chunk) for chunk in test.noun_chunks])\n",
    "print(sentence_to_chunk_pairs(test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def sentence_to_sdps(sentence, min_len=1):\n",
    "    \"\"\"Takes sentence and returns all shortest dependency paths (SDP) between pairs of noun phrase heads in a sentence\n",
    "    \n",
    "    Args:\n",
    "        sentence: a spacy Sentence\n",
    "        min_len (opt): the minimum number of words along the path (not including endpoints)\n",
    "    \n",
    "    Returns:\n",
    "        sdps: a dict with `path` and `target` fields\n",
    "                where `path` is a list of (word, dep) tuples (where dep is the dependency that word is the head of)\n",
    "                and `target` is the pair of head nouns at the endpoints of the path\n",
    "                \n",
    "    Notes:\n",
    "        There are three cases of SDPs: \n",
    "        (1) there is not dependency path between X and Y. We obviously skip these\n",
    "        (2) one nominal lies on the path to root of the other (wlog X <- ... <- Y <- ...)\n",
    "            In this case we will lose one dependency, the one where X is the tail.\n",
    "                                                                            | \n",
    "                                                                            v\n",
    "        (3) the nominals have common ancestor in the tree (wlog X <- ... <- Z -> ... -> Y)\n",
    "            In this case we will lose two dependencies, those involving X and Y.\n",
    "    \"\"\"\n",
    "    noun_pairs = sentence_to_chunk_pairs(sentence)\n",
    "    for X, Y in noun_pairs:\n",
    "        X_path = dependency_path_to_root(X)\n",
    "        Y_path = dependency_path_to_root(Y)\n",
    "        common = find_common_ancestor(X_path, Y_path) # need nouns to find case (2)\n",
    "        # now we don't want nouns for assembly\n",
    "        X_path = X_path[1:]\n",
    "        Y_path = Y_path[1:]\n",
    "        # CASE (1)\n",
    "        if not common:\n",
    "            print(\"Bad SDP: skipping\")\n",
    "            continue\n",
    "            \n",
    "        # CASE (2)\n",
    "        elif X is common:\n",
    "#             print(\"X %r is common\" % common)\n",
    "            sdp = []\n",
    "            for token in Y_path: # looks like Y <- (...) <- X <- ...\n",
    "                if token is common: # stop before X\n",
    "                    break\n",
    "                sdp.append((token.text, token.dep_))\n",
    "            sdp = list(reversed(sdp)) # flip to get -> X -> (...) -> Y\n",
    "        elif Y is common:\n",
    "#             print(\"Y %r is common\" % common)\n",
    "            sdp = []\n",
    "            for token in X_path: # looks like X <- ... <- Y \n",
    "                if token is common: # stop before Y\n",
    "                      break\n",
    "                sdp.append((token.text, token.dep_))\n",
    "    \n",
    "        # CASE (3)\n",
    "        else:\n",
    "#             print(\"Z %r is common\" % common)\n",
    "            sdp = []\n",
    "            for token in (X_path): # looks like X <- (... <- Z <-) ...\n",
    "                sdp.append((token.text, token.dep_))\n",
    "                if token is common: # keep Z this time\n",
    "                    break\n",
    "            ysdp = [] # need to keep track of seperate, then will reverse and extend later\n",
    "            for token in Y_path: # looks like (Y <- ... <-) Z <- ... \n",
    "                if token is common:\n",
    "                    break\n",
    "                ysdp.append((token.text, token.dep_))\n",
    "            sdp.extend(list(reversed(ysdp))) # looks like X <- (... <- Z -> ... ) -> Y\n",
    "            \n",
    "        if len(sdp) < min_len:\n",
    "            continue # skip ones that are too short\n",
    "        yield {'path': sdp, 'target':(X.text, Y.text)}\n",
    "    \n",
    "# test = dh.nlp(u'As a composer , Artur Cimirro is strongly influenced by the composer / pianists of different languages such as Franz Liszt , Leopold Godowsky , Ferrucio Busoni , Kaikhosru Shapurji Sorabji .')\n",
    "\n",
    "# for sdp in sentence_to_sdps(test):\n",
    "#     print(sdp['target'], sdp['path'])\n",
    "#     print('-'*80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def create_vocab_from_data(sentences, vocab_limit=None, \n",
    "                           min_count=None, dep=False, \n",
    "                           filter_oov=False, print_oov=False,\n",
    "                           oov_count=1):\n",
    "    \"\"\"Create a vocab index, inverse index, and unigram distribution over tokens from a list of spacy sentences\n",
    "    \n",
    "    if `dep`=True, return the dependencies instead of the tokens\"\"\"\n",
    "    counts = collections.Counter()\n",
    "    for sentence in sentences:\n",
    "        for token in sentence:\n",
    "            if dep:\n",
    "                counts[token.dep_] += 1\n",
    "            else:\n",
    "                if filter_oov and not token.is_oov and token.text not in [u' ', u'\\n\\n']:\n",
    "                    counts[token.text] += 1\n",
    "                elif not filter_oov and token.text not in [u' ', u'\\n\\n']:\n",
    "                    counts[token.text] += 1\n",
    "                elif print_oov:\n",
    "                    print(\"Token %r is oov\" % token.text)\n",
    "    \n",
    "    counts = counts.most_common()\n",
    "#     print(counts)\n",
    "    if not (vocab_limit or min_count):\n",
    "        vocab_limit = len(counts)\n",
    "    elif vocab_limit > len(counts):\n",
    "        print(\"Your vocab limit %i was bigger than the number of token types, now it's %i\" \n",
    "              % (vocab_limit, len(counts)))\n",
    "        vocab_limit = len(counts)\n",
    "    elif min_count:\n",
    "        # get first index of an element that doesn't meet the requency constraint\n",
    "        vocab_limit = len(counts) # never found something too small\n",
    "        for i, count in enumerate(map(lambda x:x[1], counts)):\n",
    "            if count < min_count:\n",
    "                vocab_limit = i\n",
    "                break\n",
    "    \n",
    "#     print(len(counts), vocab_limit)\n",
    "    # create the vocab in most common order\n",
    "    # include an <OOV> token and make it's count the sum of all elements that didn't make the cut\n",
    "    vocab = [ x[0] for x in counts][:vocab_limit] + [u'<OOV>']\n",
    "    if not oov_count and vocab_limit < len(vocab): # if we didn't specify a psuedocount, take the real one... probably a bad idea\n",
    "        oov_count = sum(map(lambda x:x[1], counts[vocab_limit:]))\n",
    "    freqs = [ x[1] for x in counts ][:vocab_limit] + [oov_count]\n",
    "    # calculate the empirical distribution\n",
    "    unigram_distribution = list(np.array(freqs) / np.sum(freqs, dtype=np.float32))\n",
    "    # create index and inverted index\n",
    "    vocab2int = { token:i for (i, token) in enumerate(vocab) }\n",
    "    int2vocab = { i:token for (token, i) in vocab2int.items() }\n",
    "\n",
    "    return vocab, vocab2int, int2vocab, unigram_distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vocab2idx(token, vocab2int):\n",
    "    if token in vocab2int:\n",
    "        return vocab2int[token]\n",
    "    else:\n",
    "        return len(vocab2int)-1 # OOV conversion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "FLAGS = {\n",
    "    'num_sentences': 100, # max is 31661479\n",
    "    'vocab_limit':None,\n",
    "    'min_count':3,\n",
    "    'data_dir':'data/',\n",
    "    'sentence_file':'en.tok.txt',\n",
    "    'out_prefix':'dep_rnn_'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentences = []\n",
    "for i, line in enumerate(open(os.path.join(FLAGS['data_dir'], FLAGS['sentence_file']), 'r')):\n",
    "    if i > FLAGS['num_sentences']:\n",
    "        break\n",
    "    sentences.append(dh.nlp(unicode(line.strip())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vocab, vocab2int, int2vocab, vocab_dist = create_vocab_from_data(sentences,\n",
    "                                                                 vocab_limit=FLAGS['vocab_limit'],\n",
    "                                                                 min_count=FLAGS['min_count'],\n",
    "                                                                 dep=False,\n",
    "                                                                 oov_count=1)\n",
    "dep_vocab, dep2int, int2dep, dep_dist = create_vocab_from_data(sentences,\n",
    "                                                                 vocab_limit=None,\n",
    "                                                                 min_count=0,\n",
    "                                                                 dep=True,\n",
    "                                                                 oov_count=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(FLAGS['out_prefix'] + str(FLAGS['num_sentences']), 'w') as outfile:\n",
    "    for sentence in sentences:\n",
    "        for sdp in sentence_to_sdps(sentence):\n",
    "            # convert from tokens to indices\n",
    "            sdp['path'] = [ (vocab2idx(x[0], vocab2int), vocab2idx(x[1], dep2int)) for x in sdp['path'] ]\n",
    "            sdp['target'] = [ (vocab2idx(sdp['target'][0], vocab2int), vocab2idx(sdp['target'][1], vocab2int)) ]\n",
    "            # write out the dict as json line\n",
    "            outfile.write(json.dumps(sdp) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(FLAGS['data_dir'], FLAGS['out_prefix'] + str(FLAGS['num_sentences'])+'_vocab'), 'w') as outfile:\n",
    "    for term in zip(vocab, vocab_dist):\n",
    "        outfile.write(json.dumps(term)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(FLAGS['data_dir'], FLAGS['out_prefix'] + str(FLAGS['num_sentences'])+'_dep'), 'w') as outfile:\n",
    "    for term in zip(dep_vocab, dep_dist):\n",
    "        outfile.write(json.dumps(term)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dep_rnn100                            \u001b[31men.tok.txt.gz\u001b[m\u001b[m*\r\n",
      "\u001b[31men.tok.txt\u001b[m\u001b[m*                           enwiki-latest-pages-articles.xml.bz2\r\n"
     ]
    }
   ],
   "source": [
    "ls data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = []\n",
    "with open('data/dep_rnn_100_dep', 'r') as f:\n",
    "    for line in f:\n",
    "        data.append(json.loads(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'pobj', 0.1259320629660315] [u'prep', 0.12178956089478045] [u'punct', 0.11019055509527755] [u'det', 0.09527754763877382] [u'amod', 0.07497928748964375] [u'compound', 0.0687655343827672] [u'nsubj', 0.04225352112676056] [u'ROOT', 0.04183927091963546] [u'conj', 0.04101077050538525] [u'advmod', 0.03603976801988401] [u'cc', 0.03231151615575808] [u'dobj', 0.026097763048881523] [u'aux', 0.02112676056338028] [u'poss', 0.016984258492129246] [u'auxpass', 0.015741507870753936] [u'nummod', 0.015327257663628831] [u'nsubjpass', 0.014084507042253521] [u'attr', 0.013256006628003313] [u'advcl', 0.012013256006628004] [u'appos', 0.010356255178127589] [u'relcl', 0.008285004142502071] [u'agent', 0.007042253521126761] [u'acl', 0.007042253521126761] [u'xcomp', 0.00579950289975145] [u'npadvmod', 0.004971002485501243] [u'case', 0.004971002485501243] [u'pcomp', 0.004142502071251036] [u'acomp', 0.003728251864125932] [u'ccomp', 0.0033140016570008283] [u'nmod', 0.002899751449875725] [u'oprd', 0.0024855012427506215] [u'prt', 0.0016570008285004142] [u'mark', 0.0016570008285004142] [u'expl', 0.0016570008285004142] [u'neg', 0.0012427506213753107] [u'preconj', 0.0012427506213753107] [u'dative', 0.0008285004142502071] [u'csubj', 0.00041425020712510354] [u'parataxis', 0.00041425020712510354] [u'quantmod', 0.00041425020712510354] [u'<OOV>', 0.00041425020712510354]\n"
     ]
    }
   ],
   "source": [
    "print(*data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}