Tree Structured RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recursive Neural Networks in Tensorflow\n",
    "\n",
    "My attempt to implement a more robust NN class in Tensorflow, but also to make a recursive (tree structured) network that I haven't seen an example online of yet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "import re\n",
    "from spacy.en import English\n",
    "nlp = English()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class TreeRNN(object):\n",
    "    \"\"\"A Tree Structured Recursive Neural Net, with variable number of children.\n",
    "    \n",
    "    Uses a sklearn style interface\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        # network options\n",
    "        self.num_classes = 3\n",
    "        self.embedding_size = 300\n",
    "        \n",
    "        \n",
    "        self._create_variables()\n",
    "        \n",
    "#         self.saver = tf.Saver()\n",
    "        \n",
    "        # op to initialize all the variables\n",
    "        init = tf.initialize_all_variables()\n",
    "        \n",
    "        # create a running session and start up the graph\n",
    "        # training and predicting with the graph is done from outside\n",
    "        self.session = tf.InteractiveSession()\n",
    "        self.session.run(init)\n",
    "    \n",
    "    def _create_variables(self):\n",
    "        \"\"\"Initialize the network weights\"\"\"\n",
    "        # recurrent unit weights and biases\n",
    "        self.w_input = tf.Variable(np.I(self.embedding_size))\n",
    "#         self.b_input = tf.Variable(tf.zeros(self.embedding_size))\n",
    "        \n",
    "        self.w_state = tf.Variable(np.I(self.embedding_size))\n",
    "        self.b_state = tf.Variable(tf.zeros(self.embedding_size))\n",
    "        \n",
    "        # word embedding matrix\n",
    "        self.word_embeddings = tf.Variable(tf.truncated_normal(\n",
    "                                           [self.embedding_size, self.embedding_size],\n",
    "                                           mean=0.0, stddev=1.0,\n",
    "                                           dtype=tf.float32,\n",
    "                                           random_state=0, ))\n",
    "        \n",
    "        # softmax weights and biases\n",
    "        self.softmax_w = tf.Variable(tf.random_uniform(\n",
    "                                     [self.num_classes, self.embedding_size],\n",
    "                                     min=-.5, max=.5,\n",
    "                                     dtype=tf.float32,\n",
    "                                     random_state=0))\n",
    "        self.softmax_b = tf.Variable(tf.zeros(self.num_classes))\n",
    "        \n",
    "    def _create_optimizer(self):\n",
    "        \"\"\"Initialize the optimization ops (losses will be computed dynamically)\"\"\"\n",
    "        self.optimizer = tf.train.AdagradOptimizer(1.0)\n",
    "        \n",
    "    def _recurrent_cell(self, input_vector, child_states):\n",
    "        \"\"\"Take the input vector and a list of input states (each assumed to be same size) \n",
    "        and compute the sum rnn output\n",
    "        \"\"\"\n",
    "        state_transforms = [ tf.matmul(self.w_state, input_state) for input_state in child_states ]\n",
    "        child_sum = tf.add_n(state_transforms)\n",
    "        inner_sum = tf.matmul(self.w_input, input_vector) + child_sum + self.b_state\n",
    "        output_vector = tf.tanh(inner_sum)\n",
    "        return output_vector\n",
    "        \n",
    "    def _forward_pass(self, node):\n",
    "        loss = 0\n",
    "        # compute this recursively\n",
    "        for child in node.children:\n",
    "            _, child_loss = self._forward_pass(child)\n",
    "            loss += child_loss\n",
    "        \n",
    "        # now were back at _this_ node\n",
    "        # get the vector for this node's word\n",
    "        word_vec = tf.nn.embedding_lookup(self.word_embeddings, node.word_index)\n",
    "        # get the states for all the children\n",
    "        if node.children:\n",
    "            child_states = [ child.state for child in node.children ]\n",
    "        else:\n",
    "            child_states = [ tf.zeros(embedding_size)]\n",
    "        \n",
    "        # compute the rnn over this node\n",
    "        node.state = self._recurrent_cell(word_vec, child_states)\n",
    "        \n",
    "        # measure loss as classification cross entropy for softmax layer\n",
    "        logits = tf.matmul(self.softmax_w, node.state) + self.softmax_b\n",
    "        true = tf.zeros(self.num_classes, dtype=tf.float32)\n",
    "        true[node.class_index] = 1.0\n",
    "        cross_entropy = tf.nn.softmx_cross_entropy_with_logits(true, logits)\n",
    "#         cross_entropy = -tf.reduce_sum(tf.mul(true, tf.log(1e-10 + logit)))\n",
    "        \n",
    "        node.cross_entropy = cross_entropy\n",
    "        node.loss = cross_entropy + loss\n",
    "        \n",
    "        # loss for each node also includes loss from descendants\n",
    "        return node.state, node.loss\n",
    "    \n",
    "    def partial_fit(self, tree_batch):\n",
    "        batch_size = len(tree_batch)\n",
    "        losses = []\n",
    "        for tree in tree_batch:\n",
    "            _, loss = self._forward_pass(tree)\n",
    "            losses.append(loss)\n",
    "            \n",
    "        objective = ((1.0/batch_size) * np.sum(losses) \n",
    "                     + self.lambda_reg * (\n",
    "                           tf.reduce_sum(self.softmax_w**2)\n",
    "                           + tf.reduce_sum(self.softmax_b**2)))\n",
    "    \n",
    "        # we need to perform gradient clipping explicitly\n",
    "        grads_and_vars = self.optimizer.compute_gradients(objective, \n",
    "                                                     [self.word_embeddings,\n",
    "                                                      self.w_input,\n",
    "                                                      self.w_state,\n",
    "                                                      self.b_state,\n",
    "                                                      self.word_embeddings,\n",
    "                                                      self.softmax_w,\n",
    "                                                      self.softmax_b])\n",
    "        clipped_grads_and_vars = [(tf.clip_by_norm(gv[0], 1.), gv[1]) for gv in grads_and_vars]\n",
    "        minimizer = self.optimizer.apply_gradients(clipped_grads_and_vars)\n",
    "        print \"Avg perplexity: %0.4f\" % np.exp(np.sum(losses)/len(losses))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Node(object):\n",
    "    \"\"\"Node for defining trees with adjacency lists\"\"\"\n",
    "    def __init__(self):\n",
    "        self.head = None\n",
    "        self.children = []\n",
    "        self.loss = None\n",
    "        self.state = None\n",
    "        self.cross_entropy = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_delims = [' ',',','.',';',':', '%', '\"', '$', '^']\n",
    "def split(string, delimiters=split_delims, maxsplit=0):\n",
    "    regexPattern = '|'.join(map(re.escape, delimiters))\n",
    "    return re.split(regexPattern, string, maxsplit)\n",
    "\n",
    "def convert_raw_x(line):\n",
    "    \"\"\"Convert raw line of semeval data into a useable form\n",
    "    \n",
    "    Convert to a triple of (list(raw words), e1_index, e2_index)\n",
    "    \"\"\"\n",
    "    s = line.strip()\n",
    "    s = s[s.index('\"')+1: -(s[::-1].index('\"')+1)] # get s between first \" and last \"\n",
    "    # we will assume that the first token follow the <e1> , <e2> tags are the entity words.  \n",
    "    # note this is a big assumption and hopefully phrases will be in subtrees or in heads of the parse trees\n",
    "    # TODO: this can be addressed by making it a 5-tuple with the endpoints also encoded\n",
    "    s = split(s)\n",
    "    for i in range(len(s)):\n",
    "        # deal with e1's\n",
    "        if '<e1>' in s[i]:\n",
    "            e1_index = i\n",
    "            s[i] = s[i].replace('<e1>', '')\n",
    "        if '</e1>' in s[i]:\n",
    "            #e1_index = i\n",
    "            s[i] = s[i].replace('</e1>', '')\n",
    "        # eal with e2's\n",
    "        if '<e2>' in s[i]:\n",
    "            e2_index = i\n",
    "            s[i] = s[i].replace('<e2>', '')\n",
    "        if '</e2>' in s[i]:\n",
    "            #e2_index = i\n",
    "            s[i] = s[i].replace('</e2>', '')\n",
    "        \n",
    "    # drop extraneous elements from re.split\n",
    "    # also turn it into a spacy sentence\n",
    "    s = nlp(u' '.join([ w.lower() for w in s if w is not '' ])) \n",
    "    return (s, e1_index, e2_index)\n",
    "    \n",
    "label2int = dict() # keep running dictionary of labels\n",
    "def convert_raw_y(line):\n",
    "    \"\"\"Convert raw line of semeval labels into a useable form (ints)\"\"\"\n",
    "    #print \"Raw Y: %r\" % line[:]\n",
    "    line = line.strip()\n",
    "    if line in label2int:\n",
    "        return label2int[line]\n",
    "    else:\n",
    "        label2int[line] = len(label2int.keys())\n",
    "        return label2int[line]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def load_semeval_data():\n",
    "    training_txt_file = 'SemEval2010_task8_all_data/SemEval2010_task8_training/TRAIN_FILE.TXT'\n",
    "    validation_index = 8000 - 891# len data - len valid - 1 since we start at 0\n",
    "    train = {'x':[], 'y':[]}\n",
    "    valid = {'x':[], 'y':[]}\n",
    "    text = open(training_txt_file, 'r').readlines()\n",
    "    assert len(text) // 4 == 8000\n",
    "    for cursor in range(len(text) // 4): # each 4 lines is a datum\n",
    "        if cursor < validation_index:\n",
    "            train['x'].append(convert_raw_x(text[cursor*4]))\n",
    "            train['y'].append(convert_raw_y(text[cursor*4 + 1]))\n",
    "            # ignore comments and blanks (+2, +3)\n",
    "        else:\n",
    "            valid['x'].append(convert_raw_x(text[cursor*4]))\n",
    "            valid['y'].append(convert_raw_y(text[cursor*4 + 1]))\n",
    "\n",
    "    #print train\n",
    "    #print label2int.values()\n",
    "    assert len(train['y']) == 7109 and len(valid['y']) == 891\n",
    "    assert sorted(label2int.values()) == range(19) # 2 for each 9 asymmetric relations and 1 other\n",
    "    \n",
    "    return train, valid\n",
    "    \n",
    "semeval_train, semeval_valid = load_semeval_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_semeval_to_sample(semeval_datum):\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}