semeval_rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import collections\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA, FastICA\n",
    "\n",
    "import data_handler as dh\n",
    "import semeval_data_helper as sdh\n",
    "# plot settings\n",
    "% matplotlib inline\n",
    "# print(plt.rcParams.keys())\n",
    "plt.rcParams['figure.figsize'] = (16,9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload(sdh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# print(DH.max_seq_len)\n",
    "# paths, targets = DH.readable_data(show_dep=True)\n",
    "# for p, t in zip(paths, targets) :\n",
    "#     t = t.split(\", \")\n",
    "#     print(\"%s (%s) %s\" % (t[0], p, t[1]))\n",
    "print('<X>' in DH.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class RelEmbed(object):\n",
    "    \"\"\" Encapsulation of the dependency RNN lang model\n",
    "    \n",
    "    Largely inspired by https://github.com/tensorflow/tensorflow/blob/master/tensorflow/models/rnn/ptb/ptb_word_lm.py\n",
    "    \"\"\"\n",
    "    def __init__(self, config):\n",
    "#         self.batch_size = config['batch_size']\n",
    "        self.max_num_steps = config['max_num_steps']\n",
    "        self.word_embed_size = config['word_embed_size']\n",
    "        self.dep_embed_size = config['dep_embed_size']\n",
    "        self.input_size = self.word_embed_size + self.dep_embed_size\n",
    "        self.hidden_size = 2 * self.word_embed_size #config['hidden_size']\n",
    "        self.pretrained_word_embeddings = config['pretrained_word_embeddings'] # None if we don't provide them\n",
    "        if np.any(self.pretrained_word_embeddings):\n",
    "            assert self.word_embed_size == self.pretrained_word_embeddings.shape[1]\n",
    "        self.num_classes = config['num_predict_classes']\n",
    "        self.max_grad_norm = config['max_grad_norm']\n",
    "        \n",
    "        self.vocab_size = config['vocab_size']\n",
    "        self.dep_vocab_size = config['dep_vocab_size']\n",
    "        self.name = config['model_name']\n",
    "        self.checkpoint_prefix = config['checkpoint_prefix'] + self.name\n",
    "        self.summary_prefix = config['summary_prefix'] + self.name\n",
    "        \n",
    "        self.initializer = tf.random_uniform_initializer(-1., 1.)\n",
    "        with tf.name_scope(self.name):\n",
    "            with tf.name_scope(\"Forward\"):\n",
    "                self._build_forward_graph()\n",
    "                self._build_classification_graph()\n",
    "            with tf.name_scope(\"Backward\"):\n",
    "                self._build_train_graph()\n",
    "                self._build_class_train_graph()\n",
    "            with tf.name_scope(\"Nearby\"):\n",
    "                self._build_similarity_graph()\n",
    "        \n",
    "        self._valid_accuracy = tf.Variable(0.0, trainable=False)\n",
    "        self._valid_acc_summary = tf.merge_summary([tf.scalar_summary(\"Valid_accuracy\", self._valid_accuracy)])\n",
    "\n",
    "        self.saver = tf.train.Saver(tf.all_variables())\n",
    "            \n",
    "        self.session = tf.InteractiveSession()\n",
    "        self.session.run(tf.initialize_all_variables())        \n",
    "        self.summary_writer = tf.train.SummaryWriter(self.summary_prefix, self.session.graph_def)\n",
    "        \n",
    "    def save_validation_accuracy(self, new_score):\n",
    "        assign_op = self._valid_accuracy.assign(new_score)\n",
    "        _, summary = self.session.run([assign_op, self._valid_acc_summary])\n",
    "        self.summary_writer.add_summary(summary)\n",
    "        \n",
    "    def _build_forward_graph(self):\n",
    "        # input tensor of zero padded indices to get to max_num_steps\n",
    "        # None allows for variable batch sizes\n",
    "        with tf.name_scope(\"Inputs\"):\n",
    "            self._input_phrases = tf.placeholder(tf.int32, [None, self.max_num_steps, 2]) # [batch_size, w_{1:N}, 2]\n",
    "            self._input_targets = tf.placeholder(tf.int32, [None, 2]) # [batch_size, w_x]\n",
    "            self._input_labels = tf.placeholder(tf.int32, [None, 1]) # [batch_size, from true data?] \\in {0,1}\n",
    "            self._input_lengths = tf.placeholder(tf.int32, [None, 1]) # [batch_size, N] (len of each sequence)\n",
    "            batch_size = tf.shape(self._input_lengths)[0]\n",
    "        \n",
    "        with tf.name_scope(\"Embeddings\"):\n",
    "            if np.any(self.pretrained_word_embeddings):\n",
    "                self._word_embeddings = tf.Variable(self.pretrained_word_embeddings,name=\"word_embeddings\")\n",
    "                self._left_target_embeddings = tf.Variable(self.pretrained_word_embeddings, name=\"left_target_embeddings\")\n",
    "                self._right_target_embeddings = tf.Variable(self.pretrained_word_embeddings, name=\"right_target_embeddings\")\n",
    "            else:\n",
    "                self._word_embeddings = tf.get_variable(\"word_embeddings\", \n",
    "                                                        [self.vocab_size, self.word_embed_size],\n",
    "                                                        dtype=tf.float32)\n",
    "                self._left_target_embeddings = tf.get_variable(\"left_target_embeddings\", \n",
    "                                                        [self.vocab_size, self.word_embed_size],\n",
    "                                                        dtype=tf.float32)\n",
    "                self._right_target_embeddings = tf.get_variable(\"right_target_embeddings\", \n",
    "                                                        [self.vocab_size, self.word_embed_size],\n",
    "                                                        dtype=tf.float32)\n",
    "            \n",
    "            self._dependency_embeddings = tf.get_variable(\"dependency_embeddings\", \n",
    "                                                    [self.dep_vocab_size, self.dep_embed_size],\n",
    "                                                    dtype=tf.float32)\n",
    "            # TODO: Add POS embeddings\n",
    "            \n",
    "            input_embeds = tf.nn.embedding_lookup(self._word_embeddings, \n",
    "                                                  tf.slice(self._input_phrases, [0,0,0], [-1, -1, 1]))\n",
    "            dep_embeds = tf.nn.embedding_lookup(self._dependency_embeddings,\n",
    "                                                tf.slice(self._input_phrases, [0,0,1], [-1, -1, 1]))\n",
    "            left_target_embeds = tf.nn.embedding_lookup(self._left_target_embeddings, \n",
    "                                                        tf.slice(self._input_targets, [0,0], [-1, 1]))\n",
    "            right_target_embeds = tf.nn.embedding_lookup(self._right_target_embeddings, \n",
    "                                                        tf.slice(self._input_targets, [0,1], [-1, 1]))\n",
    "#             print(tf.slice(self._input_phrases, [0,0,1], [-1, -1, 1]).get_shape(), dep_embeds.get_shape())\n",
    "#             print(left_target_embeds.get_shape(), right_target_embeds.get_shape())\n",
    "            self._target_embeds = tf.squeeze(tf.concat(2, [left_target_embeds, right_target_embeds]), [1])\n",
    "#             print(target_embeds.get_shape())\n",
    "            # TODO: Add dropout to embeddings\n",
    "        \n",
    "        with tf.name_scope(\"RNN\"):\n",
    "            # start off with a basic configuration\n",
    "            self.cell = tf.nn.rnn_cell.GRUCell(self.hidden_size, \n",
    "                                                input_size=self.input_size)\n",
    "            # TODO: Add Dropout wrapper\n",
    "            # TODO: Make it multilevel\n",
    "#             self._initial_state = self.cell.zero_state(batch_size, tf.float32)\n",
    "#             print(self._initial_state.get_shape())\n",
    "            input_words = [ tf.squeeze(input_, [1, 2]) for input_ in tf.split(1, self.max_num_steps, input_embeds)]\n",
    "            input_deps = [ tf.squeeze(input_, [1, 2]) for input_ in tf.split(1, self.max_num_steps, dep_embeds)]\n",
    "            inputs = [ tf.concat(1, [input_word, input_dep]) \n",
    "                      for (input_word, input_dep) in zip(input_words, input_deps)]\n",
    "\n",
    "            _, state = tf.nn.rnn(self.cell, inputs, \n",
    "                                 sequence_length=tf.squeeze(self._input_lengths, [1]),\n",
    "                                 dtype=tf.float32)\n",
    "#                                  initial_state=self._initial_state)\n",
    "            self._final_state = state\n",
    "            \n",
    "        with tf.name_scope(\"Loss\"):\n",
    "            flat_states = tf.reshape(state, [-1])\n",
    "            flat_target_embeds = tf.reshape(self._target_embeds, [-1])\n",
    "#             assert self.hidden_size == (self.word_embed_size), \"Hidden state must equal concated inputs\" \n",
    "            flat_logits = tf.mul(flat_states, flat_target_embeds)\n",
    "            logits = tf.reduce_sum(tf.reshape(flat_logits, tf.pack([batch_size, -1])), 1)\n",
    "            self._loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, \n",
    "                                                                    tf.to_float(self._input_labels)),\n",
    "                                        name=\"neg_sample_loss\")\n",
    "            \n",
    "        with tf.name_scope(\"Summaries\"):\n",
    "            \n",
    "            self._train_cost_summary = tf.merge_summary([tf.scalar_summary(\"Train_NEG_Loss\", self._loss)])\n",
    "            self._valid_cost_summary = tf.merge_summary([tf.scalar_summary(\"Validation_NEG_Loss\", self._loss)])\n",
    "        \n",
    "    def _build_classification_graph(self):\n",
    "        with tf.name_scope(\"Classifier\"):\n",
    "            self._softmax_input = tf.concat(1, [self._final_state, self._target_embeds], name=\"concat_input\")\n",
    "            self._softmax_w = tf.get_variable(\"softmax_w\", [self._softmax_input.get_shape()[1], self.num_classes])\n",
    "            self._softmax_b = tf.Variable(tf.zeros([self.num_classes], dtype=tf.float32), name=\"softmax_b\")\n",
    "\n",
    "            class_logits = tf.matmul(self._softmax_input, self._softmax_w) + self._softmax_b\n",
    "            self._predictions = tf.argmax(class_logits, 1, name=\"predict\")\n",
    "            self._predict_probs = tf.nn.softmax(class_logits, name=\"predict_probabilities\")\n",
    "        \n",
    "        with tf.name_scope(\"Loss\"):\n",
    "            self._class_labels = tf.placeholder(tf.int64, [None])\n",
    "            self._class_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(class_logits, \n",
    "                                                                              self._class_labels)\n",
    "            self._train_class_loss_summary = tf.merge_summary([tf.scalar_summary(\"Train_Class_Loss\", self._class_loss)])\n",
    "            self._valid_class_loss_summary = tf.merge_summary([tf.scalar_summary(\"Valid_Class_Loss\", self._class_loss)])\n",
    "\n",
    "    def _build_train_graph(self):\n",
    "        with tf.name_scope(\"Unsupervised_Trainer\"):\n",
    "            self._global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "#             self._lr = tf.Variable(1.0, trainable=False)\n",
    "            self._optimizer = tf.train.AdagradOptimizer(1.0)\n",
    "            \n",
    "            # clip and apply gradients\n",
    "            grads_and_vars = self._optimizer.compute_gradients(self._loss)\n",
    "#             for gv in grads_and_vars:\n",
    "#                 print(gv, gv[1] is self._cost)\n",
    "            clipped_grads_and_vars = [(tf.clip_by_norm(gv[0], self.max_grad_norm), gv[1]) \n",
    "                                      for gv in grads_and_vars if gv[0] is not None] # clip_by_norm doesn't like None\n",
    "            \n",
    "            with tf.name_scope(\"Summaries\"):\n",
    "                grad_summaries = []\n",
    "                for g, v in grads_and_vars:\n",
    "                    if g is not None:\n",
    "                        grad_hist_summary = tf.histogram_summary(\"{}/grad/hist\".format(v.name), g)\n",
    "                        sparsity_summary = tf.scalar_summary(\"{}/grad/sparsity\".format(v.name), tf.nn.zero_fraction(g))\n",
    "                        grad_summaries.append(grad_hist_summary)\n",
    "                        grad_summaries.append(sparsity_summary)\n",
    "                self._grad_summaries = tf.merge_summary(grad_summaries)\n",
    "            self._train_op = self._optimizer.apply_gradients(clipped_grads_and_vars, global_step=self._global_step)\n",
    "            \n",
    "    def _build_class_train_graph(self):\n",
    "        with tf.name_scope(\"Classification_Trainer\"):\n",
    "            self._class_global_step = tf.Variable(0, name=\"class_global_step\", trainable=False)\n",
    "#             self._lr = tf.Variable(1.0, trainable=False)\n",
    "            self._class_optimizer = tf.train.AdagradOptimizer(1.0)\n",
    "            \n",
    "            # clip and apply gradients\n",
    "            grads_and_vars = self._class_optimizer.compute_gradients(self._class_loss)\n",
    "#             for gv in grads_and_vars:\n",
    "#                 print(gv, gv[1] is self._cost)\n",
    "            clipped_grads_and_vars = [(tf.clip_by_norm(gv[0], self.max_grad_norm), gv[1]) \n",
    "                                      for gv in grads_and_vars if gv[0] is not None] # clip_by_norm doesn't like None\n",
    "            \n",
    "            with tf.name_scope(\"Summaries\"):\n",
    "                grad_summaries = []\n",
    "                for g, v in grads_and_vars:\n",
    "                    if g is not None:\n",
    "                        grad_hist_summary = tf.histogram_summary(\"class_{}/grad/hist\".format(v.name), g)\n",
    "                        sparsity_summary = tf.scalar_summary(\"class_{}/grad/sparsity\".format(v.name), tf.nn.zero_fraction(g))\n",
    "                        grad_summaries.append(grad_hist_summary)\n",
    "                        grad_summaries.append(sparsity_summary)\n",
    "                self._class_grad_summaries = tf.merge_summary(grad_summaries)\n",
    "            self._class_train_op = self._class_optimizer.apply_gradients(clipped_grads_and_vars, \n",
    "                                                                         global_step=self._class_global_step)\n",
    "            \n",
    "    def _build_similarity_graph(self):\n",
    "        tf.get_variable_scope().reuse_variables()\n",
    "        with tf.name_scope(\"Inputs\"):\n",
    "            # word or phrase we want similarities for\n",
    "#             self._query_word = tf.placeholder(tf.int32, [1], name=\"q_word\")\n",
    "            self._query_phrase = tf.placeholder(tf.int32, [self.max_num_steps, 2], name=\"q_phrase\")\n",
    "            self._query_length = tf.placeholder(tf.int32, [1], name=\"q_len\") # lengths for RNN\n",
    "            # words and phrases to compute similarities over\n",
    "#             self._sim_words = tf.placeholder(tf.int32, [None, 1])\n",
    "            self._sim_phrases = tf.placeholder(tf.int32, [None, self.max_num_steps, 2])\n",
    "            self._sim_lengths = tf.placeholder(tf.int32, [None, 1]) # lengths for RNN\n",
    "            sim_size = tf.shape(self._sim_lengths)[0]\n",
    "        \n",
    "        with tf.name_scope(\"Embeddings\"):\n",
    "            query_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings, \n",
    "                                                  tf.slice(self._query_phrase, [0,0], [-1, 1]))\n",
    "            query_dep_embed = tf.nn.embedding_lookup(self._dependency_embeddings,\n",
    "                                                tf.slice(self._query_phrase, [0,1], [-1, 1]))\n",
    "#             query_word_embed = tf.nn.embedding_lookup(self._word_embeddings, self._query_word)\n",
    "#             query_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings, self._query_phrase)\n",
    "#             sim_word_embed = tf.nn.embedding_lookup(self._word_embeddings, tf.squeeze(self._sim_words, [1]))\n",
    "            sim_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings, \n",
    "                                                  tf.slice(self._sim_phrases, [0, 0, 0], [-1, -1, 1]))\n",
    "            sim_dep_embed = tf.nn.embedding_lookup(self._dependency_embeddings, \n",
    "                                                  tf.slice(self._sim_phrases, [0, 0, 1], [-1, -1, 1]))\n",
    "        \n",
    "        with tf.name_scope(\"RNN\"):\n",
    "            # compute rep of a query phrase\n",
    "            query_phrase = [tf.squeeze(qw, [1]) for qw in tf.split(0, self.max_num_steps, query_phrase_embed)]\n",
    "            query_dep = [tf.squeeze(qd, [1]) for qd in tf.split(0, self.max_num_steps, query_dep_embed)]\n",
    "#             print(query_phrase[0].get_shape(), query_dep[0].get_shape())\n",
    "            query_input = [ tf.concat(1, [qw, qd]) for (qw, qd) in zip(query_phrase, query_dep)]\n",
    "            _, query_phrase_state = tf.nn.rnn(self.cell, query_input, \n",
    "                                              sequence_length=self._query_length, \n",
    "                                              dtype=tf.float32)\n",
    "            # compute reps of similarity phrases\n",
    "            sim_phrases = [tf.squeeze(qw, [1,2]) for qw in tf.split(1, self.max_num_steps, sim_phrase_embed)]\n",
    "            sim_deps = [tf.squeeze(qd, [1,2]) for qd in tf.split(1, self.max_num_steps, sim_dep_embed)]\n",
    "            sim_input = [ tf.concat(1, [qw, qd]) for (qw, qd) in zip(sim_phrases, sim_deps)]\n",
    "            _, sim_phrase_states = tf.nn.rnn(self.cell, sim_input, \n",
    "                                             sequence_length=tf.squeeze(self._sim_lengths, [1]), \n",
    "                                             dtype=tf.float32)\n",
    "            \n",
    "        with tf.name_scope(\"Similarities\"):\n",
    "            with tf.name_scope(\"Normalize\"):\n",
    "#                 print(query_phrase.get_shape())\n",
    "                query_phrase = tf.nn.l2_normalize(query_phrase_state, 1)\n",
    "#                 query_word = tf.nn.l2_normalize(query_word_embed, 1)\n",
    "                sim_phrases = tf.nn.l2_normalize(sim_phrase_states, 1)\n",
    "#                 sim_word = tf.nn.l2_normalize(sim_word_embed, 1)                \n",
    "\n",
    "            with tf.name_scope(\"Calc_distances\"):\n",
    "                # do for words\n",
    "#                 print(q)\n",
    "#                 query_word_nearby_dist = tf.matmul(query_word, sim_word, transpose_b=True)\n",
    "#                 qw_nearby_val, qw_nearby_idx = tf.nn.top_k(query_word_nearby_dist, min(1000, self.vocab_size))\n",
    "#                 self.qw_nearby_val = tf.squeeze(qw_nearby_val)\n",
    "#                 self.qw_nearby_idx = tf.squeeze(qw_nearby_idx)\n",
    "#                 self.qw_nearby_words = tf.squeeze(tf.gather(self._sim_words, qw_nearby_idx))\n",
    "\n",
    "                # do for phrases\n",
    "                query_phrase_nearby_dist = tf.matmul(query_phrase, sim_phrases, transpose_b=True)\n",
    "                qp_nearby_val, qp_nearby_idx = tf.nn.top_k(query_phrase_nearby_dist, min(1000, sim_size))\n",
    "#                 self.sanity_check = tf.squeeze(tf.matmul(query_phrase, query_phrase, transpose_b=True))\n",
    "                self.qp_nearby_val = tf.squeeze(qp_nearby_val)\n",
    "                self.qp_nearby_idx = tf.squeeze(qp_nearby_idx)\n",
    "#                 self.qp_nearby_lens = tf.squeeze(tf.gather(self._sim_lengths, qp_nearby_idx))\n",
    "            \n",
    "    def partial_class_fit(self, input_phrases, input_targets, class_labels, input_lengths):\n",
    "        \"\"\"Fit a mini-batch\n",
    "        \n",
    "        Expects a batch_x: [self.batch_size, self.max_num_steps]\n",
    "                  batch_y: the same\n",
    "                  batch_seq_lens: [self.batch_size]\n",
    "                  \n",
    "        Returns average batch perplexity\n",
    "        \"\"\"\n",
    "        loss, _, g_summaries, c_summary = self.session.run([self._class_loss, self._class_train_op, \n",
    "                                                            self._class_grad_summaries,\n",
    "                                                            self._train_class_loss_summary],\n",
    "                                                           {self._input_phrases:input_phrases,\n",
    "                                                            self._input_targets:input_targets,\n",
    "                                                            self._class_labels:class_labels,\n",
    "                                                            self._input_lengths:input_lengths})\n",
    "        self.summary_writer.add_summary(g_summaries)\n",
    "        self.summary_writer.add_summary(c_summary)\n",
    "        return loss\n",
    "    \n",
    "    def partial_unsup_fit(self, input_phrases, input_targets, input_labels, input_lengths):\n",
    "        \"\"\"Fit a mini-batch\n",
    "        \n",
    "        Expects a batch_x: [self.batch_size, self.max_num_steps]\n",
    "                  batch_y: the same\n",
    "                  batch_seq_lens: [self.batch_size]\n",
    "                  \n",
    "        Returns average batch perplexity\n",
    "        \"\"\"\n",
    "        loss, _, g_summaries, c_summary = self.session.run([self._loss, self._train_op, \n",
    "                                                            self._grad_summaries,\n",
    "                                                            self._train_cost_summary],\n",
    "                                                           {self._input_phrases:input_phrases,\n",
    "                                                            self._input_targets:input_targets,\n",
    "                                                            self._input_labels:input_labels,\n",
    "                                                            self._input_lengths:input_lengths})\n",
    "        self.summary_writer.add_summary(g_summaries)\n",
    "        self.summary_writer.add_summary(c_summary)\n",
    "        return loss\n",
    "    \n",
    "    def validation_loss(self, valid_phrases, valid_targets, valid_labels, valid_lengths):\n",
    "        \"\"\"Calculate loss on validation inputs, but don't run trainer\"\"\"\n",
    "        loss, v_summary = self.session.run([self._loss, self._valid_cost_summary],\n",
    "                                           {self._input_phrases:valid_phrases,\n",
    "                                            self._input_targets:valid_targets,\n",
    "                                            self._input_labels:valid_labels,\n",
    "                                            self._input_lengths:valid_lengths})\n",
    "        self.summary_writer.add_summary(v_summary)\n",
    "        return loss\n",
    "    \n",
    "    def validation_class_loss(self, valid_phrases, valid_targets, valid_labels, valid_lengths):\n",
    "        \"\"\"Calculate loss on validation inputs, but don't run trainer\"\"\"\n",
    "        loss, v_summary = self.session.run([self._class_loss, self._valid_class_loss_summary],\n",
    "                                           {self._input_phrases:valid_phrases,\n",
    "                                            self._input_targets:valid_targets,\n",
    "                                            self._class_labels:valid_labels,\n",
    "                                            self._input_lengths:valid_lengths})\n",
    "        self.summary_writer.add_summary(v_summary)\n",
    "        return loss\n",
    "    \n",
    "    def validation_phrase_nearby(self, q_phrase, q_phrase_len, sim_phrases, sim_phrase_lens):\n",
    "        \"\"\"Return nearby phrases from the similarity set\n",
    "        \"\"\"\n",
    "        nearby_vals, nearby_idx = self.session.run([self.qp_nearby_val, self.qp_nearby_idx],\n",
    "                                                                   {self._query_phrase:q_phrase, \n",
    "                                                                    self._query_length:q_phrase_len,\n",
    "                                                                    self._sim_phrases:sim_phrases,\n",
    "                                                                    self._sim_lengths:sim_phrase_lens})\n",
    "#         print(\"Sanity check: %r\" % sanity)\n",
    "        return nearby_vals, nearby_idx\n",
    "    \n",
    "    def embed_phrases_and_targets(self, phrases, targets, lengths):\n",
    "        phrase_reps, target_reps = self.session.run([self._final_state, self._target_embeds],\n",
    "                                                    { self._input_phrases:phrases,\n",
    "                                                      self._input_targets:targets,\n",
    "                                                      self._input_lengths:lengths})\n",
    "        return phrase_reps, target_reps\n",
    "    \n",
    "#     def validation_word_nearby(self, q_word, sim_words):\n",
    "#         \"\"\"Return nearby phrases from the similarity set\n",
    "#         \"\"\"\n",
    "#         nearby_vals, nearby_idx = self.session.run([self.qw_nearby_val, \n",
    "#                                                       self.qw_nearby_idx],\n",
    "#                                                        {self._query_word:q_word, \n",
    "#                                                         self._sim_words:sim_words})\n",
    "#         return nearby_vals, nearby_idx\n",
    "        \n",
    "    def predict(self, sequences, seq_lens, return_probs=False):\n",
    "        if return_probs:\n",
    "            predictions, distributions = self.session.run([self._predictions, self._predicted_dists],\n",
    "                                                          {self._predict_inputs:sequences,\n",
    "                                                           self._predict_lengths:seq_lens})\n",
    "            distributions = distributions.reshape([sequences.shape[0], sequences.shape[1], -1])\n",
    "            pred_list = []\n",
    "            dist_list = []\n",
    "            for i, seq_len in enumerate(seq_lens):\n",
    "                pred_list.append(list(predictions[i, :seq_len]))\n",
    "                dist_list.append([distributions[i,j,:] for j in range(seq_len)])\n",
    "            return pred_list, dist_list\n",
    "        \n",
    "        else:\n",
    "            predictions = self.session.run(self._predictions,\n",
    "                                           {self._predict_inputs:sequences,\n",
    "                                            self._predict_lengths:seq_lens})\n",
    "            pred_list = []\n",
    "            for i, seq_len in enumerate(seq_lens):\n",
    "                pred_list.append(list(predictions[i, :seq_len])) \n",
    "            return pred_list\n",
    "            \n",
    "    def checkpoint(self):\n",
    "        self.saver.save(self.session, self.checkpoint_prefix + '.ckpt', global_step=self._global_step)\n",
    "        \n",
    "    def restore(self, model_ckpt_path):\n",
    "        self.saver.restore(self.session, model_ckpt_path)\n",
    "        \n",
    "    def __repr__(self):\n",
    "        return (\"<DPNN: W:%i, D:%i, H:%i, V:%i>\" \n",
    "                % (self.word_embed_size, self.dep_embed_size, self.hidden_size, self.vocab_size))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload(dh)\n",
    "DH = dh.DataHandler('data/shuffled_wiki_sdp_50000', valid_percent=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('<X>' in DH.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%time\n",
    "# load the pretrained word embeddings\n",
    "fname = 'data/GoogleNews-vectors-negative300.bin'\n",
    "word2vec = Word2Vec.load_word2vec_format(fname, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "og_train, og_valid, og_test, label2int, int2label = sdh.load_semeval_data()\n",
    "num_classes = len(int2label.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# convert the semeval data to indices under the wiki vocab:\n",
    "train['sdps'] = DH.sentences_to_sequences(og_train['sdps'])\n",
    "valid['sdps'] = DH.sentences_to_sequences(og_valid['sdps'])\n",
    "test['sdps'] = DH.sentences_to_sequences(og_test['sdps'])\n",
    "    \n",
    "train['targets'] = DH.sentences_to_sequences(og_train['targets'])\n",
    "valid['targets'] = DH.sentences_to_sequences(og_valid['targets'])\n",
    "test['targets'] = DH.sentences_to_sequences(og_test['targets'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# the embedding matrix is started of as random uniform [-1,1]\n",
    "# then we replace everything but the OOV tokens with the approprate google vector\n",
    "word_embeddings = np.random.uniform(low=-1., high=1., size=[DH.vocab_size, 300]).astype(np.float32)\n",
    "num_found = 0\n",
    "for i, token in enumerate(DH.vocab):\n",
    "    if token in word2vec:\n",
    "        word_embeddings[i] = word2vec[token]\n",
    "        num_found += 1\n",
    "print(\"%i / %i pretrained\" % (num_found, DH.vocab_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "config = {\n",
    "    'max_num_steps':DH.max_seq_len,\n",
    "    'word_embed_size':300,\n",
    "    'dep_embed_size':25,\n",
    "    'vocab_size':DH.vocab_size,\n",
    "    'dep_vocab_size':DH.dep_size,\n",
    "    'num_predict_classes':num_classes,\n",
    "    'pretrained_word_embeddings':word_embeddings,\n",
    "    'max_grad_norm':3.,\n",
    "    'model_name':'drnn_wiki_semeval_w2v',\n",
    "    'checkpoint_prefix':'checkpoints/',\n",
    "    'summary_prefix':'tensor_summaries/'\n",
    "}\n",
    "try:\n",
    "    tf.reset_default_graph()\n",
    "except:\n",
    "    pass\n",
    "try:\n",
    "    tf.get_default_session().close()\n",
    "except:\n",
    "    pass\n",
    "drnn = RelEmbed(config)\n",
    "print(drnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_validation_test(num_nearby=20):\n",
    "    valid_phrases, valid_targets , _, valid_lens = DH.validation_batch()\n",
    "    random_index = int(random.uniform(0, len(valid_lens)))\n",
    "    query_phrase = valid_phrases[random_index]\n",
    "    query_len = valid_lens[random_index]\n",
    "    query_target = valid_targets[random_index]\n",
    "    padded_qp = np.zeros([DH.max_seq_len, 2]).astype(np.int32)\n",
    "    padded_qp[:len(query_phrase), 0] = [x[0] for x in query_phrase]\n",
    "    padded_qp[:len(query_phrase), 1] = [x[1] for x in query_phrase]    \n",
    "    dists, phrase_idx = drnn.validation_phrase_nearby(padded_qp, query_len, valid_phrases, valid_lens)\n",
    "    print(\"=\"*80)\n",
    "    print(\"Top %i closest phrases to <%s> '%s' <%s>\" \n",
    "          % (num_nearby, DH.vocab_at(query_target[0]), \n",
    "             DH.sequence_to_sentence(query_phrase, query_len), \n",
    "             DH.vocab_at(query_target[1])))\n",
    "    for i in range(num_nearby):\n",
    "        dist = dists[i]\n",
    "        phrase = valid_phrases[phrase_idx[i]]\n",
    "        len_ = valid_lens[phrase_idx[i]]\n",
    "        target = valid_targets[phrase_idx[i]]\n",
    "        print(\"%i: %0.3f : <%s> '%s' <%s>\" \n",
    "              % (i, dist, DH.vocab_at(target[0]),\n",
    "                 DH.sequence_to_sentence(phrase, len_),\n",
    "                 DH.vocab_at(target[1])))\n",
    "    print(\"=\"*80)\n",
    "#     drnn.save_validation_accuracy(frac_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def time_left(num_epochs, num_steps, fit_time, nearby_time, start_time, nearby_mod):\n",
    "    total = num_epochs*num_steps*fit_time + ((num_epochs*num_steps)/float(nearby_mod))*nearby_time\n",
    "    return total - (time() - start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "num_epochs = 1\n",
    "batch_size =50\n",
    "neg_per = 20\n",
    "num_nearby = 50\n",
    "nearby_mod = 50\n",
    "sample_power = .75\n",
    "DH.scale_vocab_dist(sample_power)\n",
    "\n",
    "# bookkeeping\n",
    "num_steps = DH.num_steps(batch_size)\n",
    "total_step = 1\n",
    "save_interval = 30 * 60 # half hour in seconds\n",
    "save_time = time()\n",
    "\n",
    "#timing stuff\n",
    "start = time()\n",
    "fit_time = 0\n",
    "nearby_time = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    offset = 0 #if epoch else 400\n",
    "    DH.shuffle_data()\n",
    "    for step , batch in enumerate(DH.batches(batch_size, offset=offset, neg_per=neg_per)):\n",
    "        if not step: step = offset\n",
    "        t0 = time()\n",
    "        loss = drnn.partial_unsup_fit(*batch)\n",
    "        fit_time = (fit_time * float(total_step) +  time() - t0) / (total_step + 1) # running average\n",
    "        if step % 10 == 0:\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            left = time_left(num_epochs, num_steps, fit_time, nearby_time, start, nearby_mod)\n",
    "            ml,sl = divmod(left, 60)\n",
    "            hl,ml = divmod(ml, 60)\n",
    "            pps = batch_size*(neg_per + 1) / fit_time \n",
    "            print(\"(%i:%i:%i) step %i/%i, epoch %i Training Loss = %1.5f :: %0.3f phrases/sec :: (%i:%i:%i) hours left\" \n",
    "                  % (h,m,s, step, num_steps, epoch, loss, pps, hl, ml, sl))\n",
    "        if (total_step-1) % nearby_mod == 0: # do one right away so we get a good timing estimate\n",
    "            t0 = time()\n",
    "            run_validation_test(num_nearby) # check out the nearby phrases in the validation set\n",
    "            valid_loss = drnn.validation_loss(*DH.validation_batch())\n",
    "            print(\"Validation loss: %0.4f\" % valid_loss)\n",
    "            nearby_time = (nearby_time * float(total_step) + time() - t0) / (total_step + 1) # running average\n",
    "\n",
    "        if (time() - save_time) > save_interval:\n",
    "            print(\"Saving model...\")\n",
    "            drnn.checkpoint()\n",
    "            save_time = time()\n",
    "        total_step +=1\n",
    "drnn.checkpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drnn.checkpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# test the embeddings\n",
    "\n",
    "### VALID ###\n",
    "# valid_phrases, valid_targets, _, valid_lens = DH.validation_batch()\n",
    "# phrase_embeds, target_embeds = drnn.embed_phrases_and_targets(valid_phrases, valid_targets, valid_lens)\n",
    "# phrase_labels, target_labels = DH.readable_data(valid=True)\n",
    "\n",
    "### TRAIN ###\n",
    "train_phrases, train_targets, _, train_lens = DH.batches(500, neg_per=0, offset=0).next()\n",
    "phrase_embeds, target_embeds = drnn.embed_phrases_and_targets(train_phrases, train_targets, train_lens)\n",
    "phrase_labels, target_labels = DH.readable_data(show_dep=False, valid=False)\n",
    "        \n",
    "phrase_embeds /= np.sqrt(np.sum(phrase_embeds**2, 1, keepdims=True))\n",
    "target_embeds /= np.sqrt(np.sum(target_embeds**2, 1, keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### JOINT ###\n",
    "start = 0\n",
    "stride = 40\n",
    "end = start + stride\n",
    "\n",
    "lowd = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "# lowd = PCA(n_components=2)\n",
    "\n",
    "joint_embeds = np.vstack([phrase_embeds[start:end], target_embeds[start:end]])\n",
    "joint_2d = lowd.fit_transform(joint_embeds)\n",
    "phrase_2d, target_2d = joint_2d[:stride], joint_2d[stride:]\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(20,16))\n",
    "for i, label in enumerate(phrase_labels[start:end]):\n",
    "    label = \"%i: %s\" % (i, label)\n",
    "    x, y = phrase_2d[i,:]\n",
    "    ax.scatter(x, y, color='b')\n",
    "    ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "                   ha='right', va='bottom')\n",
    "for i, label in enumerate(target_labels[start:end]):\n",
    "    label = \"%i: %s\" % (i, label)\n",
    "    x, y = target_2d[i,:]\n",
    "    ax.scatter(x, y, color='r')\n",
    "    ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "                   ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### PHRASE ONLY ###\n",
    "start = 0\n",
    "stride = 50\n",
    "end = start + stride\n",
    "\n",
    "lowd = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "# lowd = PCA(n_components=2)\n",
    "\n",
    "phrase_2d = lowd.fit_transform(phrase_embeds[start:end])\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(20,16))\n",
    "for i, label in enumerate(phrase_labels[start:end]):\n",
    "    label = \"%i: %s\" % (i, label)\n",
    "    x, y = phrase_2d[i,:]\n",
    "    ax.scatter(x, y, color='b')\n",
    "    ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "                   ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### TARGET ONLY ###\n",
    "start = 0\n",
    "stride = 35\n",
    "end = start + stride\n",
    "\n",
    "lowd = TSNE(perplexity=20, n_components=2, init='pca', n_iter=5000)\n",
    "# lowd = PCA(n_components=2)\n",
    "\n",
    "target_2d = lowd.fit_transform(target_embeds[start:end])\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(20,16))\n",
    "for i, label in enumerate(target_labels[start:end]):\n",
    "    label = \"%i: %s\" % (i, label)\n",
    "    x, y = target_2d[i,:]\n",
    "    ax.scatter(x, y, color='r')\n",
    "    ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "                   ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test out semeval data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "zip_train = zip(train['raws'], train['sents'], train['sdps'], train['targets'], train['labels'])\n",
    "zip_valid = zip(valid['raws'], valid['sents'], valid['sdps'], valid['targets'], valid['labels'])\n",
    "zip_test = zip(test['raws'], test['sents'], test['sdps'], test['targets'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i, (raw, _, sdp, target, label) in enumerate(zip_train):\n",
    "    if i > 5:\n",
    "        break\n",
    "    print(raw)\n",
    "    print(\"%s :: %s\" % (DH.sequence_to_sentence(sdp, show_dep=True), DH.sequence_to_sentence(target)))\n",
    "    print(int2label[label])\n",
    "    print(\"=\"*80)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_size = 50\n",
    "num_steps = len(zip_train) // batch_size\n",
    "num_steps=1\n",
    "for step in range(num_steps):\n",
    "    class_batch = DH.classification_batch(batch_size, train['sdps'], train['targets'], train['labels'], offset=step)\n",
    "    loss = drnn.partial_class_fit(*class_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from spacy.en import English\n",
    "nlp=English()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s = nlp(u\"100,000.00 ! a\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for token in s:\n",
    "    print(token, token.is_punct, token.like_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drnn.checkpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}