DataHandler.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A class to handle the reading in of data and batch generation for models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import json\n",
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class DataHandler(object):\n",
    "    \"\"\"Handler to read in data and generate data and batches for model training and evaluation\"\"\"\n",
    "    def __init__(self, data_prefix, max_sequence_len=None):\n",
    "        self._data_prefix = data_prefix\n",
    "        \n",
    "        self.read_data()\n",
    "        if max_sequence_len:\n",
    "            assert max_sequence_len >= self._max_seq_len, \"Cannot for sequence length shorter than the data yields\"\n",
    "            self._max_seq_len = max_sequence_len\n",
    "            \n",
    "    def read_data(self):\n",
    "        print(\"Creating Data objects...\")\n",
    "        # read in sdp data\n",
    "        data = []\n",
    "        with open(self._data_prefix, 'r') as f:\n",
    "            for line in f:\n",
    "                data.append(json.loads(line))\n",
    "        self._paths = [ datum['path'] for datum in data ]\n",
    "        self._max_seq_len = max([ len(path) for path in self._paths ])\n",
    "        self._targets = [ datum['target'][0] for datum in data] # targets get doubly wrapped in lists\n",
    "        \n",
    "        #make sure all of the paths have same depth and all the targets have depth 2\n",
    "#         print(self._paths)\n",
    "#         print([target for target in self._targets])\n",
    "        assert len(set(len(p) for path in self._paths for p in path)) == 1, \"Not all path tuples have same len\"\n",
    "        assert set(len(target) for target in self._targets) == set([2]), \"All target tuples must be pairs\"\n",
    "        \n",
    "        # read in vocab and distribution\n",
    "        vocab_and_dist = []\n",
    "        with open(self._data_prefix+\"_vocab\", 'r') as f:\n",
    "            for line in f:\n",
    "                vocab_and_dist.append(json.loads(line))\n",
    "        self._vocab = [x[0] for x in vocab_and_dist]\n",
    "        self._true_vocab_dist = [x[1] for x in vocab_and_dist]\n",
    "        self._vocab_dist = self._true_vocab_dist\n",
    "        self._vocab2int = {v:i for (i,v) in enumerate(self._vocab)}\n",
    "        self._int2vocab = {i:v for (v,i) in self._vocab2int.items()}\n",
    "        \n",
    "        # read in dependency vocab and distribution\n",
    "        dep_and_dist = []\n",
    "        with open(self._data_prefix+\"_dep\", 'r') as f:\n",
    "            for line in f:\n",
    "                dep_and_dist.append(json.loads(line))\n",
    "        self._dep_vocab = [x[0] for x in dep_and_dist]\n",
    "        self._true_dep_dist = [x[1] for x in dep_and_dist]\n",
    "        self._dep_dist = self._true_dep_dist\n",
    "        self._dep2int = {v:i for (i,v) in enumerate(self._dep_vocab)}\n",
    "        self._int2dep = {i:v for (v,i) in self._dep2int.items()}\n",
    "        print(\"Done creating Data objects\")\n",
    "    \n",
    "    def _sequences_to_tensor(self, list_of_lists):\n",
    "        \"\"\" Convert list of lists of either single elements or tuples into matrix of appropriate dim\"\"\"\n",
    "        lengths = np.array([len(list_) for list_ in list_of_lists]).reshape([-1, 1])\n",
    "        \n",
    "        #matrix case\n",
    "        if isinstance(list_of_lists[0][0], (int, float)):\n",
    "            matrix = np.zeros([len(list_of_lists), self._max_seq_len])\n",
    "            for i, list_ in enumerate(list_of_lists):\n",
    "                matrix[i, :len(list_)] = list_\n",
    "            return matrix, lengths\n",
    "        \n",
    "        #tensor case\n",
    "        if isinstance(list_of_lists[0][0], (tuple, list)):\n",
    "            k = len(list_of_lists[0][0]) # we asserted before that all of them were the same len\n",
    "            tensor = np.zeros([len(list_of_lists), self._max_seq_len, k])\n",
    "            for i, list_ in enumerate(list_of_lists):\n",
    "                for j in range(k):\n",
    "                    tensor[i, :len(list_), j] = [ x[j] for x in list_ ]\n",
    "            return tensor, lengths\n",
    "    \n",
    "    def _generate_batch(self, offset, batch_size, neg_per=None):\n",
    "        \"\"\"Expects the data as list of lists of indices\n",
    "\n",
    "        Converts them to matrices of indices, lang model labels, and lengths\"\"\"\n",
    "        start = offset*batch_size\n",
    "        end = start + batch_size\n",
    "        if end > len(self._paths):\n",
    "            end = len(self._paths)\n",
    "#             print(\"Not full batch\")\n",
    "        inputs = self._paths[start:end]\n",
    "        targets = np.array(self._targets[start:end])\n",
    "        print(targets.shape)\n",
    "        labels = np.ones(targets.shape[0]).reshape((-1, 1))\n",
    "        input_mat, len_vec = self._sequences_to_tensor(inputs)\n",
    "        # generate the negative samples\n",
    "        # randomly choose one index for each negative sample \n",
    "        # TODO: option to replace more than one phrase element\n",
    "        # and replace that with a random word drawn from the scaled unigram distribution\n",
    "        if neg_per:\n",
    "            negatives = []\n",
    "            neg_targets = []\n",
    "            for i, seq in enumerate(inputs):\n",
    "                for neg in range(neg_per):\n",
    "                    rand_idx = int(random.uniform(0, len(seq)))\n",
    "                    sample = self._sample_distribution(self._vocab_dist)\n",
    "#                     print(rand_idx)\n",
    "                    neg_seq = seq[:]\n",
    "#                     print(neg_seq)\n",
    "                    neg_seq[rand_idx][0] = sample\n",
    "                    negatives.append(neg_seq)\n",
    "                    neg_targets.append(targets[i])\n",
    "            neg_mat, neg_len = self._sequences_to_tensor(negatives)\n",
    "            neg_labels = np.zeros_like(neg_len)\n",
    "            print(labels.shape, neg_labels.shape)\n",
    "            all_inputs = np.vstack((input_mat, neg_mat)).astype(np.int32)\n",
    "            all_targets = np.vstack((targets, np.array(neg_targets))).astype(np.int32)\n",
    "            all_labels = np.vstack((labels, neg_labels)).astype(np.int32)\n",
    "            all_lengths = np.vstack((len_vec, neg_len)).astype(np.int32)\n",
    "        else:\n",
    "            all_inputs = input_mat.astype(np.int32)\n",
    "            all_targets = targets.astype(np.int32)\n",
    "            all_labels = labels.astype(np.int32)\n",
    "            all_lengths = len_vec.astype(np.int32)\n",
    "        return all_inputs, all_targets, all_labels, all_lengths\n",
    "    \n",
    "    def batches(self, batch_size, neg_per=5, offset=0):\n",
    "        num_steps = len(self._paths) // batch_size\n",
    "        for step in range(offset, num_steps):\n",
    "            yield self._generate_batch(step, batch_size, neg_per=neg_per)\n",
    "    \n",
    "    def scale_vocab_dist(self, power):\n",
    "        self._vocab_dist = self._distribution_to_power(self._true_vocab_dist, power)\n",
    "        \n",
    "    def scale_dep_dist(self, power):\n",
    "        self._dep_dist = self._distribution_to_power(self._true_dep_dist, power)\n",
    "        \n",
    "    def _distribution_to_power(self, distribution, power):\n",
    "        \"\"\"Return a distribution, scaled to some power\"\"\"\n",
    "        dist = [ pow(d, power) for d in distribution ]\n",
    "        dist /= np.sum(dist)\n",
    "        return dist\n",
    "    \n",
    "    def _sample_distribution(self, distribution):\n",
    "        \"\"\"Sample one element from a distribution assumed to be an array of normalized\n",
    "        probabilities.\n",
    "        \"\"\"\n",
    "        r = random.uniform(0, 1)\n",
    "        s = 0\n",
    "        for i in range(len(distribution)):\n",
    "            s += distribution[i]\n",
    "            if s >= r:\n",
    "                return i\n",
    "        return len(distribution) - 1\n",
    "    \n",
    "    @property\n",
    "    def data_prefix(self):\n",
    "        return self._data_prefix\n",
    "    \n",
    "    @property\n",
    "    def vocab(self):\n",
    "        return self._vocab\n",
    "    \n",
    "    @property\n",
    "    def dep_vocab(self):\n",
    "        return self._dep_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Data objects...\n",
      "Done creating Data objects\n"
     ]
    }
   ],
   "source": [
    "data_handler = DataHandler('data/wiki_sdp_100')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  2.11760549e-01   1.54333178e-01   1.25144722e-01   8.95462737e-02\n",
      "   7.01739473e-02   6.25782425e-02   4.03236025e-02   3.28540053e-02\n",
      "   1.75573183e-02   1.38039791e-02   9.55699360e-03   8.77174342e-03\n",
      "   8.77174342e-03   7.27025136e-03   7.27025136e-03   6.55548142e-03\n",
      "   6.55548142e-03   5.86580425e-03   5.20216840e-03   5.20216840e-03\n",
      "   5.20216840e-03   5.20216840e-03   4.56563896e-03   3.95742358e-03\n",
      "   3.95742358e-03   3.37890749e-03   3.37890749e-03   3.37890749e-03\n",
      "   3.37890749e-03   3.37890749e-03   3.37890749e-03   2.83170181e-03\n",
      "   2.83170181e-03   2.83170181e-03   2.83170181e-03   2.83170181e-03\n",
      "   2.83170181e-03   2.83170181e-03   2.83170181e-03   2.83170181e-03\n",
      "   2.31771268e-03   2.31771268e-03   2.31771268e-03   2.31771268e-03\n",
      "   2.31771268e-03   1.83924428e-03   1.83924428e-03   1.83924428e-03\n",
      "   1.83924428e-03   1.83924428e-03   1.83924428e-03   1.83924428e-03\n",
      "   1.83924428e-03   1.83924428e-03   1.39916053e-03   1.39916053e-03\n",
      "   1.39916053e-03   1.39916053e-03   1.39916053e-03   1.39916053e-03\n",
      "   1.39916053e-03   1.39916053e-03   1.39916053e-03   1.39916053e-03\n",
      "   1.25144722e-04]\n"
     ]
    }
   ],
   "source": [
    "data_handler.scale_vocab_dist(1.5)\n",
    "print(data_handler._vocab_dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 0.25)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6sAAAIXCAYAAAB6ncg/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3W2spOdZH/D/5RpXdUIjAmFHcsgeaEooqOGtMq4C4tC0\n6YZKmE+tQaIlElE+kBKJqnKKhLJ8qYhU0YIoba26qKDQQIrSuGrSOhVZtREKWSAJb97avNiyk3hL\nqiQlkahMcvfDjDezZ8/umXN25sx1nvn9pNGcueeZ577vnTl7nf88bzXGCAAAAHRyx7YHAAAAAAcJ\nqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0s1JYraoLVXWlqh6vqgcPef57q+oji9v7q+qV\nS889uWj/UFV9cJ2DB4BdpTYDMHV11HVWq+qOJI8neXWSjyW5nOSBMcaVpWXuS/LYGOPTVXUhycUx\nxn2L5/4wyTePMT65oTkAwE5RmwHYBatsWb03yRNjjKfGGM8leXuS+5cXGGN8YIzx6cXDDyS5Z+np\nWrEfAGA1ajMAk7dKobonydNLj5/J9QXvoB9I8p6lxyPJe6vqclW9/vhDBAAOUJsBmLw717myqvqO\nJK9L8q1Lza8aY3y8ql6SeWF8bIzx/nX2CwAcTm0G4KxaJax+NMnLlh6/dNF2ncWJGx5KcmH5GJgx\nxscX939cVe/MfNelGwpiVd364FkAOIYxRm17DBukNgNw5hy3Nq+yG/DlJC+vqvNVdVeSB5I8srxA\nVb0syS8n+b4xxh8std9dVS9c/PyCJK9J8ju3GPzkb295y1u2PgbzNEfzNM+p33aA2uz3wjx3dI7m\nOb3brszzJI7csjrG+FxVvTHJo5mH24fHGI9V1RvmT4+Hkvxokhcn+ZmqqiTPjTHuTXIuyTsX38ze\nmeRtY4xHTzRSACCJ2gzAbljpmNUxxn9N8ooDbf9m6efXJ7nhBA1jjD9K8g23OUYA4AC1GYCpc9r6\nU7a/v7/tIZyKXZjnLswxMc+p2ZV5wnHsyu/FLsxzF+aYmOfU7Mo8T6JOuv/wulXV6DIWAM62qsqY\n9gmWToXaDMC6nKQ227IKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQj\nrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6w\nCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8Iq\nAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsA\nAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIA\nANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAA\nQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA\n7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADttAqrVXXt\nNpvtbXs4AAAAbEmNMbY9hiRJVY1keSyVLmMD4GypqowxatvjOOuqaqjFAKzDSWpzqy2rAAAAkAir\nAAAANCSsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wC\nAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALSzUlitqgtVdaWq\nHq+qBw95/nur6iOL2/ur6pWrvhYAOD61GYCpqzHGrReouiPJ40leneRjSS4neWCMcWVpmfuSPDbG\n+HRVXUhycYxx3yqvXVrHSJbHUjlqbABwmKrKGKO2PY5NOc3arBYDsA4nqc2rbFm9N8kTY4ynxhjP\nJXl7kvuXFxhjfGCM8enFww8kuWfV1wIAx6Y2AzB5q4TVe5I8vfT4mXyh4B3mB5K854SvBQCOpjYD\nMHl3rnNlVfUdSV6X5FtPtoaLaxwNALvi0qVLuXTp0raH0dLt1uaLFy9e+3l/fz/7+/trGRcA07aO\n2rzKMav3ZX6cy4XF4zcnGWOMtx5Y7pVJfjnJhTHGHxzntYvnHLMKwFrswDGrp1ab1WIA1mFTx6xe\nTvLyqjpfVXcleSDJIwc6flnmxfD7ni+Gq74WADg2tRmAyTtyN+Axxueq6o1JHs083D48xnisqt4w\nf3o8lORHk7w4yc9UVSV5boxx781eu7HZAMAOUJsB2AVH7gZ8WuwGDMC6TH034NNiN2AA1mVTuwED\nAADAqRJWAQAAaEdYBQAAoB1hFQAAgHaEVQAAANoRVgEAAGhHWAUAAKAdYRUAAIB2hFUAAADaEVYB\nAABoR1gFAACgHWEVAACAdoRVAAAA2hFWAQAAaEdYBQAAoB1hFQAAgHaEVQAAANoRVgEAAGhHWAUA\nAKAdYRUAAIB2hFUAAADaEVYBAABoR1gFAACgHWEVAACAdoRVAAAA2hFWAQAAaEdYBQAAoB1hFQAA\ngHaEVQAAANoRVgEAAGhHWAUAAKAdYRUAAIB2hFUAAADaEVYBAABoR1gFAACgHWEVAACAdoRVAAAA\n2hFWAQAAaEdYBQAAoB1hFQAAgHaEVQAAANoRVgEAAGhHWAUAAKAdYRUAAIB2hFUAAADaEVYBAABo\nR1gFAACgHWEVAACAdoRVAAAA2hFWAQAAaEdYBQAAoB1hFQAAgHaEVQAAANoRVgEAAGhHWAUAAKAd\nYRUAAIB2hFUAAADaEVYBAABoR1gFAACgHWEVAACAdoRVAAAA2hFWAQAAaEdYBQAAoB1hFQAAgHaE\nVQAAANoRVgEAAGhHWAUAAKAdYRUAAIB2hFUAAADaEVYBAABoR1gFAACgHWEVAACAdoRVAAAA2hFW\nAQAAaEdYBQAAoB1hFQAAgHaEVQAAANoRVgEAAGhHWAUAAKAdYRUAAIB2hFUAAADaEVYBAABoR1gF\nAACgHWEVAACAdoRVAAAA2hFWAQAAaEdYBQAAoJ3WYXU220tVXXebzfa2PSwAAAA2rMYY2x5DkqSq\nRrI8llrcHxxfpcuYAeipqjLGqKOX5Faqaqi5AKzDSWpz6y2rAAAA7CZhFQAAgHZWCqtVdaGqrlTV\n41X14CHPv6KqfrWq/rSqfvjAc09W1Ueq6kNV9cF1DRwAdpnaDMDU3XnUAlV1R5KfTvLqJB9Lcrmq\n3jXGuLK02P9J8g+TfPchq/h8kv0xxifXMF4A2HlqMwC7YJUtq/cmeWKM8dQY47kkb09y//ICY4xP\njDF+I8mfHfL6WrEfAGA1ajMAk7dKobonydNLj59ZtK1qJHlvVV2uqtcfZ3AAwKHUZgAm78jdgNfg\nVWOMj1fVSzIvjI+NMd5/+KIXT2E4AEzNpUuXcunSpW0P4yxZuTZfvHjx2s/7+/vZ398/nRECcKat\nozYfeZ3VqrovycUxxoXF4zcnGWOMtx6y7FuS/MkY4ydusq6bPu86qwCsy9Svs3qatVnNBWAdNnWd\n1ctJXl5V56vqriQPJHnkVuNYGtDdVfXCxc8vSPKaJL9znAECADdQmwGYvCN3Ax5jfK6q3pjk0czD\n7cNjjMeq6g3zp8dDVXUuya8n+eIkn6+qNyX52iQvSfLO+VbT3JnkbWOMRzc1GQDYBWozALvgyN2A\nT4vdgAFYl6nvBnxa7AYMwLpsajdgAAAAOFXCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADt\nCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQj\nrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6w\nCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8Iq\nAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsA\nAAC0I6wCAADQjrAKAABAO2cyrM5me6mq626z2d62hwUAAMCa1Bhj22NIklTVSJbHUov7g+O7eXuX\nuQCwXVWVMUYdvSS3UlVDbQVgHU5Sm8/kllUAAACmTVgFAACgHWEVAACAdoRVAAAA2hFWAQAAaEdY\nBQAAoB1hFQAAgHaEVQDgpqrq2m0229v2cADYIdXlYt9VNZLlsTx/vdiD47t5e5e5ALBdJ7nwODc6\nrDartQCcxElqsy2rAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioA\nAADtCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAA\nALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA\n0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABA\nO8IqAAAA7awUVqvqQlVdqarHq+rBQ55/RVX9alX9aVX98HFeCwAcn9oMwNTVGOPWC1TdkeTxJK9O\n8rEkl5M8MMa4srTMlyU5n+S7k3xyjPETq752aR0jWR5LLe4Pju/m7UfNBYDdUFUZY9TRS55N26zN\nai0AJ3GS2rzKltV7kzwxxnhqjPFckrcnuX95gTHGJ8YYv5Hkz477WgDg2NRmACZvlbB6T5Knlx4/\ns2hbxe28FgA4nNoMwOTdue0BXO/itgcAwBl06dKlXLp0advDmKiL2x4AAGfQOmrzKmH1o0letvT4\npYu2VRzztReXfv6xFbsAYNft7+9nf3//2uMf+7HJ1xC1GYDW1lGbV9kN+HKSl1fV+aq6K8kDSR65\nxfLLB80e97UAwNHUZgAm78gtq2OMz1XVG5M8mnm4fXiM8VhVvWH+9Hioqs4l+fUkX5zk81X1piRf\nO8b4zGGv3dhsAGAHqM0A7IIjL11zWly6BoB1mfqla07LYbX53LnzuXr1qeuWO3fufJ599slTHRsA\nZ8tJarOwCsDkCKvrcZzarAYDcCubus4qAAAAnCphFQAAgHaEVQAAANoRVgEAAGhHWAUAAKAdYRUA\nAIB2hFUAAADaEVYBAABoR1gFAACgHWEVAACAdoRVAAAA2hFWAQAAaEdYBQAAoB1hFQAAgHaEVQAA\nANqZVFidzfZSVdfdZrO9bQ8LAACAY6oxxrbHkCSpqpEsj6UW9wfHd/z2LnME4HRUVcYYdfSS3Mpx\narNaC8CtnKQ2T2rLKgAAANMgrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8Iq\nAHDbZrO9VNV1t9lsb9vDAuAMqy4X8T7OhceP295ljgCcjpNceJwbraM2q8EAJCerzbasAgAA0I6w\nCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8Iq\nAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsA\nAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALSzE2F1\nNttLVV27zWZ72x4SAAAAt1BjjG2PIUlSVSNZHkst7g+Obx3tlS7zBmD9qipjjDp6SW5lHbVZvQUg\nOVlt3oktqwAAAJwtwioAAADtCKsAwMYcPG+Ec0cAsCrHrAIwOY5ZXY9N1mZ1GGC3OGYVAACASRBW\nAQAAaEdYBQAAoB1hFQAAgHaEVQAAANoRVgEAAGhHWAUAAKAdYRUAAIB2hFUAAADa2emwOpvtpaqu\n3WazvW0PCQAAgCQ1xtj2GJIkVTWS5bHU4v7g+NbRXhljpKoObQfgbKuqjDHq6CW5lU3WZvUWYLec\npDbv9JZVAAAAehJWAQAAaEdYBQAAoB1hFQAAgHaEVQAAANoRVgEAAGhHWAUAAKAdYRUAOHWz2V6q\n6tptNtvb9pAAaKa6XJR7kxcev7F9fjHyqjq0HYCz7SQXHudG26jNAEzTSWqzLasAAAC0I6wCAADQ\njrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvCKgDQxmy2l6q6dpvN9rY9JAC2pLpcgHsb\nFx6vqkPbATjbTnLhcW6kNgOwLiepzbasAgAA0M5KYbWqLlTVlap6vKoevMkyP1VVT1TVh6vqG5fa\nn6yqj1TVh6rqg+saOADsMrUZgKm786gFquqOJD+d5NVJPpbkclW9a4xxZWmZ1yb5S2OMv1xV35Lk\nXyW5b/H055PsjzE+ufbRA8AOUpsB2AWrbFm9N8kTY4ynxhjPJXl7kvsPLHN/kp9LkjHGryV5UVWd\nWzxXK/YDAKxGbQZg8lYpVPckeXrp8TOLtlst89GlZUaS91bV5ap6/UkHCgBcozYDMHlH7ga8Bq8a\nY3y8ql6SeWF8bIzx/sMXvXgKwwFgai5dupRLly5texhnidoMwEatozYfeemaqrovycUxxoXF4zcn\nGWOMty4t86+TvG+M8YuLx1eSfPsY4+qBdb0lyZ+MMX7ikH6cHh+AtZj6pWvUZgDOmk1duuZykpdX\n1fmquivJA0keObDMI0n+/mIQ9yX51BjjalXdXVUvXLS/IMlrkvzOcQYIANxAbQZg8o7cDXiM8bmq\nemOSRzMPtw+PMR6rqjfMnx4PjTHeXVXfWVW/n+SzSV63ePm5JO+cfzObO5O8bYzx6GamAgC7QW0G\nYBccuRvwabGrEQDrMvXdgE+L2gzAumxqN2AAAAA4VcIqAAAA7QirAAAAtCOsAgAA0I6wCgAAQDvC\nKgAAAO0IqwAAALQjrAIAANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA7Qir\nAAAAtCOsHjCb7aWqrrvNZnvbHhYA7Cy1GWA31Rhj22NIklTVSJbHUov7g+NbR3tljJGquqH9Zuvo\n8u8EwNGqKmOMOnpJbkVtBmBdTlKbbVkFAACgHWEVAACAdoRVAAAA2hFWAQAAaEdYBQAAoB1hFQAA\ngHaEVQAAANoRVgEAAGhHWAUAAKAdYRUAOJNms71U1XW32Wxv28MCYE1qjLHtMSRJqmoky2Opxf3B\n8a2jvTLGSFXd0H6zdXT5dwLgaFWVMUYdvSS30r0236xdzQbo5yS12ZZVAAAA2hFWAQAAaEdYXZHj\nYgAAAE6PY1YdFwMwOY5ZXQ+1GYB1ccwqAAAAkyCsAgAA0I6wCgAAQDvCKgAAAO0IqwAAALQjrAIA\nANCOsAoAAEA7wioAAADtCKsAAAC0I6wCAADQjrAKAABAO8IqAAAA7QirAAAAtCOsAgAA0I6wCgAA\nQDvCKgAAAO0IqwAAALQjrAIAANCOsAoATMpstpequu42m+1te1gAHFONMbY9hiRJVY1keSy1uD84\nvnW0V8YYqaob2o+77i7/fgB8QVVljFFHL8mtnNXafLN2NRtge05Sm21ZBQAAoB1hFQAAgHaEVQAA\nANoRVgEAAGhHWAUAdsLBswQ7QzBAb84G7GzAAJPjbMDrcVZr83H7BGDznA0YAACASRBWAQAAaEdY\nBQAAoB1h9TYdPFmDEzYAAADcPidY2lCfXf5dAXaREyytx9Rq8836BGDznGAJAACASRBWAQAAaEdY\nBQB22sHzTzj3BEAPjll1zCrA5DhmdT2mVpuP06c6DrBejlkFAFiDm53t31UAAE6PLasb6vPcufO5\nevWpay3nzp3Ps88+GQA2z5bV9Zhabd5kn13+ngLo6iS1WVh1enyAyRFW12NXarOwCrB5dgMGANiC\nVXcbtsswwOpsWbVlFWBybFldj12pzf4eANg8W1YBAM6Aw7a4OnkTwPVsWfVNKsDk2LK6HrtSm7v3\n6e8HYApsWQUAmBhbXIFdJawCADQ2vxTeuO529epTxz6pk12PgbPGbsB2AwaYHLsBr8eu1GZ92vUY\n2Dy7AZ8BTmEPAHRniyvQgS2rDfo8d+78YhefLzh37nySHNr+7LNPBoCbs2V1PXa5Nuvz8PYufzcC\nZ89JarOwegb77PKeAXQlrK6H2qzPg+2rfsHuy3XgIGF15fazXSi6vGcAXQmr66E26/N2+jzunmO3\n226vNOhNWF25/WwXCt9qAtyasLoearM+b6fPza37+H12+XsXdpkTLO2Im53C/mD7wUALALCLVr3M\nz3Hb17GOk1xyCHaFLas70Odstmc3GWCn2LK6HmqzPm+nz82te1f6vPn71uXvdzgOW1Y51GFbXI97\ngXEAAHqwNZddYcuqPldqP+5xsrbmAttky+p6qM36vJ0+N7fuXemz12elS2bg7HKCpZXbe/3y73qf\nXT6DwHQIq+uhNuvzdvrc3Lp3pc9en5VNnd35JBs61tUnp0tYXbm91y//rvd5s//8/CcCnJSwuh5q\nsz5vp8/NrXtX+tydz8pZOK/L7bYL5cLqMdp36xfxrPbpEj3ASQmr66E26/N2+tzcunelz935rOhz\ns312cZLafOemBgO36wsngVpuu/EX8fm20/6m6mbLCs4AAHD7bFnVpz5P2OfNlt3kMR23u46T9Cl8\ncxbZsroearM+b6fPza17V/rcnc+KPjfbZxcb2w24qi4k+ReZX+rm4THGWw9Z5qeSvDbJZ5N8/xjj\nw6u+drGcgqjPM9Xn5tbdq89Nhe+btQvIrMMuhFW1WZ/d+9zcunelz935rOhzs312caLaPMa45S3z\nQvb7Sc4n+aIkH07yNQeWeW2S/7L4+VuSfGDV1y6tYyRj6ZZxY9u62jPGGIe261Ofq/a5uXXr89y5\n80vP5Vrb7bafO3d+jDEObf+SLzm30T67eN/73rftIZyKxf8LmeotarM+z0Cfu1Kzpva+6XN6fXax\nGE+Oc1vlmNV7kzwxxngqSarq7UnuT3JlaZn7k/xc5iP4tap6UVWdS/KVK7wW4DqrHq983Pbn2w6u\nfx3rPqrPLmcc/MxnPpXPfvbTG+uTU6M2AzB5q4TVe5I8vfT4mcyL5FHL3LPiawEmbxsBeRt9cmrU\nZgBWctiX12flC+ZNnQ3YXy0A0IvaDLCDbv7ldX+rhNWPJnnZ0uOXLtoOLvMVhyxz1wqvXXLYP9zN\n/jFvr31+cLM+9Xm77Ztctz43se5d+Xxut09Ogdqsz/Z9bn7d0+9zVz4r+jz9Ps9KzV4lrF5O8vKq\nOp/k40keSPI9B5Z5JMkPJvnFqrovyafGGFer6hMrvDZJMiZ+1kYAWCO1GYDJOzKsjjE+V1VvTPJo\nvnCK+8eq6g3zp8dDY4x3V9V3VtXvZ356/Nfd6rUbmw0A7AC1GYBdsNJ1VgEAAOA03bHtAVTVhaq6\nUlWPV9WD2x7PulTVw1V1tap+a6ntS6rq0ar6X1X136rqRdsc4zpU1Uur6leq6ner6rer6ocW7ZOa\na1X9+ar6tar60GKu/3TRPql5Pq+q7qiq36yqRxaPJzfPqnqyqj6yeE8/uGib4jxfVFXvqKrHFp/d\nb5naPKvqqxfv428u7j9dVT80tXmeJrX5bFObpzXP56nNk5qn2rziPLcaVqvqjiQ/neRvJ/m6JN9T\nVV+zzTGt0c9mPq9lb07y38cYr0jyK0n+yamPav3+LMkPjzG+LslfT/KDi/dwUnMdY/y/JN8xxvjG\nJK9M8jeq6lWZ2DyXvCnJ7y09nuI8P59kf4zxjWOM5y/bMcV5/mSSd48x/kqSr8/8WpqTmucY4/HF\n+/hNSb45811e35mJzfO0qM2T+JyozROa5xK1eTrzVJtXnecYY2u3JPclec/S4zcneXCbY1rz/M4n\n+a2lx1eSnFv8PEtyZdtj3MCc/1OSvznluSa5O8kHk3ztFOeZ+ZlB35tkP8kji7YpzvOPknzpgbZJ\nzTPJX0zyB4e0T2qeB+b2miT/c+rz3PC/odrcYJxrnrPa3GCMtzk/tXki81SbjzfPbe8GfLMLlk/V\nl48xribJGOPZJF++5fGsVVXtJfmGJB/I/EM4qbkudr/5UJJnk1waY/xeJjjPJP88yT/O8gW5pjnP\nkeS9VXW5qn5g0Ta1eX5lkk9U1c8udsN5qKruzvTmuezvJfmFxc9Tnucmqc0TojZPY55Rm6c0T7X5\nGPPcdljddZM5u1VVvTDJf0zypjHGZ3Lj3M78XMcYnx/zXY1emuTbqmo/E5tnVf2dJFfHGB/OzS/g\nlZzxeS68asx3TfnOzHeR+7ZM7P3M/Izv35TkXy7m+tnMt5JNbZ5Jkqr6oiTfleQdi6ZJzpONm8zn\nRG3+wmKnPrA1Upun9X5GbT7WPLcdVle5qPmUXK2qc0lSVbMk/3vL41mLqroz82L482OMdy2aJznX\nJBlj/N8k707y1zK9eb4qyXdV1R8m+Q+ZH//z80mendg8M8b4+OL+jzPfRe7eTO/9fCbJ02OMX188\n/uXMC+TU5vm81yb5jTHGJxaPpzrPTVObJ0BtntQ81eZpzVNtPsY8tx1Wr13UvKruyvzC5I9seUzr\nVLn+G7BHknz/4ud/kORdB19wRv27JL83xvjJpbZJzbWqvuz5s5VV1V9I8reSfCgTm+cY40fGGC8b\nY3xV5r+PvzLG+L4k/zkTmmdV3b3Y4pCqekHmx1L8dqb3fl5N8nRVffWi6dVJfjcTm+eS78n8D7nn\nTXWem6Y2T4PaPHfm56k2T+79VJuPMc+tX2e1qi5kfkas5y9M/uNbHdCaVNUvZH4Q/JcmuZrkLZl/\nQ/SOJF+R5Kkkf3eM8altjXEdFmfd+x+Z/2cyFrcfyfwkB7+Uicy1qv5qkn+f+R84d2T+TfU/q6oX\nZ0LzXFb4axgKAAAAmUlEQVRV357kH40xvmtq86yqr8z8jHQj891x3jbG+PGpzTNJqurrk/zbJF+U\n5A+TvC7Jn8v05nl35nP5qjHGnyzaJvd+nha1+Wx/TtTm6f7uq81nf56J2pxjzHPrYRUAAAAO2vZu\nwAAAAHADYRUAAIB2hFUAAADaEVYBAABoR1gFAACgHWEVAACAdoRVAAAA2hFWAQAAaOf/AycGAkm8\nvJ1pAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114ef5350>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, (ax0, ax1) = plt.subplots(1,2, figsize=(16,9))\n",
    "ax0.bar(range(len(data_handler._vocab_dist)), data_handler._vocab_dist)\n",
    "ax1.bar(range(len(data_handler._true_vocab_dist)), data_handler._true_vocab_dist)\n",
    "ax0.set_ylim(0, .25)\n",
    "ax1.set_ylim(0, .25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n",
      "(100, 2)\n",
      "(100, 1) (500, 1)\n",
      "[(600, 16, 2), (600, 2), (600, 1), (600, 1)]\n"
     ]
    }
   ],
   "source": [
    "for batch in data_handler.batches(100):\n",
    "    print([ t.shape for t in batch])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating 60f4fb2..865390d\n",
      "Fast-forward\n",
      " wiki2sdp.py | 2 +-\n",
      " 1 file changed, 1 insertion(+), 1 deletion(-)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "From github.com:teffland/Relation-Extraction\n",
      "   60f4fb2..865390d  master     -> origin/master\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "Analyzing 50000 sentences\n",
      "================================================================================\n",
      "(0:0:0) Reading Data...\n",
      "(0:1:22) Creating vocab...\n",
      "(0:1:24) Writing data...\n",
      "(0:3:54) Writing vocab...\n",
      "================================================================================\n",
      "DONE: Created 549566 SDPs from 50000 sentences with a total vocab size of 22265\n",
      "Took a total of 0:3:54 hours\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "python wiki2sdp.py -n 50000 -m 3;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "tensorboard --port=6007 --logdir=tensor_summaries/drnn_wiki_w2v;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "drnn_embed_test1\n",
      "drnn_embed_wiki1\n",
      "drnn_embed_wiki_easy\n",
      "drnn_wiki_w2v\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "ls tensor_summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}