RelEmbed_semeval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import collections\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA, FastICA\n",
    "\n",
    "import data_handler as dh\n",
    "import semeval_data_helper as sdh\n",
    "# plot settings\n",
    "% matplotlib inline\n",
    "# print(plt.rcParams.keys())\n",
    "plt.rcParams['figure.figsize'] = (16,9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reload(sdh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reload(nn)\n",
    "import relembed as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reload(eh)\n",
    "import experiment_helper as eh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "shuffle_seed = 20\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Data objects...\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "max() arg is an empty sequence",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-a3e0efe7efdf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# reload(dh)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mDH\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataHandler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data/semeval_train_sdp_8000'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_percent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle_seed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle_seed\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# for semeval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/thomaseffland/Development/Relation-Extraction/data_handler.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data_prefix, valid_percent, max_sequence_len, shuffle_seed)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data_prefix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_prefix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_valid_percent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalid_percent\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m100.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshuffle_seed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle_seed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmax_sequence_len\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mmax_sequence_len\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_max_seq_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Cannot for sequence length shorter than the data yields\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/thomaseffland/Development/Relation-Extraction/data_handler.pyc\u001b[0m in \u001b[0;36mread_data\u001b[0;34m(self, shuffle_seed)\u001b[0m\n\u001b[1;32m     34\u001b[0m             \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# start off in random order before we do validation split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_paths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m \u001b[0mdatum\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'path'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdatum\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_max_seq_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_paths\u001b[0m \u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_targets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m \u001b[0mdatum\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'target'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdatum\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# targets get doubly wrapped in lists\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: max() arg is an empty sequence"
     ]
    }
   ],
   "source": [
    "# reload(dh)\n",
    "DH = dh.DataHandler('data/semeval_train_sdp_8000', valid_percent=10, shuffle_seed=shuffle_seed) # for semeval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Didn't find common ancestor\n",
      "1790\t\"The imams were removed from a US Airways <e1>flight</e1> awaiting departure from the Minneapolis-St. Paul <e2>airport</e2>.\"\n",
      "\n",
      "(The imams were removed from a US Airways flight awaiting departure from the Minneapolis - St . Paul airport ., flight , airport )\n",
      "Bad sentence: '1790\\t\"The imams were removed from a US Airways <e1>flight</e1> awaiting departure from the Minneapolis-St. Paul <e2>airport</e2>.\"\\r\\n'\n",
      "((The imams were removed from a US Airways flight awaiting departure from the Minneapolis - St . Paul airport ., flight , airport ), None)\n",
      "Skipping this one... '1790\\t\"The imams were removed from a US Airways <e1>flight</e1> awaiting departure from the Minneapolis-St. Paul <e2>airport</e2>.\"\\r\\n'\n",
      "(None, None, None, 4)\n",
      "Num training: 7199\n",
      "Num valididation: 800\n",
      "Didn't find common ancestor\n",
      "8310\t\"Tributes have been paid to the <e1>writer</e1> who created Goodness Gracious Me, the hit BBC television <e2>series</e2>.\"\n",
      "\n",
      "(Tributes have been paid to the writer who created Goodness Gracious Me , the hit BBC television series ., writer , series )\n",
      "Bad sentence: '8310\\t\"Tributes have been paid to the <e1>writer</e1> who created Goodness Gracious Me, the hit BBC television <e2>series</e2>.\"\\r\\n'\n",
      "((Tributes have been paid to the writer who created Goodness Gracious Me , the hit BBC television series ., writer , series ), None)\n",
      "Skipping this one... '8000\\t\"The <e1>surgeon</e1> cuts a small <e2>hole</e2> in the skull and lifts the edge of the brain to expose the nerve.\"\\r\\n'\n",
      "(None, None, None, 3)\n",
      "Num testing: 2717\n"
     ]
    }
   ],
   "source": [
    "# reload(sdh)\n",
    "train, valid, test, label2int, int2label = sdh.load_semeval_data(include_ends=False, shuffle_seed=shuffle_seed)\n",
    "num_classes = len(int2label.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# convert the semeval data to indices under the wiki vocab:\n",
    "train['sdps'] = DH.sentences_to_sequences(train['sdps'])\n",
    "valid['sdps'] = DH.sentences_to_sequences(valid['sdps'])\n",
    "test['sdps'] = DH.sentences_to_sequences(test['sdps'])\n",
    "    \n",
    "train['targets'] = DH.sentences_to_sequences(train['targets'])\n",
    "valid['targets'] = DH.sentences_to_sequences(valid['targets'])\n",
    "test['targets'] = DH.sentences_to_sequences(test['targets'])\n",
    "\n",
    "print(train['targets'][:5]) # small sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "max_seq_len = max([len(path) for path in train['sdps']+valid['sdps']+test['sdps']])\n",
    "print(max_seq_len, DH.max_seq_len)\n",
    "DH.max_seq_len = max_seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# the embedding matrix is started of as random uniform [-1,1]\n",
    "# then we replace everything but the OOV tokens with the approprate google vector\n",
    "fname = 'data/GoogleNews-vectors-negative300.bin'\n",
    "word2vec = Word2Vec.load_word2vec_format(fname, binary=True)\n",
    "\n",
    "word_embeddings = np.random.uniform(low=-1., high=1., size=[DH.vocab_size, 300]).astype(np.float32)\n",
    "num_found = 0\n",
    "for i, token in enumerate(DH.vocab):\n",
    "    if token in word2vec:\n",
    "        word_embeddings[i] = word2vec[token]\n",
    "        num_found += 1\n",
    "print(\"%i / %i pretrained\" % (num_found, DH.vocab_size))\n",
    "del word2vec # save a lot of RAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def reset_drnn(model_name='relembed', dep_embed_size=25, word_embed_size=None, max_grad_norm=3.):\n",
    "    if word_embed_size:    \n",
    "        config = {\n",
    "            'max_num_steps':DH.max_seq_len,\n",
    "            'word_embed_size':word_embed_size,\n",
    "            'dep_embed_size':dep_embed_size,\n",
    "            'bidirectional':True,\n",
    "            'hidden_layer_size':1000,\n",
    "            'vocab_size':DH.vocab_size,\n",
    "            'dep_vocab_size':DH.dep_size,\n",
    "            'num_predict_classes':num_classes,\n",
    "            'pretrained_word_embeddings':None,\n",
    "            'max_grad_norm':3.,\n",
    "            'model_name':model_name,\n",
    "            'checkpoint_prefix':'checkpoints/',\n",
    "            'summary_prefix':'tensor_summaries/'\n",
    "        }\n",
    "    else: # use pretrained google vectors\n",
    "        config = {\n",
    "            'max_num_steps':DH.max_seq_len,\n",
    "            'word_embed_size':300,\n",
    "            'dep_embed_size':dep_embed_size,\n",
    "            'bidirectional':True,\n",
    "            'hidden_layer_size':1000,\n",
    "            'vocab_size':DH.vocab_size,\n",
    "            'dep_vocab_size':DH.dep_size,\n",
    "            'num_predict_classes':num_classes,\n",
    "            'pretrained_word_embeddings':word_embeddings,\n",
    "            'max_grad_norm':3.,\n",
    "            'model_name':model_name,\n",
    "            'checkpoint_prefix':'checkpoints/',\n",
    "            'summary_prefix':'tensor_summaries/'\n",
    "        }\n",
    "    try:\n",
    "        tf.reset_default_graph()\n",
    "    except:\n",
    "        pass\n",
    "    try:\n",
    "        tf.get_default_session().close()\n",
    "    except:\n",
    "        pass\n",
    "    drnn = nn.RelEmbed(config)\n",
    "    print(drnn)\n",
    "    return drnn\n",
    "# drnn = reset_drnn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_validation_test(num_nearby=20):\n",
    "    valid_phrases, valid_targets , _, valid_lens = DH.validation_batch()\n",
    "    random_index = int(random.uniform(0, len(valid_lens)))\n",
    "    query_phrase = valid_phrases[random_index]\n",
    "    query_len = valid_lens[random_index]\n",
    "    query_target = valid_targets[random_index]\n",
    "    padded_qp = np.zeros([DH.max_seq_len, 2]).astype(np.int32)\n",
    "    padded_qp[:len(query_phrase), 0] = [x[0] for x in query_phrase]\n",
    "    padded_qp[:len(query_phrase), 1] = [x[1] for x in query_phrase]    \n",
    "    dists, phrase_idx = drnn.validation_phrase_nearby(padded_qp, query_len, valid_phrases, valid_lens)\n",
    "    print(\"=\"*80)\n",
    "    print(\"Top %i closest phrases to <%s> '%s' <%s>\" \n",
    "          % (num_nearby, DH.vocab_at(query_target[0]), \n",
    "             DH.sequence_to_sentence(query_phrase, query_len), \n",
    "             DH.vocab_at(query_target[1])))\n",
    "    for i in range(num_nearby):\n",
    "        dist = dists[i]\n",
    "        phrase = valid_phrases[phrase_idx[i]]\n",
    "        len_ = valid_lens[phrase_idx[i]]\n",
    "        target = valid_targets[phrase_idx[i]]\n",
    "        print(\"%i: %0.3f : <%s> '%s' <%s>\" \n",
    "              % (i, dist, DH.vocab_at(target[0]),\n",
    "                 DH.sequence_to_sentence(phrase, len_),\n",
    "                 DH.vocab_at(target[1])))\n",
    "    print(\"=\"*80)\n",
    "#     drnn.save_validation_accuracy(frac_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def time_left(num_epochs, num_steps, fit_time, nearby_time, start_time, nearby_mod):\n",
    "    total = num_epochs*num_steps*fit_time + ((num_epochs*num_steps)/float(nearby_mod))*nearby_time\n",
    "    return total - (time() - start_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unsupervised Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "num_epochs = 1\n",
    "batch_size =50\n",
    "neg_per = 25\n",
    "num_nearby = 50\n",
    "nearby_mod = 50\n",
    "sample_power = .75\n",
    "DH.scale_vocab_dist(sample_power)\n",
    "\n",
    "# bookkeeping\n",
    "num_steps = DH.num_steps(batch_size)\n",
    "total_step = 1\n",
    "save_interval = 30 * 60 # half hour in seconds\n",
    "save_time = time()\n",
    "\n",
    "#timing stuff\n",
    "start = time()\n",
    "fit_time = 0\n",
    "nearby_time = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    offset = 0 #if epoch else 400\n",
    "    DH.shuffle_data()\n",
    "    for step , batch in enumerate(DH.batches(batch_size, offset=offset, neg_per=neg_per)):\n",
    "        if not step: step = offset\n",
    "        t0 = time()\n",
    "        loss = drnn.partial_unsup_fit(*batch)\n",
    "        fit_time = (fit_time * float(total_step) +  time() - t0) / (total_step + 1) # running average\n",
    "        if step % 10 == 0:\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            left = time_left(num_epochs, num_steps, fit_time, nearby_time, start, nearby_mod)\n",
    "            ml,sl = divmod(left, 60)\n",
    "            hl,ml = divmod(ml, 60)\n",
    "            pps = batch_size*(neg_per + 1) / fit_time \n",
    "            print(\"(%i:%i:%i) step %i/%i, epoch %i Training Loss = %1.5f :: %0.3f phrases/sec :: (%i:%i:%i) hours left\" \n",
    "                  % (h,m,s, step, num_steps, epoch, loss, pps, hl, ml, sl))\n",
    "        if (total_step-1) % nearby_mod == 0: # do one right away so we get a good timing estimate\n",
    "            t0 = time()\n",
    "            run_validation_test(num_nearby) # check out the nearby phrases in the validation set\n",
    "            valid_loss = drnn.validation_loss(*DH.validation_batch())\n",
    "            print(\"Validation loss: %0.4f\" % valid_loss)\n",
    "            nearby_time = (nearby_time * float(total_step) + time() - t0) / (total_step + 1) # running average\n",
    "\n",
    "        if (time() - save_time) > save_interval:\n",
    "            print(\"Saving model...\")\n",
    "            drnn.checkpoint()\n",
    "            save_time = time()\n",
    "        total_step +=1\n",
    "drnn.checkpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drnn.checkpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # test the embeddings\n",
    "\n",
    "# ### VALID ###\n",
    "# # valid_phrases, valid_targets, _, valid_lens = DH.validation_batch()\n",
    "# # phrase_embeds, target_embeds = drnn.embed_phrases_and_targets(valid_phrases, valid_targets, valid_lens)\n",
    "# # phrase_labels, target_labels = DH.readable_data(valid=True)\n",
    "\n",
    "# ### TRAIN ###\n",
    "# train_phrases, train_targets, _, train_lens = DH.batches(500, neg_per=0, offset=0).next()\n",
    "# phrase_embeds, target_embeds = drnn.embed_phrases_and_targets(train_phrases, train_targets, train_lens)\n",
    "# phrase_labels, target_labels = DH.readable_data(show_dep=False, valid=False)\n",
    "        \n",
    "# phrase_embeds /= np.sqrt(np.sum(phrase_embeds**2, 1, keepdims=True))\n",
    "# target_embeds /= np.sqrt(np.sum(target_embeds**2, 1, keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ### JOINT ###\n",
    "# start = 0\n",
    "# stride = 40\n",
    "# end = start + stride\n",
    "\n",
    "# lowd = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "# # lowd = PCA(n_components=2)\n",
    "\n",
    "# joint_embeds = np.vstack([phrase_embeds[start:end], target_embeds[start:end]])\n",
    "# joint_2d = lowd.fit_transform(joint_embeds)\n",
    "# phrase_2d, target_2d = joint_2d[:stride], joint_2d[stride:]\n",
    "\n",
    "# fig, ax = plt.subplots(figsize=(20,16))\n",
    "# for i, label in enumerate(phrase_labels[start:end]):\n",
    "#     label = \"%i: %s\" % (i, label)\n",
    "#     x, y = phrase_2d[i,:]\n",
    "#     ax.scatter(x, y, color='b')\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')\n",
    "# for i, label in enumerate(target_labels[start:end]):\n",
    "#     label = \"%i: %s\" % (i, label)\n",
    "#     x, y = target_2d[i,:]\n",
    "#     ax.scatter(x, y, color='r')\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ### PHRASE ONLY ###\n",
    "# start = 0\n",
    "# stride = 50\n",
    "# end = start + stride\n",
    "\n",
    "# lowd = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "# # lowd = PCA(n_components=2)\n",
    "\n",
    "# phrase_2d = lowd.fit_transform(phrase_embeds[start:end])\n",
    "\n",
    "# fig, ax = plt.subplots(figsize=(20,16))\n",
    "# for i, label in enumerate(phrase_labels[start:end]):\n",
    "#     label = \"%i: %s\" % (i, label)\n",
    "#     x, y = phrase_2d[i,:]\n",
    "#     ax.scatter(x, y, color='b')\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ### TARGET ONLY ###\n",
    "# start = 0\n",
    "# stride = 35\n",
    "# end = start + stride\n",
    "\n",
    "# lowd = TSNE(perplexity=20, n_components=2, init='pca', n_iter=5000)\n",
    "# # lowd = PCA(n_components=2)\n",
    "\n",
    "# target_2d = lowd.fit_transform(target_embeds[start:end])\n",
    "\n",
    "# fig, ax = plt.subplots(figsize=(20,16))\n",
    "# for i, label in enumerate(target_labels[start:end]):\n",
    "#     label = \"%i: %s\" % (i, label)\n",
    "#     x, y = target_2d[i,:]\n",
    "#     ax.scatter(x, y, color='r')\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### TW2V demo ###\n",
    "start = 200\n",
    "stride = 100\n",
    "end = start + stride\n",
    "\n",
    "lowd = TSNE(perplexity=20, n_components=2, init='pca', n_iter=5000)\n",
    "# lowd = PCA(n_components=2)\n",
    "\n",
    "target_2d = lowd.fit_transform(word_embeddings[start:end])\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(28,16))\n",
    "for i, label in enumerate(DH.vocab[start:end]):\n",
    "    label = \"%s\" % (label)\n",
    "    x, y = target_2d[i,:]\n",
    "    ax.scatter(x, y, color='b')\n",
    "    ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "                   ha='right', va='bottom')\n",
    "    \n",
    "plt.savefig('word2vec_demo.png', dpi=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test out semeval data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "zip_train = zip(train['raws'], train['sents'], train['sdps'], train['targets'], train['labels'])\n",
    "zip_valid = zip(valid['raws'], valid['sents'], valid['sdps'], valid['targets'], valid['labels'])\n",
    "zip_test = zip(test['raws'], test['sents'], test['sdps'], test['targets'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i, (raw, _, sdp, target, label) in enumerate(zip_train):\n",
    "    if i > 5:\n",
    "        break\n",
    "    print(raw)\n",
    "    print(\"%s :: %s\" % (DH.sequence_to_sentence(sdp, show_dep=True), DH.sequence_to_sentence(target)))\n",
    "    print(int2label[label])\n",
    "    print(\"=\"*80)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Supervised Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "batch_size = 50\n",
    "num_steps = len(train['labels']) // batch_size\n",
    "num_epochs = 50\n",
    "display_mod = 10\n",
    "valid_mod = 50\n",
    "print(\"Num steps %i\" %num_steps)\n",
    "\n",
    "best_valid = 1000000\n",
    "best_model = None\n",
    "\n",
    "start = time()\n",
    "for epoch in range(num_epochs):\n",
    "    random.shuffle(zip_train) # shuffling should only happen once per epoch\n",
    "    _, _, sdps, targets, labels = zip(*zip_train)\n",
    "    for step in range(num_steps): # num_steps\n",
    "        class_batch = DH.classification_batch(batch_size, sdps, targets, labels, \n",
    "                                              offset=step, shuffle=False)\n",
    "        xent = drnn.partial_class_fit(*class_batch)\n",
    "        if step % display_mod == 0:   \n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            print(\"(%i:%i:%i) s %i/%i, e %i avg class xent loss = %0.4f\" % (h,m,s, step, num_steps, epoch, xent))\n",
    "        if step % valid_mod == 0:\n",
    "            valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "            valid_xent = drnn.validation_class_loss(*valid_batch)\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            print(\"=\"*80)\n",
    "            print(\"(%i:%i:%i) s %i/%i, e %i validation avg class xent loss = %0.4f\" % (h,m,s, step, num_steps, epoch, valid_xent))\n",
    "            print(\"=\"*80)\n",
    "            if valid_xent < best_valid:\n",
    "                best_valid = valid_xent\n",
    "                print(\"New best\")\n",
    "                best_model = drnn.checkpoint()\n",
    "    label_set = set(train['labels'])\n",
    "    preds = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3])\n",
    "    cm, stats = eh.confusion_matrix(preds, valid['labels'], label_set)\n",
    "    print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "    valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "    valid_xent = drnn.validation_class_loss(*valid_batch)\n",
    "    m,s = divmod(time()-start, 60)\n",
    "    h,m = divmod(m, 60)\n",
    "    print(\"=\"*80)\n",
    "    print(\"(%i:%i:%i) s %i/%i, e %i validation avg class xent loss = %0.4f\" % (h,m,s, step, num_steps, epoch, valid_xent))\n",
    "    print(\"=\"*80)\n",
    "    if valid_xent < best_valid:\n",
    "        best_valid = valid_xent\n",
    "        print(\"New best\")\n",
    "        best_model = drnn.checkpoint()\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "drnn.restore(best_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results = []\n",
    "#timing stuff\n",
    "start = time()\n",
    "fit_time = 0\n",
    "nearby_time = 0\n",
    "\n",
    "batch_size = 50\n",
    "num_cycles = 20\n",
    "num_epochs = 3\n",
    "num_class_epochs = 5\n",
    "display_mod = 10\n",
    "valid_mod = 50\n",
    "\n",
    "# hyperparameters\n",
    "\n",
    "neg_per = 15\n",
    "neg_level = 2\n",
    "num_nearby = 20\n",
    "nearby_mod = 100\n",
    "sample_power = .75\n",
    "DH.scale_vocab_dist(sample_power)\n",
    "\n",
    "# # bookkeeping\n",
    "total_step = 1\n",
    "save_interval = 30 * 60 # half hour in seconds\n",
    "save_time = time()\n",
    "\n",
    "    \n",
    "for cycle in range(num_cycles):\n",
    "    print(\"+\"*80)\n",
    "    print(\"+++ CYCLE %i +++\" % cycle)\n",
    "    print(\"+\"*80)\n",
    "\n",
    "    epoch = 0\n",
    "    class_epoch = 0\n",
    "    \n",
    "    num_steps = DH.num_steps(batch_size)\n",
    "\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print(\"=\"*80)\n",
    "        print(\"=== UNSUP EPOCH %i ===\" % epoch)\n",
    "        print(\"=\"*80)\n",
    "        offset = 0 #if epoch else 400\n",
    "        DH.shuffle_data()\n",
    "        for step , batch in enumerate(DH.batches(batch_size, offset=offset, neg_per=neg_per, neg_level=neg_level)):\n",
    "            if not step: step = offset\n",
    "            t0 = time()\n",
    "            loss = drnn.partial_unsup_fit(*batch)\n",
    "            fit_time = (fit_time * float(total_step) +  time() - t0) / (total_step + 1) # running average\n",
    "            if step % display_mod == 0:\n",
    "                m,s = divmod(time()-start, 60)\n",
    "                h,m = divmod(m, 60)\n",
    "                left = time_left(num_epochs, num_steps, fit_time, nearby_time, start, nearby_mod)\n",
    "                ml,sl = divmod(left, 60)\n",
    "                hl,ml = divmod(ml, 60)\n",
    "                pps = batch_size*(neg_per + 1) / fit_time \n",
    "                print(\"(%i:%i:%i) step %i/%i, epoch %i Training Loss = %1.5f :: %0.3f phrases/sec :: (%i:%i:%i) hours left\" \n",
    "                      % (h,m,s, step, num_steps, epoch, loss, pps, hl, ml, sl))\n",
    "            if (total_step-1) % nearby_mod == 0: # do one right away so we get a good timing estimate\n",
    "                t0 = time()\n",
    "                run_validation_test(num_nearby) # check out the nearby phrases in the validation set\n",
    "                valid_loss = drnn.validation_loss(*DH.validation_batch())\n",
    "                print(\"Validation loss: %0.4f\" % valid_loss)\n",
    "                nearby_time = (nearby_time * float(total_step) + time() - t0) / (total_step + 1) # running average\n",
    "            total_step +=1\n",
    "        valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "        label_set = set(train['labels'])\n",
    "        preds = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3])\n",
    "        cm, stats = eh.confusion_matrix(preds, valid['labels'], label_set)\n",
    "        print(\"Saving model...\")\n",
    "        model_file = drnn.checkpoint()\n",
    "        result = {'cycle':cycle, 'unsup_epoch':epoch, 'class_epoch':class_epoch, \n",
    "              'cm':cm, 'stats':stats, 'macro_f1':stats['macro_f1'], 'model':model_file}\n",
    "        results.append(result)\n",
    "        print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "\n",
    "    num_steps = len(train['labels']) // batch_size\n",
    "\n",
    "    for class_epoch in range(num_class_epochs):\n",
    "    #     class_batch = DH.classification_batch(batch_size, train['sdps'], train['targets'], train['labels'], offset=0)\n",
    "    #     random.shuffle(class_batch)\n",
    "        print(\"*\"*80)\n",
    "        print(\"*** CLASS EPOCH %i ***\" % class_epoch)\n",
    "        print(\"*\"*80)\n",
    "        for class_step in range(num_steps):\n",
    "            inputs, targets, labels, lens = DH.classification_batch(batch_size, train['sdps'], train['targets'], train['labels'], offset=class_step)\n",
    "            class_batch = zip(inputs, targets, labels, lens)\n",
    "            random.shuffle(class_batch)\n",
    "            class_batch = zip(*class_batch)\n",
    "            xent = drnn.partial_class_fit(*class_batch)\n",
    "            if class_step % display_mod == 0:   \n",
    "                m,s = divmod(time()-start, 60)\n",
    "                h,m = divmod(m, 60)\n",
    "                print(\"(%i:%i:%i) s %i/%i, e %i avg class xent loss = %0.4f\" % (h,m,s, class_step, num_steps, class_epoch, xent))\n",
    "            if class_step % valid_mod == 0:\n",
    "                valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "                valid_xent = drnn.validation_class_loss(*valid_batch)\n",
    "                m,s = divmod(time()-start, 60)\n",
    "                h,m = divmod(m, 60)\n",
    "                print(\"=\"*80)\n",
    "                print(\"(%i:%i:%i) s %i/%i, e %i validation avg class xent loss = %0.4f\" % (h,m,s, class_step, num_steps, class_epoch, valid_xent))\n",
    "                print(\"=\"*80)\n",
    "                \n",
    "        label_set = set(train['labels'])\n",
    "        preds = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3])\n",
    "        cm, stats = eh.confusion_matrix(preds, valid['labels'], label_set)\n",
    "        print(\"Saving model...\")\n",
    "        model_file = drnn.checkpoint()\n",
    "        result = {'cycle':cycle, 'unsup_epoch':epoch, 'class_epoch':class_epoch, \n",
    "              'cm':cm, 'stats':stats, 'macro_f1':stats['macro_f1'], 'model':model_file}\n",
    "        results.append(result)\n",
    "        print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "    print(\"Done\")\n",
    "best = sorted(results, key=lambda x:x['macro_f1'], reverse=True)[0]\n",
    "print(\"Best result %2.4f, C:%i, UE:%i, CE:%i\" % \n",
    "      (best['macro_f1'], best['cycle'], best['unsup_epoch'], best['class_epoch']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(best['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# write out the training and validation keys for our split\n",
    "with open('SemEval2010_task8_all_data/train_keys.txt', 'w') as f:\n",
    "    i = 1\n",
    "    for label in train['labels']:\n",
    "        f.write(\"%s\\t%s\\n\" % (i, int2label[label]))\n",
    "        i += 1\n",
    "with open('SemEval2010_task8_all_data/valid_keys.txt', 'w') as f:\n",
    "    i = len(train['labels'])\n",
    "    for label in valid['labels']:\n",
    "        f.write(\"%s\\t%s\\n\" % (i, int2label[label]))\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# check out training predictions\n",
    "valid_batch = DH.classification_batch(len(train['labels']), train['sdps'], train['targets'], train['labels'])\n",
    "\n",
    "# valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'], shuffle=False)\n",
    "preds = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3], return_probs=False)\n",
    "# write out to file and get official score\n",
    "with open('SemEval2010_task8_all_data/train_preds.txt', 'w') as f:\n",
    "    i = 1\n",
    "    for label in preds:\n",
    "        f.write(\"%s\\t%s\\n\" % (i, int2label[label]))\n",
    "        i += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "./SemEval2010_task8_all_data/SemEval2010_task8_scorer-v1.2/semeval2010_task8_scorer-v1.2.pl \\\n",
    "SemEval2010_task8_all_data/train_preds.txt SemEval2010_task8_all_data/train_keys.txt;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# check out validation predictions\n",
    "# valid_batch = DH.classification_batch(len(train['labels']), train['sdps'], train['targets'], train['labels'])\n",
    "\n",
    "valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'], shuffle=False)\n",
    "preds = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3], return_probs=False)\n",
    "# write out to file and get official score\n",
    "with open('SemEval2010_task8_all_data/valid_preds.txt', 'w') as f:\n",
    "    i = len(train['labels'])\n",
    "    for label in preds:\n",
    "        f.write(\"%s\\t%s\\n\" % (i, int2label[label]))\n",
    "        i += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "./SemEval2010_task8_all_data/SemEval2010_task8_scorer-v1.2/semeval2010_task8_scorer-v1.2.pl \\\n",
    "SemEval2010_task8_all_data/valid_preds.txt SemEval2010_task8_all_data/valid_keys.txt;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save_name = raw_input(\"Enter save name: \")\n",
    "save_name = ''\n",
    "cm, stats = eh.confusion_matrix(preds, valid['labels'], label_set, no_other=True)\n",
    "eh.plot_confusion_matrix(cm, int2label.values(), save_name=save_name, stats=stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# write out predictions for test set\n",
    "test_batch = DH.classification_batch(len(test['targets']), test['sdps'], test['targets'], \n",
    "                                     np.zeros(len(test['targets'])), shuffle=False)\n",
    "preds = drnn.predict(test_batch[0], test_batch[1], test_batch[3])\n",
    "with open('SemEval2010_task8_all_data/test_pred.txt', 'w') as f:\n",
    "    i = 8001\n",
    "    for pred in preds:\n",
    "        f.write(\"%i\\t%s\\n\" % (i, int2label[pred]))\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "./SemEval2010_task8_all_data/SemEval2010_task8_scorer-v1.2/semeval2010_task8_scorer-v1.2.pl \\\n",
    "SemEval2010_task8_all_data/test_pred.txt SemEval2010_task8_all_data/test_keys.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for i, p in enumerate(preds):\n",
    "#     print(\"%i, pred: %s, true: %s\" %(i, int2label[p], int2label[valid['labels'][i]]))\n",
    "#     taarget = DH.sequence_to_sentence(valid['targets'][i]).split(' ')\n",
    "#     sdp = DH.sequence_to_sentence(valid['sdps'][i], show_dep=True)\n",
    "#     print('<%s> \"%s\" <%s>' % (target[0], sdp, target[1]))\n",
    "#     print(valid['raws'][i])\n",
    "#     print(valid['comments'][i])\n",
    "#     print(\"=\"*80)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Directional Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save_name = raw_input(\"Enter save name: \")\n",
    "save_name = ''\n",
    "cm, stats = eh.confusion_matrix(preds, valid['labels'], label_set)\n",
    "eh.plot_confusion_matrix(cm, int2label.values(), save_name=save_name, stats=stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Non Directional evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bi_preds, bi_int2label, bi_label2int = eh.directional_to_bidirectional_labels(preds, int2label)\n",
    "bi_labels, _, _ = eh.directional_to_bidirectional_labels(valid['labels'], int2label)\n",
    "bi_labelset = set(bi_int2label.keys())\n",
    "print(bi_label2int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save_name = raw_input(\"Enter save name: \")\n",
    "bi_cm, bi_stats = eh.confusion_matrix(bi_preds, bi_labels, bi_labelset)\n",
    "eh.plot_confusion_matrix(bi_cm, bi_int2label.values(), save_name=save_name, stats=stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "random.seed(42)\n",
    "random.shuffle([0,1,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "softmax = drnn._softmax_w.eval()\n",
    "print(softmax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "eh.plot_confusion_matrix(softmax, int2label.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'], shuffle=False)\n",
    "preds, dists = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3], return_probs=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def plot_probs(dists, labels, int2label):\n",
    "    \"\"\"Plot the predicted distributions as small multiples\n",
    "\n",
    "    Args:\n",
    "        dists: list of pmfs, all the same size\n",
    "        int2label: dict of ints to labels for pmfs\n",
    "    \"\"\"\n",
    "    fig, axarr = plt.subplots(len(dists)/2 + 1, 2, sharex=True, figsize=(16, 3*len(dists)))\n",
    "    xticks = range(len(int2label.keys()))\n",
    "    for i, dist in enumerate(dists):\n",
    "        axarr[i/2, i%2].stem(dist)\n",
    "        axarr[i/2, i%2].set_xlim([-1,19])\n",
    "        axarr[i/2, i%2].set_xticks(xticks)\n",
    "#         if i % 3 == 0 and i:\n",
    "        axarr[i/2, i%2].set_xticklabels(int2label.values(), rotation=45, horizontalalignment=\"right\", x=1)\n",
    "        axarr[i/2, i%2].set_title(int2label[labels[i]])\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "num = 10\n",
    "small_dists, small_labels = dists[:num], valid['labels'][:num]\n",
    "print(small_labels)\n",
    "plot_probs(small_dists, small_labels, int2label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Active Learning Experiment\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def alternating_fit(drnn, DH, train, valid, label_set, target_neg=False, unsup_epochs=1, class_epochs=5, \n",
    "                    stop_tol=5, batch_size=50, neg_per=10, unsup=True):\n",
    "    results = []\n",
    "    #timing stuff\n",
    "    start = time()\n",
    "    fit_time = 0\n",
    "    nearby_time = 0\n",
    "\n",
    "    display_mod = 10\n",
    "    valid_mod = 50\n",
    "\n",
    "    # hyperparameters\n",
    "\n",
    "    neg_level = 2\n",
    "    num_nearby = 20\n",
    "    nearby_mod = 100\n",
    "    sample_power = .75\n",
    "    DH.scale_vocab_dist(sample_power)\n",
    "\n",
    "    # # bookkeeping\n",
    "    total_step = 1\n",
    "    save_interval = 30 * 60 # half hour in seconds\n",
    "    save_time = time()\n",
    "\n",
    "    best_valid = 1e1000\n",
    "    after_best_count = 0\n",
    "    \n",
    "    class_batch_size = min(batch_size, len(train['labels']))\n",
    "    \n",
    "    cycle = 0\n",
    "    total_unsup_count = 0\n",
    "    total_class_count = 0\n",
    "    while after_best_count < stop_tol:\n",
    "        print(\"+\"*80)\n",
    "        print(\"+++ CYCLE %i +++\" % cycle)\n",
    "        print(\"+\"*80)\n",
    "\n",
    "        epoch = 0\n",
    "        class_epoch = 0\n",
    "        total_step = 0\n",
    "\n",
    "        num_steps = DH.num_steps(batch_size)\n",
    "        if unsup:\n",
    "            for epoch in range(unsup_epochs):\n",
    "                print(\"=\"*80)\n",
    "                print(\"=== UNSUP EPOCH %i ===\" % epoch)\n",
    "                print(\"=\"*80)\n",
    "                DH.shuffle_data()\n",
    "                for step , batch in enumerate(DH.batches(batch_size, neg_per=neg_per, target_neg=target_neg, neg_level=neg_level)):\n",
    "                    t0 = time()\n",
    "                    loss = drnn.partial_unsup_fit(*batch)\n",
    "                    fit_time = (fit_time * float(total_step) +  time() - t0) / (total_step + 1) # running average\n",
    "                    if step % display_mod == 0:\n",
    "                        m,s = divmod(time()-start, 60)\n",
    "                        h,m = divmod(m, 60)\n",
    "                        left = time_left(unsup_epochs, num_steps, fit_time, nearby_time, start, nearby_mod)\n",
    "                        ml,sl = divmod(left, 60)\n",
    "                        hl,ml = divmod(ml, 60)\n",
    "                        pps = batch_size*(neg_per + 1) / fit_time \n",
    "                        print(\"(%i:%i:%i) step %i/%i, epoch %i Training Loss = %1.5f :: %0.3f phrases/sec :: (%i:%i:%i) hours left\" \n",
    "                              % (h,m,s, step, num_steps, epoch, loss, pps, hl, ml, sl))\n",
    "                    if (total_step-1) % nearby_mod == 0: # do one right away so we get a good timing estimate\n",
    "                        t0 = time()\n",
    "                        run_validation_test(num_nearby) # check out the nearby phrases in the validation set\n",
    "                        valid_loss = drnn.validation_loss(*DH.validation_batch())\n",
    "                        print(\"Validation loss: %0.4f\" % valid_loss)\n",
    "                        nearby_time = (nearby_time * float(total_step) + time() - t0) / (total_step + 1) # running average\n",
    "                    total_step +=1\n",
    "                total_unsup_count +=1\n",
    "                valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "                preds = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3])\n",
    "                cm, stats = eh.confusion_matrix(preds, valid['labels'], label_set)\n",
    "                print(\"Saving model...\")\n",
    "                model_file = drnn.checkpoint()\n",
    "                result = {'cycle':cycle, 'unsup_epoch':total_unsup_count, 'class_epoch':total_class_count, \n",
    "                      'cm':cm, 'stats':stats, 'macro_f1':stats['macro_f1'], 'model':model_file}\n",
    "                results.append(result)\n",
    "                print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "\n",
    "        \n",
    "        num_steps = len(train['labels']) // class_batch_size\n",
    "\n",
    "        for class_epoch in range(class_epochs):\n",
    "            # shuffle training\n",
    "            print(\"*\"*80)\n",
    "            print(\"*** CLASS EPOCH %i ***\" % class_epoch)\n",
    "            print(\"*\"*80)\n",
    "            for class_step in range(num_steps):\n",
    "                inputs, targets, labels, lens = DH.classification_batch(batch_size, train['sdps'], train['targets'], train['labels'], offset=class_step)\n",
    "                class_batch = zip(inputs, targets, labels, lens)\n",
    "                random.shuffle(class_batch)\n",
    "                class_batch = zip(*class_batch)\n",
    "                xent = drnn.partial_class_fit(*class_batch)\n",
    "                if class_step % display_mod == 0:   \n",
    "                    m,s = divmod(time()-start, 60)\n",
    "                    h,m = divmod(m, 60)\n",
    "                    print(\"(%i:%i:%i) s %i/%i, e %i avg class xent loss = %0.4f\" % (h,m,s, class_step, num_steps, class_epoch, xent))\n",
    "            # measure validation performance\n",
    "            valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "            valid_xent = drnn.validation_class_loss(*valid_batch)\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            print(\"=\"*80)\n",
    "            print(\"(%i:%i:%i) s %i/%i, e %i validation avg class xent loss = %0.4f\" % (h,m,s, class_step, num_steps, class_epoch, valid_xent))\n",
    "            print(\"=\"*80)\n",
    "            after_best_count += 1 # number of times since last best\n",
    "            if valid_xent < best_valid:\n",
    "                print(\"New Best!\")\n",
    "                best_valid = valid_xent\n",
    "                after_best_count = 0\n",
    "                best_model = drnn.checkpoint()\n",
    "            \n",
    "            preds = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3])\n",
    "            cm, stats = eh.confusion_matrix(preds, valid['labels'], label_set)\n",
    "            print(\"Saving model...\")\n",
    "            model_file = drnn.checkpoint()\n",
    "            result = {'cycle':cycle, 'unsup_epoch':total_unsup_count, 'class_epoch':total_class_count, \n",
    "                  'cm':cm, 'stats':stats, 'macro_f1':stats['macro_f1'], 'model':model_file}\n",
    "            results.append(result)\n",
    "            print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "            total_class_count += 1\n",
    "        cycle += 1\n",
    "    best = sorted(results, key=lambda x:x['macro_f1'], reverse=True)[0]\n",
    "    print(\"Best result %2.4f, C:%i, UE:%i, CE:%i\" % \n",
    "          (best['macro_f1'], best['cycle'], best['unsup_epoch'], best['class_epoch']))  \n",
    "    drnn.restore(best['model'])\n",
    "    return drnn, results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from scipy.stats import entropy\n",
    "label_set = set(train['labels'])\n",
    "meta_results = []\n",
    "drnn = reset_drnn()\n",
    "N = 500\n",
    "# first N examples\n",
    "small_train = {k:v[:N] for k,v in train.items()}\n",
    "rest_train = {k:v[N:] for k,v in train.items()}\n",
    "\n",
    "# fit until early_stop on limited training\n",
    "meta_batch_num = 0\n",
    "while len(small_train['labels']) < len(train['labels']):\n",
    "    print(\"%\"*80)\n",
    "    print(\"%i labeled so far (meta_cycle: %i, N: %i)\" % (len(small_train['labels']), meta_batch_num, N))\n",
    "    print(\"%\"*80)\n",
    "\n",
    "    drnn, results = alternating_fit(drnn, DH, small_train, valid, label_set, neg_per=20)\n",
    "    meta_results.append({'N':N, 'results':results})\n",
    "\n",
    "    # get all of the predictive distributions for the unknowns\n",
    "    query_batch = DH.classification_batch(len(rest_train['labels']),\n",
    "                                          rest_train['sdps'], \n",
    "                                          rest_train['targets'],\n",
    "                                          rest_train['labels'])\n",
    "    _, dists = drnn.predict(query_batch[0], query_batch[1], query_batch[3], return_probs=True)\n",
    "\n",
    "    # rank based on highest entropy\n",
    "    # then turn back into rest_train.values() style\n",
    "    ranked_train = zip(*sorted(zip(dists, *rest_train.values()), key=lambda x:entropy(x[0]), reverse=True))\n",
    "\n",
    "    # if we're at the very end we may need to make N smaller\n",
    "    if len(small_train['labels']) + N >= len(train['labels']):\n",
    "            N = len(train['labels']) - len(small_train['labels']) \n",
    "            # this should be same as what's left of rest\n",
    "            assert N == len(rest_train['labels']), \"Hm...\"\n",
    "    # skim off top N unknowns to train with and leave the rest as before\n",
    "    small_train = {k:list(v)+list(ranked_train[i+1])[:N] for i,(k,v) in enumerate(small_train.items())}\n",
    "    rest_train = {k:list(ranked_train[i+1])[N:] for i,(k,v) in enumerate(small_train.items())}\n",
    "    meta_batch_num += 1\n",
    "print(\"Done...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SUpervised only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "###### Supervised only\n",
    "from scipy.stats import entropy\n",
    "label_set = set(train['labels'])\n",
    "sup_meta_results = []\n",
    "drnn = reset_drnn(model_name='sup')\n",
    "N = 500\n",
    "# first N examples\n",
    "small_train = {k:v[:N] for k,v in train.items()}\n",
    "rest_train = {k:v[N:] for k,v in train.items()}\n",
    "\n",
    "# fit until early_stop on limited training\n",
    "meta_batch_num = 0\n",
    "while len(small_train['labels']) < len(train['labels']):\n",
    "    print(\"%\"*80)\n",
    "    print(\"%i labeled so far (meta_cycle: %i, N: %i)\" % (len(small_train['labels']), meta_batch_num, N))\n",
    "    print(\"%\"*80)\n",
    "\n",
    "    drnn, results = alternating_fit(drnn, DH, small_train, valid, label_set, neg_per=20, unsup=False)\n",
    "    sup_meta_results.append({'N':N, 'results':results})\n",
    "\n",
    "    # get all of the predictive distributions for the unknowns\n",
    "    query_batch = DH.classification_batch(len(rest_train['labels']),\n",
    "                                          rest_train['sdps'], \n",
    "                                          rest_train['targets'],\n",
    "                                          rest_train['labels'])\n",
    "    _, dists = drnn.predict(query_batch[0], query_batch[1], query_batch[3], return_probs=True)\n",
    "\n",
    "    # rank based on highest entropy\n",
    "    # then turn back into rest_train.values() style\n",
    "    ranked_train = zip(*sorted(zip(dists, *rest_train.values()), key=lambda x:entropy(x[0]), reverse=True))\n",
    "\n",
    "    # if we're at the very end we may need to make N smaller\n",
    "    if len(small_train['labels']) + N >= len(train['labels']):\n",
    "            N = len(train['labels']) - len(small_train['labels']) \n",
    "            # this should be same as what's left of rest\n",
    "            assert N == len(rest_train['labels']), \"Hm...\"\n",
    "    # skim off top N unknowns to train with and leave the rest as before\n",
    "    small_train = {k:list(v)+list(ranked_train[i+1])[:N] for i,(k,v) in enumerate(small_train.items())}\n",
    "    rest_train = {k:list(ranked_train[i+1])[N:] for i,(k,v) in enumerate(small_train.items())}\n",
    "    meta_batch_num += 1\n",
    "print(\"Done...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def best_f1(results):\n",
    "    return sorted(results, key=lambda x:x['macro_f1'], reverse=True)[0]['macro_f1']\n",
    "\n",
    "unsup_f1s = [best_f1(results['results']) for results in meta_results]\n",
    "sup_f1s = [best_f1(results['results']) for results in sup_meta_results]\n",
    "print(unsup_f1s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(train.keys())\n",
    "print(train['sents'][0][0].text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ngram lositic regression baseline on full sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "# define baseline pipelines\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "\n",
    "# Feature Extractors\n",
    "cv = CountVectorizer(\n",
    "        input=u'content', \n",
    "        encoding=u'utf-8', \n",
    "        decode_error=u'strict', \n",
    "        strip_accents='unicode', \n",
    "        lowercase=True,\n",
    "        analyzer=u'word', \n",
    "        preprocessor=None, \n",
    "        tokenizer=None, \n",
    "        stop_words='english', \n",
    "        #token_pattern=u'(?u)\\\\b\\w\\w+\\b', # one alphanumeric is a token\n",
    "        ngram_range=(1, 2), \n",
    "        max_df=.9, \n",
    "        min_df=2, \n",
    "        max_features=None, \n",
    "        vocabulary=None, \n",
    "        binary=False, \n",
    "        #dtype=type 'numpy.int64'>\n",
    "        )\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tf = TfidfTransformer(\n",
    "        norm='l2',\n",
    "        use_idf=True,\n",
    "        smooth_idf=True,\n",
    "        sublinear_tf=False\n",
    ")\n",
    "\n",
    "# Final Classifier\n",
    "lr = LogisticRegression(C=.05,\n",
    "                        fit_intercept=True,\n",
    "                        random_state=0,\n",
    "                        class_weight='balanced',\n",
    "                        multi_class='multinomial',\n",
    "                        solver='lbfgs',\n",
    "                        n_jobs=-1)\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('count', cv),\n",
    "    ('tfidf', tf),\n",
    "    ('logreg', lr)\n",
    "    ])\n",
    "\n",
    "param_grid = {\n",
    "    'count__ngram_range':[(1,1),(1,2),(1,3)],\n",
    "    'tfidf__norm':['l1', 'l2'],\n",
    "    'tfidf__use_idf':[True, False],\n",
    "    'tfidf__sublinear_tf':[True,False],\n",
    "    'logreg__C':[.001, .01, .1]\n",
    "}\n",
    "\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "grid_search = GridSearchCV(pipeline, \n",
    "                           param_grid,\n",
    "                           scoring='f1_macro',\n",
    "                           n_jobs=-1, verbose=1)\n",
    "\n",
    "print(\"Here\")\n",
    "x_data = [sent[0].text for sent in train['sents']]\n",
    "y_data = train['labels']\n",
    "print(x_data[0], y_data[0])\n",
    "grid_search.fit(np.array(x_data), y_data)\n",
    "print()\n",
    "\n",
    "print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "print(\"Best parameters set:\")\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "for param_name in sorted(param_grid.keys()):\n",
    "    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "scores = [s.mean_validation_score for s in grid_search.grid_scores_]\n",
    "plt.hist(scores, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# take the best params and then do the iterative partial fit as before\n",
    "lr = grid_search.best_estimator_\n",
    "from scipy.stats import entropy\n",
    "label_set = set(train['labels'])\n",
    "lr_meta_results = []\n",
    "# drnn = reset_drnn()\n",
    "N = 500\n",
    "\n",
    "valid_x = [sent[0].text for sent in valid['sents']]\n",
    "valid_y = valid['labels']\n",
    "\n",
    "# first N examples\n",
    "small_train = {k:v[:N] for k,v in train.items()}\n",
    "rest_train = {k:v[N:] for k,v in train.items()}\n",
    "\n",
    "# fit until early_stop on limited training\n",
    "meta_batch_num = 0\n",
    "while len(small_train['labels']) < len(train['labels']):\n",
    "    print(\"%\"*80)\n",
    "    print(\"%i labeled so far (meta_cycle: %i, N: %i)\" % (len(small_train['labels']), meta_batch_num, N))\n",
    "    print(\"%\"*80)\n",
    "\n",
    "    x_data = [sent[0].text for sent in small_train['sents']]\n",
    "    y_data = small_train['labels']\n",
    "    \n",
    "    lr.fit(np.array(x_data), y_data)\n",
    "    score = lr.score(np.array(valid_x), valid_y)\n",
    "    lr_meta_results.append(score*100)\n",
    "\n",
    "    # get all of the predictive distributions for the unknowns\n",
    "    query_x = [sent[0].text for sent in rest_train['sents']]\n",
    "#     query_y = rest_train['labels']\n",
    "    dists = lr.predict_proba(np.array(query_x))\n",
    "\n",
    "    # rank based on highest entropy\n",
    "    # then turn back into rest_train.values() style\n",
    "    ranked_train = zip(*sorted(zip(dists, *rest_train.values()), key=lambda x:entropy(x[0]), reverse=True))\n",
    "\n",
    "    # if we're at the very end we may need to make N smaller\n",
    "    if len(small_train['labels']) + N >= len(train['labels']):\n",
    "            N = len(train['labels']) - len(small_train['labels']) \n",
    "            # this should be same as what's left of rest\n",
    "            assert N == len(rest_train['labels']), \"Hm...\"\n",
    "    # skim off top N unknowns to train with and leave the rest as before\n",
    "    small_train = {k:list(v)+list(ranked_train[i+1])[:N] for i,(k,v) in enumerate(small_train.items())}\n",
    "    rest_train = {k:list(ranked_train[i+1])[N:] for i,(k,v) in enumerate(small_train.items())}\n",
    "    meta_batch_num += 1\n",
    "print(\"Done...\")\n",
    "print(lr_meta_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "N=500\n",
    "Ns = [N*i for i in range(1, len(unsup_f1s)+1)]\n",
    "fix, ax = plt.subplots(1,1)\n",
    "ax.plot(Ns, unsup_f1s, 'ro-', label=\"Semisupervised\")\n",
    "ax.plot(Ns, sup_f1s, 'b^-', label=\"Supervised\")\n",
    "ax.plot(Ns, lr_meta_results, 'g*--', label='BOW Logistic Regression')\n",
    "# ax.set_xticks(Ns)\n",
    "plt.legend(loc='best')\n",
    "plt.title('Semeval simulated active learning experiment')\n",
    "plt.xlabel('Number of training samples seen')\n",
    "plt.ylabel('Macro F1 score on validation set')\n",
    "plt.savefig('figures/active_learning_experiment.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_dists(dists, labels, int2label):\n",
    "    \"\"\"Plot the predicted distributions as small multiples\n",
    "\n",
    "    Args:\n",
    "        dists: list of pmfs, all the same size\n",
    "        int2label: dict of ints to labels for pmfs\n",
    "    \"\"\"\n",
    "    fig, axarr = plt.subplots(len(dists)/2, 2, sharex=True, figsize=(16, len(dists)))\n",
    "    xticks = range(len(int2label.keys()))\n",
    "    for i, dist in enumerate(dists):\n",
    "        pred = np.zeros_like(dist)\n",
    "        pred[np.argmax(dist)] = np.max(dist)\n",
    "        true = np.zeros_like(dist)\n",
    "        true[labels[i]] = dist[labels[i]]\n",
    "        axarr[i/2, i%2].stem(dist, 'bo-')\n",
    "        axarr[i/2, i%2].stem(pred, 'ro-')\n",
    "        axarr[i/2, i%2].stem(true, 'go-')\n",
    "\n",
    "        axarr[i/2, i%2].set_xlim([-1,19])\n",
    "        axarr[i/2, i%2].set_xticks(xticks)\n",
    "#         if i % 3 == 0 and i:\n",
    "        axarr[i/2, i%2].set_xticklabels(int2label.values(), rotation=45, horizontalalignment=\"right\", x=-2)\n",
    "        axarr[i/2, i%2].set_title(\"True(b): %s\" % int2label[labels[i]])\n",
    "        axarr[i/2, i%2].set_xlabel(\"Predicted(g): %s\" % int2label[np.argmax(dist)])\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print(len(dists))\n",
    "plot_dists(dists[:20], rest_train['labels'], int2label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sup_meta_results[0]['results'][0].keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Looking at best and worst predictions for best SEMISUP mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "best = sorted(meta_results[-1]['results'], key=lambda x:x['macro_f1'], reverse=True)[0]\n",
    "drnn.restore(best['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def abs_pred_true_diff(dist, label):\n",
    "    pred = np.argmax(dist)\n",
    "    true = dist[label]\n",
    "    return abs(pred - true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(valid.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get all of the classification distributions over the validation set\n",
    "query_batch = DH.classification_batch(len(valid['labels']),\n",
    "                                          valid['sdps'], \n",
    "                                          valid['targets'],\n",
    "                                          valid['labels'])\n",
    "_, dists = drnn.predict(query_batch[0], query_batch[1], query_batch[3], return_probs=True)\n",
    "# rank based on highest entropy\n",
    "# then turn back into rest_train.values() style\n",
    "ranked_valid = zip(*sorted(zip(dists, *valid.values()), key=lambda x:entropy(x[0]), reverse=True))\n",
    "\n",
    "# rank based on largest difference between true and predicted\n",
    "ranked_valid = zip(*sorted(zip(dists, *valid.values()), key=lambda x:abs_pred_true_diff(x[0], x[3]), reverse=True))\n",
    "\n",
    "# print(ranked_valid)\n",
    "ranked = {k:list(ranked_valid[i+1]) for i,(k,v) in enumerate(valid.items())}\n",
    "ranked['dists'] = ranked_valid[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# look at the worst (top 20)\n",
    "reload(eh)\n",
    "\n",
    "k = 20\n",
    "for i in range(k):\n",
    "    print(\"*\"*80)\n",
    "    print(ranked['raws'][i])\n",
    "    print(ranked['comments'][i])\n",
    "    print(DH.sequence_to_sentence(ranked['sdps'][i]))\n",
    "    eh.plot_dists([ranked['dists'][i]], [ranked['labels'][i]], int2label)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Looking at phrase reps by category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get the embeddings organized by label\n",
    "_, _, sdps, targets, labels = zip(*zip_valid)\n",
    "\n",
    "classes = []\n",
    "for i in label_set:\n",
    "    class_data = {'sdps':[sdp for j, sdp in enumerate(sdps) if i == labels[j]],\n",
    "                  'targets':[target for j, target in enumerate(targets) if i == labels[j]],\n",
    "                  'labels':[i for l in labels if i == l]}\n",
    "    if (class_data['labels']): # account for empty classes\n",
    "        batch = DH.classification_batch(len(class_data['labels']), \n",
    "                                        class_data['sdps'], \n",
    "                                        class_data['targets'], \n",
    "                                        class_data['labels'])\n",
    "        sdp_reps, target_reps = drnn.embed_phrases_and_targets(batch[0], batch[1], batch[3])\n",
    "        class_data['sdp_reps'] = sdp_reps\n",
    "        class_data['target_reps'] = target_reps\n",
    "    else:\n",
    "        class_data['sdp_reps'] = []\n",
    "        class_data['target_reps'] = []\n",
    "    classes.append(class_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# valid_phrases, valid_targets, valid_labels, valid_lens = DH.validation_batch()\n",
    "batch = DH.classification_batch(len(valid['labels']), \n",
    "                                    valid['sdps'], \n",
    "                                    valid['targets'], \n",
    "                                    valid['labels'])\n",
    "valid_batch = DH.validation_batch()\n",
    "# assert np.array_equal(batch[0], valid_batch[0])\n",
    "# print(batch[0], valid_batch[0])\n",
    "phrase_embeds, target_embeds = drnn.embed_phrases_and_targets(batch[0], batch[1], batch[3])\n",
    "phrase_labels, target_labels = DH.readable_data(valid=True)\n",
    "        \n",
    "phrase_embeds /= np.sqrt(np.sum(phrase_embeds**2, 1, keepdims=True))\n",
    "target_embeds /= np.sqrt(np.sum(target_embeds**2, 1, keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# just getting lots of colors\n",
    "import six\n",
    "\n",
    "from matplotlib import colors\n",
    "colors_ = list(six.iteritems(colors.cnames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### TARGET ONLY ###\n",
    "start = 100\n",
    "stride = 50\n",
    "end = start + stride\n",
    "# print(valid['labels'][start:end])\n",
    "# lowd = TSNE(perplexity=100, n_components=2, init='pca', n_iter=5000)\n",
    "lowd = PCA(n_components=2)\n",
    "\n",
    "phrase_2d = lowd.fit_transform(target_embeds[start:end])\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(20,16))\n",
    "for i, label in enumerate(target_labels[start:end]):\n",
    "    label = \"%s: %s\" % (int2label[valid['labels'][start+i]], label)\n",
    "    x, y = phrase_2d[i,:]\n",
    "    ax.scatter(x, y, color=colors_[valid['labels'][start+i]])\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### PHRASE ONLY ###\n",
    "start = 100\n",
    "stride = 50\n",
    "end = start + stride\n",
    "# print(valid['labels'][start:end])\n",
    "# lowd = TSNE(perplexity=100, n_components=2, init='pca', n_iter=5000)\n",
    "lowd = PCA(n_components=2)\n",
    "\n",
    "phrase_2d = lowd.fit_transform(phrase_embeds[start:end])\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(20,16))\n",
    "for i, label in enumerate(phrase_labels[start:end]):\n",
    "    label = \"%s: %s\" % (int2label[valid['labels'][start+i]], label)\n",
    "    x, y = phrase_2d[i,:]\n",
    "    ax.scatter(x, y, color=colors_[valid['labels'][start+i]])\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# plot all the embeddings , colored by class\n",
    "fig, ax = plt.subplots(1,1, figsize=(16,16))\n",
    "for class_data in classes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from spacy.en import English\n",
    "nlp = English()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s = u\"This is a simple sentence\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'DET', u'VERB', u'DET', u'ADJ', u'NOUN']\n"
     ]
    }
   ],
   "source": [
    "sent = nlp(s)\n",
    "print([t.pos_ for t in sent])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is a simple sentence\n"
     ]
    }
   ],
   "source": [
    "print(sent.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}