semeval_rnn_self_pretrain.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import collections\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA, FastICA\n",
    "\n",
    "import data_handler as dh\n",
    "import semeval_data_helper as sdh\n",
    "# plot settings\n",
    "% matplotlib inline\n",
    "# print(plt.rcParams.keys())\n",
    "plt.rcParams['figure.figsize'] = (16,9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reload(sdh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# print(DH.max_seq_len)\n",
    "# paths, targets = DH.readable_data(show_dep=True)\n",
    "# for p, t in zip(paths, targets) :\n",
    "#     t = t.split(\", \")\n",
    "#     print(\"%s (%s) %s\" % (t[0], p, t[1]))\n",
    "# print('<X>' in DH.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from relembed import RelEmbed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Data objects...\n",
      "Done creating Data objects\n",
      "7999 total examples :: 7199 training : 800 valid (90:10 split)\n",
      "Vocab size: 22683 Dep size: 50\n"
     ]
    }
   ],
   "source": [
    "# reload(dh)\n",
    "DH = dh.DataHandler('data/semeval_train_sdp_8000', valid_percent=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%time\n",
    "# load the pretrained word embeddings\n",
    "fname = 'data/GoogleNews-vectors-negative300.bin'\n",
    "word2vec = Word2Vec.load_word2vec_format(fname, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reload(sdh)\n",
    "train, valid, test, label2int, int2label = sdh.load_semeval_data()\n",
    "num_classes = len(int2label.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# %%bash\n",
    "# git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# convert the semeval data to indices under the wiki vocab:\n",
    "train['sdps'] = DH.sentences_to_sequences(train['sdps'])\n",
    "valid['sdps'] = DH.sentences_to_sequences(valid['sdps'])\n",
    "test['sdps'] = DH.sentences_to_sequences(test['sdps'])\n",
    "    \n",
    "train['targets'] = DH.sentences_to_sequences(train['targets'])\n",
    "valid['targets'] = DH.sentences_to_sequences(valid['targets'])\n",
    "test['targets'] = DH.sentences_to_sequences(test['targets'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "max_seq_len = max([len(path) for path in train['sdps']+valid['sdps']+test['sdps']])\n",
    "print(max_seq_len, DH.max_seq_len)\n",
    "DH.max_seq_len = max_seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# the embedding matrix is started of as random uniform [-1,1]\n",
    "# then we replace everything but the OOV tokens with the approprate google vector\n",
    "word_embeddings = np.random.uniform(low=-1., high=1., size=[DH.vocab_size, 300]).astype(np.float32)\n",
    "num_found = 0\n",
    "for i, token in enumerate(DH.vocab):\n",
    "    if token in word2vec:\n",
    "        word_embeddings[i] = word2vec[token]\n",
    "        num_found += 1\n",
    "print(\"%i / %i pretrained\" % (num_found, DH.vocab_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "config = {\n",
    "    'max_num_steps':DH.max_seq_len,\n",
    "    'word_embed_size':150,\n",
    "    'dep_embed_size':25,\n",
    "    'vocab_size':DH.vocab_size,\n",
    "    'dep_vocab_size':DH.dep_size,\n",
    "    'num_predict_classes':num_classes,\n",
    "    'pretrained_word_embeddings':None, #word_embeddings,\n",
    "    'max_grad_norm':3.,\n",
    "    'model_name':'drnn_wiki_semeval_w2v',\n",
    "    'checkpoint_prefix':'checkpoints/',\n",
    "    'summary_prefix':'tensor_summaries/'\n",
    "}\n",
    "try:\n",
    "    tf.reset_default_graph()\n",
    "except:\n",
    "    pass\n",
    "try:\n",
    "    tf.get_default_session().close()\n",
    "except:\n",
    "    pass\n",
    "drnn = RelEmbed(config)\n",
    "print(drnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_validation_test(num_nearby=20):\n",
    "    valid_phrases, valid_targets , _, valid_lens = DH.validation_batch()\n",
    "    random_index = int(random.uniform(0, len(valid_lens)))\n",
    "    query_phrase = valid_phrases[random_index]\n",
    "    query_len = valid_lens[random_index]\n",
    "    query_target = valid_targets[random_index]\n",
    "    padded_qp = np.zeros([DH.max_seq_len, 2]).astype(np.int32)\n",
    "    padded_qp[:len(query_phrase), 0] = [x[0] for x in query_phrase]\n",
    "    padded_qp[:len(query_phrase), 1] = [x[1] for x in query_phrase]    \n",
    "    dists, phrase_idx = drnn.validation_phrase_nearby(padded_qp, query_len, valid_phrases, valid_lens)\n",
    "    print(\"=\"*80)\n",
    "    print(\"Top %i closest phrases to <%s> '%s' <%s>\" \n",
    "          % (num_nearby, DH.vocab_at(query_target[0]), \n",
    "             DH.sequence_to_sentence(query_phrase, query_len), \n",
    "             DH.vocab_at(query_target[1])))\n",
    "    for i in range(num_nearby):\n",
    "        dist = dists[i]\n",
    "        phrase = valid_phrases[phrase_idx[i]]\n",
    "        len_ = valid_lens[phrase_idx[i]]\n",
    "        target = valid_targets[phrase_idx[i]]\n",
    "        print(\"%i: %0.3f : <%s> '%s' <%s>\" \n",
    "              % (i, dist, DH.vocab_at(target[0]),\n",
    "                 DH.sequence_to_sentence(phrase, len_),\n",
    "                 DH.vocab_at(target[1])))\n",
    "    print(\"=\"*80)\n",
    "#     drnn.save_validation_accuracy(frac_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def time_left(num_epochs, num_steps, fit_time, nearby_time, start_time, nearby_mod):\n",
    "    total = num_epochs*num_steps*fit_time + ((num_epochs*num_steps)/float(nearby_mod))*nearby_time\n",
    "    return total - (time() - start_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unsupervised Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "num_epochs = 1\n",
    "batch_size =50\n",
    "neg_per = 25\n",
    "num_nearby = 50\n",
    "nearby_mod = 50\n",
    "sample_power = .75\n",
    "DH.scale_vocab_dist(sample_power)\n",
    "\n",
    "# bookkeeping\n",
    "num_steps = DH.num_steps(batch_size)\n",
    "total_step = 1\n",
    "save_interval = 30 * 60 # half hour in seconds\n",
    "save_time = time()\n",
    "\n",
    "#timing stuff\n",
    "start = time()\n",
    "fit_time = 0\n",
    "nearby_time = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    offset = 0 #if epoch else 400\n",
    "    DH.shuffle_data()\n",
    "    for step , batch in enumerate(DH.batches(batch_size, offset=offset, neg_per=neg_per)):\n",
    "        if not step: step = offset\n",
    "        t0 = time()\n",
    "        loss = drnn.partial_unsup_fit(*batch)\n",
    "        fit_time = (fit_time * float(total_step) +  time() - t0) / (total_step + 1) # running average\n",
    "        if step % 10 == 0:\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            left = time_left(num_epochs, num_steps, fit_time, nearby_time, start, nearby_mod)\n",
    "            ml,sl = divmod(left, 60)\n",
    "            hl,ml = divmod(ml, 60)\n",
    "            pps = batch_size*(neg_per + 1) / fit_time \n",
    "            print(\"(%i:%i:%i) step %i/%i, epoch %i Training Loss = %1.5f :: %0.3f phrases/sec :: (%i:%i:%i) hours left\" \n",
    "                  % (h,m,s, step, num_steps, epoch, loss, pps, hl, ml, sl))\n",
    "        if (total_step-1) % nearby_mod == 0: # do one right away so we get a good timing estimate\n",
    "            t0 = time()\n",
    "            run_validation_test(num_nearby) # check out the nearby phrases in the validation set\n",
    "            valid_loss = drnn.validation_loss(*DH.validation_batch())\n",
    "            print(\"Validation loss: %0.4f\" % valid_loss)\n",
    "            nearby_time = (nearby_time * float(total_step) + time() - t0) / (total_step + 1) # running average\n",
    "\n",
    "        if (time() - save_time) > save_interval:\n",
    "            print(\"Saving model...\")\n",
    "            drnn.checkpoint()\n",
    "            save_time = time()\n",
    "        total_step +=1\n",
    "drnn.checkpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drnn.checkpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # test the embeddings\n",
    "\n",
    "# ### VALID ###\n",
    "# # valid_phrases, valid_targets, _, valid_lens = DH.validation_batch()\n",
    "# # phrase_embeds, target_embeds = drnn.embed_phrases_and_targets(valid_phrases, valid_targets, valid_lens)\n",
    "# # phrase_labels, target_labels = DH.readable_data(valid=True)\n",
    "\n",
    "# ### TRAIN ###\n",
    "# train_phrases, train_targets, _, train_lens = DH.batches(500, neg_per=0, offset=0).next()\n",
    "# phrase_embeds, target_embeds = drnn.embed_phrases_and_targets(train_phrases, train_targets, train_lens)\n",
    "# phrase_labels, target_labels = DH.readable_data(show_dep=False, valid=False)\n",
    "        \n",
    "# phrase_embeds /= np.sqrt(np.sum(phrase_embeds**2, 1, keepdims=True))\n",
    "# target_embeds /= np.sqrt(np.sum(target_embeds**2, 1, keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ### JOINT ###\n",
    "# start = 0\n",
    "# stride = 40\n",
    "# end = start + stride\n",
    "\n",
    "# lowd = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "# # lowd = PCA(n_components=2)\n",
    "\n",
    "# joint_embeds = np.vstack([phrase_embeds[start:end], target_embeds[start:end]])\n",
    "# joint_2d = lowd.fit_transform(joint_embeds)\n",
    "# phrase_2d, target_2d = joint_2d[:stride], joint_2d[stride:]\n",
    "\n",
    "# fig, ax = plt.subplots(figsize=(20,16))\n",
    "# for i, label in enumerate(phrase_labels[start:end]):\n",
    "#     label = \"%i: %s\" % (i, label)\n",
    "#     x, y = phrase_2d[i,:]\n",
    "#     ax.scatter(x, y, color='b')\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')\n",
    "# for i, label in enumerate(target_labels[start:end]):\n",
    "#     label = \"%i: %s\" % (i, label)\n",
    "#     x, y = target_2d[i,:]\n",
    "#     ax.scatter(x, y, color='r')\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ### PHRASE ONLY ###\n",
    "# start = 0\n",
    "# stride = 50\n",
    "# end = start + stride\n",
    "\n",
    "# lowd = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "# # lowd = PCA(n_components=2)\n",
    "\n",
    "# phrase_2d = lowd.fit_transform(phrase_embeds[start:end])\n",
    "\n",
    "# fig, ax = plt.subplots(figsize=(20,16))\n",
    "# for i, label in enumerate(phrase_labels[start:end]):\n",
    "#     label = \"%i: %s\" % (i, label)\n",
    "#     x, y = phrase_2d[i,:]\n",
    "#     ax.scatter(x, y, color='b')\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ### TARGET ONLY ###\n",
    "# start = 0\n",
    "# stride = 35\n",
    "# end = start + stride\n",
    "\n",
    "# lowd = TSNE(perplexity=20, n_components=2, init='pca', n_iter=5000)\n",
    "# # lowd = PCA(n_components=2)\n",
    "\n",
    "# target_2d = lowd.fit_transform(target_embeds[start:end])\n",
    "\n",
    "# fig, ax = plt.subplots(figsize=(20,16))\n",
    "# for i, label in enumerate(target_labels[start:end]):\n",
    "#     label = \"%i: %s\" % (i, label)\n",
    "#     x, y = target_2d[i,:]\n",
    "#     ax.scatter(x, y, color='r')\n",
    "#     ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### TW2V demo ###\n",
    "start = 200\n",
    "stride = 100\n",
    "end = start + stride\n",
    "\n",
    "lowd = TSNE(perplexity=20, n_components=2, init='pca', n_iter=5000)\n",
    "# lowd = PCA(n_components=2)\n",
    "\n",
    "target_2d = lowd.fit_transform(word_embeddings[start:end])\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(28,16))\n",
    "for i, label in enumerate(DH.vocab[start:end]):\n",
    "    label = \"%s\" % (label)\n",
    "    x, y = target_2d[i,:]\n",
    "    ax.scatter(x, y, color='b')\n",
    "    ax.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "                   ha='right', va='bottom')\n",
    "    \n",
    "plt.savefig('word2vec_demo.png', dpi=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test out semeval data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "zip_train = zip(train['raws'], train['sents'], train['sdps'], train['targets'], train['labels'])\n",
    "zip_valid = zip(valid['raws'], valid['sents'], valid['sdps'], valid['targets'], valid['labels'])\n",
    "zip_test = zip(test['raws'], test['sents'], test['sdps'], test['targets'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i, (raw, _, sdp, target, label) in enumerate(zip_train):\n",
    "    if i > 5:\n",
    "        break\n",
    "    print(raw)\n",
    "    print(\"%s :: %s\" % (DH.sequence_to_sentence(sdp, show_dep=True), DH.sequence_to_sentence(target)))\n",
    "    print(int2label[label])\n",
    "    print(\"=\"*80)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Supervised Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "batch_size = 50\n",
    "num_steps = len(train['labels']) // batch_size\n",
    "num_epochs = 25\n",
    "display_mod = 10\n",
    "valid_mod = 50\n",
    "print(\"Num steps %i\" %num_steps)\n",
    "\n",
    "start = time()\n",
    "for epoch in range(num_epochs):\n",
    "    random.shuffle(zip_train) # shuffling should only happen once per epoch\n",
    "    _, _, sdps, targets, labels = zip(*zip_train)\n",
    "    for step in range(num_steps): # num_steps\n",
    "        class_batch = DH.classification_batch(batch_size, sdps, targets, labels, \n",
    "                                              offset=step, shuffle=False)\n",
    "        xent = drnn.partial_class_fit(*class_batch)\n",
    "        if step % display_mod == 0:   \n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            print(\"(%i:%i:%i) s %i/%i, e %i avg class xent loss = %0.4f\" % (h,m,s, step, num_steps, epoch, xent))\n",
    "        if step % valid_mod == 0:\n",
    "            valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "            valid_xent = drnn.validation_class_loss(*valid_batch)\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            print(\"=\"*80)\n",
    "            print(\"(%i:%i:%i) s %i/%i, e %i validation avg class xent loss = %0.4f\" % (h,m,s, step, num_steps, epoch, valid_xent))\n",
    "            print(\"=\"*80)\n",
    "#             print(\"Saving model...\")\n",
    "#             drnn.checkpoint()\n",
    "    label_set = set(train['labels'])\n",
    "    preds = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3])\n",
    "    cm, stats = confusion_matrix(preds[0], valid['labels'], label_set)\n",
    "    print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "drnn.checkpoint()\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unsupervised 10 then Supervised 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "num_epochs = 5\n",
    "batch_size =50\n",
    "neg_per = 25\n",
    "num_nearby = 20\n",
    "nearby_mod = 50\n",
    "sample_power = .75\n",
    "DH.scale_vocab_dist(sample_power)\n",
    "\n",
    "# # bookkeeping\n",
    "num_steps = DH.num_steps(batch_size)\n",
    "total_step = 1\n",
    "save_interval = 30 * 60 # half hour in seconds\n",
    "save_time = time()\n",
    "\n",
    "#timing stuff\n",
    "start = time()\n",
    "fit_time = 0\n",
    "nearby_time = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    offset = 0 #if epoch else 400\n",
    "    DH.shuffle_data()\n",
    "    for step , batch in enumerate(DH.batches(batch_size, offset=offset, neg_per=neg_per)):\n",
    "        if not step: step = offset\n",
    "        t0 = time()\n",
    "        loss = drnn.partial_unsup_fit(*batch)\n",
    "        fit_time = (fit_time * float(total_step) +  time() - t0) / (total_step + 1) # running average\n",
    "        if step % 10 == 0:\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            left = time_left(num_epochs, num_steps, fit_time, nearby_time, start, nearby_mod)\n",
    "            ml,sl = divmod(left, 60)\n",
    "            hl,ml = divmod(ml, 60)\n",
    "            pps = batch_size*(neg_per + 1) / fit_time \n",
    "            print(\"(%i:%i:%i) step %i/%i, epoch %i Training Loss = %1.5f :: %0.3f phrases/sec :: (%i:%i:%i) hours left\" \n",
    "                  % (h,m,s, step, num_steps, epoch, loss, pps, hl, ml, sl))\n",
    "        if (total_step-1) % nearby_mod == 0: # do one right away so we get a good timing estimate\n",
    "            t0 = time()\n",
    "            run_validation_test(num_nearby) # check out the nearby phrases in the validation set\n",
    "            valid_loss = drnn.validation_loss(*DH.validation_batch())\n",
    "            print(\"Validation loss: %0.4f\" % valid_loss)\n",
    "            nearby_time = (nearby_time * float(total_step) + time() - t0) / (total_step + 1) # running average\n",
    "\n",
    "        if (time() - save_time) > save_interval:\n",
    "            print(\"Saving model...\")\n",
    "            drnn.checkpoint()\n",
    "            save_time = time()\n",
    "        total_step +=1\n",
    "    valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "    label_set = set(train['labels'])\n",
    "    preds, dists = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3], return_probs=True)\n",
    "    cm, stats = confusion_matrix(preds, valid['labels'], label_set)\n",
    "    print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "drnn.checkpoint()\n",
    "\n",
    "batch_size = 50\n",
    "num_steps = len(train['labels']) // batch_size\n",
    "num_epochs = 25\n",
    "display_mod = 10\n",
    "valid_mod = 50\n",
    "\n",
    "\n",
    "\n",
    "print(num_steps)\n",
    "\n",
    "start = time()\n",
    "for epoch in range(num_epochs):\n",
    "#     class_batch = DH.classification_batch(batch_size, train['sdps'], train['targets'], train['labels'], offset=0)\n",
    "#     random.shuffle(class_batch)\n",
    "\n",
    "    for step in range(10): # num_steps\n",
    "        inputs, targets, labels, lens = DH.classification_batch(batch_size, train['sdps'], train['targets'], train['labels'], offset=step)\n",
    "        class_batch = zip(inputs, targets, labels, lens)\n",
    "        random.shuffle(class_batch)\n",
    "        class_batch = zip(*class_batch)\n",
    "        xent = drnn.partial_class_fit(*class_batch)\n",
    "        if step % display_mod == 0:   \n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            print(\"(%i:%i:%i) s %i/%i, e %i avg class xent loss = %0.4f\" % (h,m,s, step, num_steps, epoch, xent))\n",
    "        if step % valid_mod == 0:\n",
    "            valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "            valid_xent = drnn.validation_class_loss(*valid_batch)\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            print(\"=\"*80)\n",
    "            print(\"(%i:%i:%i) s %i/%i, e %i validation avg class xent loss = %0.4f\" % (h,m,s, step, num_steps, epoch, valid_xent))\n",
    "            print(\"=\"*80)\n",
    "#             print(\"Saving model...\")\n",
    "#             drnn.checkpoint()\n",
    "    label_set = set(train['labels'])\n",
    "    preds, dists = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3], return_probs=True)\n",
    "    cm, stats = confusion_matrix(preds, valid['labels'], label_set)\n",
    "    print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "drnn.checkpoint()\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "for cycle in range(20):\n",
    "    # hyperparameters\n",
    "    num_epochs = 1\n",
    "    batch_size =50\n",
    "    neg_per = 25\n",
    "    num_nearby = 20\n",
    "    nearby_mod = 50\n",
    "    sample_power = .75\n",
    "    DH.scale_vocab_dist(sample_power)\n",
    "\n",
    "    # # bookkeeping\n",
    "    num_steps = DH.num_steps(batch_size)\n",
    "    total_step = 1\n",
    "    save_interval = 30 * 60 # half hour in seconds\n",
    "    save_time = time()\n",
    "\n",
    "    #timing stuff\n",
    "    start = time()\n",
    "    fit_time = 0\n",
    "    nearby_time = 0\n",
    "    for epoch in range(num_epochs):\n",
    "        offset = 0 #if epoch else 400\n",
    "        DH.shuffle_data()\n",
    "        for step , batch in enumerate(DH.batches(batch_size, offset=offset, neg_per=neg_per)):\n",
    "            if not step: step = offset\n",
    "            t0 = time()\n",
    "            loss = drnn.partial_unsup_fit(*batch)\n",
    "            fit_time = (fit_time * float(total_step) +  time() - t0) / (total_step + 1) # running average\n",
    "            if step % 10 == 0:\n",
    "                m,s = divmod(time()-start, 60)\n",
    "                h,m = divmod(m, 60)\n",
    "                left = time_left(num_epochs, num_steps, fit_time, nearby_time, start, nearby_mod)\n",
    "                ml,sl = divmod(left, 60)\n",
    "                hl,ml = divmod(ml, 60)\n",
    "                pps = batch_size*(neg_per + 1) / fit_time \n",
    "                print(\"(%i:%i:%i) step %i/%i, epoch %i Training Loss = %1.5f :: %0.3f phrases/sec :: (%i:%i:%i) hours left\" \n",
    "                      % (h,m,s, step, num_steps, epoch, loss, pps, hl, ml, sl))\n",
    "            if (total_step-1) % nearby_mod == 0: # do one right away so we get a good timing estimate\n",
    "                t0 = time()\n",
    "                run_validation_test(num_nearby) # check out the nearby phrases in the validation set\n",
    "                valid_loss = drnn.validation_loss(*DH.validation_batch())\n",
    "                print(\"Validation loss: %0.4f\" % valid_loss)\n",
    "                nearby_time = (nearby_time * float(total_step) + time() - t0) / (total_step + 1) # running average\n",
    "\n",
    "            if (time() - save_time) > save_interval:\n",
    "                print(\"Saving model...\")\n",
    "                drnn.checkpoint()\n",
    "                save_time = time()\n",
    "            total_step +=1\n",
    "        valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "        label_set = set(train['labels'])\n",
    "        preds, dists = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3], return_probs=True)\n",
    "        cm, stats = confusion_matrix(preds, valid['labels'], label_set)\n",
    "        print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "    drnn.checkpoint()\n",
    "\n",
    "    batch_size = 50\n",
    "    num_steps = len(train['labels']) // batch_size\n",
    "    num_epochs = 5\n",
    "    display_mod = 10\n",
    "    valid_mod = 50\n",
    "\n",
    "\n",
    "\n",
    "    print(num_steps)\n",
    "\n",
    "    start = time()\n",
    "    for class_epoch in range(3):\n",
    "    #     class_batch = DH.classification_batch(batch_size, train['sdps'], train['targets'], train['labels'], offset=0)\n",
    "    #     random.shuffle(class_batch)\n",
    "\n",
    "        for class_step in range(num_steps):\n",
    "            inputs, targets, labels, lens = DH.classification_batch(batch_size, train['sdps'], train['targets'], train['labels'], offset=class_step)\n",
    "            class_batch = zip(inputs, targets, labels, lens)\n",
    "            random.shuffle(class_batch)\n",
    "            class_batch = zip(*class_batch)\n",
    "            xent = drnn.partial_class_fit(*class_batch)\n",
    "            if step % display_mod == 0:   \n",
    "                m,s = divmod(time()-start, 60)\n",
    "                h,m = divmod(m, 60)\n",
    "                print(\"(%i:%i:%i) s %i/%i, e %i avg class xent loss = %0.4f\" % (h,m,s, class_step, num_steps, class_epoch, xent))\n",
    "            if step % valid_mod == 0:\n",
    "                valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "                valid_xent = drnn.validation_class_loss(*valid_batch)\n",
    "                m,s = divmod(time()-start, 60)\n",
    "                h,m = divmod(m, 60)\n",
    "                print(\"=\"*80)\n",
    "                print(\"(%i:%i:%i) s %i/%i, e %i validation avg class xent loss = %0.4f\" % (h,m,s, class_step, num_steps, class_epoch, valid_xent))\n",
    "                print(\"=\"*80)\n",
    "    #             print(\"Saving model...\")\n",
    "    #             drnn.checkpoint()\n",
    "        label_set = set(train['labels'])\n",
    "        preds, dists = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3], return_probs=True)\n",
    "        cm, stats = confusion_matrix(preds, valid['labels'], label_set)\n",
    "        print(\"Macro P: %2.4f, R: %3.4f, F1: %0.4f\" % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "    drnn.checkpoint()\n",
    "    print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# check out predictions\n",
    "# valid_batch = DH.classification_batch(len(train['labels']), train['sdps'], train['targets'], train['labels'])\n",
    "\n",
    "valid_batch = DH.classification_batch(len(valid['labels']), valid['sdps'], valid['targets'], valid['labels'])\n",
    "preds, dists = drnn.predict(valid_batch[0], valid_batch[1], valid_batch[3], return_probs=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for i, p in enumerate(preds):\n",
    "#     print(\"%i, pred: %s, true: %s\" %(i, int2label[p], int2label[valid['labels'][i]]))\n",
    "#     target = DH.sequence_to_sentence(valid['targets'][i]).split(' ')\n",
    "#     sdp = DH.sequence_to_sentence(valid['sdps'][i], show_dep=True)\n",
    "#     print('<%s> \"%s\" <%s>' % (target[0], sdp, target[1]))\n",
    "#     print(valid['raws'][i])\n",
    "#     print(valid['comments'][i])\n",
    "#     print(\"=\"*80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def confusion_matrix(preds, labels, label_set):\n",
    "    size = len(label_set)\n",
    "    matrix = np.zeros([size, size]) # rows are predictions, columns are truths\n",
    "    # fill in matrix\n",
    "    for p, l in zip(preds, labels):\n",
    "        matrix[p,l] += 1\n",
    "    # compute class specific scores\n",
    "    class_precision = np.zeros(size)\n",
    "    class_recall = np.zeros(size)\n",
    "    for label in range(size):\n",
    "        tp = matrix[label, label]\n",
    "        fp = np.sum(matrix[label, :]) - tp\n",
    "        fn = np.sum(matrix[:, label]) - tp\n",
    "        class_precision[label] = tp/float(tp + fp) if tp or fp else 0\n",
    "        class_recall[label] = tp/float(tp + fn) if tp or fn else 0\n",
    "    micro_f1 = np.array([2*(p*r)/(p+r) if p or r else 0 for (p, r) in zip(class_precision, class_recall)])\n",
    "    avg_precision = np.mean(class_precision)\n",
    "    avg_recall = np.mean(class_recall)\n",
    "    macro_f1 = (2*avg_precision*avg_recall) / (avg_precision + avg_recall)\n",
    "    stats = {'micro_precision':class_precision,\n",
    "             'micro_recall':class_recall, \n",
    "             'micro_f1':micro_f1,\n",
    "             'macro_precision':avg_precision, \n",
    "             'macro_recall':avg_recall,\n",
    "             'macro_f1':macro_f1}\n",
    "    return matrix, stats\n",
    "label_set = set(train['labels'])\n",
    "cm, stats = confusion_matrix(preds, valid['labels'], label_set)\n",
    "print(\"Macro F1: %0.4f\" % stats['macro_f1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, label_names, save_name=None, title='Normed Confusion matrix', cmap=plt.cm.Blues, stats=None):\n",
    "    fig, ax = plt.subplots(figsize=(20,20))\n",
    "    \n",
    "    # calc normalized cm\n",
    "    x, y = np.meshgrid(range(cm.shape[0]), range(cm.shape[1]))\n",
    "    cm_normalized = cm.astype('float') / cm.sum(axis=1, keepdims=True)\n",
    "    cm_normalized[np.isnan(cm_normalized)] = 0.0\n",
    "    \n",
    "    # print nonzero raw counts\n",
    "    for x_val, y_val in zip(x.flatten(), y.flatten()):\n",
    "        norm = cm_normalized[x_val, y_val]\n",
    "        c = \"%i\" % (cm.astype('int')[x_val, y_val])\n",
    "        if norm > 0.0:\n",
    "            color = 'white' if norm > .5 else 'black'\n",
    "            ax.text(y_val, x_val, c, va='center', ha='center', color=color)\n",
    "    \n",
    "    # actual plot\n",
    "    im = ax.imshow(cm_normalized, interpolation='nearest', origin='upper', cmap=cmap)\n",
    "#     divider = plt.make_axes_locatable(ax)\n",
    "#     cax = divider.append_axes(\"right\", size=\"5%\", pad=0.05)\n",
    "    plt.colorbar(im, fraction=0.046, pad=0.04)\n",
    "    \n",
    "    # set ticks and offset grid\n",
    "    tick_marks = np.arange(len(label_names))\n",
    "    tick_marks_offset = np.arange(len(label_names)) - .5\n",
    "    ax.set_xticks(tick_marks, minor=False)\n",
    "    ax.set_yticks(tick_marks, minor=False)\n",
    "    ax.set_xticks(tick_marks_offset, minor=True)\n",
    "    ax.set_yticks(tick_marks_offset, minor=True)\n",
    "    ax.grid(which='minor')\n",
    "    if stats:\n",
    "        # include micro precisio, recall, and f1\n",
    "        aug_y_labels = []\n",
    "        for i in range(len(label_names)):\n",
    "            aug = (\"%s\\nP:%0.2f, R:%0.2f, F1:%0.2f\" \n",
    "                   % (label_names[i],\n",
    "                      stats['micro_precision'][i],\n",
    "                      stats['micro_recall'][i],\n",
    "                      stats['micro_f1'][i],))\n",
    "            aug_y_labels.append(aug)\n",
    "    else:\n",
    "        aug_x_labels = label_names\n",
    "    ax.set_xticklabels(label_names, rotation=75, horizontalalignment='left', x=1)\n",
    "    ax.xaxis.tick_top()\n",
    "    ax.set_yticklabels(aug_y_labels)\n",
    "    \n",
    "    # other stuff\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('Predicted Labels', fontsize=16)\n",
    "    if stats:\n",
    "        # include macro \n",
    "        aug_x_label = (\"True Labels\\n Macro P:%0.2f, R:%0.2f, F1:%0.2f\" \n",
    "                       % (stats['macro_precision'], stats['macro_recall'], stats['macro_f1']))\n",
    "    else:\n",
    "        aug_x_label = \"True Label\"\n",
    "    plt.xlabel(aug_x_label, fontsize=16)\n",
    "    plt.title(title, y=1.12, fontsize=20)\n",
    "    if save_name:\n",
    "        plt.savefig(save_name+'.pdf')\n",
    "        \n",
    "save_name = raw_input(\"Enter save name: \")\n",
    "plot_confusion_matrix(cm, int2label.values(), save_name=save_name, stats=stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}